""" core/fetcher.py --------------- Data fetch logic for all 3 lottery sources. Each function returns: {"source", "added", "skipped", "status", "message"} All HTTP errors are caught and returned as status="error" — never raised. """ import csv import io import logging import threading import requests from datetime import datetime from db.models import get_game_by_name, insert_draw, insert_fetch_log logger = logging.getLogger(__name__) NY_API_LIMIT = 5000 # records per page POWERBALL_NY_URL = "https://data.ny.gov/resource/d6yy-54nr.json" MEGAMILLIONS_NY_URL = "https://data.ny.gov/resource/5xaw-6ayf.json" MEGAMILLIONS_TX_URL = ( "https://www.texaslottery.com/export/sites/lottery/Games/" "Mega_Millions/Winning_Numbers/download.html" ) VA_CASH5_URL = "https://www.valottery.com/api/v1/downloadall?gameId=1030" VA_MILLIONAIREFORLIFE_URL = "https://www.valottery.com/api/v1/downloadall?gameId=1075" VA_BANKAMILLION_URL = "https://www.valottery.com/api/v1/downloadall?gameId=1070" _fetch_lock = threading.Lock() def _parse_ny_date(raw): """'2024-01-01T00:00:00.000' → '2024-01-01'.""" return raw.split("T")[0].strip() def _error_result(source, added, skipped, msg): insert_fetch_log(source, added, skipped, "error", msg) return {"source": source, "added": added, "skipped": skipped, "status": "error", "message": msg} # ── Powerball NY ────────────────────────────────────────────────────────────── def fetch_powerball_ny(): """ Fetch Powerball draws from NY Open Data API. winning_numbers field: "01 13 36 61 69 07" (5 white + 1 Powerball) """ source = "powerball_ny" game = get_game_by_name("Powerball") if not game: return _error_result(source, 0, 0, "Powerball game not found in DB") game_id = game["id"] added = skipped = 0 offset = 0 try: while True: params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"} resp = requests.get(POWERBALL_NY_URL, params=params, timeout=30) resp.raise_for_status() records = resp.json() if not records: break for rec in records: raw_date = rec.get("draw_date", "") raw_nums = rec.get("winning_numbers", "").strip() multiplier = rec.get("multiplier") or None if not raw_date or not raw_nums: continue try: draw_date = _parse_ny_date(raw_date) parts = raw_nums.split() if len(parts) < 6: logger.warning("[FETCH] PB NY: unexpected numbers %r on %s", raw_nums, raw_date) continue main = [int(p) for p in parts[:5]] bonus = int(parts[5]) except (ValueError, IndexError) as e: logger.warning("[FETCH] PB NY parse error %s: %s", rec, e) continue result = insert_draw(game_id, draw_date, main, bonus, multiplier, source) if result == "inserted": added += 1 else: skipped += 1 if len(records) < NY_API_LIMIT: break offset += NY_API_LIMIT logger.info("[FETCH] Powerball NY done — added=%d skipped=%d", added, skipped) insert_fetch_log(source, added, skipped, "success") return {"source": source, "added": added, "skipped": skipped, "status": "success", "message": None} except requests.RequestException as e: logger.error("[FETCH] Powerball NY error: %s", e) return _error_result(source, added, skipped, f"Network error: {e}") # ── Mega Millions NY ────────────────────────────────────────────────────────── def fetch_megamillions_ny(): """ Fetch Mega Millions draws from NY Open Data API. winning_numbers: 5 space-separated main balls mega_ball: Mega Ball (bonus) """ source = "megamillions_ny" game = get_game_by_name("Mega Millions") if not game: return _error_result(source, 0, 0, "Mega Millions game not found in DB") game_id = game["id"] added = skipped = 0 offset = 0 try: while True: params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"} resp = requests.get(MEGAMILLIONS_NY_URL, params=params, timeout=30) resp.raise_for_status() records = resp.json() if not records: break for rec in records: raw_date = rec.get("draw_date", "") raw_nums = rec.get("winning_numbers", "").strip() mega_ball = rec.get("mega_ball") or None multiplier = rec.get("multiplier") or None if not raw_date or not raw_nums: continue try: draw_date = _parse_ny_date(raw_date) parts = raw_nums.split() main = [int(p) for p in parts[:5]] bonus = int(mega_ball) if mega_ball else None except (ValueError, IndexError) as e: logger.warning("[FETCH] MM NY parse error %s: %s", rec, e) continue result = insert_draw(game_id, draw_date, main, bonus, multiplier, source) if result == "inserted": added += 1 else: skipped += 1 if len(records) < NY_API_LIMIT: break offset += NY_API_LIMIT logger.info("[FETCH] Mega Millions NY done — added=%d skipped=%d", added, skipped) insert_fetch_log(source, added, skipped, "success") return {"source": source, "added": added, "skipped": skipped, "status": "success", "message": None} except requests.RequestException as e: logger.error("[FETCH] Mega Millions NY error: %s", e) return _error_result(source, added, skipped, f"Network error: {e}") # ── Mega Millions TX ────────────────────────────────────────────────────────── def fetch_megamillions_tx(): """ Fetch Mega Millions draws from Texas Lottery CSV. Column layout: Game Name, Month, Day, Year, Num1-5, Mega Ball, Megaplier Header row is auto-detected and skipped. """ source = "megamillions_tx" game = get_game_by_name("Mega Millions") if not game: return _error_result(source, 0, 0, "Mega Millions game not found in DB") game_id = game["id"] added = skipped = 0 try: resp = requests.get(MEGAMILLIONS_TX_URL, timeout=30) resp.raise_for_status() reader = csv.reader(io.StringIO(resp.text)) for row in reader: if not row or len(row) < 10: continue # Skip header row — month column would be "Month" (non-digit) if not row[1].strip().isdigit(): continue # Skip non-Mega Millions rows if "mega" not in row[0].lower(): continue try: month = row[1].strip().zfill(2) day = row[2].strip().zfill(2) year = row[3].strip() draw_date = f"{year}-{month}-{day}" main = [int(row[i].strip()) for i in range(4, 9)] bonus = int(row[9].strip()) multiplier = row[10].strip() if len(row) > 10 and row[10].strip() else None except (ValueError, IndexError) as e: logger.warning("[FETCH] TX CSV parse error row=%r: %s", row, e) continue result = insert_draw(game_id, draw_date, main, bonus, multiplier, source) if result == "inserted": added += 1 else: skipped += 1 logger.info("[FETCH] Mega Millions TX done — added=%d skipped=%d", added, skipped) insert_fetch_log(source, added, skipped, "success") return {"source": source, "added": added, "skipped": skipped, "status": "success", "message": None} except requests.RequestException as e: logger.error("[FETCH] Mega Millions TX error: %s", e) return _error_result(source, added, skipped, f"Network error: {e}") # ── Virginia Lottery (shared parser) ───────────────────────────────────────── def _parse_va_date(raw: str) -> str | None: """'5/22/2026' or '05/22/2026' → '2026-05-22'.""" parts = raw.strip().split("/") if len(parts) != 3: return None try: return f"{parts[2]}-{parts[0].zfill(2)}-{parts[1].zfill(2)}" except (IndexError, ValueError): return None def _fetch_va_lottery(game_name: str, source: str, url: str) -> dict: """ Fetch VA Lottery draw data from the valottery.com download API. Format per line: 'M/D/YYYY; N1,N2,...[; Label: bonus]' First line may be a 'Results for ...' header — skipped automatically. """ game = get_game_by_name(game_name) if not game: return _error_result(source, 0, 0, f"{game_name} game not found in DB") game_id = game["id"] added = skipped = 0 try: resp = requests.get(url, timeout=30) resp.raise_for_status() for line in resp.text.splitlines(): line = line.strip() if not line or line.lower().startswith("results for"): continue parts = [p.strip() for p in line.split(";")] if len(parts) < 2: continue draw_date = _parse_va_date(parts[0]) if draw_date is None: continue try: p1 = parts[1].strip() p1_lower = p1.lower() if p1_lower.startswith("day:"): # Old twice-daily format: "Day: N1,...; Night: N1,..." # Use Night draw only for consistency with modern single-draw records if len(parts) >= 3 and "night:" in parts[2].lower(): night_str = parts[2][parts[2].lower().find("night:") + 6:] numbers = [int(n.strip()) for n in night_str.split(",") if n.strip()] else: continue elif p1_lower.startswith("night:"): # Night-only historical record numbers = [int(n.strip()) for n in p1[p1_lower.find("night:") + 6:].split(",") if n.strip()] else: numbers = [int(n.strip()) for n in p1.split(",") if n.strip()] except ValueError as e: logger.warning("[FETCH] VA %s parse error %r: %s", source, line, e) continue bonus = None if len(parts) >= 3: p_last = parts[-1].strip() if "night:" not in p_last.lower() and "day:" not in p_last.lower(): colon = p_last.rfind(":") if colon >= 0: try: bonus = int(p_last[colon + 1:].strip()) except ValueError: pass result = insert_draw(game_id, draw_date, numbers, bonus=bonus, source=source) if result == "inserted": added += 1 else: skipped += 1 logger.info("[FETCH] %s done — added=%d skipped=%d", source, added, skipped) insert_fetch_log(source, added, skipped, "success") return {"source": source, "added": added, "skipped": skipped, "status": "success", "message": None} except requests.RequestException as e: logger.error("[FETCH] %s error: %s", source, e) return _error_result(source, added, skipped, f"Network error: {e}") def fetch_cash5_va(): """Fetch Cash 5 draws from VA Lottery download API.""" return _fetch_va_lottery("Cash 5", "cash5_va", VA_CASH5_URL) def fetch_millionaireforlife_va(): """Fetch Millionaire for Life draws from VA Lottery download API.""" return _fetch_va_lottery("Millionaire for Life", "millionaireforlife_va", VA_MILLIONAIREFORLIFE_URL) def fetch_bankamillion_va(): """Fetch Bank a Million draws from VA Lottery download API.""" return _fetch_va_lottery("Bank a Million", "bankamillion_va", VA_BANKAMILLION_URL) # ── fetch_all ───────────────────────────────────────────────────────────────── def fetch_all(): """ Run all 3 fetch functions and return a list of result dicts. One source erroring does not stop the others. Called by the auto-fetch thread and the manual Fetch Now button. """ logger.info("[FETCH] fetch_all() starting") results = [] for fn in ( fetch_powerball_ny, fetch_megamillions_ny, fetch_megamillions_tx, fetch_cash5_va, fetch_millionaireforlife_va, fetch_bankamillion_va, ): try: results.append(fn()) except Exception as e: logger.error("[ERROR] Unexpected error in %s: %s", fn.__name__, e, exc_info=True) results.append({"source": fn.__name__, "added": 0, "skipped": 0, "status": "error", "message": str(e)}) logger.info("[FETCH] fetch_all() complete — %d sources processed", len(results)) return results