""" core/fetcher.py --------------- Data fetch logic for all 3 lottery sources. Each function returns: {"source", "added", "skipped", "status", "message"} All HTTP errors are caught and returned as status="error" — never raised. """ import csv import io import logging import threading import requests from db.models import get_game_by_name, insert_draw, insert_fetch_log logger = logging.getLogger(__name__) NY_API_LIMIT = 5000 # records per page POWERBALL_NY_URL = "https://data.ny.gov/resource/d6yy-54nr.json" MEGAMILLIONS_NY_URL = "https://data.ny.gov/resource/5xaw-6ayf.json" MEGAMILLIONS_TX_URL = ( "https://www.texaslottery.com/export/sites/lottery/Games/" "Mega_Millions/Winning_Numbers/download.html" ) _fetch_lock = threading.Lock() def _parse_ny_date(raw): """'2024-01-01T00:00:00.000' → '2024-01-01'.""" return raw.split("T")[0].strip() def _error_result(source, added, skipped, msg): insert_fetch_log(source, added, skipped, "error", msg) return {"source": source, "added": added, "skipped": skipped, "status": "error", "message": msg} # ── Powerball NY ────────────────────────────────────────────────────────────── def fetch_powerball_ny(): """ Fetch Powerball draws from NY Open Data API. winning_numbers field: "01 13 36 61 69 07" (5 white + 1 Powerball) """ source = "powerball_ny" game = get_game_by_name("Powerball") if not game: return _error_result(source, 0, 0, "Powerball game not found in DB") game_id = game["id"] added = skipped = 0 offset = 0 try: while True: params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"} resp = requests.get(POWERBALL_NY_URL, params=params, timeout=30) resp.raise_for_status() records = resp.json() if not records: break for rec in records: raw_date = rec.get("draw_date", "") raw_nums = rec.get("winning_numbers", "").strip() multiplier = rec.get("multiplier") or None if not raw_date or not raw_nums: continue try: draw_date = _parse_ny_date(raw_date) parts = raw_nums.split() if len(parts) < 6: logger.warning("[FETCH] PB NY: unexpected numbers %r on %s", raw_nums, raw_date) continue main = [int(p) for p in parts[:5]] bonus = int(parts[5]) except (ValueError, IndexError) as e: logger.warning("[FETCH] PB NY parse error %s: %s", rec, e) continue result = insert_draw(game_id, draw_date, main, bonus, multiplier, source) if result == "inserted": added += 1 else: skipped += 1 if len(records) < NY_API_LIMIT: break offset += NY_API_LIMIT logger.info("[FETCH] Powerball NY done — added=%d skipped=%d", added, skipped) insert_fetch_log(source, added, skipped, "success") return {"source": source, "added": added, "skipped": skipped, "status": "success", "message": None} except requests.RequestException as e: logger.error("[FETCH] Powerball NY error: %s", e) return _error_result(source, added, skipped, f"Network error: {e}") # ── Mega Millions NY ────────────────────────────────────────────────────────── def fetch_megamillions_ny(): """ Fetch Mega Millions draws from NY Open Data API. winning_numbers: 5 space-separated main balls mega_ball: Mega Ball (bonus) """ source = "megamillions_ny" game = get_game_by_name("Mega Millions") if not game: return _error_result(source, 0, 0, "Mega Millions game not found in DB") game_id = game["id"] added = skipped = 0 offset = 0 try: while True: params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"} resp = requests.get(MEGAMILLIONS_NY_URL, params=params, timeout=30) resp.raise_for_status() records = resp.json() if not records: break for rec in records: raw_date = rec.get("draw_date", "") raw_nums = rec.get("winning_numbers", "").strip() mega_ball = rec.get("mega_ball") or None multiplier = rec.get("multiplier") or None if not raw_date or not raw_nums: continue try: draw_date = _parse_ny_date(raw_date) parts = raw_nums.split() main = [int(p) for p in parts[:5]] bonus = int(mega_ball) if mega_ball else None except (ValueError, IndexError) as e: logger.warning("[FETCH] MM NY parse error %s: %s", rec, e) continue result = insert_draw(game_id, draw_date, main, bonus, multiplier, source) if result == "inserted": added += 1 else: skipped += 1 if len(records) < NY_API_LIMIT: break offset += NY_API_LIMIT logger.info("[FETCH] Mega Millions NY done — added=%d skipped=%d", added, skipped) insert_fetch_log(source, added, skipped, "success") return {"source": source, "added": added, "skipped": skipped, "status": "success", "message": None} except requests.RequestException as e: logger.error("[FETCH] Mega Millions NY error: %s", e) return _error_result(source, added, skipped, f"Network error: {e}") # ── Mega Millions TX ────────────────────────────────────────────────────────── def fetch_megamillions_tx(): """ Fetch Mega Millions draws from Texas Lottery CSV. Column layout: Game Name, Month, Day, Year, Num1-5, Mega Ball, Megaplier Header row is auto-detected and skipped. """ source = "megamillions_tx" game = get_game_by_name("Mega Millions") if not game: return _error_result(source, 0, 0, "Mega Millions game not found in DB") game_id = game["id"] added = skipped = 0 try: resp = requests.get(MEGAMILLIONS_TX_URL, timeout=30) resp.raise_for_status() reader = csv.reader(io.StringIO(resp.text)) for row in reader: if not row or len(row) < 10: continue # Skip header row — month column would be "Month" (non-digit) if not row[1].strip().isdigit(): continue # Skip non-Mega Millions rows if "mega" not in row[0].lower(): continue try: month = row[1].strip().zfill(2) day = row[2].strip().zfill(2) year = row[3].strip() draw_date = f"{year}-{month}-{day}" main = [int(row[i].strip()) for i in range(4, 9)] bonus = int(row[9].strip()) multiplier = row[10].strip() if len(row) > 10 and row[10].strip() else None except (ValueError, IndexError) as e: logger.warning("[FETCH] TX CSV parse error row=%r: %s", row, e) continue result = insert_draw(game_id, draw_date, main, bonus, multiplier, source) if result == "inserted": added += 1 else: skipped += 1 logger.info("[FETCH] Mega Millions TX done — added=%d skipped=%d", added, skipped) insert_fetch_log(source, added, skipped, "success") return {"source": source, "added": added, "skipped": skipped, "status": "success", "message": None} except requests.RequestException as e: logger.error("[FETCH] Mega Millions TX error: %s", e) return _error_result(source, added, skipped, f"Network error: {e}") # ── fetch_all ───────────────────────────────────────────────────────────────── def fetch_all(): """ Run all 3 fetch functions and return a list of result dicts. One source erroring does not stop the others. Called by the auto-fetch thread and the manual Fetch Now button. """ logger.info("[FETCH] fetch_all() starting") results = [] for fn in (fetch_powerball_ny, fetch_megamillions_ny, fetch_megamillions_tx): try: results.append(fn()) except Exception as e: logger.error("[ERROR] Unexpected error in %s: %s", fn.__name__, e, exc_info=True) results.append({"source": fn.__name__, "added": 0, "skipped": 0, "status": "error", "message": str(e)}) logger.info("[FETCH] fetch_all() complete — %d sources processed", len(results)) return results