371 lines
14 KiB
Python
371 lines
14 KiB
Python
"""
|
|
core/fetcher.py
|
|
---------------
|
|
Data fetch logic for all 3 lottery sources.
|
|
Each function returns: {"source", "added", "skipped", "status", "message"}
|
|
All HTTP errors are caught and returned as status="error" — never raised.
|
|
"""
|
|
|
|
import csv
|
|
import io
|
|
import logging
|
|
import threading
|
|
|
|
import requests
|
|
from datetime import datetime
|
|
|
|
from db.models import get_game_by_name, insert_draw, insert_fetch_log
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
NY_API_LIMIT = 5000 # records per page
|
|
|
|
POWERBALL_NY_URL = "https://data.ny.gov/resource/d6yy-54nr.json"
|
|
MEGAMILLIONS_NY_URL = "https://data.ny.gov/resource/5xaw-6ayf.json"
|
|
MEGAMILLIONS_TX_URL = (
|
|
"https://www.texaslottery.com/export/sites/lottery/Games/"
|
|
"Mega_Millions/Winning_Numbers/download.html"
|
|
)
|
|
|
|
VA_CASH5_URL = "https://www.valottery.com/api/v1/downloadall?gameId=1030"
|
|
VA_MILLIONAIREFORLIFE_URL = "https://www.valottery.com/api/v1/downloadall?gameId=1075"
|
|
VA_BANKAMILLION_URL = "https://www.valottery.com/api/v1/downloadall?gameId=1070"
|
|
|
|
_fetch_lock = threading.Lock()
|
|
|
|
|
|
def _parse_ny_date(raw):
|
|
"""'2024-01-01T00:00:00.000' → '2024-01-01'."""
|
|
return raw.split("T")[0].strip()
|
|
|
|
|
|
def _error_result(source, added, skipped, msg):
|
|
insert_fetch_log(source, added, skipped, "error", msg)
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "error", "message": msg}
|
|
|
|
|
|
# ── Powerball NY ──────────────────────────────────────────────────────────────
|
|
|
|
def fetch_powerball_ny():
|
|
"""
|
|
Fetch Powerball draws from NY Open Data API.
|
|
winning_numbers field: "01 13 36 61 69 07" (5 white + 1 Powerball)
|
|
"""
|
|
source = "powerball_ny"
|
|
game = get_game_by_name("Powerball")
|
|
if not game:
|
|
return _error_result(source, 0, 0, "Powerball game not found in DB")
|
|
|
|
game_id = game["id"]
|
|
added = skipped = 0
|
|
offset = 0
|
|
|
|
try:
|
|
while True:
|
|
params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"}
|
|
resp = requests.get(POWERBALL_NY_URL, params=params, timeout=30)
|
|
resp.raise_for_status()
|
|
records = resp.json()
|
|
|
|
if not records:
|
|
break
|
|
|
|
for rec in records:
|
|
raw_date = rec.get("draw_date", "")
|
|
raw_nums = rec.get("winning_numbers", "").strip()
|
|
multiplier = rec.get("multiplier") or None
|
|
|
|
if not raw_date or not raw_nums:
|
|
continue
|
|
try:
|
|
draw_date = _parse_ny_date(raw_date)
|
|
parts = raw_nums.split()
|
|
if len(parts) < 6:
|
|
logger.warning("[FETCH] PB NY: unexpected numbers %r on %s", raw_nums, raw_date)
|
|
continue
|
|
main = [int(p) for p in parts[:5]]
|
|
bonus = int(parts[5])
|
|
except (ValueError, IndexError) as e:
|
|
logger.warning("[FETCH] PB NY parse error %s: %s", rec, e)
|
|
continue
|
|
|
|
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
|
if result == "inserted":
|
|
added += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
if len(records) < NY_API_LIMIT:
|
|
break
|
|
offset += NY_API_LIMIT
|
|
|
|
logger.info("[FETCH] Powerball NY done — added=%d skipped=%d", added, skipped)
|
|
insert_fetch_log(source, added, skipped, "success")
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "success", "message": None}
|
|
|
|
except requests.RequestException as e:
|
|
logger.error("[FETCH] Powerball NY error: %s", e)
|
|
return _error_result(source, added, skipped, f"Network error: {e}")
|
|
|
|
|
|
# ── Mega Millions NY ──────────────────────────────────────────────────────────
|
|
|
|
def fetch_megamillions_ny():
|
|
"""
|
|
Fetch Mega Millions draws from NY Open Data API.
|
|
winning_numbers: 5 space-separated main balls
|
|
mega_ball: Mega Ball (bonus)
|
|
"""
|
|
source = "megamillions_ny"
|
|
game = get_game_by_name("Mega Millions")
|
|
if not game:
|
|
return _error_result(source, 0, 0, "Mega Millions game not found in DB")
|
|
|
|
game_id = game["id"]
|
|
added = skipped = 0
|
|
offset = 0
|
|
|
|
try:
|
|
while True:
|
|
params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"}
|
|
resp = requests.get(MEGAMILLIONS_NY_URL, params=params, timeout=30)
|
|
resp.raise_for_status()
|
|
records = resp.json()
|
|
|
|
if not records:
|
|
break
|
|
|
|
for rec in records:
|
|
raw_date = rec.get("draw_date", "")
|
|
raw_nums = rec.get("winning_numbers", "").strip()
|
|
mega_ball = rec.get("mega_ball") or None
|
|
multiplier = rec.get("multiplier") or None
|
|
|
|
if not raw_date or not raw_nums:
|
|
continue
|
|
try:
|
|
draw_date = _parse_ny_date(raw_date)
|
|
parts = raw_nums.split()
|
|
main = [int(p) for p in parts[:5]]
|
|
bonus = int(mega_ball) if mega_ball else None
|
|
except (ValueError, IndexError) as e:
|
|
logger.warning("[FETCH] MM NY parse error %s: %s", rec, e)
|
|
continue
|
|
|
|
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
|
if result == "inserted":
|
|
added += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
if len(records) < NY_API_LIMIT:
|
|
break
|
|
offset += NY_API_LIMIT
|
|
|
|
logger.info("[FETCH] Mega Millions NY done — added=%d skipped=%d", added, skipped)
|
|
insert_fetch_log(source, added, skipped, "success")
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "success", "message": None}
|
|
|
|
except requests.RequestException as e:
|
|
logger.error("[FETCH] Mega Millions NY error: %s", e)
|
|
return _error_result(source, added, skipped, f"Network error: {e}")
|
|
|
|
|
|
# ── Mega Millions TX ──────────────────────────────────────────────────────────
|
|
|
|
def fetch_megamillions_tx():
|
|
"""
|
|
Fetch Mega Millions draws from Texas Lottery CSV.
|
|
Column layout: Game Name, Month, Day, Year, Num1-5, Mega Ball, Megaplier
|
|
Header row is auto-detected and skipped.
|
|
"""
|
|
source = "megamillions_tx"
|
|
game = get_game_by_name("Mega Millions")
|
|
if not game:
|
|
return _error_result(source, 0, 0, "Mega Millions game not found in DB")
|
|
|
|
game_id = game["id"]
|
|
added = skipped = 0
|
|
|
|
try:
|
|
resp = requests.get(MEGAMILLIONS_TX_URL, timeout=30)
|
|
resp.raise_for_status()
|
|
|
|
reader = csv.reader(io.StringIO(resp.text))
|
|
for row in reader:
|
|
if not row or len(row) < 10:
|
|
continue
|
|
|
|
# Skip header row — month column would be "Month" (non-digit)
|
|
if not row[1].strip().isdigit():
|
|
continue
|
|
|
|
# Skip non-Mega Millions rows
|
|
if "mega" not in row[0].lower():
|
|
continue
|
|
|
|
try:
|
|
month = row[1].strip().zfill(2)
|
|
day = row[2].strip().zfill(2)
|
|
year = row[3].strip()
|
|
draw_date = f"{year}-{month}-{day}"
|
|
|
|
main = [int(row[i].strip()) for i in range(4, 9)]
|
|
bonus = int(row[9].strip())
|
|
multiplier = row[10].strip() if len(row) > 10 and row[10].strip() else None
|
|
except (ValueError, IndexError) as e:
|
|
logger.warning("[FETCH] TX CSV parse error row=%r: %s", row, e)
|
|
continue
|
|
|
|
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
|
if result == "inserted":
|
|
added += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
logger.info("[FETCH] Mega Millions TX done — added=%d skipped=%d", added, skipped)
|
|
insert_fetch_log(source, added, skipped, "success")
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "success", "message": None}
|
|
|
|
except requests.RequestException as e:
|
|
logger.error("[FETCH] Mega Millions TX error: %s", e)
|
|
return _error_result(source, added, skipped, f"Network error: {e}")
|
|
|
|
|
|
# ── Virginia Lottery (shared parser) ─────────────────────────────────────────
|
|
|
|
def _parse_va_date(raw: str) -> str | None:
|
|
"""'5/22/2026' or '05/22/2026' → '2026-05-22'."""
|
|
parts = raw.strip().split("/")
|
|
if len(parts) != 3:
|
|
return None
|
|
try:
|
|
return f"{parts[2]}-{parts[0].zfill(2)}-{parts[1].zfill(2)}"
|
|
except (IndexError, ValueError):
|
|
return None
|
|
|
|
|
|
def _fetch_va_lottery(game_name: str, source: str, url: str) -> dict:
|
|
"""
|
|
Fetch VA Lottery draw data from the valottery.com download API.
|
|
Format per line: 'M/D/YYYY; N1,N2,...[; Label: bonus]'
|
|
First line may be a 'Results for ...' header — skipped automatically.
|
|
"""
|
|
game = get_game_by_name(game_name)
|
|
if not game:
|
|
return _error_result(source, 0, 0, f"{game_name} game not found in DB")
|
|
|
|
game_id = game["id"]
|
|
added = skipped = 0
|
|
|
|
try:
|
|
resp = requests.get(url, timeout=30)
|
|
resp.raise_for_status()
|
|
|
|
for line in resp.text.splitlines():
|
|
line = line.strip()
|
|
if not line or line.lower().startswith("results for"):
|
|
continue
|
|
|
|
parts = [p.strip() for p in line.split(";")]
|
|
if len(parts) < 2:
|
|
continue
|
|
|
|
draw_date = _parse_va_date(parts[0])
|
|
if draw_date is None:
|
|
continue
|
|
|
|
try:
|
|
p1 = parts[1].strip()
|
|
p1_lower = p1.lower()
|
|
if p1_lower.startswith("day:"):
|
|
# Old twice-daily format: "Day: N1,...; Night: N1,..."
|
|
# Use Night draw only for consistency with modern single-draw records
|
|
if len(parts) >= 3 and "night:" in parts[2].lower():
|
|
night_str = parts[2][parts[2].lower().find("night:") + 6:]
|
|
numbers = [int(n.strip()) for n in night_str.split(",") if n.strip()]
|
|
else:
|
|
continue
|
|
elif p1_lower.startswith("night:"):
|
|
# Night-only historical record
|
|
numbers = [int(n.strip()) for n in p1[p1_lower.find("night:") + 6:].split(",") if n.strip()]
|
|
else:
|
|
numbers = [int(n.strip()) for n in p1.split(",") if n.strip()]
|
|
except ValueError as e:
|
|
logger.warning("[FETCH] VA %s parse error %r: %s", source, line, e)
|
|
continue
|
|
|
|
bonus = None
|
|
if len(parts) >= 3:
|
|
p_last = parts[-1].strip()
|
|
if "night:" not in p_last.lower() and "day:" not in p_last.lower():
|
|
colon = p_last.rfind(":")
|
|
if colon >= 0:
|
|
try:
|
|
bonus = int(p_last[colon + 1:].strip())
|
|
except ValueError:
|
|
pass
|
|
|
|
result = insert_draw(game_id, draw_date, numbers, bonus=bonus, source=source)
|
|
if result == "inserted":
|
|
added += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
logger.info("[FETCH] %s done — added=%d skipped=%d", source, added, skipped)
|
|
insert_fetch_log(source, added, skipped, "success")
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "success", "message": None}
|
|
|
|
except requests.RequestException as e:
|
|
logger.error("[FETCH] %s error: %s", source, e)
|
|
return _error_result(source, added, skipped, f"Network error: {e}")
|
|
|
|
|
|
def fetch_cash5_va():
|
|
"""Fetch Cash 5 draws from VA Lottery download API."""
|
|
return _fetch_va_lottery("Cash 5", "cash5_va", VA_CASH5_URL)
|
|
|
|
|
|
def fetch_millionaireforlife_va():
|
|
"""Fetch Millionaire for Life draws from VA Lottery download API."""
|
|
return _fetch_va_lottery("Millionaire for Life", "millionaireforlife_va",
|
|
VA_MILLIONAIREFORLIFE_URL)
|
|
|
|
|
|
def fetch_bankamillion_va():
|
|
"""Fetch Bank a Million draws from VA Lottery download API."""
|
|
return _fetch_va_lottery("Bank a Million", "bankamillion_va", VA_BANKAMILLION_URL)
|
|
|
|
|
|
# ── fetch_all ─────────────────────────────────────────────────────────────────
|
|
|
|
def fetch_all():
|
|
"""
|
|
Run all 3 fetch functions and return a list of result dicts.
|
|
One source erroring does not stop the others.
|
|
Called by the auto-fetch thread and the manual Fetch Now button.
|
|
"""
|
|
logger.info("[FETCH] fetch_all() starting")
|
|
results = []
|
|
for fn in (
|
|
fetch_powerball_ny,
|
|
fetch_megamillions_ny,
|
|
fetch_megamillions_tx,
|
|
fetch_cash5_va,
|
|
fetch_millionaireforlife_va,
|
|
fetch_bankamillion_va,
|
|
):
|
|
try:
|
|
results.append(fn())
|
|
except Exception as e:
|
|
logger.error("[ERROR] Unexpected error in %s: %s", fn.__name__, e, exc_info=True)
|
|
results.append({"source": fn.__name__, "added": 0, "skipped": 0,
|
|
"status": "error", "message": str(e)})
|
|
logger.info("[FETCH] fetch_all() complete — %d sources processed", len(results))
|
|
return results
|