253 lines
9.3 KiB
Python
253 lines
9.3 KiB
Python
"""
|
|
core/fetcher.py
|
|
---------------
|
|
Data fetch logic for all 3 lottery sources.
|
|
Each function returns: {"source", "added", "skipped", "status", "message"}
|
|
All HTTP errors are caught and returned as status="error" — never raised.
|
|
"""
|
|
|
|
import csv
|
|
import io
|
|
import logging
|
|
import threading
|
|
|
|
import requests
|
|
|
|
from db.models import get_game_by_name, insert_draw, insert_fetch_log
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
NY_API_LIMIT = 5000 # records per page
|
|
|
|
POWERBALL_NY_URL = "https://data.ny.gov/resource/d6yy-54nr.json"
|
|
MEGAMILLIONS_NY_URL = "https://data.ny.gov/resource/5xaw-6ayf.json"
|
|
MEGAMILLIONS_TX_URL = (
|
|
"https://www.texaslottery.com/export/sites/lottery/Games/"
|
|
"Mega_Millions/Winning_Numbers/download.html"
|
|
)
|
|
|
|
_fetch_lock = threading.Lock()
|
|
|
|
|
|
def _parse_ny_date(raw):
|
|
"""'2024-01-01T00:00:00.000' → '2024-01-01'."""
|
|
return raw.split("T")[0].strip()
|
|
|
|
|
|
def _error_result(source, added, skipped, msg):
|
|
insert_fetch_log(source, added, skipped, "error", msg)
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "error", "message": msg}
|
|
|
|
|
|
# ── Powerball NY ──────────────────────────────────────────────────────────────
|
|
|
|
def fetch_powerball_ny():
|
|
"""
|
|
Fetch Powerball draws from NY Open Data API.
|
|
winning_numbers field: "01 13 36 61 69 07" (5 white + 1 Powerball)
|
|
"""
|
|
source = "powerball_ny"
|
|
game = get_game_by_name("Powerball")
|
|
if not game:
|
|
return _error_result(source, 0, 0, "Powerball game not found in DB")
|
|
|
|
game_id = game["id"]
|
|
added = skipped = 0
|
|
offset = 0
|
|
|
|
try:
|
|
while True:
|
|
params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"}
|
|
resp = requests.get(POWERBALL_NY_URL, params=params, timeout=30)
|
|
resp.raise_for_status()
|
|
records = resp.json()
|
|
|
|
if not records:
|
|
break
|
|
|
|
for rec in records:
|
|
raw_date = rec.get("draw_date", "")
|
|
raw_nums = rec.get("winning_numbers", "").strip()
|
|
multiplier = rec.get("multiplier") or None
|
|
|
|
if not raw_date or not raw_nums:
|
|
continue
|
|
try:
|
|
draw_date = _parse_ny_date(raw_date)
|
|
parts = raw_nums.split()
|
|
if len(parts) < 6:
|
|
logger.warning("[FETCH] PB NY: unexpected numbers %r on %s", raw_nums, raw_date)
|
|
continue
|
|
main = [int(p) for p in parts[:5]]
|
|
bonus = int(parts[5])
|
|
except (ValueError, IndexError) as e:
|
|
logger.warning("[FETCH] PB NY parse error %s: %s", rec, e)
|
|
continue
|
|
|
|
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
|
if result == "inserted":
|
|
added += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
if len(records) < NY_API_LIMIT:
|
|
break
|
|
offset += NY_API_LIMIT
|
|
|
|
logger.info("[FETCH] Powerball NY done — added=%d skipped=%d", added, skipped)
|
|
insert_fetch_log(source, added, skipped, "success")
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "success", "message": None}
|
|
|
|
except requests.RequestException as e:
|
|
logger.error("[FETCH] Powerball NY error: %s", e)
|
|
return _error_result(source, added, skipped, f"Network error: {e}")
|
|
|
|
|
|
# ── Mega Millions NY ──────────────────────────────────────────────────────────
|
|
|
|
def fetch_megamillions_ny():
|
|
"""
|
|
Fetch Mega Millions draws from NY Open Data API.
|
|
winning_numbers: 5 space-separated main balls
|
|
mega_ball: Mega Ball (bonus)
|
|
"""
|
|
source = "megamillions_ny"
|
|
game = get_game_by_name("Mega Millions")
|
|
if not game:
|
|
return _error_result(source, 0, 0, "Mega Millions game not found in DB")
|
|
|
|
game_id = game["id"]
|
|
added = skipped = 0
|
|
offset = 0
|
|
|
|
try:
|
|
while True:
|
|
params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"}
|
|
resp = requests.get(MEGAMILLIONS_NY_URL, params=params, timeout=30)
|
|
resp.raise_for_status()
|
|
records = resp.json()
|
|
|
|
if not records:
|
|
break
|
|
|
|
for rec in records:
|
|
raw_date = rec.get("draw_date", "")
|
|
raw_nums = rec.get("winning_numbers", "").strip()
|
|
mega_ball = rec.get("mega_ball") or None
|
|
multiplier = rec.get("multiplier") or None
|
|
|
|
if not raw_date or not raw_nums:
|
|
continue
|
|
try:
|
|
draw_date = _parse_ny_date(raw_date)
|
|
parts = raw_nums.split()
|
|
main = [int(p) for p in parts[:5]]
|
|
bonus = int(mega_ball) if mega_ball else None
|
|
except (ValueError, IndexError) as e:
|
|
logger.warning("[FETCH] MM NY parse error %s: %s", rec, e)
|
|
continue
|
|
|
|
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
|
if result == "inserted":
|
|
added += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
if len(records) < NY_API_LIMIT:
|
|
break
|
|
offset += NY_API_LIMIT
|
|
|
|
logger.info("[FETCH] Mega Millions NY done — added=%d skipped=%d", added, skipped)
|
|
insert_fetch_log(source, added, skipped, "success")
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "success", "message": None}
|
|
|
|
except requests.RequestException as e:
|
|
logger.error("[FETCH] Mega Millions NY error: %s", e)
|
|
return _error_result(source, added, skipped, f"Network error: {e}")
|
|
|
|
|
|
# ── Mega Millions TX ──────────────────────────────────────────────────────────
|
|
|
|
def fetch_megamillions_tx():
|
|
"""
|
|
Fetch Mega Millions draws from Texas Lottery CSV.
|
|
Column layout: Game Name, Month, Day, Year, Num1-5, Mega Ball, Megaplier
|
|
Header row is auto-detected and skipped.
|
|
"""
|
|
source = "megamillions_tx"
|
|
game = get_game_by_name("Mega Millions")
|
|
if not game:
|
|
return _error_result(source, 0, 0, "Mega Millions game not found in DB")
|
|
|
|
game_id = game["id"]
|
|
added = skipped = 0
|
|
|
|
try:
|
|
resp = requests.get(MEGAMILLIONS_TX_URL, timeout=30)
|
|
resp.raise_for_status()
|
|
|
|
reader = csv.reader(io.StringIO(resp.text))
|
|
for row in reader:
|
|
if not row or len(row) < 10:
|
|
continue
|
|
|
|
# Skip header row — month column would be "Month" (non-digit)
|
|
if not row[1].strip().isdigit():
|
|
continue
|
|
|
|
# Skip non-Mega Millions rows
|
|
if "mega" not in row[0].lower():
|
|
continue
|
|
|
|
try:
|
|
month = row[1].strip().zfill(2)
|
|
day = row[2].strip().zfill(2)
|
|
year = row[3].strip()
|
|
draw_date = f"{year}-{month}-{day}"
|
|
|
|
main = [int(row[i].strip()) for i in range(4, 9)]
|
|
bonus = int(row[9].strip())
|
|
multiplier = row[10].strip() if len(row) > 10 and row[10].strip() else None
|
|
except (ValueError, IndexError) as e:
|
|
logger.warning("[FETCH] TX CSV parse error row=%r: %s", row, e)
|
|
continue
|
|
|
|
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
|
if result == "inserted":
|
|
added += 1
|
|
else:
|
|
skipped += 1
|
|
|
|
logger.info("[FETCH] Mega Millions TX done — added=%d skipped=%d", added, skipped)
|
|
insert_fetch_log(source, added, skipped, "success")
|
|
return {"source": source, "added": added, "skipped": skipped,
|
|
"status": "success", "message": None}
|
|
|
|
except requests.RequestException as e:
|
|
logger.error("[FETCH] Mega Millions TX error: %s", e)
|
|
return _error_result(source, added, skipped, f"Network error: {e}")
|
|
|
|
|
|
# ── fetch_all ─────────────────────────────────────────────────────────────────
|
|
|
|
def fetch_all():
|
|
"""
|
|
Run all 3 fetch functions and return a list of result dicts.
|
|
One source erroring does not stop the others.
|
|
Called by the auto-fetch thread and the manual Fetch Now button.
|
|
"""
|
|
logger.info("[FETCH] fetch_all() starting")
|
|
results = []
|
|
for fn in (fetch_powerball_ny, fetch_megamillions_ny, fetch_megamillions_tx):
|
|
try:
|
|
results.append(fn())
|
|
except Exception as e:
|
|
logger.error("[ERROR] Unexpected error in %s: %s", fn.__name__, e, exc_info=True)
|
|
results.append({"source": fn.__name__, "added": 0, "skipped": 0,
|
|
"status": "error", "message": str(e)})
|
|
logger.info("[FETCH] fetch_all() complete — %d sources processed", len(results))
|
|
return results
|