05/23 Phase 2
This commit is contained in:
+252
@@ -0,0 +1,252 @@
|
||||
"""
|
||||
core/fetcher.py
|
||||
---------------
|
||||
Data fetch logic for all 3 lottery sources.
|
||||
Each function returns: {"source", "added", "skipped", "status", "message"}
|
||||
All HTTP errors are caught and returned as status="error" — never raised.
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import logging
|
||||
import threading
|
||||
|
||||
import requests
|
||||
|
||||
from db.models import get_game_by_name, insert_draw, insert_fetch_log
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NY_API_LIMIT = 5000 # records per page
|
||||
|
||||
POWERBALL_NY_URL = "https://data.ny.gov/resource/d6yy-54nr.json"
|
||||
MEGAMILLIONS_NY_URL = "https://data.ny.gov/resource/5xaw-6ayf.json"
|
||||
MEGAMILLIONS_TX_URL = (
|
||||
"https://www.texaslottery.com/export/sites/lottery/Games/"
|
||||
"Mega_Millions/Winning_Numbers/download.html"
|
||||
)
|
||||
|
||||
_fetch_lock = threading.Lock()
|
||||
|
||||
|
||||
def _parse_ny_date(raw):
|
||||
"""'2024-01-01T00:00:00.000' → '2024-01-01'."""
|
||||
return raw.split("T")[0].strip()
|
||||
|
||||
|
||||
def _error_result(source, added, skipped, msg):
|
||||
insert_fetch_log(source, added, skipped, "error", msg)
|
||||
return {"source": source, "added": added, "skipped": skipped,
|
||||
"status": "error", "message": msg}
|
||||
|
||||
|
||||
# ── Powerball NY ──────────────────────────────────────────────────────────────
|
||||
|
||||
def fetch_powerball_ny():
|
||||
"""
|
||||
Fetch Powerball draws from NY Open Data API.
|
||||
winning_numbers field: "01 13 36 61 69 07" (5 white + 1 Powerball)
|
||||
"""
|
||||
source = "powerball_ny"
|
||||
game = get_game_by_name("Powerball")
|
||||
if not game:
|
||||
return _error_result(source, 0, 0, "Powerball game not found in DB")
|
||||
|
||||
game_id = game["id"]
|
||||
added = skipped = 0
|
||||
offset = 0
|
||||
|
||||
try:
|
||||
while True:
|
||||
params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"}
|
||||
resp = requests.get(POWERBALL_NY_URL, params=params, timeout=30)
|
||||
resp.raise_for_status()
|
||||
records = resp.json()
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
for rec in records:
|
||||
raw_date = rec.get("draw_date", "")
|
||||
raw_nums = rec.get("winning_numbers", "").strip()
|
||||
multiplier = rec.get("multiplier") or None
|
||||
|
||||
if not raw_date or not raw_nums:
|
||||
continue
|
||||
try:
|
||||
draw_date = _parse_ny_date(raw_date)
|
||||
parts = raw_nums.split()
|
||||
if len(parts) < 6:
|
||||
logger.warning("[FETCH] PB NY: unexpected numbers %r on %s", raw_nums, raw_date)
|
||||
continue
|
||||
main = [int(p) for p in parts[:5]]
|
||||
bonus = int(parts[5])
|
||||
except (ValueError, IndexError) as e:
|
||||
logger.warning("[FETCH] PB NY parse error %s: %s", rec, e)
|
||||
continue
|
||||
|
||||
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
||||
if result == "inserted":
|
||||
added += 1
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
if len(records) < NY_API_LIMIT:
|
||||
break
|
||||
offset += NY_API_LIMIT
|
||||
|
||||
logger.info("[FETCH] Powerball NY done — added=%d skipped=%d", added, skipped)
|
||||
insert_fetch_log(source, added, skipped, "success")
|
||||
return {"source": source, "added": added, "skipped": skipped,
|
||||
"status": "success", "message": None}
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error("[FETCH] Powerball NY error: %s", e)
|
||||
return _error_result(source, added, skipped, f"Network error: {e}")
|
||||
|
||||
|
||||
# ── Mega Millions NY ──────────────────────────────────────────────────────────
|
||||
|
||||
def fetch_megamillions_ny():
|
||||
"""
|
||||
Fetch Mega Millions draws from NY Open Data API.
|
||||
winning_numbers: 5 space-separated main balls
|
||||
mega_ball: Mega Ball (bonus)
|
||||
"""
|
||||
source = "megamillions_ny"
|
||||
game = get_game_by_name("Mega Millions")
|
||||
if not game:
|
||||
return _error_result(source, 0, 0, "Mega Millions game not found in DB")
|
||||
|
||||
game_id = game["id"]
|
||||
added = skipped = 0
|
||||
offset = 0
|
||||
|
||||
try:
|
||||
while True:
|
||||
params = {"$limit": NY_API_LIMIT, "$offset": offset, "$order": "draw_date ASC"}
|
||||
resp = requests.get(MEGAMILLIONS_NY_URL, params=params, timeout=30)
|
||||
resp.raise_for_status()
|
||||
records = resp.json()
|
||||
|
||||
if not records:
|
||||
break
|
||||
|
||||
for rec in records:
|
||||
raw_date = rec.get("draw_date", "")
|
||||
raw_nums = rec.get("winning_numbers", "").strip()
|
||||
mega_ball = rec.get("mega_ball") or None
|
||||
multiplier = rec.get("multiplier") or None
|
||||
|
||||
if not raw_date or not raw_nums:
|
||||
continue
|
||||
try:
|
||||
draw_date = _parse_ny_date(raw_date)
|
||||
parts = raw_nums.split()
|
||||
main = [int(p) for p in parts[:5]]
|
||||
bonus = int(mega_ball) if mega_ball else None
|
||||
except (ValueError, IndexError) as e:
|
||||
logger.warning("[FETCH] MM NY parse error %s: %s", rec, e)
|
||||
continue
|
||||
|
||||
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
||||
if result == "inserted":
|
||||
added += 1
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
if len(records) < NY_API_LIMIT:
|
||||
break
|
||||
offset += NY_API_LIMIT
|
||||
|
||||
logger.info("[FETCH] Mega Millions NY done — added=%d skipped=%d", added, skipped)
|
||||
insert_fetch_log(source, added, skipped, "success")
|
||||
return {"source": source, "added": added, "skipped": skipped,
|
||||
"status": "success", "message": None}
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error("[FETCH] Mega Millions NY error: %s", e)
|
||||
return _error_result(source, added, skipped, f"Network error: {e}")
|
||||
|
||||
|
||||
# ── Mega Millions TX ──────────────────────────────────────────────────────────
|
||||
|
||||
def fetch_megamillions_tx():
|
||||
"""
|
||||
Fetch Mega Millions draws from Texas Lottery CSV.
|
||||
Column layout: Game Name, Month, Day, Year, Num1-5, Mega Ball, Megaplier
|
||||
Header row is auto-detected and skipped.
|
||||
"""
|
||||
source = "megamillions_tx"
|
||||
game = get_game_by_name("Mega Millions")
|
||||
if not game:
|
||||
return _error_result(source, 0, 0, "Mega Millions game not found in DB")
|
||||
|
||||
game_id = game["id"]
|
||||
added = skipped = 0
|
||||
|
||||
try:
|
||||
resp = requests.get(MEGAMILLIONS_TX_URL, timeout=30)
|
||||
resp.raise_for_status()
|
||||
|
||||
reader = csv.reader(io.StringIO(resp.text))
|
||||
for row in reader:
|
||||
if not row or len(row) < 10:
|
||||
continue
|
||||
|
||||
# Skip header row — month column would be "Month" (non-digit)
|
||||
if not row[1].strip().isdigit():
|
||||
continue
|
||||
|
||||
# Skip non-Mega Millions rows
|
||||
if "mega" not in row[0].lower():
|
||||
continue
|
||||
|
||||
try:
|
||||
month = row[1].strip().zfill(2)
|
||||
day = row[2].strip().zfill(2)
|
||||
year = row[3].strip()
|
||||
draw_date = f"{year}-{month}-{day}"
|
||||
|
||||
main = [int(row[i].strip()) for i in range(4, 9)]
|
||||
bonus = int(row[9].strip())
|
||||
multiplier = row[10].strip() if len(row) > 10 and row[10].strip() else None
|
||||
except (ValueError, IndexError) as e:
|
||||
logger.warning("[FETCH] TX CSV parse error row=%r: %s", row, e)
|
||||
continue
|
||||
|
||||
result = insert_draw(game_id, draw_date, main, bonus, multiplier, source)
|
||||
if result == "inserted":
|
||||
added += 1
|
||||
else:
|
||||
skipped += 1
|
||||
|
||||
logger.info("[FETCH] Mega Millions TX done — added=%d skipped=%d", added, skipped)
|
||||
insert_fetch_log(source, added, skipped, "success")
|
||||
return {"source": source, "added": added, "skipped": skipped,
|
||||
"status": "success", "message": None}
|
||||
|
||||
except requests.RequestException as e:
|
||||
logger.error("[FETCH] Mega Millions TX error: %s", e)
|
||||
return _error_result(source, added, skipped, f"Network error: {e}")
|
||||
|
||||
|
||||
# ── fetch_all ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def fetch_all():
|
||||
"""
|
||||
Run all 3 fetch functions and return a list of result dicts.
|
||||
One source erroring does not stop the others.
|
||||
Called by the auto-fetch thread and the manual Fetch Now button.
|
||||
"""
|
||||
logger.info("[FETCH] fetch_all() starting")
|
||||
results = []
|
||||
for fn in (fetch_powerball_ny, fetch_megamillions_ny, fetch_megamillions_tx):
|
||||
try:
|
||||
results.append(fn())
|
||||
except Exception as e:
|
||||
logger.error("[ERROR] Unexpected error in %s: %s", fn.__name__, e, exc_info=True)
|
||||
results.append({"source": fn.__name__, "added": 0, "skipped": 0,
|
||||
"status": "error", "message": str(e)})
|
||||
logger.info("[FETCH] fetch_all() complete — %d sources processed", len(results))
|
||||
return results
|
||||
Reference in New Issue
Block a user