05/23 update prediction accuration

.claude/settings.local.json

@@ -21,7 +21,8 @@
       "WebFetch(domain:www.valottery.com)",
       "Bash(pip install *)",
       "Bash(python -m pytest tests/ -x -q)",
-      "Bash(python -m pytest tests/test_phase22.py -v)"
+      "Bash(python -m pytest tests/test_phase22.py -v)",
+      "Bash(python -m pytest tests/test_filters.py -v)"
     ]
   }
 }

core/filters.py

@@ -0,0 +1,73 @@
+"""
+core/filters.py
+---------------
+Combination-quality filters for generated lottery tickets.
+
+A ticket is considered "weak" if it falls into a pattern that is
+statistically underrepresented in real draws:
+  • All main numbers are even  (when count >= 4)
+  • All main numbers are odd   (when count >= 4)
+  • 4 or more consecutive numbers (e.g. 12-13-14-15)
+  • Sum of main numbers is outside the historical 10th–90th percentile
+
+None of these filters improve expected-value (all draws are IID), but
+they remove tickets that players and statisticians alike would call "weak"
+and shift the distribution toward historically common patterns.
+"""
+
+import numpy as np
+
+from db.models import get_all_draws_numbers
+
+
+def _has_consecutive_run(numbers: list[int], min_len: int = 4) -> bool:
+    """Return True if `numbers` contains a run of at least min_len consecutive integers."""
+    s = sorted(numbers)
+    run = 1
+    for i in range(1, len(s)):
+        if s[i] == s[i - 1] + 1:
+            run += 1
+            if run >= min_len:
+                return True
+        else:
+            run = 1
+    return False
+
+
+def sum_range_percentiles(
+    game_id: int,
+    low_pct: float = 10.0,
+    high_pct: float = 90.0,
+) -> tuple[int, int] | None:
+    """
+    Return (low, high) sum bounds derived from historical draw data.
+    Returns None when fewer than 10 draws exist (not enough data).
+    """
+    draws = get_all_draws_numbers(game_id)
+    if len(draws) < 10:
+        return None
+    sums = [sum(d["numbers"]) for d in draws]
+    return int(np.percentile(sums, low_pct)), int(np.percentile(sums, high_pct))
+
+
+def passes_filters(
+    numbers: list[int],
+    sum_range: tuple[int, int] | None = None,
+) -> bool:
+    """
+    Return True if the ticket passes all combination-quality checks.
+    Pass a pre-computed sum_range (from sum_range_percentiles) to avoid
+    re-querying the database on every retry.
+    """
+    if len(numbers) >= 4:
+        if all(n % 2 == 0 for n in numbers):
+            return False
+        if all(n % 2 != 0 for n in numbers):
+            return False
+    if _has_consecutive_run(numbers, 4):
+        return False
+    if sum_range is not None:
+        lo, hi = sum_range
+        if not (lo <= sum(numbers) <= hi):
+            return False
+    return True

core/predictor.py

@@ -20,6 +20,9 @@ import numpy as np
 
 from db.models import get_all_draws_numbers, get_game_by_id
 from core.analyzer import frequency_analysis, gap_analysis, positional_frequency
+from core.filters import passes_filters, sum_range_percentiles
+
+_MAX_FILTER_TRIES = 50
 
 
 # ── Internal helpers ──────────────────────────────────────────────────────────
@@ -53,6 +56,46 @@ def _hot_bonus(draws, game: dict) -> int | None:
     return random.randint(1, game["bonus_max"])
 
 
+def _retry_filter(generate_fn, sum_range, max_tries: int = _MAX_FILTER_TRIES) -> list[int]:
+    """
+    Call generate_fn() up to max_tries times; return the first numbers list
+    that passes combination filters, or the last generated if none do.
+    Used for stochastic strategies where each call produces a new candidate.
+    """
+    last = generate_fn()
+    if passes_filters(last, sum_range):
+        return last
+    for _ in range(max_tries - 1):
+        attempt = generate_fn()
+        if passes_filters(attempt, sum_range):
+            return attempt
+    return last
+
+
+def _swap_filter(numbers: list[int], ranked_pool: list[int],
+                 sum_range) -> list[int]:
+    """
+    For deterministic strategies: try swapping the weakest-ranked number in
+    the ticket for the best available alternative until filters pass.
+    ranked_pool must contain candidates sorted best-first, excluding numbers
+    already in the ticket.
+    Returns the first passing combination, or the original if none found.
+    """
+    if passes_filters(numbers, sum_range):
+        return numbers
+    ticket = list(numbers)
+    for swap_idx in range(len(ticket) - 1, -1, -1):
+        original = ticket[swap_idx]
+        for alt in ranked_pool:
+            if alt not in ticket:
+                ticket[swap_idx] = alt
+                candidate = sorted(ticket)
+                if passes_filters(candidate, sum_range):
+                    return candidate
+                ticket[swap_idx] = original
+    return numbers  # graceful fallback: return original if no swap helped
+
+
 def _fill_to_count(chosen: list, game: dict, exclude: set | None = None) -> list:
     """Pad chosen with random unused numbers if fewer than main_count."""
     excl   = exclude or set()
@@ -73,6 +116,7 @@ def hot_numbers(game_id: int, last_n: int = 100, exclude: set | None = None) ->
     Top main_count most-frequent numbers from the last last_n draws.
     Bonus: most frequent historical bonus ball.
     Falls back to random if there is no history.
+    Applies combination filters; swaps the weakest pick if the ticket is weak.
     """
     excl  = exclude or set()
     game  = get_game_by_id(game_id)
@@ -86,10 +130,15 @@ def hot_numbers(game_id: int, last_n: int = 100, exclude: set | None = None) ->
     for draw in recent:
         counter.update(draw["numbers"])
 
-    top = [n for n, _ in sorted(counter.items(), key=lambda kv: (-kv[1], kv[0]))
-           if n not in excl]
-    numbers = _fill_to_count(top[: game["main_count"]], game, excl)
-    bonus   = _hot_bonus(draws, game)
+    ranked = [n for n, _ in sorted(counter.items(), key=lambda kv: (-kv[1], kv[0]))
+              if n not in excl]
+    numbers = _fill_to_count(ranked[: game["main_count"]], game, excl)
+
+    sum_range = sum_range_percentiles(game_id)
+    ranked_pool = [n for n in ranked if n not in numbers]
+    numbers = _swap_filter(numbers, ranked_pool, sum_range)
+
+    bonus = _hot_bonus(draws, game)
     return {"numbers": numbers, "bonus": bonus}
 
 
@@ -99,6 +148,7 @@ def due_numbers(game_id: int, exclude: set | None = None) -> dict:
     """
     Numbers with the largest gap (most overdue) based on historical frequency.
     Falls back to random if there is no history.
+    Applies combination filters; swaps the lowest-gap pick if the ticket is weak.
     """
     excl = exclude or set()
     game = get_game_by_id(game_id)
@@ -106,10 +156,15 @@ def due_numbers(game_id: int, exclude: set | None = None) -> dict:
     if not gaps:
         return _random_ticket(game, excl)
 
-    top = [n for n, _ in sorted(gaps.items(), key=lambda kv: (-kv[1], kv[0]))
-           if n not in excl]
-    numbers = _fill_to_count(top[: game["main_count"]], game, excl)
-    bonus   = _random_bonus(game)
+    ranked = [n for n, _ in sorted(gaps.items(), key=lambda kv: (-kv[1], kv[0]))
+              if n not in excl]
+    numbers = _fill_to_count(ranked[: game["main_count"]], game, excl)
+
+    sum_range   = sum_range_percentiles(game_id)
+    ranked_pool = [n for n in ranked if n not in numbers]
+    numbers = _swap_filter(numbers, ranked_pool, sum_range)
+
+    bonus = _random_bonus(game)
     return {"numbers": numbers, "bonus": bonus}
 
 
@@ -120,6 +175,7 @@ def weighted_random(game_id: int, exclude: set | None = None) -> dict:
     Random draw with probability proportional to historical frequency.
     Numbers that have never appeared receive a minimum weight of 1
     so they remain in contention.
+    Retries up to _MAX_FILTER_TRIES times to find a combination-quality ticket.
     """
     excl = exclude or set()
     game = get_game_by_id(game_id)
@@ -129,9 +185,15 @@ def weighted_random(game_id: int, exclude: set | None = None) -> dict:
     weights = np.array([freq.get(n, 1) for n in pool], dtype=float)
     weights /= weights.sum()
 
-    chosen = np.random.choice(pool, size=game["main_count"], replace=False, p=weights)
-    bonus  = _random_bonus(game)
-    return {"numbers": sorted(chosen.tolist()), "bonus": bonus}
+    sum_range = sum_range_percentiles(game_id)
+
+    def _generate():
+        chosen = np.random.choice(pool, size=game["main_count"], replace=False, p=weights)
+        return sorted(chosen.tolist())
+
+    numbers = _retry_filter(_generate, sum_range)
+    bonus   = _random_bonus(game)
+    return {"numbers": numbers, "bonus": bonus}
 
 
 # ── Strategy 4: Monte Carlo ───────────────────────────────────────────────────
@@ -141,6 +203,7 @@ def monte_carlo(game_id: int, simulations: int = 10_000,
     """
     Run `simulations` weighted-random draws; tally how often each number
     is selected; return the top main_count by tally count.
+    Applies combination filters; swaps the lowest-tally pick if ticket is weak.
     """
     excl = exclude or set()
     game = get_game_by_id(game_id)
@@ -155,9 +218,14 @@ def monte_carlo(game_id: int, simulations: int = 10_000,
         ticket = np.random.choice(pool, size=game["main_count"], replace=False, p=weights)
         tally.update(ticket.tolist())
 
-    top     = [n for n, _ in tally.most_common(game["main_count"])]
-    numbers = _fill_to_count(top, game, excl)
-    bonus   = _random_bonus(game)
+    ranked  = [n for n, _ in tally.most_common()]
+    numbers = _fill_to_count(ranked[: game["main_count"]], game, excl)
+
+    sum_range   = sum_range_percentiles(game_id)
+    ranked_pool = [n for n in ranked if n not in numbers]
+    numbers = _swap_filter(numbers, ranked_pool, sum_range)
+
+    bonus = _random_bonus(game)
     return {"numbers": numbers, "bonus": bonus}
 
 
@@ -167,6 +235,7 @@ def positional_pick(game_id: int, exclude: set | None = None) -> dict:
     """
     For each draw position, select the most frequently appearing number
     that has not already been chosen for a previous position.
+    Applies combination filters; swaps the lowest-positional-rank pick if weak.
     """
     excl     = exclude or set()
     game     = get_game_by_id(game_id)
@@ -177,6 +246,8 @@ def positional_pick(game_id: int, exclude: set | None = None) -> dict:
 
     selected = []
     used     = set()
+    # Also collect per-position alternates (next-best) for the swap pool
+    alternates_pool = []
 
     for pos in range(1, game["main_count"] + 1):
         freqs  = pos_freq.get(pos, {})
@@ -189,12 +260,25 @@ def positional_pick(game_id: int, exclude: set | None = None) -> dict:
             if not available:
                 available = [n for n in range(1, game["main_max"] + 1) if n not in used]
             picked = random.choice(available)
+        else:
+            # Collect alternates for this position (for potential swap)
+            for n, _ in ranked:
+                if n != picked and n not in excl:
+                    alternates_pool.append(n)
 
         selected.append(picked)
         used.add(picked)
 
+    numbers = sorted(selected)
+    sum_range = sum_range_percentiles(game_id)
+    # Alternates sorted by first-occurrence (positional best-first)
+    seen = set()
+    ranked_pool = [n for n in alternates_pool
+                   if n not in numbers and not (seen.add(n) or n in seen)]
+    numbers = _swap_filter(numbers, ranked_pool, sum_range)
+
     bonus = _random_bonus(game)
-    return {"numbers": sorted(selected), "bonus": bonus}
+    return {"numbers": numbers, "bonus": bonus}
 
 
 # ── Strategy 6: Quick Pick ────────────────────────────────────────────────────
@@ -203,6 +287,16 @@ def quick_pick(game_id: int, exclude: set | None = None) -> dict:
     """
     Pure random selection from the full number pool.
     Requires no historical draw data.
+    Retries up to _MAX_FILTER_TRIES times to find a combination-quality ticket.
     """
+    excl = exclude or set()
     game = get_game_by_id(game_id)
-    return _random_ticket(game, exclude or set())
+    sum_range = sum_range_percentiles(game_id)
+
+    def _generate():
+        pool = _safe_pool(game, excl)
+        return sorted(random.sample(pool, game["main_count"]))
+
+    numbers = _retry_filter(_generate, sum_range)
+    bonus   = _random_bonus(game)
+    return {"numbers": numbers, "bonus": bonus}

tests/test_filters.py

@@ -0,0 +1,223 @@
+"""
+tests/test_filters.py
+---------------------
+Tests for core/filters.py and the filter integration in core/predictor.py.
+"""
+
+import pytest
+from db.models import get_game_by_name, insert_draw
+from core.filters import _has_consecutive_run, sum_range_percentiles, passes_filters
+from core.predictor import (
+    hot_numbers, due_numbers, weighted_random,
+    monte_carlo, positional_pick, quick_pick,
+    _swap_filter, _retry_filter,
+)
+
+
+# ── _has_consecutive_run ──────────────────────────────────────────────────────
+
+def test_no_consecutive():
+    assert _has_consecutive_run([1, 5, 10, 20, 30]) is False
+
+
+def test_run_of_three_not_flagged():
+    assert _has_consecutive_run([1, 2, 3, 10, 20]) is False  # run=3, threshold=4
+
+
+def test_run_of_four_flagged():
+    assert _has_consecutive_run([1, 2, 3, 4, 20]) is True
+
+
+def test_run_of_five_flagged():
+    assert _has_consecutive_run([10, 11, 12, 13, 14]) is True
+
+
+def test_run_at_end():
+    assert _has_consecutive_run([5, 20, 30, 31, 32, 33]) is True
+
+
+def test_run_not_contiguous():
+    assert _has_consecutive_run([1, 3, 5, 7, 9]) is False
+
+
+# ── passes_filters — odd/even ─────────────────────────────────────────────────
+
+def test_all_even_rejected():
+    assert passes_filters([2, 14, 28, 42, 60]) is False
+
+
+def test_all_odd_rejected():
+    assert passes_filters([1, 7, 13, 29, 69]) is False
+
+
+def test_mixed_odd_even_accepted():
+    assert passes_filters([1, 14, 28, 42, 60]) is True
+
+
+def test_all_even_under_4_accepted():
+    # Odd/even filter only kicks in for 4+ numbers
+    assert passes_filters([2, 4, 6]) is True
+
+
+# ── passes_filters — consecutive ─────────────────────────────────────────────
+
+def test_four_consecutive_rejected():
+    assert passes_filters([5, 6, 7, 8, 20]) is False
+
+
+def test_three_consecutive_accepted():
+    assert passes_filters([5, 6, 7, 20, 35]) is True
+
+
+# ── passes_filters — sum range ────────────────────────────────────────────────
+
+def test_sum_in_range_accepted():
+    assert passes_filters([1, 14, 28, 42, 60], sum_range=(100, 200)) is True
+
+
+def test_sum_below_range_rejected():
+    assert passes_filters([1, 2, 3, 4, 10], sum_range=(100, 200)) is False
+
+
+def test_sum_above_range_rejected():
+    assert passes_filters([60, 62, 64, 66, 69], sum_range=(100, 200)) is False
+
+
+def test_sum_range_none_skips_check():
+    # All-even but sum would be out of range if range were set — only even check applies
+    assert passes_filters([2, 4, 6, 8, 10], sum_range=None) is False  # all-even fails
+
+
+def test_sum_at_boundary_accepted():
+    assert passes_filters([1, 14, 28, 42, 15], sum_range=(100, 100)) is True  # sum=100
+
+
+# ── sum_range_percentiles ─────────────────────────────────────────────────────
+
+def test_sum_range_none_with_few_draws(tmp_db):
+    pb = get_game_by_name("Powerball")
+    for i in range(5):
+        insert_draw(pb["id"], f"2024-01-{i+1:02d}", [1, 2, 3, 4, i+5], bonus=1)
+    result = sum_range_percentiles(pb["id"])
+    assert result is None  # < 10 draws
+
+
+def test_sum_range_returns_tuple_with_enough_draws(tmp_db):
+    pb = get_game_by_name("Powerball")
+    for i in range(20):
+        insert_draw(pb["id"], f"2024-02-{i+1:02d}", [1+i, 2+i, 3+i, 4+i, 5+i], bonus=1)
+    result = sum_range_percentiles(pb["id"])
+    assert result is not None
+    lo, hi = result
+    assert lo < hi
+    assert isinstance(lo, int)
+    assert isinstance(hi, int)
+
+
+def test_sum_range_bounds_reasonable(tmp_db):
+    pb = get_game_by_name("Powerball")
+    # Insert draws with sums ranging 15–25
+    for i in range(15, 26):
+        insert_draw(pb["id"], f"2024-03-{i-14:02d}", [1, 2, 3, 4, i-10], bonus=1)
+    lo, hi = sum_range_percentiles(pb["id"])
+    assert lo >= 10    # 10th pct of sums around 15
+    assert hi <= 25    # 90th pct of sums around 25
+
+
+# ── _swap_filter ──────────────────────────────────────────────────────────────
+
+def test_swap_filter_passes_already():
+    numbers = [1, 14, 28, 42, 60]
+    assert _swap_filter(numbers, [2, 3, 5], None) == numbers
+
+
+def test_swap_filter_fixes_all_even():
+    # all-even → swap last (weakest-ranked) for an odd
+    numbers = [2, 14, 28, 42, 60]
+    pool = [1, 3, 5, 7, 9]   # odd alternates
+    result = _swap_filter(numbers, pool, None)
+    assert passes_filters(result), f"Expected filtered result, got {result}"
+    assert len(result) == 5
+    assert len(set(result)) == 5
+
+
+def test_swap_filter_fixes_four_consecutive():
+    numbers = [10, 11, 12, 13, 25]
+    pool = [1, 5, 8, 20, 30, 35]
+    result = _swap_filter(numbers, pool, None)
+    assert passes_filters(result), f"Expected filtered result, got {result}"
+
+
+def test_swap_filter_graceful_fallback():
+    # If no swap fixes the ticket, return original
+    numbers = [2, 4, 6, 8, 10]   # all-even
+    pool = [12, 14, 16]           # all-even alternates — can't fix
+    result = _swap_filter(numbers, pool, None)
+    assert result == numbers  # graceful fallback
+
+
+# ── Filter integration in strategies ─────────────────────────────────────────
+
+def _large_draw_set(game_id):
+    """Insert 30 varied Powerball draws so sum_range_percentiles returns a range."""
+    import random as rng
+    rng.seed(42)
+    from db.models import get_game_by_id
+    game = get_game_by_id(game_id)
+    for i in range(30):
+        nums = sorted(rng.sample(range(1, game["main_max"] + 1), game["main_count"]))
+        bonus = rng.randint(1, game["bonus_max"]) if game["bonus_count"] > 0 else None
+        insert_draw(game_id, f"2020-{(i//30)+1:02d}-{(i%28)+1:02d}", nums, bonus=bonus)
+
+
+def _assert_filtered(result, game):
+    nums = result["numbers"]
+    assert len(nums) == game["main_count"]
+    assert nums == sorted(nums)
+    assert len(set(nums)) == len(nums)
+    assert all(1 <= n <= game["main_max"] for n in nums)
+    assert not (len(nums) >= 4 and all(n % 2 == 0 for n in nums)), "all-even"
+    assert not (len(nums) >= 4 and all(n % 2 != 0 for n in nums)), "all-odd"
+    assert not _has_consecutive_run(nums, 4), "4+ consecutive"
+
+
+def test_hot_numbers_filter_integration(tmp_db):
+    pb = get_game_by_name("Powerball")
+    _large_draw_set(pb["id"])
+    for _ in range(10):
+        _assert_filtered(hot_numbers(pb["id"]), pb)
+
+
+def test_due_numbers_filter_integration(tmp_db):
+    pb = get_game_by_name("Powerball")
+    _large_draw_set(pb["id"])
+    for _ in range(10):
+        _assert_filtered(due_numbers(pb["id"]), pb)
+
+
+def test_weighted_random_filter_integration(tmp_db):
+    pb = get_game_by_name("Powerball")
+    _large_draw_set(pb["id"])
+    for _ in range(20):
+        _assert_filtered(weighted_random(pb["id"]), pb)
+
+
+def test_monte_carlo_filter_integration(tmp_db):
+    pb = get_game_by_name("Powerball")
+    _large_draw_set(pb["id"])
+    for _ in range(5):
+        _assert_filtered(monte_carlo(pb["id"], simulations=500), pb)
+
+
+def test_quick_pick_filter_integration(tmp_db):
+    pb = get_game_by_name("Powerball")
+    _large_draw_set(pb["id"])
+    for _ in range(20):
+        _assert_filtered(quick_pick(pb["id"]), pb)
+
+
+def test_positional_pick_filter_integration(tmp_db):
+    pb = get_game_by_name("Powerball")
+    _large_draw_set(pb["id"])
+    for _ in range(10):
+        _assert_filtered(positional_pick(pb["id"]), pb)