Aug 27 - Add MySQL optimize and security check

app/api/inspections.py

@@ -267,8 +267,8 @@ def list_inspections():
     if user.role not in _ALLOWED_ROLES:
         return api_error('Access denied', 403)
 
-    limit  = min(int(request.args.get('limit', 50)), 200)
-    offset = max(int(request.args.get('offset', 0)), 0)
+    limit  = min(request.args.get('limit',  50, type=int) or 50, 200)
+    offset = max(request.args.get('offset', 0,  type=int) or 0,  0)
 
     query = Inspection.query
 

app/api/issues.py

@@ -149,8 +149,8 @@ def list_issues():
     if user.role not in _ALLOWED_ROLES:
         return api_error('Access denied', 403)
 
-    limit  = min(int(request.args.get('limit',  100)), 200)
-    offset = max(int(request.args.get('offset', 0)),   0)
+    limit  = min(request.args.get('limit',  100, type=int) or 100, 200)
+    offset = max(request.args.get('offset', 0,   type=int) or 0,   0)
 
     query = Issue.query
 

app/api/notifications.py

@@ -73,7 +73,7 @@ def list_notifications():
     """
     user  = g.api_user
     since = _parse_since(request.args.get('since'))
-    limit = min(int(request.args.get('limit', 50)), 50)
+    limit = min(request.args.get('limit', 50, type=int) or 50, 50)
 
     def _run_orm():
         q = Notification.query.filter_by(user_id=user.id, is_read=False)

app/api/scheduled.py

@@ -103,8 +103,8 @@ def list_scheduled():
     if user.role not in _ALLOWED_ROLES:
         return api_error('Access denied', 403)
 
-    limit  = min(int(request.args.get('limit',  100)), 200)
-    offset = max(int(request.args.get('offset', 0)),   0)
+    limit  = min(request.args.get('limit',  100, type=int) or 100, 200)
+    offset = max(request.args.get('offset', 0,   type=int) or 0,   0)
 
     query = ScheduledInspection.query.filter(ScheduledInspection.active.is_(True))
 

app/api/stats.py

@@ -111,7 +111,14 @@ def dashboard_stats():
                 )
             )
 
-    open_issues_all = open_q.all()
+    # Counts and buckets only — never a hydrated Issue. For an admin this is
+    # every open issue in the system, fetched on every iPad dashboard refresh;
+    # the full entity would drag the description TEXT and the JSON photo
+    # columns along with it. A Row exposes the same attribute names, so
+    # sla_status() below works unchanged.
+    open_issues_all = open_q.with_entities(
+        Issue.id, Issue.severity, Issue.status, Issue.reported_at
+    ).all()
     open_issues     = len(open_issues_all)
 
     # ── Severity breakdown (derived from the same open_issues_all list) ───

app/routes/dashboard.py

@@ -17,6 +17,16 @@ bp = Blueprint('dashboard', __name__)
 logger = logging.getLogger(__name__)
 
 
+# Columns the dashboard actually reads off an issue row. The cards below need
+# counts and buckets, never a hydrated Issue — loading the full entity pulls the
+# description TEXT and three JSON photo columns for every open issue in scope,
+# on every dashboard load, and registers each one in the identity map.
+# A Row exposes the same attribute names, so _handler_split() and sla_status()
+# work against these unchanged.
+_ISSUE_CARD_COLS = (Issue.id, Issue.severity, Issue.status,
+                    Issue.reported_at, Issue.handler_type)
+
+
 def _handler_split(issues):
     """Count a list of Issues by handler_type (phase35). Rows default to
     'internal' when unset. Returns a dict keyed internal/facility/vendor."""
@@ -102,7 +112,7 @@ def index():
             ))
 
     # Single query — derive count from the list to avoid hitting the DB twice
-    open_issues_all = open_issues_q.all()
+    open_issues_all = open_issues_q.with_entities(*_ISSUE_CARD_COLS).all()
     open_issues     = len(open_issues_all)
     severity_breakdown = {
         'critical': sum(1 for i in open_issues_all if i.severity == 'critical'),
@@ -224,7 +234,7 @@ def index():
     elif is_customer and not customer_facility_ids:
         all_open_issues = []
     else:
-        all_open_issues = sla_q.all()
+        all_open_issues = sla_q.with_entities(*_ISSUE_CARD_COLS).all()
     sla_breached = sum(1 for i in all_open_issues if sla_status(i) == 'breached')
     sla_at_risk  = sum(1 for i in all_open_issues if sla_status(i) == 'at_risk')
 
@@ -250,7 +260,7 @@ def index():
                 Issue.facility_id.in_(customer_facility_ids),
                 _AreaT.facility_id.in_(customer_facility_ids),
             ))
-    opened_today_all    = opened_today_q.all()
+    opened_today_all    = opened_today_q.with_entities(*_ISSUE_CARD_COLS).all()
     issues_opened_today = len(opened_today_all)
     opened_today_handler = _handler_split(opened_today_all)
 
@@ -331,7 +341,7 @@ def index():
             ))
     elif is_customer:
         unassigned_q = unassigned_q.filter(False)  # not relevant for customers
-    unassigned_all     = unassigned_q.all()
+    unassigned_all     = unassigned_q.with_entities(*_ISSUE_CARD_COLS).all()
     unassigned_open    = len(unassigned_all)
     unassigned_handler = _handler_split(unassigned_all)
 

app/routes/inspections.py

@@ -253,7 +253,8 @@ def index():
         q = q.filter(Inspection.status == status_filter)
     if contract_filter.isdigit():
         _contract_fids = [
-            f.id for f in Facility.query.filter_by(project_id=int(contract_filter)).all()
+            fid for (fid,) in db.session.query(Facility.id)
+            .filter(Facility.project_id == int(contract_filter)).all()
         ]
         q = q.filter(Inspection.facility_id.in_(_contract_fids)) if _contract_fids else q.filter(False)
     if facility_filter.isdigit():
@@ -1243,7 +1244,8 @@ def export_list_pdf():
         q = q.filter(Inspection.status == status_filter)
     if contract_filter.isdigit():
         _contract_fids = [
-            f.id for f in Facility.query.filter_by(project_id=int(contract_filter)).all()
+            fid for (fid,) in db.session.query(Facility.id)
+            .filter(Facility.project_id == int(contract_filter)).all()
         ]
         q = q.filter(Inspection.facility_id.in_(_contract_fids)) if _contract_fids else q.filter(False)
     if facility_filter.isdigit():

app/routes/issues.py

@@ -166,13 +166,14 @@ def export_list_pdf():
             date_to_filter = ''
     if contract_filter.isdigit():
         _contract_fids = [
-            f.id for f in Facility.query.filter_by(project_id=int(contract_filter)).all()
+            fid for (fid,) in db.session.query(Facility.id)
+            .filter(Facility.project_id == int(contract_filter)).all()
         ]
         q = q.filter(db.or_(
             Issue.facility_id.in_(_contract_fids),
             Area.facility_id.in_(_contract_fids),
         )) if _contract_fids else q.filter(False)
-    if facility_filter:
+    if facility_filter.isdigit():
         fid = int(facility_filter)
         q = q.filter(db.or_(Issue.facility_id == fid, Area.facility_id == fid))
     if reporter_filter.isdigit():
@@ -201,7 +202,7 @@ def export_list_pdf():
         p = db.session.get(Project, int(contract_filter))
         if p:
             filter_parts.append(f'Contract: {p.name}')
-    if facility_filter:
+    if facility_filter.isdigit():
         f = db.session.get(Facility, int(facility_filter))
         if f:
             filter_parts.append(f'Facility: {f.name}')
@@ -303,7 +304,8 @@ def index():
             date_to_filter = ''
     if contract_filter.isdigit():
         _contract_fids = [
-            f.id for f in Facility.query.filter_by(project_id=int(contract_filter)).all()
+            fid for (fid,) in db.session.query(Facility.id)
+            .filter(Facility.project_id == int(contract_filter)).all()
         ]
         q = q.filter(db.or_(
             Issue.facility_id.in_(_contract_fids),
@@ -311,7 +313,7 @@ def index():
         )) if _contract_fids else q.filter(False)
     if reporter_filter.isdigit():
         q = q.filter(Issue.reported_by == int(reporter_filter))
-    if facility_filter:
+    if facility_filter.isdigit():
         fid = int(facility_filter)
         q = q.filter(
             db.or_(
@@ -338,8 +340,9 @@ def index():
     # can render the following badge and inline unfollow button without an
     # additional query per row.
     followed_ids = {
-        f.issue_id
-        for f in IssueFollower.query.filter_by(user_id=current_user.id).all()
+        iid for (iid,) in
+        db.session.query(IssueFollower.issue_id)
+        .filter(IssueFollower.user_id == current_user.id).all()
     }
 
     # Facilities for the filter dropdown — scoped for inspectors/customers,

app/routes/reports.py

@@ -74,23 +74,29 @@ def index():
     if current_user.role in ('admin', 'director', 'project_manager'):
         inspector_filter = request.args.get('inspector_id', type=int) or None
 
-    # Pre-compute inspection ID sets used by _scope_issue to avoid join conflicts.
-    inspector_inspection_ids = []   # own inspections (inspector role)
-    filter_inspection_ids    = None # filtered inspector's inspections (admin/dir/PM)
+    # Scope issues by the relevant inspector's inspections, as a SUBQUERY rather
+    # than a materialised id list. The previous form pulled every inspection id
+    # that inspector had ever performed into Python and sent them straight back
+    # as a literal IN (1, 2, 3, ... N): the round trip is wasted, the statement
+    # grows without bound with the inspector's history, and a long enough list
+    # eventually trips max_allowed_packet. A subquery is also still a single
+    # statement, so the "avoid join conflicts" reason for pre-computing holds.
+    #
+    # IN (empty subquery) already matches nothing, so the explicit empty-list
+    # guards the old code needed are gone rather than merely moved.
+    inspector_insp_subq = None
     if is_inspector:
-        inspector_inspection_ids = [
-            row[0] for row in
+        inspector_insp_subq = (
             db.session.query(Inspection.id)
             .filter(Inspection.inspector_id == current_user.id)
-            .all()
-        ]
+            .scalar_subquery()
+        )
     elif inspector_filter:
-        filter_inspection_ids = [
-            row[0] for row in
+        inspector_insp_subq = (
             db.session.query(Inspection.id)
             .filter(Inspection.inspector_id == inspector_filter)
-            .all()
-        ]
+            .scalar_subquery()
+        )
 
     def _scope_insp(q):
         if is_inspector:
@@ -104,14 +110,8 @@ def index():
         return q
 
     def _scope_issue(q):
-        if is_inspector:
-            if not inspector_inspection_ids:
-                return q.filter(False)
-            return q.filter(Issue.inspection_id.in_(inspector_inspection_ids))
-        if filter_inspection_ids is not None:
-            if not filter_inspection_ids:
-                return q.filter(False)
-            return q.filter(Issue.inspection_id.in_(filter_inspection_ids))
+        if inspector_insp_subq is not None:
+            return q.filter(Issue.inspection_id.in_(inspector_insp_subq))
         if customer_facility_ids is not None:
             if not customer_facility_ids:
                 return q.filter(False)

app/utils/scope.py

@@ -18,6 +18,7 @@ that no facility-level scoping is required (full access applies).
 """
 
 import logging
+from app import db
 from app.models.project import CustomerAssignment
 from app.models.facility import Facility
 
@@ -43,30 +44,34 @@ def get_customer_scope(user) -> list[int] | None:
     if user.role != 'customer':
         return None  # no scoping needed for internal staff
 
-    assignments = CustomerAssignment.query.filter_by(user_id=user.id).all()
+    # Select only the two columns needed. The previous .all() built full
+    # CustomerAssignment ORM objects (and their identity-map entries) purely to
+    # read two integers off each one; this function runs on nearly every
+    # request for a customer, sometimes more than once.
+    assignments = db.session.query(
+        CustomerAssignment.project_id,
+        CustomerAssignment.facility_id,
+    ).filter(CustomerAssignment.user_id == user.id).all()
 
     if not assignments:
         return []
 
     # Separate direct facility assignments from project-level assignments
-    direct_facility_ids = {a.facility_id for a in assignments if a.facility_id}
-    project_ids         = {a.project_id  for a in assignments if not a.facility_id}
+    direct_facility_ids = {fac_id for _, fac_id in assignments if fac_id}
+    project_ids         = {proj_id for proj_id, fac_id in assignments if not fac_id}
 
     facility_ids = set(direct_facility_ids)
 
     # Single bulk query for all project-scoped facilities — replaces the
-    # previous per-assignment Facility.query loop (N+1 pattern).
+    # previous per-assignment Facility.query loop (N+1 pattern). Only the id
+    # column is read; nothing here needs a hydrated Facility.
     if project_ids:
-        project_facilities = (
-            Facility.query
-            .filter(
+        facility_ids.update(
+            fid for (fid,) in db.session.query(Facility.id).filter(
                 Facility.project_id.in_(project_ids),
                 Facility.active == True,
-            )
-            .all()
+            ).all()
         )
-        for f in project_facilities:
-            facility_ids.add(f.id)
 
     logger.debug(
         'SCOPE | customer_scope | user_id=%s username=%s facility_ids=%s',
@@ -102,16 +107,19 @@ def get_inspector_scope(user) -> list[int] | None:
 
     from app.models.inspector_assignment import InspectorAssignment
 
+    # Column-only selects — see the note in get_customer_scope(). This runs on
+    # every scoped request for both inspector roles.
     project_ids = [
-        a.project_id
-        for a in InspectorAssignment.query.filter_by(user_id=user.id).all()
+        pid for (pid,) in
+        db.session.query(InspectorAssignment.project_id)
+        .filter(InspectorAssignment.user_id == user.id).all()
     ]
 
     if not project_ids:
         return []  # strict: no assignments = no access
 
     facility_ids = [
-        f.id for f in Facility.query.filter(
+        fid for (fid,) in db.session.query(Facility.id).filter(
             Facility.project_id.in_(project_ids),
             Facility.active == True,
         ).all()

app/utils/sla.py

@@ -110,11 +110,38 @@ def send_sla_alerts():
 
     logger = logging.getLogger(__name__)
 
-    # yield_per streams rows in batches of 100 rather than loading all open
-    # issues into memory at once.  At current scale this is a no-op difference,
-    # but it prevents a memory spike if the issue count grows large.
+    # Narrow to actual CANDIDATES in SQL rather than reading every open issue
+    # and deciding in Python. This runs every 30 minutes forever, so the old
+    # form's cost grew with the whole open-issue backlog even on a quiet night
+    # where nothing was due. Three filters, each mirroring a `continue` below:
+    #
+    #   1. reported_at IS NOT NULL — the column is nullable, and sla_status()
+    #      raises TypeError on a NULL (datetime + timedelta). One such row
+    #      would abort the entire cron run, so exclude it in SQL.
+    #   2. sla_notified <> 'breached' — the highest level is already sent; the
+    #      loop skips these unconditionally.
+    #   3. old enough to be at least at-risk for its OWN severity, i.e.
+    #      reported_at <= now - (window * 0.75). A critical issue qualifies
+    #      after 3h, a low one after 90h.
+    #
+    # Anything this excludes would have hit a `continue` anyway, so the set of
+    # notifications sent is unchanged — only the rows read are.
+    now = now_eastern()
+    age_clauses = [
+        db.and_(
+            Issue.severity == severity,
+            Issue.reported_at <= now - timedelta(hours=hours * AT_RISK_THRESHOLD),
+        )
+        for severity, hours in SLA_HOURS.items()
+    ]
+
+    # yield_per streams the survivors in batches rather than materialising them
+    # all at once.
     open_issues = Issue.query.filter(
-        Issue.status.in_(['open', 'in_progress', 'pending_verification'])
+        Issue.status.in_(['open', 'in_progress', 'pending_verification']),
+        Issue.reported_at.isnot(None),
+        db.or_(Issue.sla_notified.is_(None), Issue.sla_notified != 'breached'),
+        db.or_(*age_clauses),
     ).yield_per(100)
 
     total_sent = 0