Sep 15 - Update the attendance records table to update every 20 min
This commit is contained in:
+323
-80
@@ -45,6 +45,99 @@ bp = Blueprint('attendance', __name__)
|
||||
UPPER_EMPLOYEE_ID_SQL = "UPPER(ad.employee_id)"
|
||||
|
||||
|
||||
def _attendance_filter_conditions(user_role, allowed_project_ids, allowed_location_names,
|
||||
date_from, date_to, location_filter, employee_ids, project_filter):
|
||||
"""
|
||||
Build the WHERE conditions + bound parameters for attendance_data queries.
|
||||
|
||||
Shared by the Attendance Report page and its live-update endpoint so both
|
||||
always apply the same Project Manager scope and the same user filters.
|
||||
Every value is a bound parameter. Returns (filter_conditions, query_params).
|
||||
"""
|
||||
filter_conditions = []
|
||||
query_params = {}
|
||||
|
||||
# ============================================================
|
||||
# APPLY PROJECT MANAGER FILTERS TO SQL QUERY
|
||||
# ============================================================
|
||||
if user_role == 'project_manager':
|
||||
# Filter by allowed projects
|
||||
if allowed_project_ids:
|
||||
project_placeholders = ','.join([f':project_{i}' for i in range(len(allowed_project_ids))])
|
||||
filter_conditions.append(f"qc.project_id IN ({project_placeholders})")
|
||||
for i, pid in enumerate(allowed_project_ids):
|
||||
query_params[f'project_{i}'] = pid
|
||||
|
||||
# Filter by allowed locations
|
||||
if allowed_location_names:
|
||||
location_placeholders = ','.join([f':location_{i}' for i in range(len(allowed_location_names))])
|
||||
filter_conditions.append(f"ad.location_name IN ({location_placeholders})")
|
||||
for i, loc in enumerate(allowed_location_names):
|
||||
query_params[f'location_{i}'] = loc
|
||||
# ============================================================
|
||||
# END: APPLY PROJECT MANAGER FILTERS
|
||||
# ============================================================
|
||||
|
||||
# Apply user-selected filters
|
||||
if date_from:
|
||||
filter_conditions.append("ad.check_in_date >= :date_from")
|
||||
query_params['date_from'] = date_from
|
||||
|
||||
if date_to:
|
||||
filter_conditions.append("ad.check_in_date <= :date_to")
|
||||
query_params['date_to'] = date_to
|
||||
|
||||
if location_filter:
|
||||
# Exact match — dropdown value IS the exact location_name string
|
||||
filter_conditions.append("ad.location_name = :location")
|
||||
query_params['location'] = location_filter
|
||||
|
||||
if employee_ids:
|
||||
# Expand each selected ID into every stored spelling so extra-work
|
||||
# check-ins (SP / PW / PT) are included alongside regular records.
|
||||
# Two branches: exact match on the raw column (uses the index), plus a
|
||||
# REGEXP match that catches any separator style ("1234.PW", "1234-SP").
|
||||
exact_variants, regex_patterns = expand_employee_id_filter(employee_ids)
|
||||
# Never emit an empty IN () — fall back to the raw selection if expansion
|
||||
# somehow produced nothing, so the filter can't degrade into "match all".
|
||||
if not exact_variants:
|
||||
exact_variants = list(employee_ids)
|
||||
|
||||
exact_placeholders = ', '.join([f':employee_{i}' for i in range(len(exact_variants))])
|
||||
exact_condition = f"ad.employee_id IN ({exact_placeholders})"
|
||||
for i, variant in enumerate(exact_variants):
|
||||
query_params[f'employee_{i}'] = variant
|
||||
|
||||
if regex_patterns:
|
||||
regex_clauses = ' OR '.join([
|
||||
f"{UPPER_EMPLOYEE_ID_SQL} REGEXP :employee_re_{i}"
|
||||
for i in range(len(regex_patterns))
|
||||
])
|
||||
filter_conditions.append(f"({exact_condition} OR {regex_clauses})")
|
||||
for i, pattern in enumerate(regex_patterns):
|
||||
query_params[f'employee_re_{i}'] = pattern
|
||||
else:
|
||||
filter_conditions.append(exact_condition)
|
||||
|
||||
if project_filter:
|
||||
# For standard QR records: match by the QR code's project_id directly.
|
||||
# For dynamic QR records: the dynamic QR itself may not be in any project,
|
||||
# but the employee-selected location corresponds to a standard QR in that
|
||||
# project. Match those by checking if attendance_data.location_name
|
||||
# appears in the locations of QR codes belonging to the selected project.
|
||||
filter_conditions.append(
|
||||
"(qc.project_id = :project OR "
|
||||
"(ad.is_dynamic_qr = 1 AND ad.location_name IN ("
|
||||
" SELECT DISTINCT qc2.location FROM qr_codes qc2 "
|
||||
" WHERE qc2.project_id = :project AND qc2.qr_type = 'standard' "
|
||||
" AND qc2.location IS NOT NULL AND qc2.location != ''"
|
||||
")))"
|
||||
)
|
||||
query_params['project'] = project_filter
|
||||
|
||||
return filter_conditions, query_params
|
||||
|
||||
|
||||
|
||||
@bp.route('/attendance', endpoint='attendance_report')
|
||||
@login_required
|
||||
@@ -225,93 +318,22 @@ def attendance_report():
|
||||
WHERE 1=1
|
||||
"""
|
||||
|
||||
# Prepare filter conditions and parameters
|
||||
filter_conditions = []
|
||||
query_params = {}
|
||||
|
||||
# ============================================================
|
||||
# APPLY PROJECT MANAGER FILTERS TO SQL QUERY
|
||||
# ============================================================
|
||||
if user_role == 'project_manager':
|
||||
# Filter by allowed projects
|
||||
if allowed_project_ids:
|
||||
project_placeholders = ','.join([f':project_{i}' for i in range(len(allowed_project_ids))])
|
||||
filter_conditions.append(f"qc.project_id IN ({project_placeholders})")
|
||||
for i, pid in enumerate(allowed_project_ids):
|
||||
query_params[f'project_{i}'] = pid
|
||||
|
||||
# Filter by allowed locations
|
||||
if allowed_location_names:
|
||||
location_placeholders = ','.join([f':location_{i}' for i in range(len(allowed_location_names))])
|
||||
filter_conditions.append(f"ad.location_name IN ({location_placeholders})")
|
||||
for i, loc in enumerate(allowed_location_names):
|
||||
query_params[f'location_{i}'] = loc
|
||||
# ============================================================
|
||||
# END: APPLY PROJECT MANAGER FILTERS
|
||||
# ============================================================
|
||||
|
||||
# Apply user-selected filters
|
||||
if date_from:
|
||||
filter_conditions.append("ad.check_in_date >= :date_from")
|
||||
query_params['date_from'] = date_from
|
||||
|
||||
if date_to:
|
||||
filter_conditions.append("ad.check_in_date <= :date_to")
|
||||
query_params['date_to'] = date_to
|
||||
|
||||
if location_filter:
|
||||
# Exact match — dropdown value IS the exact location_name string
|
||||
filter_conditions.append("ad.location_name = :location")
|
||||
query_params['location'] = location_filter
|
||||
# Prepare filter conditions and parameters — built by the helper shared
|
||||
# with the live-update endpoint, so the Project Manager scope and the
|
||||
# filters can never differ between the page and its live rows.
|
||||
filter_conditions, query_params = _attendance_filter_conditions(
|
||||
user_role, allowed_project_ids, allowed_location_names,
|
||||
date_from, date_to, location_filter, employee_ids, project_filter)
|
||||
|
||||
if employee_ids:
|
||||
# Expand each selected ID into every stored spelling so extra-work
|
||||
# check-ins (SP / PW / PT) are included alongside regular records.
|
||||
# Two branches: exact match on the raw column (uses the index), plus a
|
||||
# REGEXP match that catches any separator style ("1234.PW", "1234-SP").
|
||||
exact_variants, regex_patterns = expand_employee_id_filter(employee_ids)
|
||||
# Never emit an empty IN () — fall back to the raw selection if expansion
|
||||
# somehow produced nothing, so the filter can't degrade into "match all".
|
||||
if not exact_variants:
|
||||
exact_variants = list(employee_ids)
|
||||
|
||||
exact_placeholders = ', '.join([f':employee_{i}' for i in range(len(exact_variants))])
|
||||
exact_condition = f"ad.employee_id IN ({exact_placeholders})"
|
||||
for i, variant in enumerate(exact_variants):
|
||||
query_params[f'employee_{i}'] = variant
|
||||
|
||||
if regex_patterns:
|
||||
regex_clauses = ' OR '.join([
|
||||
f"{UPPER_EMPLOYEE_ID_SQL} REGEXP :employee_re_{i}"
|
||||
for i in range(len(regex_patterns))
|
||||
])
|
||||
filter_conditions.append(f"({exact_condition} OR {regex_clauses})")
|
||||
for i, pattern in enumerate(regex_patterns):
|
||||
query_params[f'employee_re_{i}'] = pattern
|
||||
else:
|
||||
filter_conditions.append(exact_condition)
|
||||
exact_variant_count = len([k for k in query_params
|
||||
if k.startswith('employee_') and not k.startswith('employee_re_')])
|
||||
logger_handler.logger.info(
|
||||
f"Attendance report filtered by employee IDs: {employee_ids} "
|
||||
f"(matching {len(exact_variants)} ID variants incl. SP/PW/PT) "
|
||||
f"(matching {exact_variant_count} ID variants incl. SP/PW/PT) "
|
||||
f"by user {session.get('username', 'unknown')}"
|
||||
)
|
||||
|
||||
if project_filter:
|
||||
# For standard QR records: match by the QR code's project_id directly.
|
||||
# For dynamic QR records: the dynamic QR itself may not be in any project,
|
||||
# but the employee-selected location corresponds to a standard QR in that
|
||||
# project. Match those by checking if attendance_data.location_name
|
||||
# appears in the locations of QR codes belonging to the selected project.
|
||||
filter_conditions.append(
|
||||
"(qc.project_id = :project OR "
|
||||
"(ad.is_dynamic_qr = 1 AND ad.location_name IN ("
|
||||
" SELECT DISTINCT qc2.location FROM qr_codes qc2 "
|
||||
" WHERE qc2.project_id = :project AND qc2.qr_type = 'standard' "
|
||||
" AND qc2.location IS NOT NULL AND qc2.location != ''"
|
||||
")))"
|
||||
)
|
||||
query_params['project'] = project_filter
|
||||
|
||||
# Combine query with filters
|
||||
if filter_conditions:
|
||||
base_query += " AND " + " AND ".join(filter_conditions)
|
||||
@@ -320,6 +342,16 @@ def attendance_report():
|
||||
ATTENDANCE_PAGE_LIMIT = 1000
|
||||
base_query += f" ORDER BY ad.check_in_date DESC, ad.check_in_time DESC LIMIT {ATTENDANCE_PAGE_LIMIT + 1}"
|
||||
|
||||
# Starting cursor for the live table: the newest attendance id right now.
|
||||
# Read BEFORE the report query, so a check-in saved in between is picked
|
||||
# up by the first poll (the browser de-duplicates by id). Primary-key MAX.
|
||||
try:
|
||||
live_latest_id = int(db.session.execute(
|
||||
text("SELECT COALESCE(MAX(id), 0) FROM attendance_data")).scalar() or 0)
|
||||
except Exception as live_error:
|
||||
logger_handler.logger.warning(f"Could not read latest attendance id for live updates: {live_error}")
|
||||
live_latest_id = None # page still renders; live updates stay off
|
||||
|
||||
logger_handler.logger.debug(f"Executing attendance query with filters: {list(query_params.keys())}")
|
||||
|
||||
# Execute query
|
||||
@@ -556,6 +588,7 @@ def attendance_report():
|
||||
attendance_records=processed_records,
|
||||
records_truncated=records_truncated,
|
||||
records_limit=ATTENDANCE_PAGE_LIMIT,
|
||||
live_latest_id=live_latest_id,
|
||||
locations=locations,
|
||||
projects=projects,
|
||||
stats=stats,
|
||||
@@ -587,6 +620,216 @@ def attendance_report():
|
||||
flash('Error loading attendance report. Please check the server logs for details.', 'error')
|
||||
return redirect(url_for('dashboard.dashboard'))
|
||||
|
||||
# ============================================================
|
||||
# LIVE UPDATES for the Attendance Report table
|
||||
# ============================================================
|
||||
# The report page polls attendance_live_updates_api(). It is designed to cost
|
||||
# almost nothing: every poll is one primary-key MAX(id) lookup, and the filtered
|
||||
# query only runs when a newer record exists, scanning only ids above since_id.
|
||||
LIVE_UPDATE_MAX_RECORDS = 200
|
||||
# Cached once the column is confirmed; information_schema is not queried per poll
|
||||
_live_has_location_accuracy = None
|
||||
|
||||
|
||||
def _live_no_store_json(payload, status=200):
|
||||
"""JSON response that browsers and proxies must never cache."""
|
||||
response = jsonify(payload)
|
||||
response.status_code = status
|
||||
response.headers['Cache-Control'] = 'no-store, no-cache, max-age=0, must-revalidate'
|
||||
response.headers['Pragma'] = 'no-cache'
|
||||
return response
|
||||
|
||||
|
||||
def _live_text(value):
|
||||
"""Text as Jinja renders it into the report (None renders as 'None')."""
|
||||
return 'None' if value is None else str(value)
|
||||
|
||||
|
||||
def _live_format_time(check_in_time):
|
||||
"""The HH:MM text the report template renders for a time / timedelta / str."""
|
||||
if not check_in_time:
|
||||
return 'N/A'
|
||||
if isinstance(check_in_time, str):
|
||||
return check_in_time
|
||||
if hasattr(check_in_time, 'strftime'):
|
||||
return check_in_time.strftime('%H:%M')
|
||||
if hasattr(check_in_time, 'total_seconds'):
|
||||
total_seconds = int(check_in_time.total_seconds())
|
||||
return f"{(total_seconds // 3600) % 24:02d}:{(total_seconds % 3600) // 60:02d}"
|
||||
return str(check_in_time)
|
||||
|
||||
|
||||
def _live_record_payload(row, has_location_accuracy):
|
||||
"""
|
||||
One attendance row in exactly the shape loadTableData() in
|
||||
static/js/attendance_report.js reads from the server-rendered table, so a
|
||||
live row sorts, filters, paginates and renders like the rows loaded with
|
||||
the page. Mirrors the <tbody> markup of templates/attendance_report.html —
|
||||
keep the two in sync.
|
||||
"""
|
||||
(record_id, employee_id, check_in_date, check_in_time, location_name, location_event,
|
||||
qr_address, checked_in_address, location_accuracy, gps_accuracy, device_info,
|
||||
created_timestamp, updated_timestamp, employee_name, verification_required,
|
||||
verification_status, is_dynamic_qr) = row
|
||||
|
||||
# Verification badge, as extractVerificationData() reads it
|
||||
if verification_required and verification_status == 'pending':
|
||||
verification = (True, 'pending')
|
||||
elif verification_status in ('approved', 'rejected'):
|
||||
verification = (True, verification_status)
|
||||
else:
|
||||
verification = (False, None)
|
||||
|
||||
# Accuracy number, as extractLocationAccuracy() parses the rendered badge text
|
||||
if verification[0]:
|
||||
accuracy = round(float(location_accuracy), 3) if location_accuracy is not None else None
|
||||
elif has_location_accuracy and location_accuracy:
|
||||
accuracy = round(float(location_accuracy), 3)
|
||||
elif gps_accuracy:
|
||||
accuracy = round(float(gps_accuracy), 1) * 0.000621371 # badge shows metres
|
||||
else:
|
||||
accuracy = None
|
||||
|
||||
if isinstance(check_in_date, str):
|
||||
date_text = check_in_date
|
||||
else:
|
||||
date_text = check_in_date.strftime('%m/%d/%Y') if check_in_date else ''
|
||||
|
||||
checked_in_text = (checked_in_address or 'N/A')[:50]
|
||||
if len(checked_in_address or '') > 50:
|
||||
checked_in_text += '...'
|
||||
device_text = device_info or ''
|
||||
device_text = device_text[:20] + ('...' if len(device_text) > 20 else '')
|
||||
|
||||
return {
|
||||
'id': str(record_id),
|
||||
'employeeId': _live_text(employee_id).strip(),
|
||||
'employeeName': (employee_name or 'Unknown Employee').strip(),
|
||||
'location': ('(No location recorded)' if location_name == 'Dynamic'
|
||||
else _live_text(location_name)).strip(),
|
||||
'event': _live_text(location_event).strip(),
|
||||
'date': date_text.strip(),
|
||||
'time': _live_format_time(check_in_time).strip(),
|
||||
'qr_address': ((qr_address[:50] + ('...' if len(qr_address) > 50 else ''))
|
||||
if qr_address else 'N/A').strip(),
|
||||
'checked_in_address': checked_in_text.strip(),
|
||||
'location_accuracy': accuracy,
|
||||
'accuracy_level': ('unknown' if accuracy is None
|
||||
else ('accurate' if accuracy < 0.3 else 'inaccurate')),
|
||||
'device': device_text.strip(),
|
||||
'isModified': bool(updated_timestamp and created_timestamp
|
||||
and updated_timestamp > created_timestamp),
|
||||
'isDynamic': bool(is_dynamic_qr) or location_name == 'Dynamic',
|
||||
'verification_required': verification[0],
|
||||
'verification_status': verification[1],
|
||||
}
|
||||
|
||||
|
||||
@bp.route('/api/attendance/live-updates', endpoint='attendance_live_updates_api')
|
||||
@login_required
|
||||
def attendance_live_updates_api():
|
||||
"""
|
||||
New attendance records for the Attendance Report's live table.
|
||||
|
||||
GET params: since_id (newest attendance id the page already knows) plus the
|
||||
page's own filters: date_from, date_to, location, employee, project.
|
||||
Returns {success, latest_id, records, has_more}; the page sends latest_id
|
||||
back as since_id on its next poll.
|
||||
"""
|
||||
global _live_has_location_accuracy
|
||||
try:
|
||||
try:
|
||||
since_id = int(request.args.get('since_id', ''))
|
||||
except (TypeError, ValueError):
|
||||
return _live_no_store_json({'success': False, 'message': 'since_id is required.'}, 400)
|
||||
|
||||
# Cheap path — taken by almost every poll: nothing newer than since_id
|
||||
latest_id = int(db.session.execute(
|
||||
text("SELECT COALESCE(MAX(id), 0) FROM attendance_data")).scalar() or 0)
|
||||
empty = {'success': True, 'latest_id': max(latest_id, since_id), 'records': [], 'has_more': False}
|
||||
if latest_id <= since_id:
|
||||
return _live_no_store_json(empty)
|
||||
|
||||
# Same Project Manager scope as the report page
|
||||
user_role = session.get('role')
|
||||
allowed_project_ids, allowed_location_names = [], []
|
||||
if user_role == 'project_manager':
|
||||
user_id = session.get('user_id')
|
||||
allowed_project_ids = [p.project_id for p in
|
||||
UserProjectPermission.query.filter_by(user_id=user_id).all()]
|
||||
allowed_location_names = [l.location_name for l in
|
||||
UserLocationPermission.query.filter_by(user_id=user_id).all()]
|
||||
if not allowed_project_ids and not allowed_location_names:
|
||||
return _live_no_store_json(empty)
|
||||
|
||||
employee_filter = request.args.get('employee', '')
|
||||
employee_ids = [e.strip() for e in employee_filter.split(',') if e.strip()]
|
||||
filter_conditions, query_params = _attendance_filter_conditions(
|
||||
user_role, allowed_project_ids, allowed_location_names,
|
||||
request.args.get('date_from', ''), request.args.get('date_to', ''),
|
||||
request.args.get('location', ''), employee_ids, request.args.get('project', ''))
|
||||
|
||||
# Only ids the page has not seen yet (a primary-key range scan)
|
||||
filter_conditions = ["ad.id > :live_since_id", "ad.id <= :live_latest_id"] + filter_conditions
|
||||
query_params['live_since_id'] = since_id
|
||||
query_params['live_latest_id'] = latest_id
|
||||
|
||||
if _live_has_location_accuracy is None and check_location_accuracy_column_exists():
|
||||
_live_has_location_accuracy = True
|
||||
has_location_accuracy = bool(_live_has_location_accuracy)
|
||||
accuracy_column = "ad.location_accuracy" if has_location_accuracy else "NULL"
|
||||
|
||||
# Same columns as the report query minus verification_photo (base64 image)
|
||||
where_clause = " AND ".join(filter_conditions)
|
||||
rows = db.session.execute(text(f"""
|
||||
SELECT
|
||||
ad.id,
|
||||
ad.employee_id,
|
||||
ad.check_in_date,
|
||||
ad.check_in_time,
|
||||
ad.location_name,
|
||||
qc.location_event,
|
||||
COALESCE(ad.qr_address, qc.location_address) AS qr_address,
|
||||
ad.address AS checked_in_address,
|
||||
{accuracy_column} AS location_accuracy,
|
||||
ad.accuracy AS gps_accuracy,
|
||||
ad.device_info,
|
||||
ad.created_timestamp,
|
||||
ad.updated_timestamp,
|
||||
CONCAT(e.firstName, ' ', e.lastName) AS employee_name,
|
||||
ad.verification_required,
|
||||
ad.verification_status,
|
||||
COALESCE(ad.is_dynamic_qr, 0) AS is_dynamic_qr
|
||||
FROM attendance_data ad
|
||||
LEFT JOIN qr_codes qc ON ad.qr_code_id = qc.id
|
||||
LEFT JOIN employee e ON CAST(ad.employee_id AS UNSIGNED) = e.id
|
||||
WHERE {where_clause}
|
||||
ORDER BY ad.id ASC
|
||||
LIMIT {LIVE_UPDATE_MAX_RECORDS + 1}
|
||||
"""), query_params).fetchall()
|
||||
|
||||
has_more = len(rows) > LIVE_UPDATE_MAX_RECORDS
|
||||
rows = rows[:LIVE_UPDATE_MAX_RECORDS]
|
||||
# Capped batch: continue after the last row sent. Otherwise every id up
|
||||
# to latest_id has been examined.
|
||||
cursor = rows[-1][0] if has_more else latest_id
|
||||
|
||||
records = []
|
||||
for row in rows:
|
||||
try:
|
||||
records.append(_live_record_payload(row, has_location_accuracy))
|
||||
except Exception as rec_error:
|
||||
logger_handler.logger.warning(f"Live updates: skipped attendance record {row[0]}: {rec_error}")
|
||||
|
||||
return _live_no_store_json({'success': True, 'latest_id': cursor,
|
||||
'records': records, 'has_more': has_more})
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
logger_handler.logger.error(f"Error loading attendance live updates: {e}", exc_info=True)
|
||||
return _live_no_store_json({'success': False, 'message': 'Could not load new records.'}, 500)
|
||||
|
||||
|
||||
@bp.route('/api/time-attendance/locations', endpoint='time_attendance_locations_api')
|
||||
@login_required
|
||||
def time_attendance_locations_api():
|
||||
|
||||
Reference in New Issue
Block a user