""" migration_attendance_indexes.py =============================== Adds the secondary indexes `attendance_data` has been missing. Without them the Attendance Report (ORDER BY date/time, filters), the check-in cooldown lookup, the check-out work-type suggestion and the Verification Review scan and sort the whole table, so they slow down as history grows. Uses pymysql directly (no Flask app / ORM import). Indexes are built with online DDL (ALGORITHM=INPLACE, LOCK=NONE) so check-ins keep working while they build; if the server refuses that, the plain ALTER TABLE is used instead. Run once on each server (LT and GOV) — before or after deploying the code: python3 tools/migration_attendance_indexes.py Safe to re-run — an index is skipped when one with the same name, or an existing index that already starts with the same columns, is present. """ import os, sys, re sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from dotenv import load_dotenv load_dotenv() import pymysql TABLE = 'attendance_data' # (index name, columns, what it speeds up) INDEXES = [ ('idx_ad_date_time', ('check_in_date', 'check_in_time'), 'Attendance Report sort and date filters'), ('idx_ad_emp_date_time', ('employee_id', 'check_in_date', 'check_in_time'), 'employee filter, check-out work-type suggestion'), ('idx_ad_qr_emp_date', ('qr_code_id', 'employee_id', 'check_in_date'), 'check-in cooldown guard'), ('idx_ad_location_name', ('location_name',), 'location filter and dropdown'), ('idx_ad_verif_status', ('verification_status',), 'Verification Review status filter'), ] def parse_db_url(url): """ Parse DATABASE_URL robustly using regex to handle special characters (including @ or : ) in the password. """ url = re.sub(r'^mysql\+pymysql://', '', url) url = re.sub(r'^mysql://', '', url) m = re.match( r'^(?P[^:]+):(?P.+)@(?P[^@:/]+)(?::(?P\d+))?/(?P[^?]+)', url ) if not m: print(f"[ERROR] Could not parse DATABASE_URL. Raw (redacted): {url[:30]}...") sys.exit(1) return { 'host': m.group('host'), 'port': int(m.group('port')) if m.group('port') else 3306, 'user': m.group('user'), 'password': m.group('password'), 'database': m.group('db'), } def get_connection(): db_url = os.environ.get('DATABASE_URL', '') if not db_url: print("[ERROR] DATABASE_URL not set in .env") sys.exit(1) params = parse_db_url(db_url) return pymysql.connect(charset='utf8mb4', autocommit=True, **params) def existing_indexes(cur): """{index_name: (col1, col2, ...)} for the table.""" cur.execute( "SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS " "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s " "ORDER BY INDEX_NAME, SEQ_IN_INDEX", (TABLE,) ) indexes = {} for index_name, column_name in cur.fetchall(): indexes.setdefault(index_name, []).append(column_name) return {name: tuple(cols) for name, cols in indexes.items()} def existing_columns(cur): cur.execute( "SELECT COLUMN_NAME FROM information_schema.COLUMNS " "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s", (TABLE,) ) return {row[0] for row in cur.fetchall()} def run(): conn = get_connection() created = skipped = failed = 0 try: with conn.cursor() as cur: columns = existing_columns(cur) if not columns: print(f"[ERROR] Table '{TABLE}' not found in this database.") sys.exit(1) for name, cols, purpose in INDEXES: indexes = existing_indexes(cur) missing = [c for c in cols if c not in columns] if missing: print(f"[SKIP] {name}: column(s) {', '.join(missing)} not present on '{TABLE}'.") skipped += 1 continue if name in indexes: print(f"[SKIP] {name}: already exists.") skipped += 1 continue covering = [n for n, existing in indexes.items() if existing[:len(cols)] == cols] if covering: print(f"[SKIP] {name}: already covered by index '{covering[0]}' {indexes[covering[0]]}.") skipped += 1 continue column_sql = ', '.join(f"`{c}`" for c in cols) print(f"[ADD] {name} ({', '.join(cols)}) — {purpose} ...") try: cur.execute( f"ALTER TABLE `{TABLE}` ADD INDEX `{name}` ({column_sql}), " f"ALGORITHM=INPLACE, LOCK=NONE" ) except pymysql.MySQLError as online_error: print(f" Online DDL not available ({online_error}); retrying with a plain ALTER TABLE ...") try: cur.execute(f"ALTER TABLE `{TABLE}` ADD INDEX `{name}` ({column_sql})") except pymysql.MySQLError as error: print(f"[FAIL] {name}: {error}") failed += 1 continue if name in existing_indexes(cur): print(f"[OK] {name} created.") created += 1 else: print(f"[FAIL] {name} was not created — check DB permissions.") failed += 1 finally: conn.close() print(f"\nDone: {created} created, {skipped} skipped, {failed} failed.") if failed: sys.exit(1) if __name__ == '__main__': run()