Sep 16 - Optimize code, part 1
This commit is contained in:
@@ -0,0 +1,163 @@
|
||||
"""
|
||||
migration_attendance_indexes.py
|
||||
===============================
|
||||
Adds the secondary indexes `attendance_data` has been missing.
|
||||
|
||||
Without them the Attendance Report (ORDER BY date/time, filters), the check-in
|
||||
cooldown lookup, the check-out work-type suggestion and the Verification Review
|
||||
scan and sort the whole table, so they slow down as history grows.
|
||||
|
||||
Uses pymysql directly (no Flask app / ORM import). Indexes are built with online
|
||||
DDL (ALGORITHM=INPLACE, LOCK=NONE) so check-ins keep working while they build;
|
||||
if the server refuses that, the plain ALTER TABLE is used instead.
|
||||
|
||||
Run once on each server (LT and GOV) — before or after deploying the code:
|
||||
python3 tools/migration_attendance_indexes.py
|
||||
|
||||
Safe to re-run — an index is skipped when one with the same name, or an
|
||||
existing index that already starts with the same columns, is present.
|
||||
"""
|
||||
|
||||
import os, sys, re
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
import pymysql
|
||||
|
||||
TABLE = 'attendance_data'
|
||||
|
||||
# (index name, columns, what it speeds up)
|
||||
INDEXES = [
|
||||
('idx_ad_date_time', ('check_in_date', 'check_in_time'),
|
||||
'Attendance Report sort and date filters'),
|
||||
('idx_ad_emp_date_time', ('employee_id', 'check_in_date', 'check_in_time'),
|
||||
'employee filter, check-out work-type suggestion'),
|
||||
('idx_ad_qr_emp_date', ('qr_code_id', 'employee_id', 'check_in_date'),
|
||||
'check-in cooldown guard'),
|
||||
('idx_ad_location_name', ('location_name',),
|
||||
'location filter and dropdown'),
|
||||
('idx_ad_verif_status', ('verification_status',),
|
||||
'Verification Review status filter'),
|
||||
]
|
||||
|
||||
|
||||
def parse_db_url(url):
|
||||
"""
|
||||
Parse DATABASE_URL robustly using regex to handle special characters
|
||||
(including @ or : ) in the password.
|
||||
"""
|
||||
url = re.sub(r'^mysql\+pymysql://', '', url)
|
||||
url = re.sub(r'^mysql://', '', url)
|
||||
m = re.match(
|
||||
r'^(?P<user>[^:]+):(?P<password>.+)@(?P<host>[^@:/]+)(?::(?P<port>\d+))?/(?P<db>[^?]+)',
|
||||
url
|
||||
)
|
||||
if not m:
|
||||
print(f"[ERROR] Could not parse DATABASE_URL. Raw (redacted): {url[:30]}...")
|
||||
sys.exit(1)
|
||||
return {
|
||||
'host': m.group('host'),
|
||||
'port': int(m.group('port')) if m.group('port') else 3306,
|
||||
'user': m.group('user'),
|
||||
'password': m.group('password'),
|
||||
'database': m.group('db'),
|
||||
}
|
||||
|
||||
|
||||
def get_connection():
|
||||
db_url = os.environ.get('DATABASE_URL', '')
|
||||
if not db_url:
|
||||
print("[ERROR] DATABASE_URL not set in .env")
|
||||
sys.exit(1)
|
||||
params = parse_db_url(db_url)
|
||||
return pymysql.connect(charset='utf8mb4', autocommit=True, **params)
|
||||
|
||||
|
||||
def existing_indexes(cur):
|
||||
"""{index_name: (col1, col2, ...)} for the table."""
|
||||
cur.execute(
|
||||
"SELECT INDEX_NAME, COLUMN_NAME FROM information_schema.STATISTICS "
|
||||
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s "
|
||||
"ORDER BY INDEX_NAME, SEQ_IN_INDEX",
|
||||
(TABLE,)
|
||||
)
|
||||
indexes = {}
|
||||
for index_name, column_name in cur.fetchall():
|
||||
indexes.setdefault(index_name, []).append(column_name)
|
||||
return {name: tuple(cols) for name, cols in indexes.items()}
|
||||
|
||||
|
||||
def existing_columns(cur):
|
||||
cur.execute(
|
||||
"SELECT COLUMN_NAME FROM information_schema.COLUMNS "
|
||||
"WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s",
|
||||
(TABLE,)
|
||||
)
|
||||
return {row[0] for row in cur.fetchall()}
|
||||
|
||||
|
||||
def run():
|
||||
conn = get_connection()
|
||||
created = skipped = failed = 0
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
columns = existing_columns(cur)
|
||||
if not columns:
|
||||
print(f"[ERROR] Table '{TABLE}' not found in this database.")
|
||||
sys.exit(1)
|
||||
|
||||
for name, cols, purpose in INDEXES:
|
||||
indexes = existing_indexes(cur)
|
||||
|
||||
missing = [c for c in cols if c not in columns]
|
||||
if missing:
|
||||
print(f"[SKIP] {name}: column(s) {', '.join(missing)} not present on '{TABLE}'.")
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
if name in indexes:
|
||||
print(f"[SKIP] {name}: already exists.")
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
covering = [n for n, existing in indexes.items() if existing[:len(cols)] == cols]
|
||||
if covering:
|
||||
print(f"[SKIP] {name}: already covered by index '{covering[0]}' {indexes[covering[0]]}.")
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
column_sql = ', '.join(f"`{c}`" for c in cols)
|
||||
print(f"[ADD] {name} ({', '.join(cols)}) — {purpose} ...")
|
||||
try:
|
||||
cur.execute(
|
||||
f"ALTER TABLE `{TABLE}` ADD INDEX `{name}` ({column_sql}), "
|
||||
f"ALGORITHM=INPLACE, LOCK=NONE"
|
||||
)
|
||||
except pymysql.MySQLError as online_error:
|
||||
print(f" Online DDL not available ({online_error}); retrying with a plain ALTER TABLE ...")
|
||||
try:
|
||||
cur.execute(f"ALTER TABLE `{TABLE}` ADD INDEX `{name}` ({column_sql})")
|
||||
except pymysql.MySQLError as error:
|
||||
print(f"[FAIL] {name}: {error}")
|
||||
failed += 1
|
||||
continue
|
||||
|
||||
if name in existing_indexes(cur):
|
||||
print(f"[OK] {name} created.")
|
||||
created += 1
|
||||
else:
|
||||
print(f"[FAIL] {name} was not created — check DB permissions.")
|
||||
failed += 1
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
print(f"\nDone: {created} created, {skipped} skipped, {failed} failed.")
|
||||
if failed:
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
Reference in New Issue
Block a user