Sep 11 - Reupload the code
This commit is contained in:
@@ -0,0 +1,121 @@
|
||||
"""
|
||||
Database Migration Script for Project Manager Permissions (MySQL Version)
|
||||
==========================================================================
|
||||
|
||||
This script creates the necessary tables for Project Manager role permissions.
|
||||
It adds support for assigning specific projects and locations to Project Managers.
|
||||
|
||||
Tables Created:
|
||||
1. user_project_permissions: Links users to projects they can access
|
||||
2. user_location_permissions: Links users to locations they can access
|
||||
|
||||
Run this script ONCE after backing up your database.
|
||||
"""
|
||||
|
||||
from flask import Flask
|
||||
from flask_sqlalchemy import SQLAlchemy
|
||||
from sqlalchemy import text
|
||||
import os
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
# Initialize Flask app for migration
|
||||
app = Flask(__name__)
|
||||
app.config['SQLALCHEMY_DATABASE_URI'] = os.environ.get('DATABASE_URL', 'mysql://user:pass@localhost/qr_management')
|
||||
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
|
||||
db = SQLAlchemy(app)
|
||||
|
||||
def run_migration():
|
||||
"""Execute the migration to add project manager permission tables"""
|
||||
|
||||
with app.app_context():
|
||||
print("\n" + "="*70)
|
||||
print("PROJECT MANAGER PERMISSIONS MIGRATION (MySQL)")
|
||||
print("="*70 + "\n")
|
||||
|
||||
try:
|
||||
# Check if tables already exist
|
||||
print("🔍 Checking if migration is needed...")
|
||||
|
||||
result = db.session.execute(text("""
|
||||
SELECT TABLE_NAME
|
||||
FROM information_schema.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME IN ('user_project_permissions', 'user_location_permissions')
|
||||
"""))
|
||||
existing_tables = [row[0] for row in result.fetchall()]
|
||||
|
||||
if len(existing_tables) == 2:
|
||||
print("✅ Migration tables already exist. No action needed.")
|
||||
return True
|
||||
|
||||
if 'user_project_permissions' in existing_tables:
|
||||
print("⚠️ user_project_permissions table already exists, skipping...")
|
||||
else:
|
||||
# Create user_project_permissions table
|
||||
print("\n📝 Creating user_project_permissions table...")
|
||||
db.session.execute(text("""
|
||||
CREATE TABLE user_project_permissions (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
user_id INT NOT NULL,
|
||||
project_id INT NOT NULL,
|
||||
created_date DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE,
|
||||
FOREIGN KEY (project_id) REFERENCES projects (id) ON DELETE CASCADE,
|
||||
UNIQUE KEY unique_user_project (user_id, project_id),
|
||||
INDEX idx_user_id (user_id),
|
||||
INDEX idx_project_id (project_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
||||
"""))
|
||||
print("✅ user_project_permissions table created successfully")
|
||||
|
||||
if 'user_location_permissions' in existing_tables:
|
||||
print("⚠️ user_location_permissions table already exists, skipping...")
|
||||
else:
|
||||
# Create user_location_permissions table
|
||||
print("\n📝 Creating user_location_permissions table...")
|
||||
db.session.execute(text("""
|
||||
CREATE TABLE user_location_permissions (
|
||||
id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
user_id INT NOT NULL,
|
||||
location_name VARCHAR(200) NOT NULL,
|
||||
created_date DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE CASCADE,
|
||||
UNIQUE KEY unique_user_location (user_id, location_name),
|
||||
INDEX idx_user_id (user_id)
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci
|
||||
"""))
|
||||
print("✅ user_location_permissions table created successfully")
|
||||
|
||||
# Commit all changes
|
||||
db.session.commit()
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("✅ MIGRATION COMPLETED SUCCESSFULLY")
|
||||
print("="*70)
|
||||
print("\nNext Steps:")
|
||||
print("1. The tables are ready for use")
|
||||
print("2. You can now assign projects and locations to Project Managers")
|
||||
print("3. Restart your application\n")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
db.session.rollback()
|
||||
print(f"\n❌ Migration failed: {e}")
|
||||
print("Please check your database and try again.")
|
||||
return False
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("\n⚠️ IMPORTANT: Backup your database before running this migration!")
|
||||
response = input("Continue with migration? (yes/no): ")
|
||||
|
||||
if response.lower() == 'yes':
|
||||
success = run_migration()
|
||||
if success:
|
||||
print("\n✅ Migration completed. You can now restart your application.")
|
||||
else:
|
||||
print("\n❌ Migration failed. Please check the errors above.")
|
||||
else:
|
||||
print("\n❌ Migration cancelled.")
|
||||
@@ -0,0 +1,150 @@
|
||||
"""
|
||||
Migration: Dynamic QR Code Support
|
||||
====================================
|
||||
Applies the following database changes required for the Dynamic QR feature:
|
||||
|
||||
1. Adds qr_type column to qr_codes (VARCHAR 20, default 'standard')
|
||||
2. Creates qr_code_locations table
|
||||
3. Makes qr_codes.location nullable (was NOT NULL)
|
||||
4. Makes qr_codes.location_address nullable (was NOT NULL)
|
||||
|
||||
Usage (run once from the project root):
|
||||
python tools/migration_dynamic_qr_locations.py
|
||||
|
||||
Fully idempotent — safe to run multiple times without side effects.
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from app import app, db
|
||||
|
||||
|
||||
def run_migration():
|
||||
with app.app_context():
|
||||
from sqlalchemy import text, inspect as sa_inspect
|
||||
|
||||
inspector = sa_inspect(db.engine)
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Step 1: Add qr_type column to qr_codes if it does not exist yet
|
||||
# ----------------------------------------------------------------
|
||||
existing_cols = {c['name'] for c in inspector.get_columns('qr_codes')}
|
||||
|
||||
if 'qr_type' not in existing_cols:
|
||||
with db.engine.connect() as conn:
|
||||
conn.execute(text(
|
||||
"ALTER TABLE qr_codes "
|
||||
"ADD COLUMN qr_type VARCHAR(20) NOT NULL DEFAULT 'standard'"
|
||||
))
|
||||
conn.commit()
|
||||
print("✅ Added column: qr_codes.qr_type (default = 'standard')")
|
||||
else:
|
||||
print("ℹ️ Column qr_codes.qr_type already exists — skipped.")
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Step 2: Create qr_code_locations table if it does not exist yet
|
||||
# ----------------------------------------------------------------
|
||||
existing_tables = set(inspector.get_table_names())
|
||||
|
||||
if 'qr_code_locations' not in existing_tables:
|
||||
from models.qrcode import QRCodeLocation
|
||||
QRCodeLocation.__table__.create(db.engine, checkfirst=True)
|
||||
print("✅ Created table: qr_code_locations")
|
||||
else:
|
||||
print("ℹ️ Table qr_code_locations already exists — skipped.")
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Step 3: Make qr_codes.location and location_address nullable
|
||||
# Dynamic QR codes have no single fixed location/address so these
|
||||
# columns must allow NULL.
|
||||
# ----------------------------------------------------------------
|
||||
# Re-inspect to get current column definitions
|
||||
inspector2 = sa_inspect(db.engine)
|
||||
col_map = {c['name']: c for c in inspector2.get_columns('qr_codes')}
|
||||
|
||||
location_nullable = col_map.get('location', {}).get('nullable', True)
|
||||
loc_addr_nullable = col_map.get('location_address', {}).get('nullable', True)
|
||||
|
||||
if not location_nullable or not loc_addr_nullable:
|
||||
with db.engine.connect() as conn:
|
||||
if not location_nullable:
|
||||
conn.execute(text(
|
||||
"ALTER TABLE qr_codes "
|
||||
"MODIFY COLUMN location VARCHAR(100) NULL"
|
||||
))
|
||||
print("✅ Made qr_codes.location nullable")
|
||||
else:
|
||||
print("ℹ️ qr_codes.location already nullable — skipped.")
|
||||
|
||||
if not loc_addr_nullable:
|
||||
conn.execute(text(
|
||||
"ALTER TABLE qr_codes "
|
||||
"MODIFY COLUMN location_address TEXT NULL"
|
||||
))
|
||||
print("✅ Made qr_codes.location_address nullable")
|
||||
else:
|
||||
print("ℹ️ qr_codes.location_address already nullable — skipped.")
|
||||
|
||||
conn.commit()
|
||||
else:
|
||||
print("ℹ️ qr_codes.location and location_address already nullable — skipped.")
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Step 4: Add qr_address column to attendance_data if absent
|
||||
# Stores the selected location's address for dynamic QR check-ins
|
||||
# ----------------------------------------------------------------
|
||||
inspector3 = sa_inspect(db.engine)
|
||||
att_cols = {c['name'] for c in inspector3.get_columns('attendance_data')}
|
||||
if 'qr_address' not in att_cols:
|
||||
with db.engine.connect() as conn:
|
||||
conn.execute(text(
|
||||
"ALTER TABLE attendance_data ADD COLUMN qr_address TEXT NULL"
|
||||
))
|
||||
conn.commit()
|
||||
print("\u2705 Added column: attendance_data.qr_address")
|
||||
else:
|
||||
print("\u2139\ufe0f Column attendance_data.qr_address already exists \u2014 skipped.")
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Step 5: Add is_dynamic_qr column to attendance_data if absent
|
||||
# Flags records that came from a Dynamic QR code scan
|
||||
# ----------------------------------------------------------------
|
||||
if 'is_dynamic_qr' not in att_cols:
|
||||
with db.engine.connect() as conn:
|
||||
conn.execute(text(
|
||||
"ALTER TABLE attendance_data "
|
||||
"ADD COLUMN is_dynamic_qr TINYINT(1) NOT NULL DEFAULT 0"
|
||||
))
|
||||
conn.commit()
|
||||
print("\u2705 Added column: attendance_data.is_dynamic_qr")
|
||||
else:
|
||||
print("\u2139\ufe0f Column attendance_data.is_dynamic_qr already exists \u2014 skipped.")
|
||||
|
||||
# ----------------------------------------------------------------
|
||||
# Step 6: Backfill is_dynamic_qr=1 on old broken records
|
||||
# Records created before this migration where location_name='Dynamic'
|
||||
# are legacy dynamic QR check-ins that were saved with the wrong name.
|
||||
# Mark them so the badge shows correctly in the attendance report.
|
||||
# ----------------------------------------------------------------
|
||||
with db.engine.connect() as conn:
|
||||
result = conn.execute(text(
|
||||
"UPDATE attendance_data "
|
||||
"SET is_dynamic_qr = 1 "
|
||||
"WHERE location_name = 'Dynamic' AND is_dynamic_qr = 0"
|
||||
))
|
||||
conn.commit()
|
||||
updated = result.rowcount
|
||||
if updated > 0:
|
||||
print(f"\u2705 Backfilled is_dynamic_qr=1 on {updated} legacy 'Dynamic' record(s)")
|
||||
else:
|
||||
print("\u2139\ufe0f No legacy 'Dynamic' records to backfill.")
|
||||
|
||||
print("\nMigration complete.")
|
||||
print("All existing QR codes remain fully unaffected (qr_type = 'standard').")
|
||||
print("All existing attendance records default to is_dynamic_qr = 0 (False).")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run_migration()
|
||||
@@ -0,0 +1,94 @@
|
||||
"""
|
||||
migration_legacy_attendance_remote_indexes.py
|
||||
================================================
|
||||
Adds missing indexes to the REMOTE legacy database (the one described by
|
||||
QrCodeLtServices.sql — contract / employee / locations / records) so the
|
||||
Legacy Attendance feature's live queries don't full-table-scan on every
|
||||
page load.
|
||||
|
||||
As shipped, that schema has NO index on:
|
||||
records.employeeId, records.locationId, records.time, records.contractId
|
||||
employee.id (only the meaningless auto-increment `index` is indexed)
|
||||
locations.location
|
||||
|
||||
Adding an index does not change or risk any existing data — it only
|
||||
speeds up reads. Safe to re-run: each ALTER is skipped if the index
|
||||
already exists.
|
||||
|
||||
Uses pymysql directly (never SQLAlchemy ORM), consistent with every other
|
||||
migration script in tools/ — this connects to REMOTE_DB_* (the legacy
|
||||
server), not the app's own local database.
|
||||
|
||||
Run once per server (LT and GOV each point at their own legacy DB):
|
||||
python3 tools/migration_legacy_attendance_remote_indexes.py
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
import pymysql
|
||||
|
||||
# (table, column, index_name)
|
||||
INDEXES_TO_ADD = [
|
||||
('records', 'employeeId', 'idx_records_employeeId'),
|
||||
('records', 'locationId', 'idx_records_locationId'),
|
||||
('records', 'time', 'idx_records_time'),
|
||||
('records', 'contractId', 'idx_records_contractId'),
|
||||
('employee', 'id', 'idx_employee_id'),
|
||||
('locations', 'location', 'idx_locations_location'),
|
||||
]
|
||||
|
||||
|
||||
def get_connection():
|
||||
host = os.environ.get('REMOTE_DB_HOST', '')
|
||||
port = int(os.environ.get('REMOTE_DB_PORT', '3306'))
|
||||
user = os.environ.get('REMOTE_DB_USERNAME', '')
|
||||
password = os.environ.get('REMOTE_DB_PASSWORD', '')
|
||||
database = os.environ.get('REMOTE_DB_NAME', '')
|
||||
|
||||
if not host or not database:
|
||||
print("[ERROR] REMOTE_DB_HOST / REMOTE_DB_NAME not set in .env — aborting.")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"[INFO] Connecting to legacy DB {user}@{host}:{port}/{database} ...")
|
||||
return pymysql.connect(
|
||||
host=host, port=port, user=user, password=password, database=database,
|
||||
charset='utf8mb4', connect_timeout=10
|
||||
)
|
||||
|
||||
|
||||
def index_exists(cursor, table, index_name):
|
||||
cursor.execute("""
|
||||
SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = %s AND INDEX_NAME = %s
|
||||
""", (table, index_name))
|
||||
return cursor.fetchone()[0] > 0
|
||||
|
||||
|
||||
def main():
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
for table, column, index_name in INDEXES_TO_ADD:
|
||||
if index_exists(cur, table, index_name):
|
||||
print(f"[SKIP] {table}.{index_name} already exists")
|
||||
continue
|
||||
print(f"[ADD] {table}.{index_name} ON ({column}) ...")
|
||||
cur.execute(f"ALTER TABLE `{table}` ADD INDEX `{index_name}` (`{column}`)")
|
||||
conn.commit()
|
||||
print(f"[OK] {table}.{index_name} created")
|
||||
print("[DONE] Legacy database indexes are up to date.")
|
||||
except pymysql.MySQLError as e:
|
||||
print(f"[ERROR] {e}")
|
||||
sys.exit(1)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
"""
|
||||
migration_photo_verification_toggle.py
|
||||
=======================================
|
||||
Adds `photo_verification_enabled` column to the `qr_codes` table.
|
||||
|
||||
Uses pymysql directly to avoid SQLAlchemy ORM loading the model
|
||||
(which would fail if the column doesn't exist yet).
|
||||
|
||||
Default: 1 (True) for all existing rows — preserves current behaviour.
|
||||
|
||||
Run once on each server (LT and GOV):
|
||||
python3 tools/migration_photo_verification_toggle.py
|
||||
|
||||
Safe to re-run — skips if column already exists.
|
||||
"""
|
||||
|
||||
import os, sys, re
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
import pymysql
|
||||
|
||||
TABLE = 'qr_codes'
|
||||
COLUMN = 'photo_verification_enabled'
|
||||
|
||||
|
||||
def parse_db_url(url):
|
||||
"""
|
||||
Parse DATABASE_URL robustly using regex to handle special characters
|
||||
(including @ or : ) in the password.
|
||||
|
||||
Supports:
|
||||
mysql+pymysql://user:pass@host:port/dbname
|
||||
mysql+pymysql://user:pass@host/dbname
|
||||
"""
|
||||
# Strip driver prefix
|
||||
url = re.sub(r'^mysql\+pymysql://', '', url)
|
||||
url = re.sub(r'^mysql://', '', url)
|
||||
|
||||
# Split credentials from host/db on the LAST @ before the host
|
||||
# Pattern: user:password@host[:port]/dbname[?...]
|
||||
m = re.match(
|
||||
r'^(?P<user>[^:]+):(?P<password>.+)@(?P<host>[^@:/]+)(?::(?P<port>\d+))?/(?P<db>[^?]+)',
|
||||
url
|
||||
)
|
||||
if not m:
|
||||
print(f"[ERROR] Could not parse DATABASE_URL. Raw (redacted): {url[:30]}...")
|
||||
sys.exit(1)
|
||||
|
||||
return {
|
||||
'host': m.group('host'),
|
||||
'port': int(m.group('port')) if m.group('port') else 3306,
|
||||
'user': m.group('user'),
|
||||
'password': m.group('password'),
|
||||
'database': m.group('db'),
|
||||
}
|
||||
|
||||
|
||||
def get_connection():
|
||||
db_url = os.environ.get('DATABASE_URL', '')
|
||||
if not db_url:
|
||||
print("[ERROR] DATABASE_URL not set in .env")
|
||||
sys.exit(1)
|
||||
|
||||
params = parse_db_url(db_url)
|
||||
return pymysql.connect(
|
||||
host=params['host'],
|
||||
port=params['port'],
|
||||
user=params['user'],
|
||||
password=params['password'],
|
||||
database=params['database'],
|
||||
charset='utf8mb4',
|
||||
autocommit=False,
|
||||
)
|
||||
|
||||
|
||||
def run():
|
||||
conn = get_connection()
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
# Check if column already exists
|
||||
cur.execute(
|
||||
"SELECT COUNT(*) FROM information_schema.COLUMNS "
|
||||
"WHERE TABLE_SCHEMA = DATABASE() "
|
||||
f"AND TABLE_NAME = '{TABLE}' "
|
||||
f"AND COLUMN_NAME = '{COLUMN}'"
|
||||
)
|
||||
exists = cur.fetchone()[0] > 0
|
||||
|
||||
if exists:
|
||||
print(f"[SKIP] Column '{COLUMN}' already exists on '{TABLE}'. Nothing to do.")
|
||||
return
|
||||
|
||||
print(f"[ADD] Adding column '{COLUMN}' to '{TABLE}' ...")
|
||||
cur.execute(
|
||||
f"ALTER TABLE `{TABLE}` "
|
||||
f"ADD COLUMN `{COLUMN}` TINYINT(1) NOT NULL DEFAULT 1"
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
# Verify
|
||||
cur.execute(
|
||||
"SELECT COUNT(*) FROM information_schema.COLUMNS "
|
||||
"WHERE TABLE_SCHEMA = DATABASE() "
|
||||
f"AND TABLE_NAME = '{TABLE}' "
|
||||
f"AND COLUMN_NAME = '{COLUMN}'"
|
||||
)
|
||||
if cur.fetchone()[0] > 0:
|
||||
print(f"[OK] Column '{COLUMN}' added. All existing rows default to 1 (enabled).")
|
||||
else:
|
||||
print(f"[FAIL] Column was not created — check DB permissions.")
|
||||
sys.exit(1)
|
||||
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run()
|
||||
@@ -0,0 +1,600 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
==============================================================================
|
||||
Time Attendance Database Optimization Script
|
||||
==============================================================================
|
||||
|
||||
Standalone script for optimizing the time_attendance table.
|
||||
Can be run manually or as a cronjob.
|
||||
|
||||
Usage:
|
||||
python optimize_time_attendance_db.py --action optimize
|
||||
python optimize_time_attendance_db.py --action analyze
|
||||
python optimize_time_attendance_db.py --action archive --days 365
|
||||
python optimize_time_attendance_db.py --action cleanup --days 90
|
||||
python optimize_time_attendance_db.py --action report
|
||||
python optimize_time_attendance_db.py --action all
|
||||
|
||||
Requirements:
|
||||
- Must be run from the application directory
|
||||
- Database credentials must be configured in config.py or environment
|
||||
|
||||
Author: Database Optimization Team
|
||||
Date: 2025-10-14
|
||||
==============================================================================
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
from datetime import datetime, timedelta
|
||||
import logging
|
||||
|
||||
# Add the application directory to the path
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
# Import Flask app and database
|
||||
try:
|
||||
from app import app, db
|
||||
from models.time_attendance import TimeAttendance
|
||||
from sqlalchemy import text
|
||||
except ImportError as e:
|
||||
print(f"❌ Error: Cannot import required modules: {e}")
|
||||
print(" Make sure this script is in the same directory as app.py")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
class TimeAttendanceOptimizer:
|
||||
"""Standalone optimizer for time_attendance table"""
|
||||
|
||||
def __init__(self, verbose=True):
|
||||
self.verbose = verbose
|
||||
self.setup_logging()
|
||||
|
||||
def setup_logging(self):
|
||||
"""Setup logging configuration"""
|
||||
log_format = '%(asctime)s - %(levelname)s - %(message)s'
|
||||
logging.basicConfig(
|
||||
level=logging.INFO if self.verbose else logging.WARNING,
|
||||
format=log_format
|
||||
)
|
||||
self.logger = logging.getLogger(__name__)
|
||||
|
||||
def log(self, message, level='info'):
|
||||
"""Log message with appropriate level"""
|
||||
if level == 'info':
|
||||
self.logger.info(message)
|
||||
if self.verbose:
|
||||
print(f"ℹ️ {message}")
|
||||
elif level == 'success':
|
||||
self.logger.info(message)
|
||||
if self.verbose:
|
||||
print(f"✅ {message}")
|
||||
elif level == 'warning':
|
||||
self.logger.warning(message)
|
||||
if self.verbose:
|
||||
print(f"⚠️ {message}")
|
||||
elif level == 'error':
|
||||
self.logger.error(message)
|
||||
if self.verbose:
|
||||
print(f"❌ {message}")
|
||||
|
||||
def create_indexes(self):
|
||||
"""Create optimized indexes for time_attendance table"""
|
||||
self.log("Creating optimized indexes...", 'info')
|
||||
|
||||
indexes = [
|
||||
{
|
||||
'name': 'idx_ta_employee_date_time',
|
||||
'columns': 'employee_id, attendance_date DESC, attendance_time DESC',
|
||||
'purpose': 'Employee-based queries with date filtering'
|
||||
},
|
||||
{
|
||||
'name': 'idx_ta_date_location_employee',
|
||||
'columns': 'attendance_date DESC, location_name, employee_id',
|
||||
'purpose': 'Date and location filtering'
|
||||
},
|
||||
{
|
||||
'name': 'idx_ta_project_date_employee',
|
||||
'columns': 'project_id, attendance_date DESC, employee_id',
|
||||
'purpose': 'Project-based attendance queries'
|
||||
},
|
||||
{
|
||||
'name': 'idx_ta_batch_date',
|
||||
'columns': 'import_batch_id, attendance_date DESC',
|
||||
'purpose': 'Import batch management'
|
||||
},
|
||||
{
|
||||
'name': 'idx_ta_action_date_employee',
|
||||
'columns': 'action_description, attendance_date DESC, employee_id',
|
||||
'purpose': 'Action-based analytics'
|
||||
},
|
||||
{
|
||||
'name': 'idx_ta_date_time_desc',
|
||||
'columns': 'attendance_date DESC, attendance_time DESC, id DESC',
|
||||
'purpose': 'Recent records retrieval'
|
||||
},
|
||||
{
|
||||
'name': 'idx_ta_location_action_date',
|
||||
'columns': 'location_name, action_description, attendance_date DESC',
|
||||
'purpose': 'Location-based action analysis'
|
||||
},
|
||||
]
|
||||
|
||||
created = 0
|
||||
skipped = 0
|
||||
failed = 0
|
||||
|
||||
for idx in indexes:
|
||||
try:
|
||||
# Check if index exists
|
||||
check_query = f"""
|
||||
SELECT COUNT(*) as count
|
||||
FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'time_attendance'
|
||||
AND INDEX_NAME = '{idx['name']}'
|
||||
"""
|
||||
result = db.session.execute(text(check_query)).fetchone()
|
||||
|
||||
if result and result.count > 0:
|
||||
self.log(f"Index {idx['name']} already exists - skipped", 'info')
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
# Create index
|
||||
create_query = f"""
|
||||
CREATE INDEX {idx['name']}
|
||||
ON time_attendance ({idx['columns']})
|
||||
"""
|
||||
|
||||
self.log(f"Creating index: {idx['name']}", 'info')
|
||||
db.session.execute(text(create_query))
|
||||
db.session.commit()
|
||||
|
||||
self.log(f"Created index: {idx['name']} - {idx['purpose']}", 'success')
|
||||
created += 1
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"Failed to create index {idx['name']}: {str(e)[:100]}", 'warning')
|
||||
failed += 1
|
||||
db.session.rollback()
|
||||
continue
|
||||
|
||||
self.log(f"Index creation complete: {created} created, {skipped} skipped, {failed} failed", 'success')
|
||||
return {'created': created, 'skipped': skipped, 'failed': failed}
|
||||
|
||||
def analyze_table(self):
|
||||
"""Analyze time_attendance table statistics"""
|
||||
self.log("Analyzing table statistics...", 'info')
|
||||
|
||||
try:
|
||||
stats_query = """
|
||||
SELECT
|
||||
COUNT(*) as total_records,
|
||||
COUNT(DISTINCT employee_id) as unique_employees,
|
||||
COUNT(DISTINCT location_name) as unique_locations,
|
||||
COUNT(DISTINCT DATE(attendance_date)) as unique_dates,
|
||||
COUNT(DISTINCT import_batch_id) as unique_batches,
|
||||
COUNT(DISTINCT project_id) as unique_projects,
|
||||
MIN(attendance_date) as earliest_date,
|
||||
MAX(attendance_date) as latest_date,
|
||||
COUNT(CASE WHEN recorded_address IS NOT NULL THEN 1 END) as records_with_address
|
||||
FROM time_attendance
|
||||
"""
|
||||
|
||||
result = db.session.execute(text(stats_query)).fetchone()
|
||||
|
||||
# Get table size
|
||||
size_query = """
|
||||
SELECT
|
||||
ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024, 2) as size_mb,
|
||||
ROUND(DATA_LENGTH / 1024 / 1024, 2) as data_mb,
|
||||
ROUND(INDEX_LENGTH / 1024 / 1024, 2) as index_mb
|
||||
FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = 'time_attendance'
|
||||
"""
|
||||
|
||||
size_result = db.session.execute(text(size_query)).fetchone()
|
||||
|
||||
if result:
|
||||
date_range_days = (result.latest_date - result.earliest_date).days if result.latest_date and result.earliest_date else 0
|
||||
|
||||
stats = {
|
||||
'total_records': result.total_records,
|
||||
'unique_employees': result.unique_employees,
|
||||
'unique_locations': result.unique_locations,
|
||||
'unique_dates': result.unique_dates,
|
||||
'unique_batches': result.unique_batches,
|
||||
'unique_projects': result.unique_projects,
|
||||
'earliest_date': result.earliest_date,
|
||||
'latest_date': result.latest_date,
|
||||
'date_range_days': date_range_days,
|
||||
'records_with_address': result.records_with_address,
|
||||
'table_size_mb': size_result.size_mb if size_result else 0,
|
||||
'data_size_mb': size_result.data_mb if size_result else 0,
|
||||
'index_size_mb': size_result.index_mb if size_result else 0
|
||||
}
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("📊 TIME ATTENDANCE TABLE STATISTICS")
|
||||
print("="*60)
|
||||
print(f"Total Records: {stats['total_records']:>15,}")
|
||||
print(f"Unique Employees: {stats['unique_employees']:>15,}")
|
||||
print(f"Unique Locations: {stats['unique_locations']:>15,}")
|
||||
print(f"Unique Dates: {stats['unique_dates']:>15,}")
|
||||
print(f"Import Batches: {stats['unique_batches']:>15,}")
|
||||
print(f"Projects: {stats['unique_projects']:>15,}")
|
||||
print(f"Date Range: {stats['date_range_days']:>15,} days")
|
||||
print(f"Earliest Date: {stats['earliest_date']:>15}")
|
||||
print(f"Latest Date: {stats['latest_date']:>15}")
|
||||
print(f"Records w/ Address: {stats['records_with_address']:>15,}")
|
||||
print(f"\nTable Size: {stats['table_size_mb']:>15.2f} MB")
|
||||
print(f" Data Size: {stats['data_size_mb']:>15.2f} MB")
|
||||
print(f" Index Size: {stats['index_size_mb']:>15.2f} MB")
|
||||
print("="*60 + "\n")
|
||||
|
||||
return stats
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"Error analyzing table: {e}", 'error')
|
||||
return None
|
||||
|
||||
def optimize_table(self):
|
||||
"""Run MySQL OPTIMIZE TABLE and ANALYZE TABLE"""
|
||||
self.log("Optimizing table structure...", 'info')
|
||||
|
||||
try:
|
||||
# Analyze table
|
||||
self.log("Running ANALYZE TABLE...", 'info')
|
||||
db.session.execute(text("ANALYZE TABLE time_attendance"))
|
||||
db.session.commit()
|
||||
self.log("ANALYZE TABLE completed", 'success')
|
||||
|
||||
# Optimize table
|
||||
self.log("Running OPTIMIZE TABLE (this may take a while)...", 'info')
|
||||
db.session.execute(text("OPTIMIZE TABLE time_attendance"))
|
||||
db.session.commit()
|
||||
self.log("OPTIMIZE TABLE completed", 'success')
|
||||
|
||||
return {'success': True}
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"Error optimizing table: {e}", 'error')
|
||||
db.session.rollback()
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def create_archive_table(self):
|
||||
"""Create archive table if it doesn't exist"""
|
||||
try:
|
||||
create_query = """
|
||||
CREATE TABLE IF NOT EXISTS time_attendance_archive
|
||||
LIKE time_attendance
|
||||
"""
|
||||
db.session.execute(text(create_query))
|
||||
db.session.commit()
|
||||
return True
|
||||
except Exception as e:
|
||||
self.log(f"Error creating archive table: {e}", 'error')
|
||||
db.session.rollback()
|
||||
return False
|
||||
|
||||
def archive_old_records(self, days=365, execute=False):
|
||||
"""Archive records older than specified days"""
|
||||
self.log(f"Archive process for records older than {days} days...", 'info')
|
||||
|
||||
cutoff_date = datetime.now() - timedelta(days=days)
|
||||
|
||||
try:
|
||||
# Count records to archive
|
||||
count_query = """
|
||||
SELECT COUNT(*) as count
|
||||
FROM time_attendance
|
||||
WHERE attendance_date < :cutoff_date
|
||||
"""
|
||||
result = db.session.execute(
|
||||
text(count_query),
|
||||
{'cutoff_date': cutoff_date.date()}
|
||||
).fetchone()
|
||||
|
||||
records_to_archive = result.count if result else 0
|
||||
|
||||
print(f"\n📦 Archive Summary:")
|
||||
print(f" Cutoff Date: {cutoff_date.date()}")
|
||||
print(f" Records to Archive: {records_to_archive:,}")
|
||||
|
||||
if records_to_archive == 0:
|
||||
self.log("No records to archive", 'info')
|
||||
return {'records_archived': 0}
|
||||
|
||||
if not execute:
|
||||
print(f"\n⚠️ DRY RUN MODE - No records will be archived")
|
||||
print(f" Use --execute flag to actually archive records\n")
|
||||
return {'records_archived': 0, 'dry_run': True}
|
||||
|
||||
# Create archive table
|
||||
if not self.create_archive_table():
|
||||
return {'success': False, 'error': 'Failed to create archive table'}
|
||||
|
||||
# Archive records in batches
|
||||
self.log("Starting archive process...", 'info')
|
||||
batch_size = 10000
|
||||
total_archived = 0
|
||||
|
||||
while total_archived < records_to_archive:
|
||||
# Insert to archive
|
||||
archive_query = """
|
||||
INSERT INTO time_attendance_archive
|
||||
SELECT * FROM time_attendance
|
||||
WHERE attendance_date < :cutoff_date
|
||||
LIMIT :batch_size
|
||||
"""
|
||||
|
||||
db.session.execute(
|
||||
text(archive_query),
|
||||
{'cutoff_date': cutoff_date.date(), 'batch_size': batch_size}
|
||||
)
|
||||
|
||||
# Delete from main table
|
||||
delete_query = """
|
||||
DELETE FROM time_attendance
|
||||
WHERE attendance_date < :cutoff_date
|
||||
LIMIT :batch_size
|
||||
"""
|
||||
|
||||
result = db.session.execute(
|
||||
text(delete_query),
|
||||
{'cutoff_date': cutoff_date.date(), 'batch_size': batch_size}
|
||||
)
|
||||
|
||||
rows_affected = result.rowcount
|
||||
|
||||
if rows_affected == 0:
|
||||
break
|
||||
|
||||
db.session.commit()
|
||||
total_archived += rows_affected
|
||||
|
||||
self.log(f"Archived {total_archived:,} / {records_to_archive:,} records...", 'info')
|
||||
|
||||
# Safety limit
|
||||
if total_archived >= 100000:
|
||||
self.log("Reached safety limit of 100,000 records per run", 'warning')
|
||||
break
|
||||
|
||||
self.log(f"Archive complete: {total_archived:,} records archived", 'success')
|
||||
return {'records_archived': total_archived, 'success': True}
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"Error during archive: {e}", 'error')
|
||||
db.session.rollback()
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def cleanup_old_records(self, days=90, execute=False):
|
||||
"""Delete records older than specified days"""
|
||||
self.log(f"Cleanup process for records older than {days} days...", 'info')
|
||||
|
||||
cutoff_date = datetime.now() - timedelta(days=days)
|
||||
|
||||
try:
|
||||
# Count records to delete
|
||||
count_query = """
|
||||
SELECT COUNT(*) as count
|
||||
FROM time_attendance
|
||||
WHERE import_date < :cutoff_date
|
||||
"""
|
||||
result = db.session.execute(
|
||||
text(count_query),
|
||||
{'cutoff_date': cutoff_date}
|
||||
).fetchone()
|
||||
|
||||
records_to_delete = result.count if result else 0
|
||||
|
||||
print(f"\n🗑️ Cleanup Summary:")
|
||||
print(f" Cutoff Date: {cutoff_date.date()}")
|
||||
print(f" Records to Delete: {records_to_delete:,}")
|
||||
|
||||
if records_to_delete == 0:
|
||||
self.log("No records to delete", 'info')
|
||||
return {'records_deleted': 0}
|
||||
|
||||
if not execute:
|
||||
print(f"\n⚠️ DRY RUN MODE - No records will be deleted")
|
||||
print(f" Use --execute flag to actually delete records\n")
|
||||
return {'records_deleted': 0, 'dry_run': True}
|
||||
|
||||
# Delete records
|
||||
delete_query = """
|
||||
DELETE FROM time_attendance
|
||||
WHERE import_date < :cutoff_date
|
||||
"""
|
||||
|
||||
self.log("Deleting old records...", 'info')
|
||||
db.session.execute(
|
||||
text(delete_query),
|
||||
{'cutoff_date': cutoff_date}
|
||||
)
|
||||
db.session.commit()
|
||||
|
||||
self.log(f"Cleanup complete: {records_to_delete:,} records deleted", 'success')
|
||||
return {'records_deleted': records_to_delete, 'success': True}
|
||||
|
||||
except Exception as e:
|
||||
self.log(f"Error during cleanup: {e}", 'error')
|
||||
db.session.rollback()
|
||||
return {'success': False, 'error': str(e)}
|
||||
|
||||
def generate_report(self):
|
||||
"""Generate comprehensive optimization report"""
|
||||
print("\n" + "="*60)
|
||||
print("📋 TIME ATTENDANCE OPTIMIZATION REPORT")
|
||||
print("="*60)
|
||||
print(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
|
||||
|
||||
# Get statistics
|
||||
stats = self.analyze_table()
|
||||
|
||||
if stats:
|
||||
# Generate recommendations
|
||||
print("\n" + "="*60)
|
||||
print("💡 RECOMMENDATIONS")
|
||||
print("="*60)
|
||||
|
||||
recommendations = []
|
||||
|
||||
if stats['total_records'] > 100000:
|
||||
recommendations.append({
|
||||
'priority': 'HIGH',
|
||||
'type': 'Indexing',
|
||||
'message': f"Table has {stats['total_records']:,} records. Run index optimization."
|
||||
})
|
||||
|
||||
if stats['total_records'] > 500000:
|
||||
recommendations.append({
|
||||
'priority': 'HIGH',
|
||||
'type': 'Archiving',
|
||||
'message': f"Consider archiving records older than 365 days."
|
||||
})
|
||||
|
||||
if stats['table_size_mb'] > 500:
|
||||
recommendations.append({
|
||||
'priority': 'MEDIUM',
|
||||
'type': 'Optimization',
|
||||
'message': f"Table size is {stats['table_size_mb']:.2f} MB. Run OPTIMIZE TABLE."
|
||||
})
|
||||
|
||||
if stats['date_range_days'] > 365:
|
||||
recommendations.append({
|
||||
'priority': 'MEDIUM',
|
||||
'type': 'Data Retention',
|
||||
'message': f"Data spans {stats['date_range_days']} days. Implement retention policy."
|
||||
})
|
||||
|
||||
if stats['index_size_mb'] > stats['data_size_mb'] * 1.5:
|
||||
recommendations.append({
|
||||
'priority': 'LOW',
|
||||
'type': 'Index Review',
|
||||
'message': f"Index size ({stats['index_size_mb']:.2f} MB) is large. Review index usage."
|
||||
})
|
||||
|
||||
if not recommendations:
|
||||
print("✓ No major issues found. Database is well optimized.\n")
|
||||
else:
|
||||
for rec in recommendations:
|
||||
priority_icon = "🔴" if rec['priority'] == 'HIGH' else "🟡" if rec['priority'] == 'MEDIUM' else "🟢"
|
||||
print(f"{priority_icon} [{rec['priority']}] {rec['type']}")
|
||||
print(f" {rec['message']}\n")
|
||||
|
||||
print("="*60)
|
||||
|
||||
# Suggested actions
|
||||
print("\n💻 SUGGESTED ACTIONS:")
|
||||
print("-"*60)
|
||||
if stats['total_records'] > 100000:
|
||||
print("• python optimize_time_attendance_db.py --action optimize")
|
||||
if stats['total_records'] > 500000:
|
||||
print("• python optimize_time_attendance_db.py --action archive --days 365 --execute")
|
||||
if stats['date_range_days'] > 180:
|
||||
print("• python optimize_time_attendance_db.py --action cleanup --days 90 --execute")
|
||||
print("="*60 + "\n")
|
||||
|
||||
|
||||
def main():
|
||||
"""Main function to handle command line arguments"""
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Time Attendance Database Optimization Tool',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
%(prog)s --action optimize # Create indexes and optimize table
|
||||
%(prog)s --action analyze # Analyze table statistics
|
||||
%(prog)s --action archive --days 365 # Archive records older than 365 days (dry run)
|
||||
%(prog)s --action archive --days 365 --execute # Actually archive records
|
||||
%(prog)s --action cleanup --days 90 --execute # Delete records older than 90 days
|
||||
%(prog)s --action report # Generate optimization report
|
||||
%(prog)s --action all # Run full optimization (indexes + analyze + optimize)
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--action',
|
||||
required=True,
|
||||
choices=['optimize', 'analyze', 'archive', 'cleanup', 'report', 'all', 'indexes'],
|
||||
help='Action to perform'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--days',
|
||||
type=int,
|
||||
default=365,
|
||||
help='Number of days for archive/cleanup (default: 365)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--execute',
|
||||
action='store_true',
|
||||
help='Actually execute archive/cleanup (otherwise dry run)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--quiet',
|
||||
action='store_true',
|
||||
help='Suppress verbose output'
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Create optimizer instance
|
||||
optimizer = TimeAttendanceOptimizer(verbose=not args.quiet)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("🔧 TIME ATTENDANCE DATABASE OPTIMIZER")
|
||||
print("="*60)
|
||||
print(f"Action: {args.action.upper()}")
|
||||
print(f"Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
|
||||
print("="*60 + "\n")
|
||||
|
||||
try:
|
||||
with app.app_context():
|
||||
if args.action == 'indexes':
|
||||
optimizer.create_indexes()
|
||||
|
||||
elif args.action == 'optimize':
|
||||
optimizer.create_indexes()
|
||||
optimizer.optimize_table()
|
||||
|
||||
elif args.action == 'analyze':
|
||||
optimizer.analyze_table()
|
||||
|
||||
elif args.action == 'archive':
|
||||
optimizer.archive_old_records(days=args.days, execute=args.execute)
|
||||
|
||||
elif args.action == 'cleanup':
|
||||
optimizer.cleanup_old_records(days=args.days, execute=args.execute)
|
||||
|
||||
elif args.action == 'report':
|
||||
optimizer.generate_report()
|
||||
|
||||
elif args.action == 'all':
|
||||
optimizer.create_indexes()
|
||||
optimizer.analyze_table()
|
||||
optimizer.optimize_table()
|
||||
print("\n✅ Full optimization complete!")
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ OPTIMIZATION COMPLETED SUCCESSFULLY")
|
||||
print("="*60 + "\n")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ Operation cancelled by user")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user