From 4a4947a7424ba069e9390a533dd21377093b8d8e Mon Sep 17 00:00:00 2001 From: NguyenND Date: Tue, 14 Oct 2025 17:38:21 -0400 Subject: [PATCH] Updated functions --- app.py | 12 +- .../time_attendance_duplicate_review.html | 1 + templates/time_attendance_invalid_review.html | 1 + tools/optimize_time_attendance_db.py | 600 ++++++++++++++++++ 4 files changed, 611 insertions(+), 3 deletions(-) create mode 100644 tools/optimize_time_attendance_db.py diff --git a/app.py b/app.py index 8fb992c..959c70b 100644 --- a/app.py +++ b/app.py @@ -6676,10 +6676,13 @@ def import_time_attendance(): if duplicate_analysis['duplicate_records'] > 0: print(f"⚠️ Found {duplicate_analysis['duplicate_records']} duplicates") + # Get project_id from form + project_id = request.form.get('project_id') # Show duplicate review page return render_template('time_attendance_duplicate_review.html', - analysis=duplicate_analysis, - filename=filename) + analysis=duplicate_analysis, + filename=filename, + project_id=project_id) else: print("✅ No duplicates found") flash('No duplicates found. Proceeding with import.', 'info') @@ -6691,10 +6694,13 @@ def import_time_attendance(): if invalid_analysis['invalid_rows'] > 0: print(f"⚠️ Found {invalid_analysis['invalid_rows']} invalid rows") + # Get project_id from form + project_id = request.form.get('project_id') # Show invalid row review page return render_template('time_attendance_invalid_review.html', analysis=invalid_analysis, - filename=filename) + filename=filename, + project_id=project_id) else: print("✅ All rows are valid") flash('All rows are valid. Proceeding with import.', 'info') diff --git a/templates/time_attendance_duplicate_review.html b/templates/time_attendance_duplicate_review.html index 4001575..b0abdc9 100644 --- a/templates/time_attendance_duplicate_review.html +++ b/templates/time_attendance_duplicate_review.html @@ -370,6 +370,7 @@ +
{% for duplicate in analysis.duplicates %} diff --git a/templates/time_attendance_invalid_review.html b/templates/time_attendance_invalid_review.html index d7b31f0..6b13c18 100644 --- a/templates/time_attendance_invalid_review.html +++ b/templates/time_attendance_invalid_review.html @@ -327,6 +327,7 @@ +
{% for invalid in analysis.invalid_details %} diff --git a/tools/optimize_time_attendance_db.py b/tools/optimize_time_attendance_db.py new file mode 100644 index 0000000..83174d1 --- /dev/null +++ b/tools/optimize_time_attendance_db.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +""" +============================================================================== +Time Attendance Database Optimization Script +============================================================================== + +Standalone script for optimizing the time_attendance table. +Can be run manually or as a cronjob. + +Usage: + python optimize_time_attendance_db.py --action optimize + python optimize_time_attendance_db.py --action analyze + python optimize_time_attendance_db.py --action archive --days 365 + python optimize_time_attendance_db.py --action cleanup --days 90 + python optimize_time_attendance_db.py --action report + python optimize_time_attendance_db.py --action all + +Requirements: + - Must be run from the application directory + - Database credentials must be configured in config.py or environment + +Author: Database Optimization Team +Date: 2025-10-14 +============================================================================== +""" + +import sys +import os +import argparse +from datetime import datetime, timedelta +import logging + +# Add the application directory to the path +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +# Import Flask app and database +try: + from app import app, db + from models.time_attendance import TimeAttendance + from sqlalchemy import text +except ImportError as e: + print(f"❌ Error: Cannot import required modules: {e}") + print(" Make sure this script is in the same directory as app.py") + sys.exit(1) + + +class TimeAttendanceOptimizer: + """Standalone optimizer for time_attendance table""" + + def __init__(self, verbose=True): + self.verbose = verbose + self.setup_logging() + + def setup_logging(self): + """Setup logging configuration""" + log_format = '%(asctime)s - %(levelname)s - %(message)s' + logging.basicConfig( + level=logging.INFO if self.verbose else logging.WARNING, + format=log_format + ) + self.logger = logging.getLogger(__name__) + + def log(self, message, level='info'): + """Log message with appropriate level""" + if level == 'info': + self.logger.info(message) + if self.verbose: + print(f"ℹ️ {message}") + elif level == 'success': + self.logger.info(message) + if self.verbose: + print(f"✅ {message}") + elif level == 'warning': + self.logger.warning(message) + if self.verbose: + print(f"⚠️ {message}") + elif level == 'error': + self.logger.error(message) + if self.verbose: + print(f"❌ {message}") + + def create_indexes(self): + """Create optimized indexes for time_attendance table""" + self.log("Creating optimized indexes...", 'info') + + indexes = [ + { + 'name': 'idx_ta_employee_date_time', + 'columns': 'employee_id, attendance_date DESC, attendance_time DESC', + 'purpose': 'Employee-based queries with date filtering' + }, + { + 'name': 'idx_ta_date_location_employee', + 'columns': 'attendance_date DESC, location_name, employee_id', + 'purpose': 'Date and location filtering' + }, + { + 'name': 'idx_ta_project_date_employee', + 'columns': 'project_id, attendance_date DESC, employee_id', + 'purpose': 'Project-based attendance queries' + }, + { + 'name': 'idx_ta_batch_date', + 'columns': 'import_batch_id, attendance_date DESC', + 'purpose': 'Import batch management' + }, + { + 'name': 'idx_ta_action_date_employee', + 'columns': 'action_description, attendance_date DESC, employee_id', + 'purpose': 'Action-based analytics' + }, + { + 'name': 'idx_ta_date_time_desc', + 'columns': 'attendance_date DESC, attendance_time DESC, id DESC', + 'purpose': 'Recent records retrieval' + }, + { + 'name': 'idx_ta_location_action_date', + 'columns': 'location_name, action_description, attendance_date DESC', + 'purpose': 'Location-based action analysis' + }, + ] + + created = 0 + skipped = 0 + failed = 0 + + for idx in indexes: + try: + # Check if index exists + check_query = f""" + SELECT COUNT(*) as count + FROM INFORMATION_SCHEMA.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'time_attendance' + AND INDEX_NAME = '{idx['name']}' + """ + result = db.session.execute(text(check_query)).fetchone() + + if result and result.count > 0: + self.log(f"Index {idx['name']} already exists - skipped", 'info') + skipped += 1 + continue + + # Create index + create_query = f""" + CREATE INDEX {idx['name']} + ON time_attendance ({idx['columns']}) + """ + + self.log(f"Creating index: {idx['name']}", 'info') + db.session.execute(text(create_query)) + db.session.commit() + + self.log(f"Created index: {idx['name']} - {idx['purpose']}", 'success') + created += 1 + + except Exception as e: + self.log(f"Failed to create index {idx['name']}: {str(e)[:100]}", 'warning') + failed += 1 + db.session.rollback() + continue + + self.log(f"Index creation complete: {created} created, {skipped} skipped, {failed} failed", 'success') + return {'created': created, 'skipped': skipped, 'failed': failed} + + def analyze_table(self): + """Analyze time_attendance table statistics""" + self.log("Analyzing table statistics...", 'info') + + try: + stats_query = """ + SELECT + COUNT(*) as total_records, + COUNT(DISTINCT employee_id) as unique_employees, + COUNT(DISTINCT location_name) as unique_locations, + COUNT(DISTINCT DATE(attendance_date)) as unique_dates, + COUNT(DISTINCT import_batch_id) as unique_batches, + COUNT(DISTINCT project_id) as unique_projects, + MIN(attendance_date) as earliest_date, + MAX(attendance_date) as latest_date, + COUNT(CASE WHEN recorded_address IS NOT NULL THEN 1 END) as records_with_address + FROM time_attendance + """ + + result = db.session.execute(text(stats_query)).fetchone() + + # Get table size + size_query = """ + SELECT + ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024, 2) as size_mb, + ROUND(DATA_LENGTH / 1024 / 1024, 2) as data_mb, + ROUND(INDEX_LENGTH / 1024 / 1024, 2) as index_mb + FROM INFORMATION_SCHEMA.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'time_attendance' + """ + + size_result = db.session.execute(text(size_query)).fetchone() + + if result: + date_range_days = (result.latest_date - result.earliest_date).days if result.latest_date and result.earliest_date else 0 + + stats = { + 'total_records': result.total_records, + 'unique_employees': result.unique_employees, + 'unique_locations': result.unique_locations, + 'unique_dates': result.unique_dates, + 'unique_batches': result.unique_batches, + 'unique_projects': result.unique_projects, + 'earliest_date': result.earliest_date, + 'latest_date': result.latest_date, + 'date_range_days': date_range_days, + 'records_with_address': result.records_with_address, + 'table_size_mb': size_result.size_mb if size_result else 0, + 'data_size_mb': size_result.data_mb if size_result else 0, + 'index_size_mb': size_result.index_mb if size_result else 0 + } + + print("\n" + "="*60) + print("📊 TIME ATTENDANCE TABLE STATISTICS") + print("="*60) + print(f"Total Records: {stats['total_records']:>15,}") + print(f"Unique Employees: {stats['unique_employees']:>15,}") + print(f"Unique Locations: {stats['unique_locations']:>15,}") + print(f"Unique Dates: {stats['unique_dates']:>15,}") + print(f"Import Batches: {stats['unique_batches']:>15,}") + print(f"Projects: {stats['unique_projects']:>15,}") + print(f"Date Range: {stats['date_range_days']:>15,} days") + print(f"Earliest Date: {stats['earliest_date']:>15}") + print(f"Latest Date: {stats['latest_date']:>15}") + print(f"Records w/ Address: {stats['records_with_address']:>15,}") + print(f"\nTable Size: {stats['table_size_mb']:>15.2f} MB") + print(f" Data Size: {stats['data_size_mb']:>15.2f} MB") + print(f" Index Size: {stats['index_size_mb']:>15.2f} MB") + print("="*60 + "\n") + + return stats + + except Exception as e: + self.log(f"Error analyzing table: {e}", 'error') + return None + + def optimize_table(self): + """Run MySQL OPTIMIZE TABLE and ANALYZE TABLE""" + self.log("Optimizing table structure...", 'info') + + try: + # Analyze table + self.log("Running ANALYZE TABLE...", 'info') + db.session.execute(text("ANALYZE TABLE time_attendance")) + db.session.commit() + self.log("ANALYZE TABLE completed", 'success') + + # Optimize table + self.log("Running OPTIMIZE TABLE (this may take a while)...", 'info') + db.session.execute(text("OPTIMIZE TABLE time_attendance")) + db.session.commit() + self.log("OPTIMIZE TABLE completed", 'success') + + return {'success': True} + + except Exception as e: + self.log(f"Error optimizing table: {e}", 'error') + db.session.rollback() + return {'success': False, 'error': str(e)} + + def create_archive_table(self): + """Create archive table if it doesn't exist""" + try: + create_query = """ + CREATE TABLE IF NOT EXISTS time_attendance_archive + LIKE time_attendance + """ + db.session.execute(text(create_query)) + db.session.commit() + return True + except Exception as e: + self.log(f"Error creating archive table: {e}", 'error') + db.session.rollback() + return False + + def archive_old_records(self, days=365, execute=False): + """Archive records older than specified days""" + self.log(f"Archive process for records older than {days} days...", 'info') + + cutoff_date = datetime.now() - timedelta(days=days) + + try: + # Count records to archive + count_query = """ + SELECT COUNT(*) as count + FROM time_attendance + WHERE attendance_date < :cutoff_date + """ + result = db.session.execute( + text(count_query), + {'cutoff_date': cutoff_date.date()} + ).fetchone() + + records_to_archive = result.count if result else 0 + + print(f"\n📦 Archive Summary:") + print(f" Cutoff Date: {cutoff_date.date()}") + print(f" Records to Archive: {records_to_archive:,}") + + if records_to_archive == 0: + self.log("No records to archive", 'info') + return {'records_archived': 0} + + if not execute: + print(f"\n⚠️ DRY RUN MODE - No records will be archived") + print(f" Use --execute flag to actually archive records\n") + return {'records_archived': 0, 'dry_run': True} + + # Create archive table + if not self.create_archive_table(): + return {'success': False, 'error': 'Failed to create archive table'} + + # Archive records in batches + self.log("Starting archive process...", 'info') + batch_size = 10000 + total_archived = 0 + + while total_archived < records_to_archive: + # Insert to archive + archive_query = """ + INSERT INTO time_attendance_archive + SELECT * FROM time_attendance + WHERE attendance_date < :cutoff_date + LIMIT :batch_size + """ + + db.session.execute( + text(archive_query), + {'cutoff_date': cutoff_date.date(), 'batch_size': batch_size} + ) + + # Delete from main table + delete_query = """ + DELETE FROM time_attendance + WHERE attendance_date < :cutoff_date + LIMIT :batch_size + """ + + result = db.session.execute( + text(delete_query), + {'cutoff_date': cutoff_date.date(), 'batch_size': batch_size} + ) + + rows_affected = result.rowcount + + if rows_affected == 0: + break + + db.session.commit() + total_archived += rows_affected + + self.log(f"Archived {total_archived:,} / {records_to_archive:,} records...", 'info') + + # Safety limit + if total_archived >= 100000: + self.log("Reached safety limit of 100,000 records per run", 'warning') + break + + self.log(f"Archive complete: {total_archived:,} records archived", 'success') + return {'records_archived': total_archived, 'success': True} + + except Exception as e: + self.log(f"Error during archive: {e}", 'error') + db.session.rollback() + return {'success': False, 'error': str(e)} + + def cleanup_old_records(self, days=90, execute=False): + """Delete records older than specified days""" + self.log(f"Cleanup process for records older than {days} days...", 'info') + + cutoff_date = datetime.now() - timedelta(days=days) + + try: + # Count records to delete + count_query = """ + SELECT COUNT(*) as count + FROM time_attendance + WHERE import_date < :cutoff_date + """ + result = db.session.execute( + text(count_query), + {'cutoff_date': cutoff_date} + ).fetchone() + + records_to_delete = result.count if result else 0 + + print(f"\n🗑️ Cleanup Summary:") + print(f" Cutoff Date: {cutoff_date.date()}") + print(f" Records to Delete: {records_to_delete:,}") + + if records_to_delete == 0: + self.log("No records to delete", 'info') + return {'records_deleted': 0} + + if not execute: + print(f"\n⚠️ DRY RUN MODE - No records will be deleted") + print(f" Use --execute flag to actually delete records\n") + return {'records_deleted': 0, 'dry_run': True} + + # Delete records + delete_query = """ + DELETE FROM time_attendance + WHERE import_date < :cutoff_date + """ + + self.log("Deleting old records...", 'info') + db.session.execute( + text(delete_query), + {'cutoff_date': cutoff_date} + ) + db.session.commit() + + self.log(f"Cleanup complete: {records_to_delete:,} records deleted", 'success') + return {'records_deleted': records_to_delete, 'success': True} + + except Exception as e: + self.log(f"Error during cleanup: {e}", 'error') + db.session.rollback() + return {'success': False, 'error': str(e)} + + def generate_report(self): + """Generate comprehensive optimization report""" + print("\n" + "="*60) + print("📋 TIME ATTENDANCE OPTIMIZATION REPORT") + print("="*60) + print(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") + + # Get statistics + stats = self.analyze_table() + + if stats: + # Generate recommendations + print("\n" + "="*60) + print("💡 RECOMMENDATIONS") + print("="*60) + + recommendations = [] + + if stats['total_records'] > 100000: + recommendations.append({ + 'priority': 'HIGH', + 'type': 'Indexing', + 'message': f"Table has {stats['total_records']:,} records. Run index optimization." + }) + + if stats['total_records'] > 500000: + recommendations.append({ + 'priority': 'HIGH', + 'type': 'Archiving', + 'message': f"Consider archiving records older than 365 days." + }) + + if stats['table_size_mb'] > 500: + recommendations.append({ + 'priority': 'MEDIUM', + 'type': 'Optimization', + 'message': f"Table size is {stats['table_size_mb']:.2f} MB. Run OPTIMIZE TABLE." + }) + + if stats['date_range_days'] > 365: + recommendations.append({ + 'priority': 'MEDIUM', + 'type': 'Data Retention', + 'message': f"Data spans {stats['date_range_days']} days. Implement retention policy." + }) + + if stats['index_size_mb'] > stats['data_size_mb'] * 1.5: + recommendations.append({ + 'priority': 'LOW', + 'type': 'Index Review', + 'message': f"Index size ({stats['index_size_mb']:.2f} MB) is large. Review index usage." + }) + + if not recommendations: + print("✓ No major issues found. Database is well optimized.\n") + else: + for rec in recommendations: + priority_icon = "🔴" if rec['priority'] == 'HIGH' else "🟡" if rec['priority'] == 'MEDIUM' else "🟢" + print(f"{priority_icon} [{rec['priority']}] {rec['type']}") + print(f" {rec['message']}\n") + + print("="*60) + + # Suggested actions + print("\n💻 SUGGESTED ACTIONS:") + print("-"*60) + if stats['total_records'] > 100000: + print("• python optimize_time_attendance_db.py --action optimize") + if stats['total_records'] > 500000: + print("• python optimize_time_attendance_db.py --action archive --days 365 --execute") + if stats['date_range_days'] > 180: + print("• python optimize_time_attendance_db.py --action cleanup --days 90 --execute") + print("="*60 + "\n") + + +def main(): + """Main function to handle command line arguments""" + parser = argparse.ArgumentParser( + description='Time Attendance Database Optimization Tool', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s --action optimize # Create indexes and optimize table + %(prog)s --action analyze # Analyze table statistics + %(prog)s --action archive --days 365 # Archive records older than 365 days (dry run) + %(prog)s --action archive --days 365 --execute # Actually archive records + %(prog)s --action cleanup --days 90 --execute # Delete records older than 90 days + %(prog)s --action report # Generate optimization report + %(prog)s --action all # Run full optimization (indexes + analyze + optimize) + """ + ) + + parser.add_argument( + '--action', + required=True, + choices=['optimize', 'analyze', 'archive', 'cleanup', 'report', 'all', 'indexes'], + help='Action to perform' + ) + + parser.add_argument( + '--days', + type=int, + default=365, + help='Number of days for archive/cleanup (default: 365)' + ) + + parser.add_argument( + '--execute', + action='store_true', + help='Actually execute archive/cleanup (otherwise dry run)' + ) + + parser.add_argument( + '--quiet', + action='store_true', + help='Suppress verbose output' + ) + + args = parser.parse_args() + + # Create optimizer instance + optimizer = TimeAttendanceOptimizer(verbose=not args.quiet) + + print("\n" + "="*60) + print("🔧 TIME ATTENDANCE DATABASE OPTIMIZER") + print("="*60) + print(f"Action: {args.action.upper()}") + print(f"Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("="*60 + "\n") + + try: + with app.app_context(): + if args.action == 'indexes': + optimizer.create_indexes() + + elif args.action == 'optimize': + optimizer.create_indexes() + optimizer.optimize_table() + + elif args.action == 'analyze': + optimizer.analyze_table() + + elif args.action == 'archive': + optimizer.archive_old_records(days=args.days, execute=args.execute) + + elif args.action == 'cleanup': + optimizer.cleanup_old_records(days=args.days, execute=args.execute) + + elif args.action == 'report': + optimizer.generate_report() + + elif args.action == 'all': + optimizer.create_indexes() + optimizer.analyze_table() + optimizer.optimize_table() + print("\n✅ Full optimization complete!") + + print("\n" + "="*60) + print("✅ OPTIMIZATION COMPLETED SUCCESSFULLY") + print("="*60 + "\n") + + except KeyboardInterrupt: + print("\n\n⚠️ Operation cancelled by user") + sys.exit(1) + except Exception as e: + print(f"\n❌ Error: {e}") + import traceback + traceback.print_exc() + sys.exit(1) + + +if __name__ == '__main__': + main() \ No newline at end of file