Updated functions

app.py

@@ -6676,10 +6676,13 @@ def import_time_attendance():
                     
                     if duplicate_analysis['duplicate_records'] > 0:
                         print(f"⚠️ Found {duplicate_analysis['duplicate_records']} duplicates")
+                        # Get project_id from form
+                        project_id = request.form.get('project_id')
                         # Show duplicate review page
                         return render_template('time_attendance_duplicate_review.html',
                                             analysis=duplicate_analysis,
-                                             filename=filename)
+                                            filename=filename,
+                                            project_id=project_id)
                     else:
                         print("✅ No duplicates found")
                         flash('No duplicates found. Proceeding with import.', 'info')
@@ -6691,10 +6694,13 @@ def import_time_attendance():
                     
                     if invalid_analysis['invalid_rows'] > 0:
                         print(f"⚠️ Found {invalid_analysis['invalid_rows']} invalid rows")
+                        # Get project_id from form
+                        project_id = request.form.get('project_id')
                         # Show invalid row review page
                         return render_template('time_attendance_invalid_review.html',
                                             analysis=invalid_analysis,
-                                            filename=filename)
+                                            filename=filename,
+                                            project_id=project_id)
                     else:
                         print("✅ All rows are valid")
                         flash('All rows are valid. Proceeding with import.', 'info')

templates/time_attendance_duplicate_review.html

@@ -370,6 +370,7 @@
     <input type="hidden" name="analyze_duplicates" value="false">
     <input type="hidden" name="skip_duplicates" value="true">
     <input type="hidden" name="import_source" value="Import with Duplicates - {{ filename }}">
+    <input type="hidden" name="project_id" value="{{ project_id }}">
     
     <div class="duplicates-container">
       {% for duplicate in analysis.duplicates %}

templates/time_attendance_invalid_review.html

@@ -327,6 +327,7 @@
     <input type="hidden" name="validate_only" value="false">
     <input type="hidden" name="skip_duplicates" value="true">
     <input type="hidden" name="import_source" value="Import (Skipped Invalid Rows) - {{ filename }}">
+    <input type="hidden" name="project_id" value="{{ project_id }}">
     
     <div class="invalid-container">
       {% for invalid in analysis.invalid_details %}

tools/optimize_time_attendance_db.py

@@ -0,0 +1,600 @@
+#!/usr/bin/env python3
+"""
+==============================================================================
+Time Attendance Database Optimization Script
+==============================================================================
+
+Standalone script for optimizing the time_attendance table.
+Can be run manually or as a cronjob.
+
+Usage:
+    python optimize_time_attendance_db.py --action optimize
+    python optimize_time_attendance_db.py --action analyze
+    python optimize_time_attendance_db.py --action archive --days 365
+    python optimize_time_attendance_db.py --action cleanup --days 90
+    python optimize_time_attendance_db.py --action report
+    python optimize_time_attendance_db.py --action all
+
+Requirements:
+    - Must be run from the application directory
+    - Database credentials must be configured in config.py or environment
+
+Author: Database Optimization Team
+Date: 2025-10-14
+==============================================================================
+"""
+
+import sys
+import os
+import argparse
+from datetime import datetime, timedelta
+import logging
+
+# Add the application directory to the path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# Import Flask app and database
+try:
+    from app import app, db
+    from models.time_attendance import TimeAttendance
+    from sqlalchemy import text
+except ImportError as e:
+    print(f"❌ Error: Cannot import required modules: {e}")
+    print("   Make sure this script is in the same directory as app.py")
+    sys.exit(1)
+
+
+class TimeAttendanceOptimizer:
+    """Standalone optimizer for time_attendance table"""
+    
+    def __init__(self, verbose=True):
+        self.verbose = verbose
+        self.setup_logging()
+    
+    def setup_logging(self):
+        """Setup logging configuration"""
+        log_format = '%(asctime)s - %(levelname)s - %(message)s'
+        logging.basicConfig(
+            level=logging.INFO if self.verbose else logging.WARNING,
+            format=log_format
+        )
+        self.logger = logging.getLogger(__name__)
+    
+    def log(self, message, level='info'):
+        """Log message with appropriate level"""
+        if level == 'info':
+            self.logger.info(message)
+            if self.verbose:
+                print(f"ℹ️  {message}")
+        elif level == 'success':
+            self.logger.info(message)
+            if self.verbose:
+                print(f"✅ {message}")
+        elif level == 'warning':
+            self.logger.warning(message)
+            if self.verbose:
+                print(f"⚠️  {message}")
+        elif level == 'error':
+            self.logger.error(message)
+            if self.verbose:
+                print(f"❌ {message}")
+    
+    def create_indexes(self):
+        """Create optimized indexes for time_attendance table"""
+        self.log("Creating optimized indexes...", 'info')
+        
+        indexes = [
+            {
+                'name': 'idx_ta_employee_date_time',
+                'columns': 'employee_id, attendance_date DESC, attendance_time DESC',
+                'purpose': 'Employee-based queries with date filtering'
+            },
+            {
+                'name': 'idx_ta_date_location_employee',
+                'columns': 'attendance_date DESC, location_name, employee_id',
+                'purpose': 'Date and location filtering'
+            },
+            {
+                'name': 'idx_ta_project_date_employee',
+                'columns': 'project_id, attendance_date DESC, employee_id',
+                'purpose': 'Project-based attendance queries'
+            },
+            {
+                'name': 'idx_ta_batch_date',
+                'columns': 'import_batch_id, attendance_date DESC',
+                'purpose': 'Import batch management'
+            },
+            {
+                'name': 'idx_ta_action_date_employee',
+                'columns': 'action_description, attendance_date DESC, employee_id',
+                'purpose': 'Action-based analytics'
+            },
+            {
+                'name': 'idx_ta_date_time_desc',
+                'columns': 'attendance_date DESC, attendance_time DESC, id DESC',
+                'purpose': 'Recent records retrieval'
+            },
+            {
+                'name': 'idx_ta_location_action_date',
+                'columns': 'location_name, action_description, attendance_date DESC',
+                'purpose': 'Location-based action analysis'
+            },
+        ]
+        
+        created = 0
+        skipped = 0
+        failed = 0
+        
+        for idx in indexes:
+            try:
+                # Check if index exists
+                check_query = f"""
+                    SELECT COUNT(*) as count 
+                    FROM INFORMATION_SCHEMA.STATISTICS 
+                    WHERE TABLE_SCHEMA = DATABASE() 
+                    AND TABLE_NAME = 'time_attendance' 
+                    AND INDEX_NAME = '{idx['name']}'
+                """
+                result = db.session.execute(text(check_query)).fetchone()
+                
+                if result and result.count > 0:
+                    self.log(f"Index {idx['name']} already exists - skipped", 'info')
+                    skipped += 1
+                    continue
+                
+                # Create index
+                create_query = f"""
+                    CREATE INDEX {idx['name']} 
+                    ON time_attendance ({idx['columns']})
+                """
+                
+                self.log(f"Creating index: {idx['name']}", 'info')
+                db.session.execute(text(create_query))
+                db.session.commit()
+                
+                self.log(f"Created index: {idx['name']} - {idx['purpose']}", 'success')
+                created += 1
+                
+            except Exception as e:
+                self.log(f"Failed to create index {idx['name']}: {str(e)[:100]}", 'warning')
+                failed += 1
+                db.session.rollback()
+                continue
+        
+        self.log(f"Index creation complete: {created} created, {skipped} skipped, {failed} failed", 'success')
+        return {'created': created, 'skipped': skipped, 'failed': failed}
+    
+    def analyze_table(self):
+        """Analyze time_attendance table statistics"""
+        self.log("Analyzing table statistics...", 'info')
+        
+        try:
+            stats_query = """
+                SELECT 
+                    COUNT(*) as total_records,
+                    COUNT(DISTINCT employee_id) as unique_employees,
+                    COUNT(DISTINCT location_name) as unique_locations,
+                    COUNT(DISTINCT DATE(attendance_date)) as unique_dates,
+                    COUNT(DISTINCT import_batch_id) as unique_batches,
+                    COUNT(DISTINCT project_id) as unique_projects,
+                    MIN(attendance_date) as earliest_date,
+                    MAX(attendance_date) as latest_date,
+                    COUNT(CASE WHEN recorded_address IS NOT NULL THEN 1 END) as records_with_address
+                FROM time_attendance
+            """
+            
+            result = db.session.execute(text(stats_query)).fetchone()
+            
+            # Get table size
+            size_query = """
+                SELECT 
+                    ROUND((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024, 2) as size_mb,
+                    ROUND(DATA_LENGTH / 1024 / 1024, 2) as data_mb,
+                    ROUND(INDEX_LENGTH / 1024 / 1024, 2) as index_mb
+                FROM INFORMATION_SCHEMA.TABLES
+                WHERE TABLE_SCHEMA = DATABASE()
+                AND TABLE_NAME = 'time_attendance'
+            """
+            
+            size_result = db.session.execute(text(size_query)).fetchone()
+            
+            if result:
+                date_range_days = (result.latest_date - result.earliest_date).days if result.latest_date and result.earliest_date else 0
+                
+                stats = {
+                    'total_records': result.total_records,
+                    'unique_employees': result.unique_employees,
+                    'unique_locations': result.unique_locations,
+                    'unique_dates': result.unique_dates,
+                    'unique_batches': result.unique_batches,
+                    'unique_projects': result.unique_projects,
+                    'earliest_date': result.earliest_date,
+                    'latest_date': result.latest_date,
+                    'date_range_days': date_range_days,
+                    'records_with_address': result.records_with_address,
+                    'table_size_mb': size_result.size_mb if size_result else 0,
+                    'data_size_mb': size_result.data_mb if size_result else 0,
+                    'index_size_mb': size_result.index_mb if size_result else 0
+                }
+                
+                print("\n" + "="*60)
+                print("📊 TIME ATTENDANCE TABLE STATISTICS")
+                print("="*60)
+                print(f"Total Records:        {stats['total_records']:>15,}")
+                print(f"Unique Employees:     {stats['unique_employees']:>15,}")
+                print(f"Unique Locations:     {stats['unique_locations']:>15,}")
+                print(f"Unique Dates:         {stats['unique_dates']:>15,}")
+                print(f"Import Batches:       {stats['unique_batches']:>15,}")
+                print(f"Projects:             {stats['unique_projects']:>15,}")
+                print(f"Date Range:           {stats['date_range_days']:>15,} days")
+                print(f"Earliest Date:        {stats['earliest_date']:>15}")
+                print(f"Latest Date:          {stats['latest_date']:>15}")
+                print(f"Records w/ Address:   {stats['records_with_address']:>15,}")
+                print(f"\nTable Size:           {stats['table_size_mb']:>15.2f} MB")
+                print(f"  Data Size:          {stats['data_size_mb']:>15.2f} MB")
+                print(f"  Index Size:         {stats['index_size_mb']:>15.2f} MB")
+                print("="*60 + "\n")
+                
+                return stats
+            
+        except Exception as e:
+            self.log(f"Error analyzing table: {e}", 'error')
+            return None
+    
+    def optimize_table(self):
+        """Run MySQL OPTIMIZE TABLE and ANALYZE TABLE"""
+        self.log("Optimizing table structure...", 'info')
+        
+        try:
+            # Analyze table
+            self.log("Running ANALYZE TABLE...", 'info')
+            db.session.execute(text("ANALYZE TABLE time_attendance"))
+            db.session.commit()
+            self.log("ANALYZE TABLE completed", 'success')
+            
+            # Optimize table
+            self.log("Running OPTIMIZE TABLE (this may take a while)...", 'info')
+            db.session.execute(text("OPTIMIZE TABLE time_attendance"))
+            db.session.commit()
+            self.log("OPTIMIZE TABLE completed", 'success')
+            
+            return {'success': True}
+            
+        except Exception as e:
+            self.log(f"Error optimizing table: {e}", 'error')
+            db.session.rollback()
+            return {'success': False, 'error': str(e)}
+    
+    def create_archive_table(self):
+        """Create archive table if it doesn't exist"""
+        try:
+            create_query = """
+                CREATE TABLE IF NOT EXISTS time_attendance_archive 
+                LIKE time_attendance
+            """
+            db.session.execute(text(create_query))
+            db.session.commit()
+            return True
+        except Exception as e:
+            self.log(f"Error creating archive table: {e}", 'error')
+            db.session.rollback()
+            return False
+    
+    def archive_old_records(self, days=365, execute=False):
+        """Archive records older than specified days"""
+        self.log(f"Archive process for records older than {days} days...", 'info')
+        
+        cutoff_date = datetime.now() - timedelta(days=days)
+        
+        try:
+            # Count records to archive
+            count_query = """
+                SELECT COUNT(*) as count 
+                FROM time_attendance 
+                WHERE attendance_date < :cutoff_date
+            """
+            result = db.session.execute(
+                text(count_query),
+                {'cutoff_date': cutoff_date.date()}
+            ).fetchone()
+            
+            records_to_archive = result.count if result else 0
+            
+            print(f"\n📦 Archive Summary:")
+            print(f"   Cutoff Date:       {cutoff_date.date()}")
+            print(f"   Records to Archive: {records_to_archive:,}")
+            
+            if records_to_archive == 0:
+                self.log("No records to archive", 'info')
+                return {'records_archived': 0}
+            
+            if not execute:
+                print(f"\n⚠️  DRY RUN MODE - No records will be archived")
+                print(f"   Use --execute flag to actually archive records\n")
+                return {'records_archived': 0, 'dry_run': True}
+            
+            # Create archive table
+            if not self.create_archive_table():
+                return {'success': False, 'error': 'Failed to create archive table'}
+            
+            # Archive records in batches
+            self.log("Starting archive process...", 'info')
+            batch_size = 10000
+            total_archived = 0
+            
+            while total_archived < records_to_archive:
+                # Insert to archive
+                archive_query = """
+                    INSERT INTO time_attendance_archive 
+                    SELECT * FROM time_attendance 
+                    WHERE attendance_date < :cutoff_date 
+                    LIMIT :batch_size
+                """
+                
+                db.session.execute(
+                    text(archive_query),
+                    {'cutoff_date': cutoff_date.date(), 'batch_size': batch_size}
+                )
+                
+                # Delete from main table
+                delete_query = """
+                    DELETE FROM time_attendance 
+                    WHERE attendance_date < :cutoff_date 
+                    LIMIT :batch_size
+                """
+                
+                result = db.session.execute(
+                    text(delete_query),
+                    {'cutoff_date': cutoff_date.date(), 'batch_size': batch_size}
+                )
+                
+                rows_affected = result.rowcount
+                
+                if rows_affected == 0:
+                    break
+                
+                db.session.commit()
+                total_archived += rows_affected
+                
+                self.log(f"Archived {total_archived:,} / {records_to_archive:,} records...", 'info')
+                
+                # Safety limit
+                if total_archived >= 100000:
+                    self.log("Reached safety limit of 100,000 records per run", 'warning')
+                    break
+            
+            self.log(f"Archive complete: {total_archived:,} records archived", 'success')
+            return {'records_archived': total_archived, 'success': True}
+            
+        except Exception as e:
+            self.log(f"Error during archive: {e}", 'error')
+            db.session.rollback()
+            return {'success': False, 'error': str(e)}
+    
+    def cleanup_old_records(self, days=90, execute=False):
+        """Delete records older than specified days"""
+        self.log(f"Cleanup process for records older than {days} days...", 'info')
+        
+        cutoff_date = datetime.now() - timedelta(days=days)
+        
+        try:
+            # Count records to delete
+            count_query = """
+                SELECT COUNT(*) as count 
+                FROM time_attendance 
+                WHERE import_date < :cutoff_date
+            """
+            result = db.session.execute(
+                text(count_query),
+                {'cutoff_date': cutoff_date}
+            ).fetchone()
+            
+            records_to_delete = result.count if result else 0
+            
+            print(f"\n🗑️  Cleanup Summary:")
+            print(f"   Cutoff Date:        {cutoff_date.date()}")
+            print(f"   Records to Delete:  {records_to_delete:,}")
+            
+            if records_to_delete == 0:
+                self.log("No records to delete", 'info')
+                return {'records_deleted': 0}
+            
+            if not execute:
+                print(f"\n⚠️  DRY RUN MODE - No records will be deleted")
+                print(f"   Use --execute flag to actually delete records\n")
+                return {'records_deleted': 0, 'dry_run': True}
+            
+            # Delete records
+            delete_query = """
+                DELETE FROM time_attendance 
+                WHERE import_date < :cutoff_date
+            """
+            
+            self.log("Deleting old records...", 'info')
+            db.session.execute(
+                text(delete_query),
+                {'cutoff_date': cutoff_date}
+            )
+            db.session.commit()
+            
+            self.log(f"Cleanup complete: {records_to_delete:,} records deleted", 'success')
+            return {'records_deleted': records_to_delete, 'success': True}
+            
+        except Exception as e:
+            self.log(f"Error during cleanup: {e}", 'error')
+            db.session.rollback()
+            return {'success': False, 'error': str(e)}
+    
+    def generate_report(self):
+        """Generate comprehensive optimization report"""
+        print("\n" + "="*60)
+        print("📋 TIME ATTENDANCE OPTIMIZATION REPORT")
+        print("="*60)
+        print(f"Generated: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n")
+        
+        # Get statistics
+        stats = self.analyze_table()
+        
+        if stats:
+            # Generate recommendations
+            print("\n" + "="*60)
+            print("💡 RECOMMENDATIONS")
+            print("="*60)
+            
+            recommendations = []
+            
+            if stats['total_records'] > 100000:
+                recommendations.append({
+                    'priority': 'HIGH',
+                    'type': 'Indexing',
+                    'message': f"Table has {stats['total_records']:,} records. Run index optimization."
+                })
+            
+            if stats['total_records'] > 500000:
+                recommendations.append({
+                    'priority': 'HIGH',
+                    'type': 'Archiving',
+                    'message': f"Consider archiving records older than 365 days."
+                })
+            
+            if stats['table_size_mb'] > 500:
+                recommendations.append({
+                    'priority': 'MEDIUM',
+                    'type': 'Optimization',
+                    'message': f"Table size is {stats['table_size_mb']:.2f} MB. Run OPTIMIZE TABLE."
+                })
+            
+            if stats['date_range_days'] > 365:
+                recommendations.append({
+                    'priority': 'MEDIUM',
+                    'type': 'Data Retention',
+                    'message': f"Data spans {stats['date_range_days']} days. Implement retention policy."
+                })
+            
+            if stats['index_size_mb'] > stats['data_size_mb'] * 1.5:
+                recommendations.append({
+                    'priority': 'LOW',
+                    'type': 'Index Review',
+                    'message': f"Index size ({stats['index_size_mb']:.2f} MB) is large. Review index usage."
+                })
+            
+            if not recommendations:
+                print("✓ No major issues found. Database is well optimized.\n")
+            else:
+                for rec in recommendations:
+                    priority_icon = "🔴" if rec['priority'] == 'HIGH' else "🟡" if rec['priority'] == 'MEDIUM' else "🟢"
+                    print(f"{priority_icon} [{rec['priority']}] {rec['type']}")
+                    print(f"   {rec['message']}\n")
+            
+            print("="*60)
+            
+            # Suggested actions
+            print("\n💻 SUGGESTED ACTIONS:")
+            print("-"*60)
+            if stats['total_records'] > 100000:
+                print("• python optimize_time_attendance_db.py --action optimize")
+            if stats['total_records'] > 500000:
+                print("• python optimize_time_attendance_db.py --action archive --days 365 --execute")
+            if stats['date_range_days'] > 180:
+                print("• python optimize_time_attendance_db.py --action cleanup --days 90 --execute")
+            print("="*60 + "\n")
+
+
+def main():
+    """Main function to handle command line arguments"""
+    parser = argparse.ArgumentParser(
+        description='Time Attendance Database Optimization Tool',
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  %(prog)s --action optimize              # Create indexes and optimize table
+  %(prog)s --action analyze               # Analyze table statistics
+  %(prog)s --action archive --days 365    # Archive records older than 365 days (dry run)
+  %(prog)s --action archive --days 365 --execute  # Actually archive records
+  %(prog)s --action cleanup --days 90 --execute   # Delete records older than 90 days
+  %(prog)s --action report                # Generate optimization report
+  %(prog)s --action all                   # Run full optimization (indexes + analyze + optimize)
+        """
+    )
+    
+    parser.add_argument(
+        '--action',
+        required=True,
+        choices=['optimize', 'analyze', 'archive', 'cleanup', 'report', 'all', 'indexes'],
+        help='Action to perform'
+    )
+    
+    parser.add_argument(
+        '--days',
+        type=int,
+        default=365,
+        help='Number of days for archive/cleanup (default: 365)'
+    )
+    
+    parser.add_argument(
+        '--execute',
+        action='store_true',
+        help='Actually execute archive/cleanup (otherwise dry run)'
+    )
+    
+    parser.add_argument(
+        '--quiet',
+        action='store_true',
+        help='Suppress verbose output'
+    )
+    
+    args = parser.parse_args()
+    
+    # Create optimizer instance
+    optimizer = TimeAttendanceOptimizer(verbose=not args.quiet)
+    
+    print("\n" + "="*60)
+    print("🔧 TIME ATTENDANCE DATABASE OPTIMIZER")
+    print("="*60)
+    print(f"Action: {args.action.upper()}")
+    print(f"Date: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
+    print("="*60 + "\n")
+    
+    try:
+        with app.app_context():
+            if args.action == 'indexes':
+                optimizer.create_indexes()
+            
+            elif args.action == 'optimize':
+                optimizer.create_indexes()
+                optimizer.optimize_table()
+            
+            elif args.action == 'analyze':
+                optimizer.analyze_table()
+            
+            elif args.action == 'archive':
+                optimizer.archive_old_records(days=args.days, execute=args.execute)
+            
+            elif args.action == 'cleanup':
+                optimizer.cleanup_old_records(days=args.days, execute=args.execute)
+            
+            elif args.action == 'report':
+                optimizer.generate_report()
+            
+            elif args.action == 'all':
+                optimizer.create_indexes()
+                optimizer.analyze_table()
+                optimizer.optimize_table()
+                print("\n✅ Full optimization complete!")
+            
+            print("\n" + "="*60)
+            print("✅ OPTIMIZATION COMPLETED SUCCESSFULLY")
+            print("="*60 + "\n")
+            
+    except KeyboardInterrupt:
+        print("\n\n⚠️  Operation cancelled by user")
+        sys.exit(1)
+    except Exception as e:
+        print(f"\n❌ Error: {e}")
+        import traceback
+        traceback.print_exc()
+        sys.exit(1)
+
+
+if __name__ == '__main__':
+    main()