Optimize system

advanced_security_middleware.py

@@ -0,0 +1,463 @@
+# File: advanced_security_middleware.py
+# Enhanced security middleware for QR Attendance System
+
+from functools import wraps
+from flask import request, session, jsonify, current_app, g
+import hashlib
+import secrets
+import jwt
+from datetime import datetime, timedelta
+import re
+from collections import defaultdict, deque
+import time
+import hmac
+import base64
+import os
+
+# Try to import cryptography, fallback if not available
+try:
+    from cryptography.fernet import Fernet
+    HAS_CRYPTOGRAPHY = True
+except ImportError:
+    HAS_CRYPTOGRAPHY = False
+
+class SecurityManager:
+    """
+    Advanced security manager for QR Attendance System
+    """
+    
+    def __init__(self, app=None, db=None, logger_handler=None):
+        self.app = app
+        self.db = db
+        self.logger_handler = logger_handler
+        
+        # Security tracking
+        self.failed_attempts = defaultdict(lambda: deque(maxlen=10))
+        self.suspicious_ips = defaultdict(int)
+        self.session_tokens = {}
+        
+        # Security configuration
+        self.max_failed_attempts = 5
+        self.lockout_duration = 900  # 15 minutes
+        self.session_timeout = 3600  # 1 hour
+        
+        if app:
+            self.init_app(app, db, logger_handler)
+    
+    def init_app(self, app, db, logger_handler):
+        """Initialize security manager with Flask app"""
+        self.app = app
+        self.db = db
+        self.logger_handler = logger_handler
+        
+        # Generate encryption key for sensitive data
+        self.setup_encryption()
+        
+        # Register security middleware
+        app.before_request(self.security_check)
+        
+        # Register security routes
+        self.register_security_routes()
+    
+    def setup_encryption(self):
+        """Setup encryption for sensitive data"""
+        if HAS_CRYPTOGRAPHY:
+            encryption_key = self.app.config.get('ENCRYPTION_KEY')
+            if not encryption_key:
+                # Generate a new key (should be stored securely in production)
+                encryption_key = Fernet.generate_key()
+                if self.logger_handler:
+                    self.logger_handler.logger.warning(
+                        "Generated new encryption key - store this securely!"
+                    )
+            
+            self.cipher = Fernet(encryption_key)
+        else:
+            self.cipher = None
+            if self.logger_handler:
+                self.logger_handler.logger.warning(
+                    "Cryptography not available - encryption features disabled"
+                )
+    
+    def security_check(self):
+        """Comprehensive security check before each request"""
+        client_ip = self.get_client_ip()
+        
+        # Check for suspicious activity
+        if self.is_suspicious_request():
+            self.log_security_event('suspicious_request', {
+                'ip': client_ip,
+                'user_agent': request.headers.get('User-Agent', ''),
+                'endpoint': request.endpoint,
+                'method': request.method
+            })
+            return jsonify({'error': 'Request blocked for security reasons'}), 403
+        
+        # Validate session security
+        if 'user_id' in session:
+            if not self.validate_session_security():
+                session.clear()
+                return jsonify({'error': 'Session security validation failed'}), 401
+        
+        # Check for SQL injection attempts
+        if self.detect_sql_injection():
+            self.log_security_event('sql_injection_attempt', {
+                'ip': client_ip,
+                'query_params': dict(request.args),
+                'form_data': dict(request.form) if request.form else {}
+            })
+            return jsonify({'error': 'Malicious request detected'}), 403
+        
+        # Rate limiting for authentication endpoints
+        if request.endpoint in ['login', 'register', 'reset_password']:
+            if self.is_auth_rate_limited():
+                return jsonify({
+                    'error': 'Too many attempts, please try again later'
+                }), 429
+    
+    def get_client_ip(self):
+        """Get real client IP address"""
+        # Check for forwarded headers (in case behind proxy/CDN)
+        forwarded_ips = request.headers.getlist('X-Forwarded-For')
+        if forwarded_ips:
+            return forwarded_ips[0].split(',')[0].strip()
+        
+        return request.headers.get('X-Real-IP') or request.remote_addr
+    
+    def is_suspicious_request(self):
+        """Detect suspicious request patterns"""
+        client_ip = self.get_client_ip()
+        user_agent = request.headers.get('User-Agent', '').lower()
+        
+        # Check for common attack patterns
+        suspicious_patterns = [
+            r'<script', r'javascript:', r'vbscript:', r'onload=', r'onerror=',
+            r'union\s+select', r'drop\s+table', r'insert\s+into',
+            r'\.\./\.\./.*etc/passwd', r'cmd\.exe', r'/bin/bash'
+        ]
+        
+        request_data = str(request.args) + str(request.form) + request.path
+        
+        for pattern in suspicious_patterns:
+            if re.search(pattern, request_data, re.IGNORECASE):
+                self.suspicious_ips[client_ip] += 1
+                return True
+        
+        # Check for suspicious user agents
+        bot_patterns = ['bot', 'crawler', 'spider', 'scraper', 'scanner']
+        if any(pattern in user_agent for pattern in bot_patterns):
+            if request.endpoint not in ['static', 'favicon']:
+                return True
+        
+        # Check request frequency (basic rate limiting)
+        current_time = time.time()
+        if not hasattr(g, 'request_history'):
+            g.request_history = deque(maxlen=50)
+        
+        g.request_history.append(current_time)
+        recent_requests = [t for t in g.request_history if current_time - t < 60]
+        
+        if len(recent_requests) > 30:  # More than 30 requests per minute
+            return True
+        
+        return False
+    
+    def validate_session_security(self):
+        """Validate session security and integrity"""
+        try:
+            user_id = session.get('user_id')
+            session_token = session.get('security_token')
+            
+            if not user_id or not session_token:
+                return False
+            
+            # Check if session token matches stored token
+            stored_token = self.session_tokens.get(user_id)
+            if not stored_token or not hmac.compare_digest(session_token, stored_token['token']):
+                return False
+            
+            # Check session timeout
+            if time.time() - stored_token['created'] > self.session_timeout:
+                del self.session_tokens[user_id]
+                return False
+            
+            # Check if session IP matches (optional security measure)
+            if self.app.config.get('STRICT_SESSION_IP', False):
+                if stored_token['ip'] != self.get_client_ip():
+                    self.log_security_event('session_ip_mismatch', {
+                        'user_id': user_id,
+                        'original_ip': stored_token['ip'],
+                        'current_ip': self.get_client_ip()
+                    })
+                    return False
+            
+            return True
+            
+        except Exception as e:
+            if self.logger_handler:
+                self.logger_handler.logger.error(f"Session validation error: {e}")
+            return False
+    
+    def detect_sql_injection(self):
+        """Detect potential SQL injection attempts"""
+        sql_patterns = [
+            r"union\s+select", r"drop\s+table", r"insert\s+into",
+            r"delete\s+from", r"update\s+set", r"exec\s*\(",
+            r"sp_executesql", r"xp_cmdshell", r";\s*--",
+            r"'\s*or\s*'", r'"\s*or\s*"', r"1\s*=\s*1"
+        ]
+        
+        # Check all request parameters
+        check_data = []
+        check_data.extend(request.args.values())
+        check_data.extend(request.form.values())
+        
+        if request.json:
+            check_data.extend(str(v) for v in request.json.values() if isinstance(v, (str, int, float)))
+        
+        for data in check_data:
+            data_str = str(data).lower()
+            for pattern in sql_patterns:
+                if re.search(pattern, data_str, re.IGNORECASE):
+                    return True
+        
+        return False
+    
+    def is_auth_rate_limited(self):
+        """Check if authentication endpoint is rate limited"""
+        client_ip = self.get_client_ip()
+        current_time = time.time()
+        
+        # Clean old attempts
+        self.failed_attempts[client_ip] = deque([
+            attempt for attempt in self.failed_attempts[client_ip]
+            if current_time - attempt < 900  # Keep attempts from last 15 minutes
+        ], maxlen=10)
+        
+        return len(self.failed_attempts[client_ip]) >= self.max_failed_attempts
+    
+    def record_failed_attempt(self, identifier):
+        """Record a failed authentication attempt"""
+        client_ip = self.get_client_ip()
+        current_time = time.time()
+        
+        self.failed_attempts[client_ip].append(current_time)
+        
+        self.log_security_event('authentication_failure', {
+            'ip': client_ip,
+            'identifier': identifier,
+            'attempts': len(self.failed_attempts[client_ip])
+        })
+    
+    def create_secure_session(self, user_id):
+        """Create a secure session with additional security measures"""
+        # Generate secure session token
+        session_token = secrets.token_urlsafe(32)
+        
+        # Store session information
+        self.session_tokens[user_id] = {
+            'token': session_token,
+            'created': time.time(),
+            'ip': self.get_client_ip(),
+            'user_agent': request.headers.get('User-Agent', '')[:200]
+        }
+        
+        # Set session data
+        session['security_token'] = session_token
+        session['login_time'] = datetime.utcnow().isoformat()
+        
+        # Clear any failed attempts for this IP
+        client_ip = self.get_client_ip()
+        if client_ip in self.failed_attempts:
+            del self.failed_attempts[client_ip]
+        
+        self.log_security_event('secure_session_created', {
+            'user_id': user_id,
+            'ip': client_ip
+        })
+    
+    def encrypt_sensitive_data(self, data):
+        """Encrypt sensitive data before storage"""
+        if not self.cipher:
+            return data  # Return as-is if encryption not available
+        
+        try:
+            if isinstance(data, str):
+                data = data.encode('utf-8')
+            
+            encrypted_data = self.cipher.encrypt(data)
+            return base64.b64encode(encrypted_data).decode('utf-8')
+            
+        except Exception as e:
+            if self.logger_handler:
+                self.logger_handler.logger.error(f"Encryption error: {e}")
+            return data
+    
+    def decrypt_sensitive_data(self, encrypted_data):
+        """Decrypt sensitive data"""
+        if not self.cipher:
+            return encrypted_data  # Return as-is if encryption not available
+        
+        try:
+            encrypted_bytes = base64.b64decode(encrypted_data.encode('utf-8'))
+            decrypted_data = self.cipher.decrypt(encrypted_bytes)
+            return decrypted_data.decode('utf-8')
+            
+        except Exception as e:
+            if self.logger_handler:
+                self.logger_handler.logger.error(f"Decryption error: {e}")
+            return encrypted_data
+    
+    def log_security_event(self, event_type, details):
+        """Log security events for monitoring"""
+        try:
+            security_log = {
+                'event_type': event_type,
+                'timestamp': datetime.utcnow().isoformat(),
+                'ip': self.get_client_ip(),
+                'user_agent': request.headers.get('User-Agent', ''),
+                'endpoint': request.endpoint,
+                'method': request.method,
+                'details': details
+            }
+            
+            if self.logger_handler:
+                self.logger_handler.log_security_event(
+                    event_type=event_type,
+                    description=f"Security event: {event_type}",
+                    additional_data=security_log
+                )
+            
+        except Exception as e:
+            if self.logger_handler:
+                self.logger_handler.logger.error(f"Security logging error: {e}")
+    
+    def register_security_routes(self):
+        """Register security monitoring API endpoints"""
+        
+        @self.app.route('/api/security/status')
+        def security_status():
+            """Get current security status"""
+            try:
+                # Admin only endpoint
+                if not session.get('user_id') or session.get('role') != 'admin':
+                    return jsonify({'error': 'Access denied'}), 403
+                
+                current_time = time.time()
+                
+                # Count active suspicious IPs
+                suspicious_count = len([
+                    ip for ip, count in self.suspicious_ips.items() 
+                    if count > 3
+                ])
+                
+                # Count recent failed attempts
+                recent_failures = sum(
+                    len([
+                        attempt for attempt in attempts 
+                        if current_time - attempt < 300  # Last 5 minutes
+                    ])
+                    for attempts in self.failed_attempts.values()
+                )
+                
+                # Count active sessions
+                active_sessions = len([
+                    token for token in self.session_tokens.values()
+                    if current_time - token['created'] < self.session_timeout
+                ])
+                
+                return jsonify({
+                    'suspicious_ips': suspicious_count,
+                    'recent_failed_attempts': recent_failures,
+                    'active_sessions': active_sessions,
+                    'rate_limited_ips': len(self.failed_attempts),
+                    'security_status': 'normal' if suspicious_count < 5 else 'elevated'
+                })
+                
+            except Exception as e:
+                if self.logger_handler:
+                    self.logger_handler.logger.error(f"Security status error: {e}")
+                return jsonify({'error': 'Failed to get security status'}), 500
+        
+        @self.app.route('/api/security/clear-blocks', methods=['POST'])
+        def clear_security_blocks():
+            """Clear security blocks (admin only)"""
+            try:
+                if not session.get('user_id') or session.get('role') != 'admin':
+                    return jsonify({'error': 'Access denied'}), 403
+                
+                # Clear failed attempts
+                cleared_ips = len(self.failed_attempts)
+                self.failed_attempts.clear()
+                
+                # Clear suspicious IPs
+                cleared_suspicious = len(self.suspicious_ips)
+                self.suspicious_ips.clear()
+                
+                self.log_security_event('security_blocks_cleared', {
+                    'admin_user': session.get('username'),
+                    'cleared_failed_attempts': cleared_ips,
+                    'cleared_suspicious_ips': cleared_suspicious
+                })
+                
+                return jsonify({
+                    'message': 'Security blocks cleared successfully',
+                    'cleared_failed_attempts': cleared_ips,
+                    'cleared_suspicious_ips': cleared_suspicious
+                })
+                
+            except Exception as e:
+                if self.logger_handler:
+                    self.logger_handler.logger.error(f"Clear blocks error: {e}")
+                return jsonify({'error': 'Failed to clear security blocks'}), 500
+
+def enhanced_login_required(f):
+    """
+    Enhanced login required decorator with security checks
+    """
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        if 'user_id' not in session:
+            return jsonify({'error': 'Authentication required'}), 401
+        
+        # Additional security validation
+        if not session.get('security_token'):
+            session.clear()
+            return jsonify({'error': 'Session security validation failed'}), 401
+        
+        # Check session timeout
+        login_time_str = session.get('login_time')
+        if login_time_str:
+            try:
+                login_time = datetime.fromisoformat(login_time_str)
+                if datetime.utcnow() - login_time > timedelta(hours=8):
+                    session.clear()
+                    return jsonify({'error': 'Session expired'}), 401
+            except ValueError:
+                session.clear()
+                return jsonify({'error': 'Invalid session data'}), 401
+        
+        return f(*args, **kwargs)
+    return decorated_function
+
+def csrf_protect(f):
+    """
+    CSRF protection decorator
+    """
+    @wraps(f)
+    def decorated_function(*args, **kwargs):
+        if request.method == 'POST':
+            token = request.form.get('csrf_token') or request.headers.get('X-CSRF-Token')
+            expected_token = session.get('csrf_token')
+            
+            if not token or not expected_token or not hmac.compare_digest(token, expected_token):
+                return jsonify({'error': 'CSRF token validation failed'}), 403
+        
+        return f(*args, **kwargs)
+    return decorated_function
+
+def generate_csrf_token():
+    """Generate CSRF token for forms"""
+    if 'csrf_token' not in session:
+        session['csrf_token'] = secrets.token_urlsafe(32)
+    return session['csrf_token']

app.py

@@ -18,6 +18,8 @@ from payroll_excel_exporter import PayrollExcelExporter
 # Load environment variables in .env
 load_dotenv()
 from turnstile_utils import turnstile_utils
+from db_performance_optimization import initialize_performance_optimizations
+from app_performance_middleware import PerformanceMonitor
 
 # Initialize Flask application
 app = Flask(__name__)
@@ -1621,6 +1623,102 @@ def dashboard():
         flash('Error loading dashboard. Please try again.', 'error')
         return redirect(url_for('login'))
 
+@app.route('/api/dashboard/stats')
+@login_required
+def dashboard_stats_api():
+    """API endpoint for dashboard statistics"""
+    try:
+        # Get current stats
+        total_qr_codes = QRCode.query.filter_by(active_status=True).count()
+        
+        # Today's check-ins
+        today = datetime.utcnow().date()
+        today_checkins = AttendanceData.query.filter(
+            AttendanceData.check_in_date == today
+        ).count()
+        
+        # Active projects
+        active_projects = Project.query.filter_by(active_status=True).count()
+        
+        # Unique locations
+        unique_locations = db.session.query(
+            AttendanceData.location_name
+        ).distinct().count()
+        
+        # Calculate trends (compared to last month)
+        last_month = datetime.utcnow() - timedelta(days=30)
+        
+        # QR codes trend
+        old_qr_count = QRCode.query.filter(
+            QRCode.created_date <= last_month,
+            QRCode.active_status == True
+        ).count()
+        qr_change = ((total_qr_codes - old_qr_count) / max(old_qr_count, 1)) * 100
+        
+        # Check-ins trend (yesterday)
+        yesterday = today - timedelta(days=1)
+        yesterday_checkins = AttendanceData.query.filter(
+            AttendanceData.check_in_date == yesterday
+        ).count()
+        checkin_change = ((today_checkins - yesterday_checkins) / max(yesterday_checkins, 1)) * 100
+        
+        return jsonify({
+            'success': True,
+            'total_qr_codes': total_qr_codes,
+            'today_checkins': today_checkins,
+            'active_projects': active_projects,
+            'unique_locations': unique_locations,
+            'qr_change': round(qr_change, 1),
+            'checkin_change': round(checkin_change, 1),
+            'project_change': 0,  # You can calculate this based on your needs
+            'location_change': 0   # You can calculate this based on your needs
+        })
+        
+    except Exception as e:
+        logger_handler.log_database_error('dashboard_stats_api', e)
+        return jsonify({
+            'success': False,
+            'error': 'Failed to fetch dashboard statistics'
+        }), 500
+
+@app.route('/api/dashboard/realtime')
+@login_required
+def dashboard_realtime_api():
+    """API endpoint for real-time dashboard data"""
+    try:
+        # Get recent activity (last 10 check-ins)
+        recent_activity = db.session.query(
+            AttendanceData.employee_id,
+            AttendanceData.location_name,
+            AttendanceData.check_in_time,
+            AttendanceData.check_in_date
+        ).order_by(
+            AttendanceData.check_in_date.desc(),
+            AttendanceData.check_in_time.desc()
+        ).limit(10).all()
+        
+        activity_data = [
+            {
+                'employee_id': activity.employee_id,
+                'location': activity.location_name,
+                'time': activity.check_in_time.strftime('%H:%M'),
+                'date': activity.check_in_date.strftime('%Y-%m-%d')
+            }
+            for activity in recent_activity
+        ]
+        
+        return jsonify({
+            'success': True,
+            'recent_activity': activity_data
+        })
+        
+    except Exception as e:
+        logger_handler.log_database_error('dashboard_realtime_api', e)
+        return jsonify({
+            'success': False,
+            'error': 'Failed to fetch real-time data'
+        }), 500
+    
 # USER MANAGEMENT ROUTES
 @app.route('/profile', methods=['GET', 'POST'])
 @login_required
@@ -6048,6 +6146,7 @@ def employee_detail(employee_index):
     
 create_location_logging_routes(app, db, logger_handler)
 
+
 # Jinja2 filters for better template functionality
 @app.template_filter('days_since')
 def days_since_filter(date):
@@ -6440,22 +6539,16 @@ if __name__ == '__main__':
             # Initialize database and logging
             create_tables()
             
-            # Add performance optimizations
+            # Initialize performance optimizations
-            create_performance_indexes()
+            print("🚀 Initializing performance optimizations...")
-            create_audit_triggers()
+            cached_query = initialize_performance_optimizations(app, db, logger_handler)
+            performance_monitor = PerformanceMonitor(app, db, logger_handler)
+
+            if cached_query:
+                print("✅ Performance optimizations completed successfully")
+            else:
+                print("⚠️ Performance optimizations completed with warnings")
 
-            # Log optimization completion
-            logger_handler.log_system_event(
-                event_type="database_optimization_complete",
-                description="Database performance optimization completed successfully",
-                severity="INFO",
-                additional_data={
-                    "indexes_created": True,
-                    "triggers_created": True,
-                    "optimization_timestamp": datetime.utcnow().isoformat()
-                }
-            )
-            print("🚀 Database optimization completed successfully")
             # Log application startup
             logger_handler.logger.info("QR Attendance Management System started successfully")
 

app_performance_middleware.py

@@ -0,0 +1,347 @@
+# File: app_performance_middleware.py
+# Advanced performance middleware for QR Attendance System
+
+from functools import wraps
+from flask import request, g, jsonify, current_app
+import time
+import threading
+import queue
+from datetime import datetime, timedelta
+from collections import defaultdict, deque
+import gc
+import psutil
+import os
+
+class PerformanceMonitor:
+    """
+    Advanced performance monitoring and optimization middleware
+    """
+    
+    def __init__(self, app=None, db=None, logger_handler=None):
+        self.app = app
+        self.db = db
+        self.logger_handler = logger_handler
+        
+        # Performance metrics storage
+        self.request_times = deque(maxlen=1000)  # Keep last 1000 requests
+        self.slow_queries = deque(maxlen=100)
+        self.error_rates = defaultdict(int)
+        self.endpoint_stats = defaultdict(lambda: {'count': 0, 'total_time': 0, 'errors': 0})
+        
+        # Rate limiting storage
+        self.rate_limit_storage = defaultdict(lambda: {'requests': deque(), 'blocked_until': None})
+        
+        # Background task queue
+        self.task_queue = queue.Queue()
+        self.background_worker = None
+        
+        if app:
+            self.init_app(app, db, logger_handler)
+    
+    def init_app(self, app, db, logger_handler):
+        """Initialize performance monitoring with Flask app"""
+        self.app = app
+        self.db = db
+        self.logger_handler = logger_handler
+        
+        # Register before/after request handlers
+        app.before_request(self.before_request)
+        app.after_request(self.after_request)
+        
+        # Start background worker
+        self.start_background_worker()
+        
+        # Register performance monitoring routes
+        self.register_performance_routes()
+    
+    def before_request(self):
+        """Performance monitoring before each request"""
+        g.start_time = time.time()
+        g.request_id = f"{int(time.time())}-{threading.get_ident()}"
+        
+        # Rate limiting check
+        if self.is_rate_limited():
+            return jsonify({
+                'error': 'Rate limit exceeded',
+                'retry_after': 60
+            }), 429
+        
+        # Memory usage monitoring
+        self.monitor_memory_usage()
+    
+    def after_request(self, response):
+        """Performance monitoring after each request"""
+        if hasattr(g, 'start_time'):
+            request_time = time.time() - g.start_time
+            
+            # Record request metrics
+            self.record_request_metrics(request_time, response.status_code)
+            
+            # Log slow requests
+            if request_time > 2.0:  # Requests taking more than 2 seconds
+                self.log_slow_request(request_time)
+            
+            # Add performance headers
+            response.headers['X-Response-Time'] = f"{request_time:.3f}s"
+            response.headers['X-Request-ID'] = getattr(g, 'request_id', 'unknown')
+        
+        return response
+    
+    def record_request_metrics(self, request_time, status_code):
+        """Record request performance metrics"""
+        endpoint = request.endpoint or 'unknown'
+        
+        # Store request time
+        self.request_times.append({
+            'endpoint': endpoint,
+            'time': request_time,
+            'status': status_code,
+            'timestamp': datetime.utcnow()
+        })
+        
+        # Update endpoint statistics
+        self.endpoint_stats[endpoint]['count'] += 1
+        self.endpoint_stats[endpoint]['total_time'] += request_time
+        
+        if status_code >= 400:
+            self.endpoint_stats[endpoint]['errors'] += 1
+            self.error_rates[status_code] += 1
+    
+    def is_rate_limited(self):
+        """Check if current request should be rate limited"""
+        client_ip = request.environ.get('REMOTE_ADDR', 'unknown')
+        current_time = time.time()
+        
+        # Clean up old requests
+        client_data = self.rate_limit_storage[client_ip]
+        client_data['requests'] = deque([
+            req_time for req_time in client_data['requests'] 
+            if current_time - req_time < 60  # 1 minute window
+        ], maxlen=100)
+        
+        # Check if currently blocked
+        if client_data['blocked_until'] and current_time < client_data['blocked_until']:
+            return True
+        
+        # Add current request
+        client_data['requests'].append(current_time)
+        
+        # Check rate limit (100 requests per minute)
+        if len(client_data['requests']) > 100:
+            client_data['blocked_until'] = current_time + 300  # Block for 5 minutes
+            self.logger_handler.logger.warning(f"Rate limit exceeded for IP: {client_ip}")
+            return True
+        
+        return False
+    
+    def monitor_memory_usage(self):
+        """Monitor application memory usage"""
+        # Get memory usage every 10 requests (approximately)
+        import random
+        if random.randint(1, 10) == 1:
+            process = psutil.Process(os.getpid())
+            memory_info = process.memory_info()
+            memory_mb = memory_info.rss / 1024 / 1024
+            
+            if memory_mb > 1000:  # More than 1GB
+                self.logger_handler.logger.warning(f"High memory usage: {memory_mb:.1f}MB")
+                
+                # Force garbage collection
+                gc.collect()
+                
+                # Queue background cleanup task
+                self.task_queue.put({
+                    'type': 'memory_cleanup',
+                    'timestamp': datetime.utcnow()
+                })
+    
+    def log_slow_request(self, request_time):
+        """Log slow requests for optimization"""
+        slow_request_data = {
+            'endpoint': request.endpoint,
+            'method': request.method,
+            'time': request_time,
+            'args': dict(request.args),
+            'timestamp': datetime.utcnow()
+        }
+        
+        self.slow_queries.append(slow_request_data)
+        
+        self.logger_handler.logger.warning(
+            f"Slow request: {request.method} {request.endpoint} - {request_time:.3f}s"
+        )
+    
+    def start_background_worker(self):
+        """Start background worker for performance tasks"""
+        def worker():
+            while True:
+                try:
+                    task = self.task_queue.get(timeout=30)
+                    self.process_background_task(task)
+                    self.task_queue.task_done()
+                except queue.Empty:
+                    continue
+                except Exception as e:
+                    if self.logger_handler:
+                        self.logger_handler.logger.error(f"Background worker error: {e}")
+        
+        self.background_worker = threading.Thread(target=worker, daemon=True)
+        self.background_worker.start()
+    
+    def process_background_task(self, task):
+        """Process background performance tasks"""
+        task_type = task.get('type')
+        
+        if task_type == 'memory_cleanup':
+            self.perform_memory_cleanup()
+        elif task_type == 'performance_analysis':
+            self.perform_performance_analysis()
+        elif task_type == 'database_optimization':
+            self.optimize_database_connections()
+    
+    def perform_memory_cleanup(self):
+        """Perform memory cleanup operations"""
+        try:
+            # Clear old metrics
+            cutoff_time = datetime.utcnow() - timedelta(hours=1)
+            
+            # Clean request times
+            self.request_times = deque([
+                req for req in self.request_times 
+                if req['timestamp'] > cutoff_time
+            ], maxlen=1000)
+            
+            # Clean slow queries
+            self.slow_queries = deque([
+                query for query in self.slow_queries 
+                if query['timestamp'] > cutoff_time
+            ], maxlen=100)
+            
+            # Clean rate limit storage
+            current_time = time.time()
+            for ip, data in list(self.rate_limit_storage.items()):
+                if not data['requests'] and (
+                    not data['blocked_until'] or current_time > data['blocked_until']
+                ):
+                    del self.rate_limit_storage[ip]
+            
+            # Force garbage collection
+            gc.collect()
+            
+            self.logger_handler.logger.info("Memory cleanup completed")
+            
+        except Exception as e:
+            self.logger_handler.logger.error(f"Memory cleanup failed: {e}")
+    
+    def register_performance_routes(self):
+        """Register performance monitoring API endpoints"""
+        
+        @self.app.route('/api/performance/stats')
+        def performance_stats():
+            """Get current performance statistics"""
+            try:
+                # Calculate average response times
+                recent_requests = [
+                    req for req in self.request_times 
+                    if req['timestamp'] > datetime.utcnow() - timedelta(minutes=5)
+                ]
+                
+                avg_response_time = (
+                    sum(req['time'] for req in recent_requests) / len(recent_requests)
+                    if recent_requests else 0
+                )
+                
+                # Get endpoint statistics
+                endpoint_performance = {}
+                for endpoint, stats in self.endpoint_stats.items():
+                    endpoint_performance[endpoint] = {
+                        'avg_response_time': stats['total_time'] / stats['count'] if stats['count'] > 0 else 0,
+                        'total_requests': stats['count'],
+                        'error_rate': stats['errors'] / stats['count'] if stats['count'] > 0 else 0
+                    }
+                
+                # Get memory info
+                process = psutil.Process(os.getpid())
+                memory_info = process.memory_info()
+                
+                return jsonify({
+                    'avg_response_time': round(avg_response_time, 3),
+                    'total_requests': len(self.request_times),
+                    'slow_requests': len(self.slow_queries),
+                    'memory_usage_mb': round(memory_info.rss / 1024 / 1024, 1),
+                    'endpoint_performance': endpoint_performance,
+                    'error_rates': dict(self.error_rates)
+                })
+                
+            except Exception as e:
+                self.logger_handler.logger.error(f"Performance stats error: {e}")
+                return jsonify({'error': 'Failed to get performance stats'}), 500
+        
+        @self.app.route('/api/performance/slow-requests')
+        def slow_requests():
+            """Get recent slow requests for analysis"""
+            try:
+                slow_request_list = [
+                    {
+                        'endpoint': req['endpoint'],
+                        'method': req.get('method', 'GET'),
+                        'time': round(req['time'], 3),
+                        'timestamp': req['timestamp'].isoformat()
+                    }
+                    for req in list(self.slow_queries)[-20:]  # Last 20 slow requests
+                ]
+                
+                return jsonify({
+                    'slow_requests': slow_request_list,
+                    'total_slow_requests': len(self.slow_queries)
+                })
+                
+            except Exception as e:
+                self.logger_handler.logger.error(f"Slow requests API error: {e}")
+                return jsonify({'error': 'Failed to get slow requests'}), 500
+
+def performance_optimization_decorator(threshold=1.0):
+    """
+    Decorator to monitor and optimize specific function performance
+    """
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            start_time = time.time()
+            
+            try:
+                result = func(*args, **kwargs)
+                execution_time = time.time() - start_time
+                
+                if execution_time > threshold:
+                    print(f"⚠️  Slow function: {func.__name__} took {execution_time:.3f}s")
+                
+                return result
+                
+            except Exception as e:
+                execution_time = time.time() - start_time
+                print(f"❌ Function error: {func.__name__} failed after {execution_time:.3f}s - {e}")
+                raise
+                
+        return wrapper
+    return decorator
+
+def optimize_database_queries():
+    """
+    Database query optimization decorator
+    """
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            # Enable query logging for this function
+            query_start = time.time()
+            
+            result = func(*args, **kwargs)
+            
+            query_time = time.time() - query_start
+            if query_time > 0.5:  # Queries taking more than 500ms
+                print(f"🐌 Slow query in {func.__name__}: {query_time:.3f}s")
+            
+            return result
+        return wrapper
+    return decorator

db_performance_optimization.py

@@ -0,0 +1,217 @@
+# File: db_performance_optimization_fixed.py
+# Fixed version compatible with your existing AppLogger
+
+from sqlalchemy import text
+from datetime import datetime, timedelta
+import logging
+
+def create_advanced_performance_indexes(db, logger_handler):
+    """
+    Create advanced performance indexes for optimal query performance
+    Compatible with existing AppLogger
+    """
+    try:
+        # Critical indexes for attendance data
+        performance_indexes = [
+            # Composite index for attendance queries by date range and employee
+            "CREATE INDEX IF NOT EXISTS idx_attendance_employee_date ON attendance_data(employee_id, check_in_date)",
+            
+            # Index for location-based queries
+            "CREATE INDEX IF NOT EXISTS idx_attendance_location_date ON attendance_data(location_name, check_in_date)",
+            
+            # Index for time-based analytics
+            "CREATE INDEX IF NOT EXISTS idx_attendance_datetime ON attendance_data(check_in_date, check_in_time)",
+            
+            # QR Code performance indexes
+            "CREATE INDEX IF NOT EXISTS idx_qrcode_project_active ON qr_codes(project_id, active_status)",
+            
+            # User authentication indexes
+            "CREATE INDEX IF NOT EXISTS idx_users_username_active ON users(username, active_status)",
+            "CREATE INDEX IF NOT EXISTS idx_users_role_active ON users(role, active_status)",
+            
+            # Project management indexes
+            "CREATE INDEX IF NOT EXISTS idx_projects_active_name ON projects(active_status, name)",
+            
+            # Employee search optimization
+            "CREATE INDEX IF NOT EXISTS idx_employee_search ON employee(firstName, lastName, id)",
+        ]
+        
+        indexes_created = 0
+        for index_sql in performance_indexes:
+            try:
+                db.session.execute(text(index_sql))
+                logger_handler.logger.info(f"Created index: {index_sql[:50]}...")
+                indexes_created += 1
+            except Exception as e:
+                logger_handler.logger.warning(f"Index creation skipped: {str(e)[:100]}")
+        
+        db.session.commit()
+        
+        # Log success using compatible method
+        logger_handler.logger.info(f"Performance optimization complete: {indexes_created} indexes created")
+        
+        return True
+        
+    except Exception as e:
+        db.session.rollback()
+        # Use compatible logging method
+        logger_handler.log_database_error('performance_optimization', e)
+        return False
+
+def optimize_database_configuration(db, logger_handler):
+    """
+    Optimize database configuration for better performance
+    """
+    try:
+        optimization_queries = [
+            # Query cache optimization (MySQL specific)
+            "SET SESSION query_cache_type = ON",
+            
+            # Connection optimization
+            "SET SESSION wait_timeout = 28800",
+            "SET SESSION interactive_timeout = 28800",
+        ]
+        
+        optimizations_applied = 0
+        for query in optimization_queries:
+            try:
+                db.session.execute(text(query))
+                optimizations_applied += 1
+            except Exception as e:
+                # Some settings may require specific privileges
+                logger_handler.logger.debug(f"Configuration skip: {str(e)[:50]}")
+        
+        logger_handler.logger.info(f"Database configuration optimization completed: {optimizations_applied} optimizations applied")
+        
+    except Exception as e:
+        logger_handler.log_database_error('database_configuration', e)
+
+def create_database_maintenance_routine(app, db, logger_handler):
+    """
+    Create automated database maintenance routine
+    """
+    @app.cli.command()
+    def db_maintenance():
+        """Run database maintenance tasks"""
+        try:
+            with app.app_context():
+                logger_handler.logger.info("Starting database maintenance routine")
+                
+                # Optimize all tables
+                maintenance_queries = [
+                    "OPTIMIZE TABLE attendance_data",
+                    "OPTIMIZE TABLE qr_codes", 
+                    "OPTIMIZE TABLE projects",
+                    "OPTIMIZE TABLE users",
+                    "OPTIMIZE TABLE employee",
+                ]
+                
+                successful_optimizations = 0
+                for query in maintenance_queries:
+                    try:
+                        db.session.execute(text(query))
+                        logger_handler.logger.info(f"Executed: {query}")
+                        successful_optimizations += 1
+                    except Exception as e:
+                        logger_handler.logger.warning(f"Maintenance query failed: {query} - {str(e)}")
+                
+                db.session.commit()
+                
+                logger_handler.logger.info(f"Database maintenance completed: {successful_optimizations} tables optimized")
+                print("✅ Database maintenance completed successfully")
+                
+        except Exception as e:
+            logger_handler.log_database_error('database_maintenance', e)
+            print(f"❌ Database maintenance failed: {e}")
+
+def implement_caching_strategy(app, db, logger_handler):
+    """
+    Implement intelligent caching strategy for improved performance
+    """
+    from functools import wraps
+    import hashlib
+    
+    # Simple in-memory cache
+    cache_storage = {}
+    cache_ttl = {}
+    
+    def cached_query(ttl=300):  # 5 minutes default TTL
+        """Decorator for caching database queries"""
+        def decorator(func):
+            @wraps(func)
+            def wrapper(*args, **kwargs):
+                # Create cache key
+                cache_key = f"{func.__name__}_{hashlib.md5(str(args + tuple(kwargs.items())).encode()).hexdigest()}"
+                current_time = datetime.utcnow().timestamp()
+                
+                # Check if cached result exists and is still valid
+                if cache_key in cache_storage:
+                    if current_time - cache_ttl.get(cache_key, 0) < ttl:
+                        logger_handler.logger.debug(f"Cache hit for {func.__name__}")
+                        return cache_storage[cache_key]
+                
+                # Execute function and cache result
+                result = func(*args, **kwargs)
+                cache_storage[cache_key] = result
+                cache_ttl[cache_key] = current_time
+                
+                logger_handler.logger.debug(f"Cache miss for {func.__name__} - result cached")
+                return result
+            return wrapper
+        return decorator
+    
+    # Clean up expired cache entries periodically
+    def cleanup_cache():
+        current_time = datetime.utcnow().timestamp()
+        expired_keys = [
+            key for key, timestamp in cache_ttl.items() 
+            if current_time - timestamp > 300  # 5 minutes
+        ]
+        
+        for key in expired_keys:
+            cache_storage.pop(key, None)
+            cache_ttl.pop(key, None)
+        
+        if expired_keys:
+            logger_handler.logger.debug(f"Cleaned up {len(expired_keys)} expired cache entries")
+    
+    # Schedule cache cleanup
+    @app.before_request
+    def before_request_cache_cleanup():
+        # Cleanup cache every 100 requests (approximately)
+        import random
+        if random.randint(1, 100) == 1:
+            cleanup_cache()
+    
+    logger_handler.logger.info("Caching strategy implemented successfully")
+    return cached_query
+
+def initialize_performance_optimizations(app, db, logger_handler):
+    """
+    Initialize all performance optimizations with compatibility
+    """
+    try:
+        logger_handler.logger.info("Starting performance optimization...")
+        
+        # Create advanced indexes
+        index_success = create_advanced_performance_indexes(db, logger_handler)
+        
+        # Optimize database configuration
+        optimize_database_configuration(db, logger_handler)
+        
+        # Create maintenance routines
+        create_database_maintenance_routine(app, db, logger_handler)
+        
+        # Implement caching
+        cached_query = implement_caching_strategy(app, db, logger_handler)
+        
+        if index_success:
+            logger_handler.logger.info("✅ Performance optimization completed successfully")
+        else:
+            logger_handler.logger.warning("⚠️ Performance optimization completed with some issues")
+            
+        return cached_query
+        
+    except Exception as e:
+        logger_handler.log_database_error('performance_initialization', e)
+        return None