diff --git a/advanced_security_middleware.py b/advanced_security_middleware.py new file mode 100644 index 0000000..981b848 --- /dev/null +++ b/advanced_security_middleware.py @@ -0,0 +1,463 @@ +# File: advanced_security_middleware.py +# Enhanced security middleware for QR Attendance System + +from functools import wraps +from flask import request, session, jsonify, current_app, g +import hashlib +import secrets +import jwt +from datetime import datetime, timedelta +import re +from collections import defaultdict, deque +import time +import hmac +import base64 +import os + +# Try to import cryptography, fallback if not available +try: + from cryptography.fernet import Fernet + HAS_CRYPTOGRAPHY = True +except ImportError: + HAS_CRYPTOGRAPHY = False + +class SecurityManager: + """ + Advanced security manager for QR Attendance System + """ + + def __init__(self, app=None, db=None, logger_handler=None): + self.app = app + self.db = db + self.logger_handler = logger_handler + + # Security tracking + self.failed_attempts = defaultdict(lambda: deque(maxlen=10)) + self.suspicious_ips = defaultdict(int) + self.session_tokens = {} + + # Security configuration + self.max_failed_attempts = 5 + self.lockout_duration = 900 # 15 minutes + self.session_timeout = 3600 # 1 hour + + if app: + self.init_app(app, db, logger_handler) + + def init_app(self, app, db, logger_handler): + """Initialize security manager with Flask app""" + self.app = app + self.db = db + self.logger_handler = logger_handler + + # Generate encryption key for sensitive data + self.setup_encryption() + + # Register security middleware + app.before_request(self.security_check) + + # Register security routes + self.register_security_routes() + + def setup_encryption(self): + """Setup encryption for sensitive data""" + if HAS_CRYPTOGRAPHY: + encryption_key = self.app.config.get('ENCRYPTION_KEY') + if not encryption_key: + # Generate a new key (should be stored securely in production) + encryption_key = Fernet.generate_key() + if self.logger_handler: + self.logger_handler.logger.warning( + "Generated new encryption key - store this securely!" + ) + + self.cipher = Fernet(encryption_key) + else: + self.cipher = None + if self.logger_handler: + self.logger_handler.logger.warning( + "Cryptography not available - encryption features disabled" + ) + + def security_check(self): + """Comprehensive security check before each request""" + client_ip = self.get_client_ip() + + # Check for suspicious activity + if self.is_suspicious_request(): + self.log_security_event('suspicious_request', { + 'ip': client_ip, + 'user_agent': request.headers.get('User-Agent', ''), + 'endpoint': request.endpoint, + 'method': request.method + }) + return jsonify({'error': 'Request blocked for security reasons'}), 403 + + # Validate session security + if 'user_id' in session: + if not self.validate_session_security(): + session.clear() + return jsonify({'error': 'Session security validation failed'}), 401 + + # Check for SQL injection attempts + if self.detect_sql_injection(): + self.log_security_event('sql_injection_attempt', { + 'ip': client_ip, + 'query_params': dict(request.args), + 'form_data': dict(request.form) if request.form else {} + }) + return jsonify({'error': 'Malicious request detected'}), 403 + + # Rate limiting for authentication endpoints + if request.endpoint in ['login', 'register', 'reset_password']: + if self.is_auth_rate_limited(): + return jsonify({ + 'error': 'Too many attempts, please try again later' + }), 429 + + def get_client_ip(self): + """Get real client IP address""" + # Check for forwarded headers (in case behind proxy/CDN) + forwarded_ips = request.headers.getlist('X-Forwarded-For') + if forwarded_ips: + return forwarded_ips[0].split(',')[0].strip() + + return request.headers.get('X-Real-IP') or request.remote_addr + + def is_suspicious_request(self): + """Detect suspicious request patterns""" + client_ip = self.get_client_ip() + user_agent = request.headers.get('User-Agent', '').lower() + + # Check for common attack patterns + suspicious_patterns = [ + r' 30: # More than 30 requests per minute + return True + + return False + + def validate_session_security(self): + """Validate session security and integrity""" + try: + user_id = session.get('user_id') + session_token = session.get('security_token') + + if not user_id or not session_token: + return False + + # Check if session token matches stored token + stored_token = self.session_tokens.get(user_id) + if not stored_token or not hmac.compare_digest(session_token, stored_token['token']): + return False + + # Check session timeout + if time.time() - stored_token['created'] > self.session_timeout: + del self.session_tokens[user_id] + return False + + # Check if session IP matches (optional security measure) + if self.app.config.get('STRICT_SESSION_IP', False): + if stored_token['ip'] != self.get_client_ip(): + self.log_security_event('session_ip_mismatch', { + 'user_id': user_id, + 'original_ip': stored_token['ip'], + 'current_ip': self.get_client_ip() + }) + return False + + return True + + except Exception as e: + if self.logger_handler: + self.logger_handler.logger.error(f"Session validation error: {e}") + return False + + def detect_sql_injection(self): + """Detect potential SQL injection attempts""" + sql_patterns = [ + r"union\s+select", r"drop\s+table", r"insert\s+into", + r"delete\s+from", r"update\s+set", r"exec\s*\(", + r"sp_executesql", r"xp_cmdshell", r";\s*--", + r"'\s*or\s*'", r'"\s*or\s*"', r"1\s*=\s*1" + ] + + # Check all request parameters + check_data = [] + check_data.extend(request.args.values()) + check_data.extend(request.form.values()) + + if request.json: + check_data.extend(str(v) for v in request.json.values() if isinstance(v, (str, int, float))) + + for data in check_data: + data_str = str(data).lower() + for pattern in sql_patterns: + if re.search(pattern, data_str, re.IGNORECASE): + return True + + return False + + def is_auth_rate_limited(self): + """Check if authentication endpoint is rate limited""" + client_ip = self.get_client_ip() + current_time = time.time() + + # Clean old attempts + self.failed_attempts[client_ip] = deque([ + attempt for attempt in self.failed_attempts[client_ip] + if current_time - attempt < 900 # Keep attempts from last 15 minutes + ], maxlen=10) + + return len(self.failed_attempts[client_ip]) >= self.max_failed_attempts + + def record_failed_attempt(self, identifier): + """Record a failed authentication attempt""" + client_ip = self.get_client_ip() + current_time = time.time() + + self.failed_attempts[client_ip].append(current_time) + + self.log_security_event('authentication_failure', { + 'ip': client_ip, + 'identifier': identifier, + 'attempts': len(self.failed_attempts[client_ip]) + }) + + def create_secure_session(self, user_id): + """Create a secure session with additional security measures""" + # Generate secure session token + session_token = secrets.token_urlsafe(32) + + # Store session information + self.session_tokens[user_id] = { + 'token': session_token, + 'created': time.time(), + 'ip': self.get_client_ip(), + 'user_agent': request.headers.get('User-Agent', '')[:200] + } + + # Set session data + session['security_token'] = session_token + session['login_time'] = datetime.utcnow().isoformat() + + # Clear any failed attempts for this IP + client_ip = self.get_client_ip() + if client_ip in self.failed_attempts: + del self.failed_attempts[client_ip] + + self.log_security_event('secure_session_created', { + 'user_id': user_id, + 'ip': client_ip + }) + + def encrypt_sensitive_data(self, data): + """Encrypt sensitive data before storage""" + if not self.cipher: + return data # Return as-is if encryption not available + + try: + if isinstance(data, str): + data = data.encode('utf-8') + + encrypted_data = self.cipher.encrypt(data) + return base64.b64encode(encrypted_data).decode('utf-8') + + except Exception as e: + if self.logger_handler: + self.logger_handler.logger.error(f"Encryption error: {e}") + return data + + def decrypt_sensitive_data(self, encrypted_data): + """Decrypt sensitive data""" + if not self.cipher: + return encrypted_data # Return as-is if encryption not available + + try: + encrypted_bytes = base64.b64decode(encrypted_data.encode('utf-8')) + decrypted_data = self.cipher.decrypt(encrypted_bytes) + return decrypted_data.decode('utf-8') + + except Exception as e: + if self.logger_handler: + self.logger_handler.logger.error(f"Decryption error: {e}") + return encrypted_data + + def log_security_event(self, event_type, details): + """Log security events for monitoring""" + try: + security_log = { + 'event_type': event_type, + 'timestamp': datetime.utcnow().isoformat(), + 'ip': self.get_client_ip(), + 'user_agent': request.headers.get('User-Agent', ''), + 'endpoint': request.endpoint, + 'method': request.method, + 'details': details + } + + if self.logger_handler: + self.logger_handler.log_security_event( + event_type=event_type, + description=f"Security event: {event_type}", + additional_data=security_log + ) + + except Exception as e: + if self.logger_handler: + self.logger_handler.logger.error(f"Security logging error: {e}") + + def register_security_routes(self): + """Register security monitoring API endpoints""" + + @self.app.route('/api/security/status') + def security_status(): + """Get current security status""" + try: + # Admin only endpoint + if not session.get('user_id') or session.get('role') != 'admin': + return jsonify({'error': 'Access denied'}), 403 + + current_time = time.time() + + # Count active suspicious IPs + suspicious_count = len([ + ip for ip, count in self.suspicious_ips.items() + if count > 3 + ]) + + # Count recent failed attempts + recent_failures = sum( + len([ + attempt for attempt in attempts + if current_time - attempt < 300 # Last 5 minutes + ]) + for attempts in self.failed_attempts.values() + ) + + # Count active sessions + active_sessions = len([ + token for token in self.session_tokens.values() + if current_time - token['created'] < self.session_timeout + ]) + + return jsonify({ + 'suspicious_ips': suspicious_count, + 'recent_failed_attempts': recent_failures, + 'active_sessions': active_sessions, + 'rate_limited_ips': len(self.failed_attempts), + 'security_status': 'normal' if suspicious_count < 5 else 'elevated' + }) + + except Exception as e: + if self.logger_handler: + self.logger_handler.logger.error(f"Security status error: {e}") + return jsonify({'error': 'Failed to get security status'}), 500 + + @self.app.route('/api/security/clear-blocks', methods=['POST']) + def clear_security_blocks(): + """Clear security blocks (admin only)""" + try: + if not session.get('user_id') or session.get('role') != 'admin': + return jsonify({'error': 'Access denied'}), 403 + + # Clear failed attempts + cleared_ips = len(self.failed_attempts) + self.failed_attempts.clear() + + # Clear suspicious IPs + cleared_suspicious = len(self.suspicious_ips) + self.suspicious_ips.clear() + + self.log_security_event('security_blocks_cleared', { + 'admin_user': session.get('username'), + 'cleared_failed_attempts': cleared_ips, + 'cleared_suspicious_ips': cleared_suspicious + }) + + return jsonify({ + 'message': 'Security blocks cleared successfully', + 'cleared_failed_attempts': cleared_ips, + 'cleared_suspicious_ips': cleared_suspicious + }) + + except Exception as e: + if self.logger_handler: + self.logger_handler.logger.error(f"Clear blocks error: {e}") + return jsonify({'error': 'Failed to clear security blocks'}), 500 + +def enhanced_login_required(f): + """ + Enhanced login required decorator with security checks + """ + @wraps(f) + def decorated_function(*args, **kwargs): + if 'user_id' not in session: + return jsonify({'error': 'Authentication required'}), 401 + + # Additional security validation + if not session.get('security_token'): + session.clear() + return jsonify({'error': 'Session security validation failed'}), 401 + + # Check session timeout + login_time_str = session.get('login_time') + if login_time_str: + try: + login_time = datetime.fromisoformat(login_time_str) + if datetime.utcnow() - login_time > timedelta(hours=8): + session.clear() + return jsonify({'error': 'Session expired'}), 401 + except ValueError: + session.clear() + return jsonify({'error': 'Invalid session data'}), 401 + + return f(*args, **kwargs) + return decorated_function + +def csrf_protect(f): + """ + CSRF protection decorator + """ + @wraps(f) + def decorated_function(*args, **kwargs): + if request.method == 'POST': + token = request.form.get('csrf_token') or request.headers.get('X-CSRF-Token') + expected_token = session.get('csrf_token') + + if not token or not expected_token or not hmac.compare_digest(token, expected_token): + return jsonify({'error': 'CSRF token validation failed'}), 403 + + return f(*args, **kwargs) + return decorated_function + +def generate_csrf_token(): + """Generate CSRF token for forms""" + if 'csrf_token' not in session: + session['csrf_token'] = secrets.token_urlsafe(32) + return session['csrf_token'] \ No newline at end of file diff --git a/app.py b/app.py index a8abe60..9428984 100644 --- a/app.py +++ b/app.py @@ -18,6 +18,8 @@ from payroll_excel_exporter import PayrollExcelExporter # Load environment variables in .env load_dotenv() from turnstile_utils import turnstile_utils +from db_performance_optimization import initialize_performance_optimizations +from app_performance_middleware import PerformanceMonitor # Initialize Flask application app = Flask(__name__) @@ -1621,6 +1623,102 @@ def dashboard(): flash('Error loading dashboard. Please try again.', 'error') return redirect(url_for('login')) +@app.route('/api/dashboard/stats') +@login_required +def dashboard_stats_api(): + """API endpoint for dashboard statistics""" + try: + # Get current stats + total_qr_codes = QRCode.query.filter_by(active_status=True).count() + + # Today's check-ins + today = datetime.utcnow().date() + today_checkins = AttendanceData.query.filter( + AttendanceData.check_in_date == today + ).count() + + # Active projects + active_projects = Project.query.filter_by(active_status=True).count() + + # Unique locations + unique_locations = db.session.query( + AttendanceData.location_name + ).distinct().count() + + # Calculate trends (compared to last month) + last_month = datetime.utcnow() - timedelta(days=30) + + # QR codes trend + old_qr_count = QRCode.query.filter( + QRCode.created_date <= last_month, + QRCode.active_status == True + ).count() + qr_change = ((total_qr_codes - old_qr_count) / max(old_qr_count, 1)) * 100 + + # Check-ins trend (yesterday) + yesterday = today - timedelta(days=1) + yesterday_checkins = AttendanceData.query.filter( + AttendanceData.check_in_date == yesterday + ).count() + checkin_change = ((today_checkins - yesterday_checkins) / max(yesterday_checkins, 1)) * 100 + + return jsonify({ + 'success': True, + 'total_qr_codes': total_qr_codes, + 'today_checkins': today_checkins, + 'active_projects': active_projects, + 'unique_locations': unique_locations, + 'qr_change': round(qr_change, 1), + 'checkin_change': round(checkin_change, 1), + 'project_change': 0, # You can calculate this based on your needs + 'location_change': 0 # You can calculate this based on your needs + }) + + except Exception as e: + logger_handler.log_database_error('dashboard_stats_api', e) + return jsonify({ + 'success': False, + 'error': 'Failed to fetch dashboard statistics' + }), 500 + +@app.route('/api/dashboard/realtime') +@login_required +def dashboard_realtime_api(): + """API endpoint for real-time dashboard data""" + try: + # Get recent activity (last 10 check-ins) + recent_activity = db.session.query( + AttendanceData.employee_id, + AttendanceData.location_name, + AttendanceData.check_in_time, + AttendanceData.check_in_date + ).order_by( + AttendanceData.check_in_date.desc(), + AttendanceData.check_in_time.desc() + ).limit(10).all() + + activity_data = [ + { + 'employee_id': activity.employee_id, + 'location': activity.location_name, + 'time': activity.check_in_time.strftime('%H:%M'), + 'date': activity.check_in_date.strftime('%Y-%m-%d') + } + for activity in recent_activity + ] + + return jsonify({ + 'success': True, + 'recent_activity': activity_data + }) + + except Exception as e: + logger_handler.log_database_error('dashboard_realtime_api', e) + return jsonify({ + 'success': False, + 'error': 'Failed to fetch real-time data' + }), 500 + # USER MANAGEMENT ROUTES @app.route('/profile', methods=['GET', 'POST']) @login_required @@ -6048,6 +6146,7 @@ def employee_detail(employee_index): create_location_logging_routes(app, db, logger_handler) + # Jinja2 filters for better template functionality @app.template_filter('days_since') def days_since_filter(date): @@ -6440,22 +6539,16 @@ if __name__ == '__main__': # Initialize database and logging create_tables() - # Add performance optimizations - create_performance_indexes() - create_audit_triggers() + # Initialize performance optimizations + print("🚀 Initializing performance optimizations...") + cached_query = initialize_performance_optimizations(app, db, logger_handler) + performance_monitor = PerformanceMonitor(app, db, logger_handler) + + if cached_query: + print("✅ Performance optimizations completed successfully") + else: + print("⚠️ Performance optimizations completed with warnings") - # Log optimization completion - logger_handler.log_system_event( - event_type="database_optimization_complete", - description="Database performance optimization completed successfully", - severity="INFO", - additional_data={ - "indexes_created": True, - "triggers_created": True, - "optimization_timestamp": datetime.utcnow().isoformat() - } - ) - print("🚀 Database optimization completed successfully") # Log application startup logger_handler.logger.info("QR Attendance Management System started successfully") diff --git a/app_performance_middleware.py b/app_performance_middleware.py new file mode 100644 index 0000000..73c9823 --- /dev/null +++ b/app_performance_middleware.py @@ -0,0 +1,347 @@ +# File: app_performance_middleware.py +# Advanced performance middleware for QR Attendance System + +from functools import wraps +from flask import request, g, jsonify, current_app +import time +import threading +import queue +from datetime import datetime, timedelta +from collections import defaultdict, deque +import gc +import psutil +import os + +class PerformanceMonitor: + """ + Advanced performance monitoring and optimization middleware + """ + + def __init__(self, app=None, db=None, logger_handler=None): + self.app = app + self.db = db + self.logger_handler = logger_handler + + # Performance metrics storage + self.request_times = deque(maxlen=1000) # Keep last 1000 requests + self.slow_queries = deque(maxlen=100) + self.error_rates = defaultdict(int) + self.endpoint_stats = defaultdict(lambda: {'count': 0, 'total_time': 0, 'errors': 0}) + + # Rate limiting storage + self.rate_limit_storage = defaultdict(lambda: {'requests': deque(), 'blocked_until': None}) + + # Background task queue + self.task_queue = queue.Queue() + self.background_worker = None + + if app: + self.init_app(app, db, logger_handler) + + def init_app(self, app, db, logger_handler): + """Initialize performance monitoring with Flask app""" + self.app = app + self.db = db + self.logger_handler = logger_handler + + # Register before/after request handlers + app.before_request(self.before_request) + app.after_request(self.after_request) + + # Start background worker + self.start_background_worker() + + # Register performance monitoring routes + self.register_performance_routes() + + def before_request(self): + """Performance monitoring before each request""" + g.start_time = time.time() + g.request_id = f"{int(time.time())}-{threading.get_ident()}" + + # Rate limiting check + if self.is_rate_limited(): + return jsonify({ + 'error': 'Rate limit exceeded', + 'retry_after': 60 + }), 429 + + # Memory usage monitoring + self.monitor_memory_usage() + + def after_request(self, response): + """Performance monitoring after each request""" + if hasattr(g, 'start_time'): + request_time = time.time() - g.start_time + + # Record request metrics + self.record_request_metrics(request_time, response.status_code) + + # Log slow requests + if request_time > 2.0: # Requests taking more than 2 seconds + self.log_slow_request(request_time) + + # Add performance headers + response.headers['X-Response-Time'] = f"{request_time:.3f}s" + response.headers['X-Request-ID'] = getattr(g, 'request_id', 'unknown') + + return response + + def record_request_metrics(self, request_time, status_code): + """Record request performance metrics""" + endpoint = request.endpoint or 'unknown' + + # Store request time + self.request_times.append({ + 'endpoint': endpoint, + 'time': request_time, + 'status': status_code, + 'timestamp': datetime.utcnow() + }) + + # Update endpoint statistics + self.endpoint_stats[endpoint]['count'] += 1 + self.endpoint_stats[endpoint]['total_time'] += request_time + + if status_code >= 400: + self.endpoint_stats[endpoint]['errors'] += 1 + self.error_rates[status_code] += 1 + + def is_rate_limited(self): + """Check if current request should be rate limited""" + client_ip = request.environ.get('REMOTE_ADDR', 'unknown') + current_time = time.time() + + # Clean up old requests + client_data = self.rate_limit_storage[client_ip] + client_data['requests'] = deque([ + req_time for req_time in client_data['requests'] + if current_time - req_time < 60 # 1 minute window + ], maxlen=100) + + # Check if currently blocked + if client_data['blocked_until'] and current_time < client_data['blocked_until']: + return True + + # Add current request + client_data['requests'].append(current_time) + + # Check rate limit (100 requests per minute) + if len(client_data['requests']) > 100: + client_data['blocked_until'] = current_time + 300 # Block for 5 minutes + self.logger_handler.logger.warning(f"Rate limit exceeded for IP: {client_ip}") + return True + + return False + + def monitor_memory_usage(self): + """Monitor application memory usage""" + # Get memory usage every 10 requests (approximately) + import random + if random.randint(1, 10) == 1: + process = psutil.Process(os.getpid()) + memory_info = process.memory_info() + memory_mb = memory_info.rss / 1024 / 1024 + + if memory_mb > 1000: # More than 1GB + self.logger_handler.logger.warning(f"High memory usage: {memory_mb:.1f}MB") + + # Force garbage collection + gc.collect() + + # Queue background cleanup task + self.task_queue.put({ + 'type': 'memory_cleanup', + 'timestamp': datetime.utcnow() + }) + + def log_slow_request(self, request_time): + """Log slow requests for optimization""" + slow_request_data = { + 'endpoint': request.endpoint, + 'method': request.method, + 'time': request_time, + 'args': dict(request.args), + 'timestamp': datetime.utcnow() + } + + self.slow_queries.append(slow_request_data) + + self.logger_handler.logger.warning( + f"Slow request: {request.method} {request.endpoint} - {request_time:.3f}s" + ) + + def start_background_worker(self): + """Start background worker for performance tasks""" + def worker(): + while True: + try: + task = self.task_queue.get(timeout=30) + self.process_background_task(task) + self.task_queue.task_done() + except queue.Empty: + continue + except Exception as e: + if self.logger_handler: + self.logger_handler.logger.error(f"Background worker error: {e}") + + self.background_worker = threading.Thread(target=worker, daemon=True) + self.background_worker.start() + + def process_background_task(self, task): + """Process background performance tasks""" + task_type = task.get('type') + + if task_type == 'memory_cleanup': + self.perform_memory_cleanup() + elif task_type == 'performance_analysis': + self.perform_performance_analysis() + elif task_type == 'database_optimization': + self.optimize_database_connections() + + def perform_memory_cleanup(self): + """Perform memory cleanup operations""" + try: + # Clear old metrics + cutoff_time = datetime.utcnow() - timedelta(hours=1) + + # Clean request times + self.request_times = deque([ + req for req in self.request_times + if req['timestamp'] > cutoff_time + ], maxlen=1000) + + # Clean slow queries + self.slow_queries = deque([ + query for query in self.slow_queries + if query['timestamp'] > cutoff_time + ], maxlen=100) + + # Clean rate limit storage + current_time = time.time() + for ip, data in list(self.rate_limit_storage.items()): + if not data['requests'] and ( + not data['blocked_until'] or current_time > data['blocked_until'] + ): + del self.rate_limit_storage[ip] + + # Force garbage collection + gc.collect() + + self.logger_handler.logger.info("Memory cleanup completed") + + except Exception as e: + self.logger_handler.logger.error(f"Memory cleanup failed: {e}") + + def register_performance_routes(self): + """Register performance monitoring API endpoints""" + + @self.app.route('/api/performance/stats') + def performance_stats(): + """Get current performance statistics""" + try: + # Calculate average response times + recent_requests = [ + req for req in self.request_times + if req['timestamp'] > datetime.utcnow() - timedelta(minutes=5) + ] + + avg_response_time = ( + sum(req['time'] for req in recent_requests) / len(recent_requests) + if recent_requests else 0 + ) + + # Get endpoint statistics + endpoint_performance = {} + for endpoint, stats in self.endpoint_stats.items(): + endpoint_performance[endpoint] = { + 'avg_response_time': stats['total_time'] / stats['count'] if stats['count'] > 0 else 0, + 'total_requests': stats['count'], + 'error_rate': stats['errors'] / stats['count'] if stats['count'] > 0 else 0 + } + + # Get memory info + process = psutil.Process(os.getpid()) + memory_info = process.memory_info() + + return jsonify({ + 'avg_response_time': round(avg_response_time, 3), + 'total_requests': len(self.request_times), + 'slow_requests': len(self.slow_queries), + 'memory_usage_mb': round(memory_info.rss / 1024 / 1024, 1), + 'endpoint_performance': endpoint_performance, + 'error_rates': dict(self.error_rates) + }) + + except Exception as e: + self.logger_handler.logger.error(f"Performance stats error: {e}") + return jsonify({'error': 'Failed to get performance stats'}), 500 + + @self.app.route('/api/performance/slow-requests') + def slow_requests(): + """Get recent slow requests for analysis""" + try: + slow_request_list = [ + { + 'endpoint': req['endpoint'], + 'method': req.get('method', 'GET'), + 'time': round(req['time'], 3), + 'timestamp': req['timestamp'].isoformat() + } + for req in list(self.slow_queries)[-20:] # Last 20 slow requests + ] + + return jsonify({ + 'slow_requests': slow_request_list, + 'total_slow_requests': len(self.slow_queries) + }) + + except Exception as e: + self.logger_handler.logger.error(f"Slow requests API error: {e}") + return jsonify({'error': 'Failed to get slow requests'}), 500 + +def performance_optimization_decorator(threshold=1.0): + """ + Decorator to monitor and optimize specific function performance + """ + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + start_time = time.time() + + try: + result = func(*args, **kwargs) + execution_time = time.time() - start_time + + if execution_time > threshold: + print(f"⚠️ Slow function: {func.__name__} took {execution_time:.3f}s") + + return result + + except Exception as e: + execution_time = time.time() - start_time + print(f"❌ Function error: {func.__name__} failed after {execution_time:.3f}s - {e}") + raise + + return wrapper + return decorator + +def optimize_database_queries(): + """ + Database query optimization decorator + """ + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + # Enable query logging for this function + query_start = time.time() + + result = func(*args, **kwargs) + + query_time = time.time() - query_start + if query_time > 0.5: # Queries taking more than 500ms + print(f"🐌 Slow query in {func.__name__}: {query_time:.3f}s") + + return result + return wrapper + return decorator \ No newline at end of file diff --git a/db_performance_optimization.py b/db_performance_optimization.py new file mode 100644 index 0000000..7f73910 --- /dev/null +++ b/db_performance_optimization.py @@ -0,0 +1,217 @@ +# File: db_performance_optimization_fixed.py +# Fixed version compatible with your existing AppLogger + +from sqlalchemy import text +from datetime import datetime, timedelta +import logging + +def create_advanced_performance_indexes(db, logger_handler): + """ + Create advanced performance indexes for optimal query performance + Compatible with existing AppLogger + """ + try: + # Critical indexes for attendance data + performance_indexes = [ + # Composite index for attendance queries by date range and employee + "CREATE INDEX IF NOT EXISTS idx_attendance_employee_date ON attendance_data(employee_id, check_in_date)", + + # Index for location-based queries + "CREATE INDEX IF NOT EXISTS idx_attendance_location_date ON attendance_data(location_name, check_in_date)", + + # Index for time-based analytics + "CREATE INDEX IF NOT EXISTS idx_attendance_datetime ON attendance_data(check_in_date, check_in_time)", + + # QR Code performance indexes + "CREATE INDEX IF NOT EXISTS idx_qrcode_project_active ON qr_codes(project_id, active_status)", + + # User authentication indexes + "CREATE INDEX IF NOT EXISTS idx_users_username_active ON users(username, active_status)", + "CREATE INDEX IF NOT EXISTS idx_users_role_active ON users(role, active_status)", + + # Project management indexes + "CREATE INDEX IF NOT EXISTS idx_projects_active_name ON projects(active_status, name)", + + # Employee search optimization + "CREATE INDEX IF NOT EXISTS idx_employee_search ON employee(firstName, lastName, id)", + ] + + indexes_created = 0 + for index_sql in performance_indexes: + try: + db.session.execute(text(index_sql)) + logger_handler.logger.info(f"Created index: {index_sql[:50]}...") + indexes_created += 1 + except Exception as e: + logger_handler.logger.warning(f"Index creation skipped: {str(e)[:100]}") + + db.session.commit() + + # Log success using compatible method + logger_handler.logger.info(f"Performance optimization complete: {indexes_created} indexes created") + + return True + + except Exception as e: + db.session.rollback() + # Use compatible logging method + logger_handler.log_database_error('performance_optimization', e) + return False + +def optimize_database_configuration(db, logger_handler): + """ + Optimize database configuration for better performance + """ + try: + optimization_queries = [ + # Query cache optimization (MySQL specific) + "SET SESSION query_cache_type = ON", + + # Connection optimization + "SET SESSION wait_timeout = 28800", + "SET SESSION interactive_timeout = 28800", + ] + + optimizations_applied = 0 + for query in optimization_queries: + try: + db.session.execute(text(query)) + optimizations_applied += 1 + except Exception as e: + # Some settings may require specific privileges + logger_handler.logger.debug(f"Configuration skip: {str(e)[:50]}") + + logger_handler.logger.info(f"Database configuration optimization completed: {optimizations_applied} optimizations applied") + + except Exception as e: + logger_handler.log_database_error('database_configuration', e) + +def create_database_maintenance_routine(app, db, logger_handler): + """ + Create automated database maintenance routine + """ + @app.cli.command() + def db_maintenance(): + """Run database maintenance tasks""" + try: + with app.app_context(): + logger_handler.logger.info("Starting database maintenance routine") + + # Optimize all tables + maintenance_queries = [ + "OPTIMIZE TABLE attendance_data", + "OPTIMIZE TABLE qr_codes", + "OPTIMIZE TABLE projects", + "OPTIMIZE TABLE users", + "OPTIMIZE TABLE employee", + ] + + successful_optimizations = 0 + for query in maintenance_queries: + try: + db.session.execute(text(query)) + logger_handler.logger.info(f"Executed: {query}") + successful_optimizations += 1 + except Exception as e: + logger_handler.logger.warning(f"Maintenance query failed: {query} - {str(e)}") + + db.session.commit() + + logger_handler.logger.info(f"Database maintenance completed: {successful_optimizations} tables optimized") + print("✅ Database maintenance completed successfully") + + except Exception as e: + logger_handler.log_database_error('database_maintenance', e) + print(f"❌ Database maintenance failed: {e}") + +def implement_caching_strategy(app, db, logger_handler): + """ + Implement intelligent caching strategy for improved performance + """ + from functools import wraps + import hashlib + + # Simple in-memory cache + cache_storage = {} + cache_ttl = {} + + def cached_query(ttl=300): # 5 minutes default TTL + """Decorator for caching database queries""" + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + # Create cache key + cache_key = f"{func.__name__}_{hashlib.md5(str(args + tuple(kwargs.items())).encode()).hexdigest()}" + current_time = datetime.utcnow().timestamp() + + # Check if cached result exists and is still valid + if cache_key in cache_storage: + if current_time - cache_ttl.get(cache_key, 0) < ttl: + logger_handler.logger.debug(f"Cache hit for {func.__name__}") + return cache_storage[cache_key] + + # Execute function and cache result + result = func(*args, **kwargs) + cache_storage[cache_key] = result + cache_ttl[cache_key] = current_time + + logger_handler.logger.debug(f"Cache miss for {func.__name__} - result cached") + return result + return wrapper + return decorator + + # Clean up expired cache entries periodically + def cleanup_cache(): + current_time = datetime.utcnow().timestamp() + expired_keys = [ + key for key, timestamp in cache_ttl.items() + if current_time - timestamp > 300 # 5 minutes + ] + + for key in expired_keys: + cache_storage.pop(key, None) + cache_ttl.pop(key, None) + + if expired_keys: + logger_handler.logger.debug(f"Cleaned up {len(expired_keys)} expired cache entries") + + # Schedule cache cleanup + @app.before_request + def before_request_cache_cleanup(): + # Cleanup cache every 100 requests (approximately) + import random + if random.randint(1, 100) == 1: + cleanup_cache() + + logger_handler.logger.info("Caching strategy implemented successfully") + return cached_query + +def initialize_performance_optimizations(app, db, logger_handler): + """ + Initialize all performance optimizations with compatibility + """ + try: + logger_handler.logger.info("Starting performance optimization...") + + # Create advanced indexes + index_success = create_advanced_performance_indexes(db, logger_handler) + + # Optimize database configuration + optimize_database_configuration(db, logger_handler) + + # Create maintenance routines + create_database_maintenance_routine(app, db, logger_handler) + + # Implement caching + cached_query = implement_caching_strategy(app, db, logger_handler) + + if index_success: + logger_handler.logger.info("✅ Performance optimization completed successfully") + else: + logger_handler.logger.warning("⚠️ Performance optimization completed with some issues") + + return cached_query + + except Exception as e: + logger_handler.log_database_error('performance_initialization', e) + return None \ No newline at end of file