Database optimize scripts
This commit is contained in:
@@ -0,0 +1,97 @@
|
||||
# quick_fix_tables.py
|
||||
"""
|
||||
Quick Fix for Missing Maintenance Tables
|
||||
======================================
|
||||
|
||||
Run this script to quickly create the missing tables that are causing the cleanup error.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from sqlalchemy import text
|
||||
|
||||
# Add current directory to path
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from app import app, db, logger_handler
|
||||
|
||||
def quick_create_missing_tables():
|
||||
"""Quickly create the missing tables for maintenance system"""
|
||||
|
||||
print("🔧 Quick Fix: Creating Missing Maintenance Tables")
|
||||
print("=" * 55)
|
||||
|
||||
with app.app_context():
|
||||
try:
|
||||
# Create attendance_audit table
|
||||
print("📋 Creating attendance_audit table...")
|
||||
|
||||
audit_sql = """
|
||||
CREATE TABLE IF NOT EXISTS attendance_audit (
|
||||
audit_id INT AUTO_INCREMENT PRIMARY KEY,
|
||||
record_id INT NOT NULL,
|
||||
action_type ENUM('INSERT', 'UPDATE', 'DELETE') NOT NULL,
|
||||
old_values JSON,
|
||||
new_values JSON,
|
||||
changed_by VARCHAR(100),
|
||||
change_timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
ip_address VARCHAR(45),
|
||||
INDEX idx_audit_timestamp (change_timestamp DESC),
|
||||
INDEX idx_audit_record (record_id)
|
||||
) ENGINE=InnoDB
|
||||
"""
|
||||
|
||||
db.session.execute(text(audit_sql))
|
||||
print(" ✅ attendance_audit table created")
|
||||
|
||||
# Create attendance_statistics_cache table
|
||||
print("📋 Creating attendance_statistics_cache table...")
|
||||
|
||||
cache_sql = """
|
||||
CREATE TABLE IF NOT EXISTS attendance_statistics_cache (
|
||||
cache_key VARCHAR(255) PRIMARY KEY,
|
||||
cache_data JSON NOT NULL,
|
||||
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
||||
expires_at TIMESTAMP,
|
||||
INDEX idx_cache_expires (expires_at),
|
||||
INDEX idx_cache_updated (last_updated DESC)
|
||||
) ENGINE=InnoDB
|
||||
"""
|
||||
|
||||
db.session.execute(text(cache_sql))
|
||||
print(" ✅ attendance_statistics_cache table created")
|
||||
|
||||
# Commit the changes
|
||||
db.session.commit()
|
||||
|
||||
print("\n✅ SUCCESS: Missing tables created successfully!")
|
||||
print("\n🧪 Testing the tables...")
|
||||
|
||||
# Test the tables
|
||||
test_queries = [
|
||||
"SELECT COUNT(*) FROM attendance_audit",
|
||||
"SELECT COUNT(*) FROM attendance_statistics_cache"
|
||||
]
|
||||
|
||||
for query in test_queries:
|
||||
try:
|
||||
result = db.session.execute(text(query)).fetchone()
|
||||
table_name = query.split("FROM ")[1]
|
||||
print(f" ✅ {table_name}: OK (rows: {result[0]})")
|
||||
except Exception as e:
|
||||
print(f" ❌ Test failed: {e}")
|
||||
|
||||
print(f"\n🎉 QUICK FIX COMPLETED!")
|
||||
print("\nNow you can run:")
|
||||
print(" python maintenance_cli.py cleanup")
|
||||
print(" python maintenance_cli.py health-check")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error creating tables: {e}")
|
||||
db.session.rollback()
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
quick_create_missing_tables()
|
||||
@@ -0,0 +1,186 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple Health Check - Windows Compatible
|
||||
=======================================
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from datetime import datetime
|
||||
from sqlalchemy import text
|
||||
|
||||
# Add current directory to path
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
def health_check():
|
||||
"""Run database health check"""
|
||||
try:
|
||||
from app import app, db, logger_handler
|
||||
|
||||
print("Database Health Check - Windows Compatible")
|
||||
print("=" * 50)
|
||||
|
||||
with app.app_context():
|
||||
health_status = {
|
||||
'overall': 'healthy',
|
||||
'issues': [],
|
||||
'warnings': [],
|
||||
'recommendations': []
|
||||
}
|
||||
|
||||
# Check database connectivity
|
||||
try:
|
||||
result = db.session.execute(text("SELECT 1 as test")).fetchone()
|
||||
if result and result.test == 1:
|
||||
print("[OK] Database connectivity: WORKING")
|
||||
else:
|
||||
print("[ERROR] Database connectivity: FAILED - Unexpected result")
|
||||
health_status['issues'].append("Database connectivity test failed")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Database connectivity: FAILED - {e}")
|
||||
health_status['overall'] = 'critical'
|
||||
health_status['issues'].append(f"Database connectivity failed: {e}")
|
||||
|
||||
# Check critical tables exist
|
||||
critical_tables = ['attendance_data', 'qr_codes', 'users', 'projects', 'log_events']
|
||||
existing_tables = []
|
||||
|
||||
print("\nTable Status:")
|
||||
for table in critical_tables:
|
||||
try:
|
||||
# Use SHOW TABLES for better MySQL compatibility
|
||||
table_check = db.session.execute(text(f"SHOW TABLES LIKE '{table}'")).fetchone()
|
||||
if table_check:
|
||||
# Get row count
|
||||
count_result = db.session.execute(text(f"SELECT COUNT(*) FROM {table}")).fetchone()
|
||||
row_count = count_result[0] if count_result else 0
|
||||
print(f"[OK] {table}: EXISTS ({row_count:,} rows)")
|
||||
existing_tables.append(table)
|
||||
else:
|
||||
print(f"[MISSING] {table}: NOT FOUND")
|
||||
health_status['issues'].append(f"Critical table {table} is missing")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] {table}: ERROR - {e}")
|
||||
health_status['issues'].append(f"Could not check table {table}: {e}")
|
||||
|
||||
# Check maintenance tables
|
||||
maintenance_tables = ['attendance_audit', 'attendance_statistics_cache']
|
||||
print("\nMaintenance Tables:")
|
||||
for table in maintenance_tables:
|
||||
try:
|
||||
table_check = db.session.execute(text(f"SHOW TABLES LIKE '{table}'")).fetchone()
|
||||
if table_check:
|
||||
count_result = db.session.execute(text(f"SELECT COUNT(*) FROM {table}")).fetchone()
|
||||
row_count = count_result[0] if count_result else 0
|
||||
print(f"[OK] {table}: EXISTS ({row_count:,} rows)")
|
||||
else:
|
||||
print(f"[MISSING] {table}: NOT FOUND")
|
||||
health_status['warnings'].append(f"Maintenance table {table} is missing")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] {table}: ERROR - {e}")
|
||||
|
||||
# Check for indexes on existing tables
|
||||
print("\nIndex Analysis:")
|
||||
missing_indexes = 0
|
||||
for table in existing_tables:
|
||||
try:
|
||||
# Check if table has performance indexes
|
||||
index_check = db.session.execute(text(f"""
|
||||
SELECT COUNT(*) as index_count
|
||||
FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = '{table}'
|
||||
AND INDEX_NAME != 'PRIMARY'
|
||||
""")).fetchone()
|
||||
|
||||
index_count = index_check.index_count if index_check else 0
|
||||
if index_count == 0:
|
||||
print(f"[WARNING] {table}: No performance indexes")
|
||||
missing_indexes += 1
|
||||
else:
|
||||
print(f"[OK] {table}: {index_count} indexes")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ERROR] {table} indexes: {e}")
|
||||
|
||||
if missing_indexes > 0:
|
||||
health_status['warnings'].append(f"{missing_indexes} tables have no performance indexes")
|
||||
health_status['recommendations'].append("Add performance indexes")
|
||||
|
||||
# Check triggers
|
||||
print("\nTrigger Status:")
|
||||
try:
|
||||
trigger_count_query = """
|
||||
SELECT COUNT(*) as trigger_count
|
||||
FROM INFORMATION_SCHEMA.TRIGGERS
|
||||
WHERE TRIGGER_SCHEMA = DATABASE()
|
||||
"""
|
||||
|
||||
trigger_result = db.session.execute(text(trigger_count_query)).fetchone()
|
||||
trigger_count = trigger_result.trigger_count if trigger_result else 0
|
||||
|
||||
if trigger_count > 0:
|
||||
print(f"[OK] Database triggers: {trigger_count} found")
|
||||
else:
|
||||
print("[WARNING] Database triggers: None found")
|
||||
health_status['warnings'].append("No database triggers found")
|
||||
health_status['recommendations'].append("Consider adding audit triggers")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Trigger check failed: {e}")
|
||||
health_status['warnings'].append("Trigger check failed")
|
||||
|
||||
# Determine overall health
|
||||
if len(health_status['issues']) > 0:
|
||||
health_status['overall'] = 'critical'
|
||||
elif len(health_status['warnings']) > 3:
|
||||
health_status['overall'] = 'warning'
|
||||
|
||||
# Print summary
|
||||
print(f"\n{'='*50}")
|
||||
print(f"OVERALL HEALTH: {health_status['overall'].upper()}")
|
||||
print(f"{'='*50}")
|
||||
|
||||
# Report issues
|
||||
if health_status['issues']:
|
||||
print("\nCRITICAL ISSUES:")
|
||||
for i, issue in enumerate(health_status['issues'], 1):
|
||||
print(f"{i}. {issue}")
|
||||
|
||||
if health_status['warnings']:
|
||||
print("\nWARNINGS:")
|
||||
for i, warning in enumerate(health_status['warnings'], 1):
|
||||
print(f"{i}. {warning}")
|
||||
|
||||
if health_status['recommendations']:
|
||||
print("\nRECOMMENDATIONS:")
|
||||
for i, rec in enumerate(health_status['recommendations'], 1):
|
||||
print(f"{i}. {rec}")
|
||||
|
||||
# Provide actionable next steps
|
||||
print("\nNEXT STEPS:")
|
||||
if health_status['overall'] == 'critical':
|
||||
print("1. Address critical issues above")
|
||||
print("2. Verify database connection and table creation")
|
||||
print("3. Check Flask app initialization")
|
||||
elif health_status['overall'] == 'warning':
|
||||
print("1. Add performance indexes: python simple_maintenance.py add-indexes")
|
||||
print("2. Set up audit triggers for data tracking")
|
||||
print("3. Run regular maintenance")
|
||||
else:
|
||||
print("1. System is healthy!")
|
||||
print("2. Run regular maintenance: python simple_maintenance.py cleanup")
|
||||
print("3. Monitor performance metrics")
|
||||
|
||||
return health_status
|
||||
|
||||
except ImportError as e:
|
||||
print(f"[ERROR] Could not import Flask app: {e}")
|
||||
print("Make sure you're running this from your Flask app directory")
|
||||
return {'overall': 'critical', 'issues': [f"Import error: {e}"]}
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Health check failed: {e}")
|
||||
return {'overall': 'critical', 'issues': [f"Health check error: {e}"]}
|
||||
|
||||
if __name__ == "__main__":
|
||||
health_check()
|
||||
@@ -0,0 +1,526 @@
|
||||
# fixed_simple_maintenance.py
|
||||
"""
|
||||
Fixed Simple Maintenance Script
|
||||
==============================
|
||||
|
||||
Fixes SQL parameter binding issues and datetime deprecation warnings.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from sqlalchemy import text
|
||||
|
||||
# Add current directory to path
|
||||
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
def cleanup_data(dry_run=False):
|
||||
"""Clean up old data from maintenance tables - FIXED VERSION"""
|
||||
try:
|
||||
from app import app, db
|
||||
|
||||
print("Data Cleanup - Fixed Version")
|
||||
print("=" * 35)
|
||||
|
||||
with app.app_context():
|
||||
# Define cleanup operations
|
||||
# (table_name, date_column, retention_days, description)
|
||||
tables_to_clean = [
|
||||
('attendance_audit', 'change_timestamp', 90, 'Audit trail records'),
|
||||
('attendance_statistics_cache', 'expires_at', 1, 'Expired cache entries'),
|
||||
('log_events', 'created_timestamp', 180, 'Old system logs')
|
||||
]
|
||||
|
||||
total_records_to_clean = 0
|
||||
total_records_cleaned = 0
|
||||
|
||||
print(f"Retention policy:")
|
||||
for table, col, days, desc in tables_to_clean:
|
||||
print(f" - {desc}: {days} days")
|
||||
|
||||
print("\nAnalyzing tables...\n")
|
||||
|
||||
for table_name, date_column, retention_days, description in tables_to_clean:
|
||||
try:
|
||||
# Check if table exists
|
||||
table_check = db.session.execute(text(f"SHOW TABLES LIKE '{table_name}'")).fetchone()
|
||||
|
||||
if not table_check:
|
||||
print(f"[SKIP] {table_name}: Table doesn't exist")
|
||||
continue
|
||||
|
||||
# Calculate cutoff date (using timezone-aware datetime)
|
||||
cutoff_date = datetime.now(timezone.utc) - timedelta(days=retention_days)
|
||||
cutoff_str = cutoff_date.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# Count total records in table
|
||||
total_count_query = f"SELECT COUNT(*) as total FROM {table_name}"
|
||||
total_result = db.session.execute(text(total_count_query)).fetchone()
|
||||
total_records = total_result.total if total_result else 0
|
||||
|
||||
# Count records to clean - Fixed SQL syntax
|
||||
count_query = f"""
|
||||
SELECT COUNT(*) as count
|
||||
FROM {table_name}
|
||||
WHERE {date_column} < :cutoff_date
|
||||
"""
|
||||
count_result = db.session.execute(
|
||||
text(count_query),
|
||||
{'cutoff_date': cutoff_str}
|
||||
).fetchone()
|
||||
|
||||
records_to_clean = count_result.count if count_result else 0
|
||||
records_to_keep = total_records - records_to_clean
|
||||
|
||||
if records_to_clean > 0:
|
||||
print(f"[{table_name}]")
|
||||
print(f" Total records: {total_records:,}")
|
||||
print(f" Records to clean: {records_to_clean:,}")
|
||||
print(f" Records to keep: {records_to_keep:,}")
|
||||
print(f" Cutoff date: {cutoff_str}")
|
||||
|
||||
if dry_run:
|
||||
print(f" [DRY RUN] Would delete {records_to_clean:,} records")
|
||||
else:
|
||||
# Perform cleanup - Fixed SQL syntax
|
||||
delete_query = f"""
|
||||
DELETE FROM {table_name}
|
||||
WHERE {date_column} < :cutoff_date
|
||||
"""
|
||||
result = db.session.execute(
|
||||
text(delete_query),
|
||||
{'cutoff_date': cutoff_str}
|
||||
)
|
||||
actual_deleted = result.rowcount
|
||||
print(f" [CLEANED] Deleted {actual_deleted:,} records")
|
||||
total_records_cleaned += actual_deleted
|
||||
|
||||
total_records_to_clean += records_to_clean
|
||||
print() # Empty line for readability
|
||||
|
||||
else:
|
||||
print(f"[{table_name}]")
|
||||
print(f" Total records: {total_records:,}")
|
||||
print(f" No old records to clean (all newer than {retention_days} days)")
|
||||
print()
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Error processing {table_name}: {e}")
|
||||
print(f" Error type: {type(e).__name__}")
|
||||
print()
|
||||
|
||||
# Summary
|
||||
print("=" * 35)
|
||||
if dry_run:
|
||||
if total_records_to_clean > 0:
|
||||
print(f"[DRY RUN SUMMARY]")
|
||||
print(f"Would clean {total_records_to_clean:,} total records")
|
||||
print(f"No changes made to database")
|
||||
else:
|
||||
print(f"[DRY RUN SUMMARY]")
|
||||
print(f"No records need cleaning")
|
||||
else:
|
||||
if total_records_cleaned > 0:
|
||||
# Commit changes
|
||||
db.session.commit()
|
||||
print(f"[CLEANUP COMPLETED]")
|
||||
print(f"Successfully cleaned {total_records_cleaned:,} total records")
|
||||
else:
|
||||
print(f"[NO CLEANUP NEEDED]")
|
||||
print(f"All records are within retention policy")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Cleanup failed: {e}")
|
||||
print(f"Error type: {type(e).__name__}")
|
||||
try:
|
||||
db.session.rollback()
|
||||
print("Database changes rolled back")
|
||||
except:
|
||||
pass
|
||||
|
||||
def add_performance_indexes():
|
||||
"""Add basic performance indexes - FIXED VERSION"""
|
||||
try:
|
||||
from app import app, db
|
||||
|
||||
print("Adding Performance Indexes - Fixed Version")
|
||||
print("=" * 45)
|
||||
|
||||
with app.app_context():
|
||||
# Define critical indexes with better error handling
|
||||
indexes_to_create = [
|
||||
{
|
||||
'name': 'idx_attendance_date_employee',
|
||||
'table': 'attendance_data',
|
||||
'columns': 'check_in_date DESC, employee_id',
|
||||
'purpose': 'Optimize attendance queries by date and employee'
|
||||
},
|
||||
{
|
||||
'name': 'idx_attendance_qr_date',
|
||||
'table': 'attendance_data',
|
||||
'columns': 'qr_code_id, check_in_date DESC',
|
||||
'purpose': 'Optimize queries by QR code and date'
|
||||
},
|
||||
{
|
||||
'name': 'idx_attendance_location_date',
|
||||
'table': 'attendance_data',
|
||||
'columns': 'location_name, check_in_date DESC',
|
||||
'purpose': 'Optimize queries by location'
|
||||
},
|
||||
{
|
||||
'name': 'idx_qrcode_project_active',
|
||||
'table': 'qr_codes',
|
||||
'columns': 'project_id, active_status',
|
||||
'purpose': 'Optimize project-based QR code queries'
|
||||
},
|
||||
{
|
||||
'name': 'idx_users_username_active',
|
||||
'table': 'users',
|
||||
'columns': 'username, active_status',
|
||||
'purpose': 'Optimize user authentication'
|
||||
},
|
||||
{
|
||||
'name': 'idx_log_events_timestamp_category',
|
||||
'table': 'log_events',
|
||||
'columns': 'created_timestamp DESC, event_category',
|
||||
'purpose': 'Optimize log queries and filtering'
|
||||
}
|
||||
]
|
||||
|
||||
indexes_created = 0
|
||||
indexes_skipped = 0
|
||||
indexes_failed = 0
|
||||
|
||||
for index_def in indexes_to_create:
|
||||
print(f"\nProcessing {index_def['name']}...")
|
||||
|
||||
try:
|
||||
# Check if table exists first
|
||||
table_check = db.session.execute(
|
||||
text(f"SHOW TABLES LIKE :table_name"),
|
||||
{'table_name': index_def['table']}
|
||||
).fetchone()
|
||||
|
||||
if not table_check:
|
||||
print(f" [SKIP] Table {index_def['table']} doesn't exist")
|
||||
indexes_skipped += 1
|
||||
continue
|
||||
|
||||
# Check if index already exists
|
||||
index_check = db.session.execute(text("""
|
||||
SELECT COUNT(*) as count
|
||||
FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = :table_name
|
||||
AND INDEX_NAME = :index_name
|
||||
"""), {
|
||||
'table_name': index_def['table'],
|
||||
'index_name': index_def['name']
|
||||
}).fetchone()
|
||||
|
||||
if index_check and index_check.count > 0:
|
||||
print(f" [EXISTS] Index already exists")
|
||||
indexes_skipped += 1
|
||||
continue
|
||||
|
||||
# Create the index
|
||||
create_sql = f"""
|
||||
CREATE INDEX {index_def['name']}
|
||||
ON {index_def['table']} ({index_def['columns']})
|
||||
"""
|
||||
|
||||
db.session.execute(text(create_sql))
|
||||
|
||||
print(f" [CREATED] Successfully created index")
|
||||
print(f" Purpose: {index_def['purpose']}")
|
||||
indexes_created += 1
|
||||
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
if 'duplicate' in error_msg or 'already exists' in error_msg:
|
||||
print(f" [EXISTS] Index already exists (detected in error)")
|
||||
indexes_skipped += 1
|
||||
else:
|
||||
print(f" [ERROR] Failed to create index: {e}")
|
||||
indexes_failed += 1
|
||||
|
||||
# Commit changes if any indexes were created
|
||||
if indexes_created > 0:
|
||||
db.session.commit()
|
||||
|
||||
# Summary
|
||||
print(f"\n{'=' * 45}")
|
||||
print(f"INDEX CREATION SUMMARY")
|
||||
print(f"{'=' * 45}")
|
||||
print(f"Created: {indexes_created}")
|
||||
print(f"Skipped (already exist): {indexes_skipped}")
|
||||
print(f"Failed: {indexes_failed}")
|
||||
print(f"Total processed: {len(indexes_to_create)}")
|
||||
|
||||
if indexes_created > 0:
|
||||
print(f"\n[SUCCESS] Created {indexes_created} new performance indexes")
|
||||
print("These indexes will improve query performance for:")
|
||||
print("- Attendance filtering by date/employee/location")
|
||||
print("- User authentication")
|
||||
print("- QR code project queries")
|
||||
print("- Log event filtering")
|
||||
elif indexes_skipped == len(indexes_to_create):
|
||||
print(f"\n[INFO] All indexes already exist - no action needed")
|
||||
else:
|
||||
print(f"\n[WARNING] Some indexes could not be created")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Index creation failed: {e}")
|
||||
try:
|
||||
db.session.rollback()
|
||||
except:
|
||||
pass
|
||||
|
||||
def show_table_info():
|
||||
"""Show detailed information about database tables"""
|
||||
try:
|
||||
from app import app, db
|
||||
|
||||
print("Database Table Information - Detailed")
|
||||
print("=" * 45)
|
||||
|
||||
with app.app_context():
|
||||
# Get all tables
|
||||
tables_result = db.session.execute(text("SHOW TABLES")).fetchall()
|
||||
table_names = [row[0] for row in tables_result]
|
||||
|
||||
print(f"Database: {db.engine.url.database}")
|
||||
print(f"Total tables: {len(table_names)}\n")
|
||||
|
||||
# Header for table info
|
||||
print(f"{'Table Name':<25} {'Rows':<10} {'Size (MB)':<10} {'Indexes':<8} {'Type'}")
|
||||
print("-" * 70)
|
||||
|
||||
total_size = 0
|
||||
total_rows = 0
|
||||
|
||||
for table in sorted(table_names):
|
||||
try:
|
||||
# Get row count
|
||||
count_result = db.session.execute(text(f"SELECT COUNT(*) FROM {table}")).fetchone()
|
||||
row_count = count_result[0] if count_result else 0
|
||||
|
||||
# Get table size and type
|
||||
info_query = f"""
|
||||
SELECT
|
||||
ROUND(((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024), 2) as size_mb,
|
||||
ENGINE as engine_type
|
||||
FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '{table}'
|
||||
"""
|
||||
info_result = db.session.execute(text(info_query)).fetchone()
|
||||
size_mb = info_result.size_mb if info_result and info_result.size_mb else 0
|
||||
engine_type = info_result.engine_type if info_result else 'Unknown'
|
||||
|
||||
# Get index count
|
||||
index_query = f"""
|
||||
SELECT COUNT(*) as index_count
|
||||
FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = '{table}'
|
||||
AND INDEX_NAME != 'PRIMARY'
|
||||
"""
|
||||
index_result = db.session.execute(text(index_query)).fetchone()
|
||||
index_count = index_result.index_count if index_result else 0
|
||||
|
||||
# Determine table category
|
||||
table_type = "System"
|
||||
if table in ['attendance_data', 'qr_codes', 'users', 'projects']:
|
||||
table_type = "Core"
|
||||
elif table in ['attendance_audit', 'attendance_statistics_cache', 'log_events']:
|
||||
table_type = "Maintenance"
|
||||
elif table in ['customers', 'tenant_configs']:
|
||||
table_type = "Config"
|
||||
|
||||
print(f"{table:<25} {row_count:<10,} {size_mb:<10.1f} {index_count:<8} {table_type}")
|
||||
|
||||
total_size += size_mb
|
||||
total_rows += row_count
|
||||
|
||||
except Exception as e:
|
||||
print(f"{table:<25} {'ERROR':<10} {'N/A':<10} {'N/A':<8} - {str(e)[:20]}")
|
||||
|
||||
# Summary
|
||||
print("-" * 70)
|
||||
print(f"{'TOTALS':<25} {total_rows:<10,} {total_size:<10.1f}")
|
||||
|
||||
# Additional insights
|
||||
print(f"\nTable Categories:")
|
||||
core_tables = [t for t in table_names if t in ['attendance_data', 'qr_codes', 'users', 'projects']]
|
||||
maintenance_tables = [t for t in table_names if t in ['attendance_audit', 'attendance_statistics_cache', 'log_events']]
|
||||
|
||||
print(f" Core tables: {len(core_tables)} ({', '.join(core_tables)})")
|
||||
print(f" Maintenance tables: {len(maintenance_tables)} ({', '.join(maintenance_tables)})")
|
||||
print(f" Other tables: {len(table_names) - len(core_tables) - len(maintenance_tables)}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Failed to get table info: {e}")
|
||||
|
||||
def analyze_performance():
|
||||
"""Analyze database performance and suggest optimizations"""
|
||||
try:
|
||||
from app import app, db
|
||||
|
||||
print("Database Performance Analysis")
|
||||
print("=" * 35)
|
||||
|
||||
with app.app_context():
|
||||
# Check for tables without indexes
|
||||
tables_to_check = ['attendance_data', 'qr_codes', 'users', 'log_events']
|
||||
|
||||
print("Index Coverage Analysis:")
|
||||
print("-" * 25)
|
||||
|
||||
tables_needing_indexes = []
|
||||
|
||||
for table in tables_to_check:
|
||||
try:
|
||||
# Check if table exists
|
||||
table_check = db.session.execute(
|
||||
text(f"SHOW TABLES LIKE :table_name"),
|
||||
{'table_name': table}
|
||||
).fetchone()
|
||||
|
||||
if not table_check:
|
||||
continue
|
||||
|
||||
# Count non-primary indexes
|
||||
index_query = f"""
|
||||
SELECT COUNT(*) as index_count
|
||||
FROM INFORMATION_SCHEMA.STATISTICS
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_NAME = '{table}'
|
||||
AND INDEX_NAME != 'PRIMARY'
|
||||
"""
|
||||
index_result = db.session.execute(text(index_query)).fetchone()
|
||||
index_count = index_result.index_count if index_result else 0
|
||||
|
||||
# Get table size for impact assessment
|
||||
size_query = f"""
|
||||
SELECT
|
||||
TABLE_ROWS,
|
||||
ROUND(((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024), 2) as size_mb
|
||||
FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '{table}'
|
||||
"""
|
||||
size_result = db.session.execute(text(size_query)).fetchone()
|
||||
row_count = size_result.TABLE_ROWS if size_result else 0
|
||||
size_mb = size_result.size_mb if size_result and size_result.size_mb else 0
|
||||
|
||||
status = "GOOD" if index_count > 0 else "NEEDS INDEXES"
|
||||
priority = "HIGH" if row_count > 1000 and index_count == 0 else "MEDIUM"
|
||||
|
||||
print(f"{table:<20} {index_count:<3} indexes {row_count:<8,} rows {status}")
|
||||
|
||||
if index_count == 0 and row_count > 100:
|
||||
tables_needing_indexes.append({
|
||||
'table': table,
|
||||
'rows': row_count,
|
||||
'size_mb': size_mb,
|
||||
'priority': priority
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
print(f"{table:<20} ERROR - {e}")
|
||||
|
||||
# Recommendations
|
||||
if tables_needing_indexes:
|
||||
print(f"\nPerformance Recommendations:")
|
||||
print("-" * 28)
|
||||
|
||||
for table_info in sorted(tables_needing_indexes, key=lambda x: x['rows'], reverse=True):
|
||||
print(f"• {table_info['table']}: Add performance indexes")
|
||||
print(f" Rows: {table_info['rows']:,}, Priority: {table_info['priority']}")
|
||||
|
||||
print(f"\nTo add indexes: python fixed_simple_maintenance.py add-indexes")
|
||||
else:
|
||||
print(f"\n[GOOD] All major tables have performance indexes")
|
||||
|
||||
# Check for large tables that might need optimization
|
||||
print(f"\nLarge Table Analysis:")
|
||||
print("-" * 22)
|
||||
|
||||
large_tables_query = """
|
||||
SELECT
|
||||
TABLE_NAME,
|
||||
TABLE_ROWS,
|
||||
ROUND(((DATA_LENGTH + INDEX_LENGTH) / 1024 / 1024), 2) as size_mb
|
||||
FROM INFORMATION_SCHEMA.TABLES
|
||||
WHERE TABLE_SCHEMA = DATABASE()
|
||||
AND TABLE_ROWS > 1000
|
||||
ORDER BY TABLE_ROWS DESC
|
||||
"""
|
||||
|
||||
large_tables = db.session.execute(text(large_tables_query)).fetchall()
|
||||
|
||||
if large_tables:
|
||||
for table in large_tables:
|
||||
print(f"{table.TABLE_NAME:<20} {table.TABLE_ROWS:<8,} rows {table.size_mb:<6.1f} MB")
|
||||
|
||||
if any(table.TABLE_ROWS > 10000 for table in large_tables):
|
||||
print(f"\nTip: Consider regular cleanup for tables with >10k rows")
|
||||
else:
|
||||
print("No large tables found (all have <1000 rows)")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Performance analysis failed: {e}")
|
||||
|
||||
def main():
|
||||
"""Main function with enhanced commands"""
|
||||
if len(sys.argv) < 2:
|
||||
print("Fixed Simple Maintenance Script")
|
||||
print("=" * 35)
|
||||
print("Available commands:")
|
||||
print(" cleanup [--dry-run] - Clean up old data")
|
||||
print(" add-indexes - Add performance indexes")
|
||||
print(" table-info - Show detailed table information")
|
||||
print(" analyze-performance - Analyze database performance")
|
||||
print(" health-check - Run health check")
|
||||
print("")
|
||||
print("Examples:")
|
||||
print(" python fixed_simple_maintenance.py cleanup --dry-run")
|
||||
print(" python fixed_simple_maintenance.py add-indexes")
|
||||
print(" python fixed_simple_maintenance.py table-info")
|
||||
return
|
||||
|
||||
command = sys.argv[1].lower()
|
||||
|
||||
try:
|
||||
if command == 'cleanup':
|
||||
dry_run = '--dry-run' in sys.argv
|
||||
cleanup_data(dry_run=dry_run)
|
||||
|
||||
elif command == 'add-indexes':
|
||||
add_performance_indexes()
|
||||
|
||||
elif command == 'table-info':
|
||||
show_table_info()
|
||||
|
||||
elif command == 'analyze-performance':
|
||||
analyze_performance()
|
||||
|
||||
elif command == 'health-check':
|
||||
# Run health check from the other script
|
||||
try:
|
||||
from simple_health_check import health_check
|
||||
health_check()
|
||||
except ImportError:
|
||||
print("[ERROR] simple_health_check.py not found")
|
||||
print("Run: python windows_compatible_fix.py first")
|
||||
|
||||
else:
|
||||
print(f"Unknown command: {command}")
|
||||
print("Use: cleanup, add-indexes, table-info, analyze-performance, or health-check")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print("\n[CANCELLED] Operation cancelled by user")
|
||||
except Exception as e:
|
||||
print(f"[ERROR] Command failed: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user