Add some tools

This commit is contained in:
Nguyen Ngo
2025-08-04 17:29:48 -04:00
parent 87a3b846b1
commit 1cd4080bfa
5 changed files with 3849 additions and 61 deletions
+60 -58
View File
@@ -123,6 +123,7 @@ class AttendanceData(db.Model):
latitude = db.Column(db.Float, nullable=True)
longitude = db.Column(db.Float, nullable=True)
accuracy = db.Column(db.Float, nullable=True)
location_accuracy = db.Column(db.Float, nullable=True)
altitude = db.Column(db.Float, nullable=True)
location_source = db.Column(db.String(50), default='manual')
address = db.Column(db.String(500), nullable=True)
@@ -664,7 +665,8 @@ def process_location_data(location_data):
def process_location_data_enhanced(form_data):
"""
Enhanced location data processing with better validation and error handling
Enhanced processing of location data from form submission
Validates and cleans location data for storage
"""
processed = {
'latitude': None,
@@ -676,49 +678,51 @@ def process_location_data_enhanced(form_data):
}
try:
# Enhanced latitude validation
# Process latitude
if form_data.get('latitude') and form_data['latitude'] not in ['null', '', 'undefined']:
lat = float(form_data['latitude'])
if -90 <= lat <= 90:
processed['latitude'] = round(lat, 6) # 6 decimal precision
if -90 <= lat <= 90: # Valid latitude range
processed['latitude'] = lat
else:
print(f"⚠️ Invalid latitude range: {lat}")
print(f"⚠️ Invalid latitude: {lat}")
# Enhanced longitude validation
# Process longitude
if form_data.get('longitude') and form_data['longitude'] not in ['null', '', 'undefined']:
lng = float(form_data['longitude'])
if -180 <= lng <= 180:
processed['longitude'] = round(lng, 6) # 6 decimal precision
if -180 <= lng <= 180: # Valid longitude range
processed['longitude'] = lng
else:
print(f"⚠️ Invalid longitude range: {lng}")
print(f"⚠️ Invalid longitude: {lng}")
# Enhanced accuracy validation
# Process GPS accuracy
if form_data.get('accuracy') and form_data['accuracy'] not in ['null', '', 'undefined']:
acc = float(form_data['accuracy'])
if 0 <= acc <= 50000: # Reasonable accuracy range in meters
processed['accuracy'] = round(acc, 1)
if acc >= 0: # Accuracy should be positive
processed['accuracy'] = acc
else:
print(f"⚠️ Invalid accuracy value: {acc}")
print(f"⚠️ Invalid GPS accuracy: {acc}")
# Enhanced altitude validation
# Process altitude
if form_data.get('altitude') and form_data['altitude'] not in ['null', '', 'undefined']:
alt = float(form_data['altitude'])
if -1000 <= alt <= 10000: # Reasonable altitude range in meters
processed['altitude'] = round(alt, 1)
else:
print(f"⚠️ Invalid altitude value: {alt}")
processed['altitude'] = alt
# Enhanced address processing
# Process address (limit length for database storage)
if form_data.get('address'):
address = form_data['address'].strip()
if len(address) > 0:
if address and address not in ['null', '', 'undefined']:
processed['address'] = address[:500] # Limit to 500 characters
print(f"✅ Enhanced location data processed: {processed}")
print(f"📍 Processed location data:")
print(f" Coordinates: {processed['latitude']}, {processed['longitude']}")
print(f" GPS Accuracy: {processed['accuracy']}m")
print(f" Source: {processed['source']}")
print(f" Address: {processed['address'][:100] if processed['address'] else 'None'}...")
except (ValueError, TypeError) as e:
print(f"⚠️ Error in enhanced location data processing: {e}")
return processed
except Exception as e:
print(f"❌ Error processing location data: {e}")
return processed
def migrate_to_enhanced_location_accuracy():
@@ -1861,7 +1865,8 @@ def qr_destination(qr_url):
@app.route('/qr/<string:qr_url>/checkin', methods=['POST'])
def qr_checkin(qr_url):
"""
Enhanced staff check-in with improved location accuracy calculation
Enhanced staff check-in with location accuracy calculation
Calculates distance between QR code address and actual check-in location
"""
try:
print(f"\n🚀 STARTING ENHANCED CHECK-IN PROCESS")
@@ -1882,7 +1887,7 @@ def qr_checkin(qr_url):
print(f" Location: {qr_code.location}")
print(f" QR Address: {qr_code.location_address}")
# Get form data
# Get and validate employee ID
employee_id = request.form.get('employee_id', '').strip()
if not employee_id:
@@ -1906,7 +1911,7 @@ def qr_checkin(qr_url):
'message': f'You have already checked in today at {existing_checkin.check_in_time.strftime("%H:%M")}.'
}), 400
# Process location data with enhanced validation
# Process location data
location_data = process_location_data_enhanced(request.form)
# Get device and network info
@@ -1919,7 +1924,7 @@ def qr_checkin(qr_url):
print(f"📍 Location Data: {location_data}")
# Create attendance record
print(f"\n💾 CREATING ENHANCED ATTENDANCE RECORD:")
print(f"\n💾 CREATING ATTENDANCE RECORD:")
attendance = AttendanceData(
qr_code_id=qr_code.id,
@@ -1941,11 +1946,12 @@ def qr_checkin(qr_url):
print(f"✅ Created base attendance record")
# ENHANCED LOCATION ACCURACY CALCULATION
print(f"\n🎯 CALCULATING ENHANCED LOCATION ACCURACY...")
# CRITICAL: LOCATION ACCURACY CALCULATION
print(f"\n🎯 CALCULATING LOCATION ACCURACY...")
location_accuracy = None
try:
# Calculate accuracy using existing enhanced function
location_accuracy = calculate_location_accuracy_enhanced(
qr_address=qr_code.location_address,
checkin_address=location_data['address'],
@@ -1954,20 +1960,40 @@ def qr_checkin(qr_url):
)
if location_accuracy is not None:
# Store the calculated accuracy in the database
attendance.location_accuracy = location_accuracy
accuracy_level = get_location_accuracy_level_enhanced(location_accuracy)
print(f"Enhanced location accuracy set: {location_accuracy:.4f} miles ({accuracy_level})")
print(f"Location accuracy calculated: {location_accuracy:.4f} miles ({accuracy_level})")
else:
print(f"⚠️ Could not calculate enhanced location accuracy")
print(f"⚠️ Could not calculate location accuracy")
except Exception as e:
print(f"❌ Error in enhanced location accuracy calculation: {e}")
print(f"❌ Error in location accuracy calculation: {e}")
# Continue with check-in even if accuracy calculation fails
# Save to database
try:
db.session.add(attendance)
db.session.commit()
print(f"✅ Successfully saved enhanced attendance record with ID: {attendance.id}")
print(f"✅ Successfully saved attendance record with ID: {attendance.id}")
# Prepare success response
response_data = {
'success': True,
'message': f'Successfully checked in at {datetime.now().strftime("%H:%M")}',
'employee_id': employee_id.upper(),
'location': qr_code.location,
'timestamp': datetime.now().strftime("%Y-%m-%d %H:%M:%S")
}
# Include location accuracy in response if calculated
if location_accuracy is not None:
response_data['location_accuracy'] = {
'distance_miles': round(location_accuracy, 4),
'level': get_location_accuracy_level_enhanced(location_accuracy)
}
return jsonify(response_data), 200
except Exception as e:
print(f"❌ Database error: {e}")
@@ -1977,32 +2003,8 @@ def qr_checkin(qr_url):
'message': 'Database error occurred. Please try again.'
}), 500
# Build enhanced success response
response_data = {
'success': True,
'message': 'Enhanced check-in successful!',
'data': {
'employee_id': attendance.employee_id,
'location': attendance.location_name,
'check_in_time': attendance.check_in_time.strftime('%H:%M'),
'has_gps': attendance.latitude is not None and attendance.longitude is not None,
'location_accuracy': f"{location_accuracy:.4f} miles" if location_accuracy else "Not calculated",
'accuracy_level': get_location_accuracy_level_enhanced(location_accuracy) if location_accuracy else "unknown",
'location_source': location_data['source'],
'enhanced_features': True
}
}
print(f"✅ ENHANCED CHECK-IN COMPLETED SUCCESSFULLY!")
print(f" Employee: {attendance.employee_id}")
print(f" Location: {attendance.location_name}")
print(f" Accuracy: {location_accuracy:.4f} miles" if location_accuracy else "Not calculated")
print(f" Time: {attendance.check_in_time}")
return jsonify(response_data)
except Exception as e:
print(f"Critical error in enhanced check-in: {e}")
print(f"Unexpected error in check-in process: {e}")
return jsonify({
'success': False,
'message': 'An unexpected error occurred. Please try again.'
+2857
View File
File diff suppressed because it is too large Load Diff
+384
View File
@@ -0,0 +1,384 @@
#!/usr/bin/env python3
"""
Location Accuracy Calculator Script for Existing Database Records
This script calculates location accuracy for all existing attendance records
in the database where location_accuracy is NULL or needs recalculation.
Usage:
python calculate_location_accuracy.py [options]
Options:
--dry-run Show what would be updated without making changes
--force-recalc Recalculate accuracy for all records (even existing ones)
--batch-size N Process records in batches of N (default: 100)
--specific-date Process only records from specific date (YYYY-MM-DD)
--help Show this help message
Author: Attendance System Location Enhancement
Version: 1.0
"""
import sys
import os
import argparse
from datetime import datetime, date
import time
# Add your app directory to path for imports
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
# Import your Flask app and models
try:
from app import app, db, AttendanceData, QRCode
from app import (
calculate_location_accuracy_enhanced,
get_location_accuracy_level_enhanced,
get_coordinates_from_address,
calculate_distance_miles
)
except ImportError as e:
print(f"❌ Error importing app modules: {e}")
print("Make sure this script is in the same directory as your app.py file")
sys.exit(1)
# Configuration
DEFAULT_BATCH_SIZE = 100
GEOCODING_DELAY = 0.1 # Delay between geocoding requests to avoid rate limits
class LocationAccuracyCalculator:
"""Main class for calculating location accuracy for existing records"""
def __init__(self, dry_run=False, force_recalc=False, batch_size=DEFAULT_BATCH_SIZE):
self.dry_run = dry_run
self.force_recalc = force_recalc
self.batch_size = batch_size
self.stats = {
'total_records': 0,
'processed': 0,
'updated': 0,
'skipped': 0,
'errors': 0,
'accuracy_calculated': 0,
'accuracy_failed': 0
}
def run(self, specific_date=None):
"""Main execution method"""
print("🚀 LOCATION ACCURACY CALCULATOR - STARTING")
print("=" * 60)
print(f"Mode: {'DRY RUN' if self.dry_run else 'LIVE UPDATE'}")
print(f"Force Recalculation: {'YES' if self.force_recalc else 'NO'}")
print(f"Batch Size: {self.batch_size}")
if specific_date:
print(f"Target Date: {specific_date}")
print("=" * 60)
with app.app_context():
try:
# Get records to process
records = self._get_records_to_process(specific_date)
self.stats['total_records'] = len(records)
if not records:
print("️ No records found to process")
return
print(f"📊 Found {len(records)} records to process")
# Process records in batches
self._process_records_in_batches(records)
# Final database commit to ensure all changes are saved
if not self.dry_run:
try:
db.session.commit()
print(f"\n💾 Final database commit completed successfully")
except Exception as e:
print(f"❌ Final commit error: {e}")
db.session.rollback()
# Print final statistics
self._print_final_stats()
except Exception as e:
print(f"❌ Fatal error: {e}")
import traceback
traceback.print_exc()
def _get_records_to_process(self, specific_date=None):
"""Get attendance records that need location accuracy calculation"""
query = db.session.query(AttendanceData).join(QRCode)
if specific_date:
query = query.filter(AttendanceData.check_in_date == specific_date)
if not self.force_recalc:
# Only get records where location_accuracy is NULL
query = query.filter(AttendanceData.location_accuracy.is_(None))
# Order by date and time for consistent processing
query = query.order_by(
AttendanceData.check_in_date.desc(),
AttendanceData.check_in_time.desc()
)
return query.all()
def _process_records_in_batches(self, records):
"""Process records in configurable batches"""
total_batches = (len(records) + self.batch_size - 1) // self.batch_size
for batch_num in range(total_batches):
start_idx = batch_num * self.batch_size
end_idx = min(start_idx + self.batch_size, len(records))
batch_records = records[start_idx:end_idx]
print(f"\n📦 BATCH {batch_num + 1}/{total_batches} - Processing records {start_idx + 1}-{end_idx}")
print("-" * 50)
self._process_batch(batch_records, batch_num + 1)
# Small delay between batches to avoid overwhelming external services
if batch_num < total_batches - 1:
time.sleep(0.5)
def _process_batch(self, records, batch_num):
"""Process a single batch of records"""
batch_updates = []
records_to_update = []
for idx, record in enumerate(records, 1):
try:
result = self._process_single_record(record, batch_num, idx)
if result:
batch_updates.append(result)
records_to_update.append(record)
except Exception as e:
print(f"❌ Error processing record {record.id}: {e}")
self.stats['errors'] += 1
# Save updates to database
if batch_updates and not self.dry_run:
try:
# Update each record in the database
for i, update_data in enumerate(batch_updates):
record = records_to_update[i]
record.location_accuracy = update_data['accuracy']
# Mark the record as modified
db.session.merge(record)
# Commit all changes in this batch
db.session.commit()
print(f"✅ Successfully saved {len(batch_updates)} location accuracy updates to database")
except Exception as e:
print(f"❌ Database commit error: {e}")
db.session.rollback()
self.stats['errors'] += len(batch_updates)
# Reset the updated count since commit failed
self.stats['updated'] -= len(batch_updates)
self.stats['accuracy_calculated'] -= len(batch_updates)
def _process_single_record(self, record, batch_num, record_idx):
"""Process a single attendance record"""
self.stats['processed'] += 1
# Skip if already has accuracy and not forcing recalculation
if record.location_accuracy is not None and not self.force_recalc:
print(f"⏭️ [{batch_num}.{record_idx}] Record {record.id}: Already has accuracy ({record.location_accuracy:.4f} mi)")
self.stats['skipped'] += 1
return None
# Get QR code information
qr_code = record.qr_code
if not qr_code:
print(f"⚠️ [{batch_num}.{record_idx}] Record {record.id}: No QR code found")
self.stats['skipped'] += 1
return None
print(f"🔄 [{batch_num}.{record_idx}] Processing: Employee {record.employee_id} | {record.check_in_date} | {qr_code.location}")
# Calculate location accuracy
location_accuracy = self._calculate_accuracy_for_record(record, qr_code)
if location_accuracy is not None:
accuracy_level = get_location_accuracy_level_enhanced(location_accuracy)
# Always update the record object, database save happens in batch processing
if not self.dry_run:
# Update the record's location_accuracy field
record.location_accuracy = location_accuracy
print(f"✅ [{batch_num}.{record_idx}] Accuracy set: {location_accuracy:.4f} miles ({accuracy_level}) - Will save to database")
else:
print(f"✅ [{batch_num}.{record_idx}] Accuracy calculated: {location_accuracy:.4f} miles ({accuracy_level}) - DRY RUN")
self.stats['updated'] += 1
self.stats['accuracy_calculated'] += 1
return {
'record_id': record.id,
'accuracy': location_accuracy,
'level': accuracy_level
}
else:
print(f"⚠️ [{batch_num}.{record_idx}] Could not calculate accuracy")
self.stats['accuracy_failed'] += 1
return None
def _calculate_accuracy_for_record(self, record, qr_code):
"""Calculate location accuracy for a specific record"""
try:
# Add small delay to avoid overwhelming geocoding services
time.sleep(GEOCODING_DELAY)
return calculate_location_accuracy_enhanced(
qr_address=qr_code.location_address,
checkin_address=record.address,
checkin_lat=record.latitude,
checkin_lng=record.longitude
)
except Exception as e:
print(f" ❌ Calculation error: {e}")
return None
def _print_final_stats(self):
"""Print comprehensive final statistics"""
print("\n" + "=" * 60)
print("📊 FINAL STATISTICS")
print("=" * 60)
print(f"Total Records Found: {self.stats['total_records']:,}")
print(f"Records Processed: {self.stats['processed']:,}")
print(f"Records Updated: {self.stats['updated']:,}")
print(f"Records Skipped: {self.stats['skipped']:,}")
print(f"Errors Encountered: {self.stats['errors']:,}")
print("-" * 40)
print(f"Accuracy Calculated: {self.stats['accuracy_calculated']:,}")
print(f"Accuracy Failed: {self.stats['accuracy_failed']:,}")
if self.stats['processed'] > 0:
success_rate = (self.stats['accuracy_calculated'] / self.stats['processed']) * 100
print(f"Success Rate: {success_rate:.1f}%")
print("=" * 60)
if self.dry_run:
print("🔍 DRY RUN COMPLETED - No changes were made to the database")
else:
print("✅ LIVE UPDATE COMPLETED - Database has been updated")
# Verify database updates
self._verify_database_updates()
print("=" * 60)
def _verify_database_updates(self):
"""Verify that the database updates were actually saved"""
try:
print("\n🔍 VERIFYING DATABASE UPDATES...")
# Count records with location_accuracy that were just updated
updated_count = db.session.query(AttendanceData).filter(
AttendanceData.location_accuracy.isnot(None)
).count()
print(f"📊 Database verification:")
print(f" Total records with location_accuracy: {updated_count:,}")
if self.stats['updated'] > 0:
print(f" Expected updates from this run: {self.stats['updated']:,}")
# Get a sample of recently updated records to verify
sample_records = db.session.query(AttendanceData).filter(
AttendanceData.location_accuracy.isnot(None)
).limit(3).all()
if sample_records:
print(f" Sample updated records:")
for record in sample_records:
print(f" • Record {record.id}: {record.location_accuracy:.4f} miles")
else:
print(" ⚠️ No sample records found - verification inconclusive")
print("✅ Database verification completed")
except Exception as e:
print(f"❌ Error during database verification: {e}")
def main():
"""Main entry point with command line argument parsing"""
parser = argparse.ArgumentParser(
description="Calculate location accuracy for existing attendance records",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
python calculate_location_accuracy.py --dry-run
python calculate_location_accuracy.py --force-recalc --batch-size 50
python calculate_location_accuracy.py --specific-date 2025-01-15
python calculate_location_accuracy.py --dry-run --specific-date 2025-01-15
"""
)
parser.add_argument(
'--dry-run',
action='store_true',
help='Show what would be updated without making changes'
)
parser.add_argument(
'--force-recalc',
action='store_true',
help='Recalculate accuracy for all records (even existing ones)'
)
parser.add_argument(
'--batch-size',
type=int,
default=DEFAULT_BATCH_SIZE,
help=f'Process records in batches of N (default: {DEFAULT_BATCH_SIZE})'
)
parser.add_argument(
'--specific-date',
type=str,
help='Process only records from specific date (YYYY-MM-DD format)'
)
args = parser.parse_args()
# Validate specific date if provided
specific_date = None
if args.specific_date:
try:
specific_date = datetime.strptime(args.specific_date, '%Y-%m-%d').date()
except ValueError:
print("❌ Invalid date format. Use YYYY-MM-DD (e.g., 2025-01-15)")
sys.exit(1)
# Validate batch size
if args.batch_size < 1:
print("❌ Batch size must be at least 1")
sys.exit(1)
# Create and run calculator
calculator = LocationAccuracyCalculator(
dry_run=args.dry_run,
force_recalc=args.force_recalc,
batch_size=args.batch_size
)
try:
calculator.run(specific_date=specific_date)
except KeyboardInterrupt:
print("\n⏹️ Operation cancelled by user")
sys.exit(0)
except Exception as e:
print(f"\n❌ Unexpected error: {e}")
sys.exit(1)
if __name__ == '__main__':
main()
+372
View File
@@ -0,0 +1,372 @@
#!/usr/bin/env python3
"""
Location Accuracy Database Utilities
Additional utility functions for managing location accuracy data
in your attendance system database.
Usage:
python location_accuracy_utils.py [command] [options]
Commands:
check-schema Verify database schema for location accuracy
add-field Add location_accuracy field to attendance_data table
stats Show location accuracy statistics
export-report Export detailed location accuracy report
verify-data Verify data integrity for location accuracy
Author: Attendance System Enhancement Team
"""
import sys
import os
import csv
from datetime import datetime, date
import json
# Add your app directory to path
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
try:
from app import app, db, AttendanceData, QRCode
from app import get_location_accuracy_level_enhanced
from sqlalchemy import text, inspect
except ImportError as e:
print(f"❌ Error importing modules: {e}")
sys.exit(1)
class LocationAccuracyUtils:
"""Utility class for location accuracy database management"""
def __init__(self):
self.app = app
def check_database_schema(self):
"""Check if the database schema supports location accuracy"""
print("🔍 CHECKING DATABASE SCHEMA")
print("=" * 50)
with self.app.app_context():
inspector = inspect(db.engine)
# Check attendance_data table
try:
columns = inspector.get_columns('attendance_data')
column_names = [col['name'] for col in columns]
print("📋 Attendance Data Table Columns:")
required_fields = [
'id', 'qr_code_id', 'employee_id', 'check_in_date', 'check_in_time',
'latitude', 'longitude', 'address', 'location_accuracy'
]
for field in required_fields:
status = "" if field in column_names else ""
print(f" {status} {field}")
# Check for location_accuracy field specifically
has_accuracy_field = 'location_accuracy' in column_names
if has_accuracy_field:
# Get field details
accuracy_col = next((col for col in columns if col['name'] == 'location_accuracy'), None)
if accuracy_col:
print(f"\n📊 location_accuracy field details:")
print(f" Type: {accuracy_col['type']}")
print(f" Nullable: {accuracy_col['nullable']}")
print(f" Default: {accuracy_col.get('default', 'None')}")
return has_accuracy_field
except Exception as e:
print(f"❌ Error checking schema: {e}")
return False
def add_location_accuracy_field(self):
"""Add location_accuracy field to attendance_data table if missing"""
print("🔧 ADDING LOCATION ACCURACY FIELD")
print("=" * 50)
with self.app.app_context():
try:
# Check if field already exists
if self.check_database_schema():
print("️ location_accuracy field already exists")
return True
print(" Adding location_accuracy field...")
# Add the field
db.session.execute(text("""
ALTER TABLE attendance_data
ADD COLUMN location_accuracy FLOAT
"""))
db.session.commit()
print("✅ Successfully added location_accuracy field")
# Verify addition
if self.check_database_schema():
print("✅ Field addition verified")
return True
else:
print("❌ Field addition verification failed")
return False
except Exception as e:
print(f"❌ Error adding field: {e}")
db.session.rollback()
return False
def show_location_accuracy_stats(self):
"""Show comprehensive statistics about location accuracy data"""
print("📊 LOCATION ACCURACY STATISTICS")
print("=" * 60)
with self.app.app_context():
try:
# Basic counts
total_records = db.session.query(AttendanceData).count()
records_with_location = db.session.query(AttendanceData).filter(
AttendanceData.latitude.isnot(None),
AttendanceData.longitude.isnot(None)
).count()
records_with_accuracy = db.session.query(AttendanceData).filter(
AttendanceData.location_accuracy.isnot(None)
).count()
print(f"Total Records: {total_records:,}")
print(f"Records with GPS Data: {records_with_location:,}")
print(f"Records with Accuracy: {records_with_accuracy:,}")
if total_records > 0:
gps_percentage = (records_with_location / total_records) * 100
accuracy_percentage = (records_with_accuracy / total_records) * 100
print(f"GPS Coverage: {gps_percentage:.1f}%")
print(f"Accuracy Coverage: {accuracy_percentage:.1f}%")
# Accuracy level distribution
if records_with_accuracy > 0:
print("\n📈 ACCURACY LEVEL DISTRIBUTION")
print("-" * 40)
accuracy_records = db.session.query(AttendanceData.location_accuracy).filter(
AttendanceData.location_accuracy.isnot(None)
).all()
levels = {}
total_distance = 0
for record in accuracy_records:
accuracy = record[0]
level = get_location_accuracy_level_enhanced(accuracy)
levels[level] = levels.get(level, 0) + 1
total_distance += accuracy
for level in ['excellent', 'very_good', 'good', 'fair', 'poor', 'very_poor']:
count = levels.get(level, 0)
percentage = (count / records_with_accuracy) * 100 if records_with_accuracy > 0 else 0
print(f"{level.replace('_', ' ').title():12} {count:6,} ({percentage:5.1f}%)")
avg_accuracy = total_distance / len(accuracy_records)
print(f"\nAverage Distance: {avg_accuracy:.4f} miles")
# Recent activity
print("\n📅 RECENT ACTIVITY (Last 7 Days)")
print("-" * 40)
seven_days_ago = date.today().replace(day=date.today().day - 7) if date.today().day > 7 else date.today().replace(month=date.today().month - 1, day=date.today().day + 30 - 7)
recent_records = db.session.query(AttendanceData).filter(
AttendanceData.check_in_date >= seven_days_ago
).count()
recent_with_accuracy = db.session.query(AttendanceData).filter(
AttendanceData.check_in_date >= seven_days_ago,
AttendanceData.location_accuracy.isnot(None)
).count()
print(f"Recent Check-ins: {recent_records:,}")
print(f"Recent with Accuracy: {recent_with_accuracy:,}")
if recent_records > 0:
recent_percentage = (recent_with_accuracy / recent_records) * 100
print(f"Recent Accuracy Rate: {recent_percentage:.1f}%")
except Exception as e:
print(f"❌ Error generating statistics: {e}")
def export_location_accuracy_report(self, output_file=None):
"""Export detailed location accuracy report to CSV"""
if not output_file:
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
output_file = f"location_accuracy_report_{timestamp}.csv"
print(f"📄 EXPORTING LOCATION ACCURACY REPORT")
print(f"Output File: {output_file}")
print("=" * 60)
with self.app.app_context():
try:
# Query all records with their QR code information
query = db.session.query(AttendanceData, QRCode).join(
QRCode, AttendanceData.qr_code_id == QRCode.id
).order_by(AttendanceData.check_in_date.desc(), AttendanceData.check_in_time.desc())
records = query.all()
print(f"Found {len(records)} records to export")
# Write CSV file
with open(output_file, 'w', newline='', encoding='utf-8') as csvfile:
fieldnames = [
'attendance_id', 'employee_id', 'check_in_date', 'check_in_time',
'location_name', 'qr_address', 'checkin_address',
'checkin_latitude', 'checkin_longitude', 'gps_accuracy_meters',
'location_accuracy_miles', 'accuracy_level',
'has_gps_data', 'has_location_accuracy', 'device_info'
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
writer.writeheader()
for attendance, qr_code in records:
accuracy_level = 'unknown'
if attendance.location_accuracy:
accuracy_level = get_location_accuracy_level_enhanced(attendance.location_accuracy)
writer.writerow({
'attendance_id': attendance.id,
'employee_id': attendance.employee_id,
'check_in_date': attendance.check_in_date.isoformat(),
'check_in_time': attendance.check_in_time.isoformat(),
'location_name': attendance.location_name,
'qr_address': qr_code.location_address,
'checkin_address': attendance.address or 'Not captured',
'checkin_latitude': attendance.latitude,
'checkin_longitude': attendance.longitude,
'gps_accuracy_meters': attendance.accuracy,
'location_accuracy_miles': attendance.location_accuracy,
'accuracy_level': accuracy_level,
'has_gps_data': attendance.latitude is not None and attendance.longitude is not None,
'has_location_accuracy': attendance.location_accuracy is not None,
'device_info': attendance.device_info or 'Unknown'
})
print(f"✅ Report exported successfully to {output_file}")
return output_file
except Exception as e:
print(f"❌ Error exporting report: {e}")
return None
def verify_data_integrity(self):
"""Verify data integrity for location accuracy calculations"""
print("🔍 VERIFYING DATA INTEGRITY")
print("=" * 50)
with self.app.app_context():
try:
issues = []
# Check for records with coordinates but no address
no_address_with_coords = db.session.query(AttendanceData).filter(
AttendanceData.latitude.isnot(None),
AttendanceData.longitude.isnot(None),
AttendanceData.address.is_(None)
).count()
if no_address_with_coords > 0:
issues.append(f"{no_address_with_coords} records have GPS coordinates but no address")
# Check for invalid coordinates
invalid_coords = db.session.query(AttendanceData).filter(
db.or_(
AttendanceData.latitude < -90,
AttendanceData.latitude > 90,
AttendanceData.longitude < -180,
AttendanceData.longitude > 180
)
).count()
if invalid_coords > 0:
issues.append(f"{invalid_coords} records have invalid GPS coordinates")
# Check for QR codes without addresses
qr_no_address = db.session.query(QRCode).filter(
db.or_(
QRCode.location_address.is_(None),
QRCode.location_address == ''
)
).count()
if qr_no_address > 0:
issues.append(f"{qr_no_address} QR codes have no address defined")
# Check for impossible accuracy values
extreme_accuracy = db.session.query(AttendanceData).filter(
db.or_(
AttendanceData.location_accuracy < 0,
AttendanceData.location_accuracy > 1000 # More than 1000 miles seems extreme
)
).count()
if extreme_accuracy > 0:
issues.append(f"{extreme_accuracy} records have extreme location accuracy values")
# Report results
if issues:
print("⚠️ Data integrity issues found:")
for issue in issues:
print(f"{issue}")
else:
print("✅ No data integrity issues found")
return len(issues) == 0
except Exception as e:
print(f"❌ Error during verification: {e}")
return False
def main():
"""Main entry point for utility commands"""
if len(sys.argv) < 2:
print("""
Usage: python location_accuracy_utils.py [command]
Commands:
check-schema Check database schema
add-field Add location_accuracy field
stats Show statistics
export-report Export CSV report
verify-data Verify data integrity
help Show this help
""")
sys.exit(1)
command = sys.argv[1].lower()
utils = LocationAccuracyUtils()
if command == 'check-schema':
utils.check_database_schema()
elif command == 'add-field':
utils.add_location_accuracy_field()
elif command == 'stats':
utils.show_location_accuracy_stats()
elif command == 'export-report':
output_file = sys.argv[2] if len(sys.argv) > 2 else None
utils.export_location_accuracy_report(output_file)
elif command == 'verify-data':
utils.verify_data_integrity()
elif command == 'help':
print(__doc__)
else:
print(f"❌ Unknown command: {command}")
print("Use 'help' to see available commands")
sys.exit(1)
if __name__ == '__main__':
main()
+173
View File
@@ -0,0 +1,173 @@
#!/usr/bin/env python3
"""
Database Update Verification Script
Quick script to verify that location accuracy calculations
are being properly saved to the database.
Usage:
python verify_database_updates.py
This script will:
1. Check database connection
2. Count records with location_accuracy
3. Show sample records
4. Display recent updates
"""
import sys
import os
from datetime import datetime, date, timedelta
# Add your app directory to path
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
try:
from app import app, db, AttendanceData, QRCode
from app import get_location_accuracy_level_enhanced
except ImportError as e:
print(f"❌ Error importing modules: {e}")
sys.exit(1)
def verify_database_updates():
"""Comprehensive verification of database updates"""
print("🔍 DATABASE UPDATE VERIFICATION")
print("=" * 50)
with app.app_context():
try:
# Basic counts
total_records = db.session.query(AttendanceData).count()
records_with_accuracy = db.session.query(AttendanceData).filter(
AttendanceData.location_accuracy.isnot(None)
).count()
print(f"📊 Database Status:")
print(f" Total attendance records: {total_records:,}")
print(f" Records with location_accuracy: {records_with_accuracy:,}")
if total_records > 0:
coverage = (records_with_accuracy / total_records) * 100
print(f" Coverage: {coverage:.1f}%")
# Show sample records with accuracy
if records_with_accuracy > 0:
print(f"\n📝 Sample Records with Location Accuracy:")
print("-" * 40)
sample_records = db.session.query(AttendanceData, QRCode).join(
QRCode, AttendanceData.qr_code_id == QRCode.id
).filter(
AttendanceData.location_accuracy.isnot(None)
).order_by(AttendanceData.id.desc()).limit(5).all()
for attendance, qr_code in sample_records:
accuracy_level = get_location_accuracy_level_enhanced(attendance.location_accuracy)
print(f" • Record {attendance.id}: Employee {attendance.employee_id}")
print(f" Date: {attendance.check_in_date} | Location: {attendance.location_name}")
print(f" Accuracy: {attendance.location_accuracy:.4f} miles ({accuracy_level})")
print(f" QR Address: {qr_code.location_address[:50]}...")
if attendance.address:
print(f" Check-in Address: {attendance.address[:50]}...")
print()
# Check for recent updates (if any exist)
print(f"📅 Recent Activity Check:")
print("-" * 30)
yesterday = date.today() - timedelta(days=1)
recent_records = db.session.query(AttendanceData).filter(
AttendanceData.check_in_date >= yesterday
).count()
recent_with_accuracy = db.session.query(AttendanceData).filter(
AttendanceData.check_in_date >= yesterday,
AttendanceData.location_accuracy.isnot(None)
).count()
print(f" Recent records (last 24h): {recent_records}")
print(f" Recent with accuracy: {recent_with_accuracy}")
# Accuracy level distribution
if records_with_accuracy > 0:
print(f"\n📈 Accuracy Level Distribution:")
print("-" * 35)
accuracy_records = db.session.query(AttendanceData.location_accuracy).filter(
AttendanceData.location_accuracy.isnot(None)
).all()
levels = {}
for record in accuracy_records:
accuracy = record[0]
level = get_location_accuracy_level_enhanced(accuracy)
levels[level] = levels.get(level, 0) + 1
for level in ['excellent', 'very_good', 'good', 'fair', 'poor', 'very_poor']:
count = levels.get(level, 0)
if count > 0:
percentage = (count / records_with_accuracy) * 100
print(f" {level.replace('_', ' ').title():12} {count:6,} ({percentage:5.1f}%)")
# Database integrity checks
print(f"\n🔍 Database Integrity Checks:")
print("-" * 35)
# Check for invalid accuracy values
invalid_accuracy = db.session.query(AttendanceData).filter(
AttendanceData.location_accuracy < 0
).count()
extreme_accuracy = db.session.query(AttendanceData).filter(
AttendanceData.location_accuracy > 100
).count()
print(f" Invalid accuracy values (< 0): {invalid_accuracy}")
print(f" Extreme accuracy values (> 100 mi): {extreme_accuracy}")
if invalid_accuracy == 0 and extreme_accuracy == 0:
print(" ✅ All accuracy values are within reasonable ranges")
print(f"\n✅ Database verification completed successfully")
except Exception as e:
print(f"❌ Error during verification: {e}")
import traceback
traceback.print_exc()
def test_database_connection():
"""Test basic database connectivity"""
print("🔌 Testing Database Connection...")
with app.app_context():
try:
# Simple query to test connection
count = db.session.query(AttendanceData).count()
print(f"✅ Database connection successful")
print(f" Found {count:,} attendance records")
return True
except Exception as e:
print(f"❌ Database connection failed: {e}")
return False
def main():
"""Main execution function"""
print("DATABASE UPDATE VERIFICATION TOOL")
print("=" * 50)
print(f"Timestamp: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
print()
# Test connection first
if not test_database_connection():
print("Cannot proceed without database connection")
sys.exit(1)
print()
# Run verification
verify_database_updates()
print(f"\nVerification completed at {datetime.now().strftime('%H:%M:%S')}")
if __name__ == '__main__':
main()