From 3648c5dddac2df15b0be437296fa200629040b42 Mon Sep 17 00:00:00 2001 From: NguyenND Date: Mon, 6 Oct 2025 17:23:30 -0400 Subject: [PATCH] Updated Time Attendance functionality --- app.py | 246 +++++++- static/css/time_attendance.css | 28 + templates/time_attendance_batch_detail.html | 211 +++++++ templates/time_attendance_dashboard.html | 32 +- templates/time_attendance_import.html | 662 +++++++++----------- time_attendance_import_service.py | 398 ++++++++++-- 6 files changed, 1147 insertions(+), 430 deletions(-) create mode 100644 templates/time_attendance_batch_detail.html diff --git a/app.py b/app.py index e874233..d098ef4 100644 --- a/app.py +++ b/app.py @@ -1,6 +1,7 @@ from flask import Flask, render_template, request, redirect, url_for, flash, session, jsonify, send_file from flask_sqlalchemy import SQLAlchemy from werkzeug.security import generate_password_hash, check_password_hash +from werkzeug.utils import secure_filename from functools import wraps from datetime import datetime, date, time, timedelta from sqlalchemy import text @@ -15,6 +16,7 @@ from logger_handler import AppLogger, log_user_activity, log_database_operations from single_checkin_calculator import SingleCheckInCalculator from payroll_excel_exporter import PayrollExcelExporter from enhanced_payroll_excel_exporter import EnhancedPayrollExcelExporter +from time_attendance_import_service import TimeAttendanceImportService # Load environment variables in .env load_dotenv() @@ -6542,7 +6544,7 @@ def time_attendance_dashboard(): @admin_required @log_database_operations('time_attendance_import') def import_time_attendance(): - """Import time attendance data from Excel file""" + """Enhanced import time attendance data from Excel file""" if request.method == 'POST': try: # Check if file is uploaded @@ -6564,12 +6566,19 @@ def import_time_attendance(): filename = secure_filename(file.filename) temp_path = os.path.join(app.config.get('UPLOAD_FOLDER', '/tmp'), f"temp_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{filename}") + + # Ensure upload directory exists + os.makedirs(os.path.dirname(temp_path), exist_ok=True) file.save(temp_path) try: - # Initialize import service + # Initialize enhanced import service import_service = TimeAttendanceImportService(db, logger_handler) + # Get import options + skip_duplicates = request.form.get('skip_duplicates', 'true').lower() == 'true' + validate_only = request.form.get('validate_only', 'false').lower() == 'true' + # Validate file first validation_result = import_service.validate_excel_file(temp_path) @@ -6578,12 +6587,24 @@ def import_time_attendance(): return render_template('time_attendance_import.html', validation_result=validation_result) + # Show warnings if any + if validation_result['warnings']: + for warning in validation_result['warnings']: + flash(warning, 'warning') + + # If validate only, return validation results + if validate_only: + flash(f"File validation successful! Found {validation_result['valid_rows']} valid records.", 'success') + return render_template('time_attendance_import.html', + validation_result=validation_result) + # Proceed with import import_source = request.form.get('import_source', f"Manual Import - {filename}") import_result = import_service.import_from_excel( temp_path, created_by=session['user_id'], - import_source=import_source + import_source=import_source, + skip_duplicates=skip_duplicates ) if import_result['success']: @@ -6591,12 +6612,17 @@ def import_time_attendance(): logger_handler.logger.info( f"User {session['username']} successfully imported time attendance data - " f"Batch: {import_result['batch_id']}, " - f"Records: {import_result['imported_records']}/{import_result['total_records']}" + f"Records: {import_result['imported_records']}/{import_result['total_records']}, " + f"Duplicates: {import_result['duplicate_records']}, " + f"Failed: {import_result['failed_records']}" ) flash(f"Import successful! Imported {import_result['imported_records']} records " f"out of {import_result['total_records']} total records.", 'success') + if import_result['duplicate_records'] > 0: + flash(f"Skipped {import_result['duplicate_records']} duplicate records.", 'info') + if import_result['failed_records'] > 0: flash(f"Note: {import_result['failed_records']} records failed to import. " f"Check the error details below.", 'warning') @@ -6604,14 +6630,19 @@ def import_time_attendance(): return render_template('time_attendance_import_result.html', import_result=import_result) else: - flash(f"Import failed: {'; '.join(import_result['errors'])}", 'error') + flash(f"Import failed: {'; '.join(import_result['errors'][:3])}", 'error') + if len(import_result['errors']) > 3: + flash(f"...and {len(import_result['errors']) - 3} more errors", 'warning') return render_template('time_attendance_import.html', import_result=import_result) finally: # Clean up temporary file if os.path.exists(temp_path): - os.remove(temp_path) + try: + os.remove(temp_path) + except Exception as cleanup_error: + logger_handler.logger.warning(f"Failed to cleanup temp file: {cleanup_error}") except Exception as e: logger_handler.log_database_error('time_attendance_import', e) @@ -6620,6 +6651,209 @@ def import_time_attendance(): return render_template('time_attendance_import.html') +@app.route('/time-attendance/import/validate', methods=['POST']) +@admin_required +def validate_import_file(): + """AJAX endpoint to validate Excel file before import""" + try: + if 'file' not in request.files: + return jsonify({'success': False, 'message': 'No file provided'}), 400 + + file = request.files['file'] + if file.filename == '': + return jsonify({'success': False, 'message': 'No file selected'}), 400 + + # Validate file extension + if not file.filename.lower().endswith(('.xlsx', '.xls')): + return jsonify({'success': False, 'message': 'Invalid file format'}), 400 + + # Save temporarily + filename = secure_filename(file.filename) + temp_path = os.path.join(app.config.get('UPLOAD_FOLDER', '/tmp'), + f"validate_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{filename}") + + os.makedirs(os.path.dirname(temp_path), exist_ok=True) + file.save(temp_path) + + try: + # Validate file + import_service = TimeAttendanceImportService(db, logger_handler) + validation_result = import_service.validate_excel_file(temp_path) + + return jsonify({ + 'success': True, + 'validation': validation_result + }) + + finally: + # Cleanup + if os.path.exists(temp_path): + os.remove(temp_path) + + except Exception as e: + logger_handler.logger.error(f"Validation error: {e}") + return jsonify({ + 'success': False, + 'message': f'Validation failed: {str(e)}' + }), 500 + +@app.route('/time-attendance/import/batch/') +@login_required +@log_user_activity('view_import_batch') +def view_import_batch(batch_id): + """View details of a specific import batch""" + try: + import_service = TimeAttendanceImportService(db, logger_handler) + batch_summary = import_service.get_import_summary(batch_id) + + if not batch_summary: + flash('Import batch not found.', 'error') + return redirect(url_for('time_attendance_dashboard')) + + return render_template('time_attendance_batch_detail.html', + batch_summary=batch_summary) + + except Exception as e: + logger_handler.logger.error(f"Error viewing batch {batch_id}: {e}") + flash('Error loading batch details.', 'error') + return redirect(url_for('time_attendance_dashboard')) + + +@app.route('/time-attendance/import/batch//delete', methods=['POST']) +@admin_required +@log_database_operations('delete_import_batch') +def delete_import_batch(batch_id): + """Delete an entire import batch""" + try: + import_service = TimeAttendanceImportService(db, logger_handler) + result = import_service.delete_import_batch(batch_id, deleted_by=session['user_id']) + + if result['success']: + flash(result['message'], 'success') + logger_handler.logger.info( + f"User {session['username']} deleted import batch {batch_id} - " + f"{result['deleted_count']} records removed" + ) + else: + flash(result['message'], 'error') + + return redirect(url_for('time_attendance_dashboard')) + + except Exception as e: + logger_handler.logger.error(f"Error deleting batch {batch_id}: {e}") + flash('Error deleting import batch.', 'error') + return redirect(url_for('time_attendance_dashboard')) + + +@app.route('/time-attendance/import/download-template') +@login_required +def download_import_template(): + """Download Excel template for time attendance import""" + try: + import io + from openpyxl import Workbook + from openpyxl.styles import Font, PatternFill, Alignment + from flask import send_file + + # Create workbook + wb = Workbook() + ws = wb.active + ws.title = "Time Attendance Template" + + # Define headers + headers = ['ID', 'Name', 'Platform', 'Date', 'Time', 'Location Name', + 'Action Description', 'Event Description', 'Recorded Address'] + + # Style headers + header_fill = PatternFill(start_color="4472C4", end_color="4472C4", fill_type="solid") + header_font = Font(bold=True, color="FFFFFF") + + for col_num, header in enumerate(headers, 1): + cell = ws.cell(row=1, column=col_num) + cell.value = header + cell.fill = header_fill + cell.font = header_font + cell.alignment = Alignment(horizontal='center') + + # Add sample data rows + sample_data = [ + ['12345', 'John Doe', 'iPhone - iOS', '2025-10-06', '09:00:00', + 'HQ Suite 210', 'Check In', 'Morning Entry', '123 Main Street'], + ['12345', 'John Doe', 'iPhone - iOS', '2025-10-06', '17:30:00', + 'HQ Suite 210', 'Check Out', 'Evening Exit', '123 Main Street'], + ['67890', 'Jane Smith', 'Android', '2025-10-06', '08:45:00', + 'Branch Office', 'Check In', 'Morning Entry', '456 Oak Avenue'], + ] + + for row_num, row_data in enumerate(sample_data, 2): + for col_num, value in enumerate(row_data, 1): + ws.cell(row=row_num, column=col_num, value=value) + + # Adjust column widths + for col in ws.columns: + max_length = 0 + col_letter = col[0].column_letter + for cell in col: + try: + if len(str(cell.value)) > max_length: + max_length = len(str(cell.value)) + except: + pass + adjusted_width = min(max_length + 2, 50) + ws.column_dimensions[col_letter].width = adjusted_width + + # Add instructions sheet + ws_instructions = wb.create_sheet("Instructions") + instructions = [ + ["Time Attendance Import Template - Instructions"], + [""], + ["Required Columns:"], + ["- ID: Employee ID (required)"], + ["- Name: Employee full name (required)"], + ["- Date: Attendance date in YYYY-MM-DD format (required)"], + ["- Time: Attendance time in HH:MM:SS format (required)"], + ["- Location Name: Location where attendance was recorded (required)"], + ["- Action Description: Type of action (e.g., Check In, Check Out) (required)"], + [""], + ["Optional Columns:"], + ["- Platform: Device platform (e.g., iPhone - iOS, Android)"], + ["- Event Description: Additional event details"], + ["- Recorded Address: Physical address where attendance was recorded"], + [""], + ["Important Notes:"], + ["- Do not modify the header row"], + ["- Ensure all required fields have values"], + ["- Date format must be YYYY-MM-DD (e.g., 2025-10-06)"], + ["- Time format must be HH:MM:SS (e.g., 09:00:00)"], + ["- Remove the sample data rows before importing your actual data"], + ["- Duplicate records will be automatically detected and skipped"], + ] + + for row_num, instruction in enumerate(instructions, 1): + ws_instructions.cell(row=row_num, column=1, value=instruction[0]) + + ws_instructions.column_dimensions['A'].width = 80 + + # Save to bytes + output = io.BytesIO() + wb.save(output) + output.seek(0) + + # Log download + logger_handler.logger.info(f"User {session['username']} downloaded import template") + + return send_file( + output, + mimetype='application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + as_attachment=True, + download_name=f'time_attendance_template_{datetime.now().strftime("%Y%m%d")}.xlsx' + ) + + except Exception as e: + logger_handler.logger.error(f"Error generating template: {e}") + flash('Error generating template file.', 'error') + return redirect(url_for('import_time_attendance')) + @app.route('/time-attendance/records') @login_required @log_user_activity('time_attendance_records_view') diff --git a/static/css/time_attendance.css b/static/css/time_attendance.css index 4d870ee..f4aec01 100644 --- a/static/css/time_attendance.css +++ b/static/css/time_attendance.css @@ -554,6 +554,34 @@ body.has-sidebar .time-attendance-page { background: #fecaca; } +/* Enhanced action buttons */ +.action-btn.btn-info { + background: #4299e1; + color: white; +} + +.action-btn.btn-info:hover { + background: #3182ce; +} + +.action-btn.btn-delete { + background: #f56565; + color: white; +} + +.action-btn.btn-delete:hover { + background: #e53e3e; +} + +/* Progress indicator animation */ +@keyframes spin { + to { transform: rotate(360deg); } +} + +.spinner { + animation: spin 0.8s linear infinite; +} + /* Pagination */ .pagination-section { padding: 1.5rem; diff --git a/templates/time_attendance_batch_detail.html b/templates/time_attendance_batch_detail.html new file mode 100644 index 0000000..b94e952 --- /dev/null +++ b/templates/time_attendance_batch_detail.html @@ -0,0 +1,211 @@ +{% extends "base_authenticated.html" %} +{% block title %}Import Batch Details - {{ COMPANY_NAME }}{% endblock %} + +{% block extra_head %} + + +{% endblock %} + +{% block page_title %}Import Batch Details{% endblock %} + +{% block content %} +
+ +
+
+ + + Back to Dashboard + +
+ +
+

+ + Import Batch Details +

+

+ Batch ID: {{ batch_summary.batch_id }} +

+
+ +
+ + + View All Records + + {% if session.role == 'admin' %} + + {% endif %} +
+
+ + +
+
+
{{ batch_summary.total_records }}
+
Total Records
+
+ +
+
{{ batch_summary.unique_employees }}
+
Unique Employees
+
+ +
+
{{ batch_summary.unique_locations }}
+
Locations
+
+ +
+
{{ batch_summary.date_range.start.strftime('%Y-%m-%d') }}
+
Start Date
+
+ +
+
{{ batch_summary.date_range.end.strftime('%Y-%m-%d') }}
+
End Date
+
+
+ + +
+

+ + Import Information +

+

Import Date: {{ batch_summary.import_date.strftime('%Y-%m-%d %H:%M:%S') if batch_summary.import_date else 'Unknown' }}

+

Import Source: {{ batch_summary.import_source or 'Not specified' }}

+

Batch ID: {{ batch_summary.batch_id }}

+
+ + +
+

+ + Actions Breakdown +

+
+ {% for action, count in batch_summary.actions.items() %} +
+ {{ action }} + {{ count }} records +
+ {% endfor %} +
+
+ + +
+

+ + Employee Summary +

+
+ {% for emp_id, emp_data in batch_summary.employee_summary.items() %} +
+ {{ emp_data.name }} (ID: {{ emp_id }}) + {{ emp_data.count }} records +
+ {% endfor %} +
+
+
+ + +{% endblock %} \ No newline at end of file diff --git a/templates/time_attendance_dashboard.html b/templates/time_attendance_dashboard.html index b07629f..fffa677 100644 --- a/templates/time_attendance_dashboard.html +++ b/templates/time_attendance_dashboard.html @@ -257,9 +257,19 @@
+ class="action-btn btn-view" title="View Records"> + + + + {% if session.role == 'admin' %} + + {% endif %}
@@ -430,5 +440,25 @@ function deleteRecord(recordId, employeeName, date) { form.submit(); } } + +function deleteBatch(batchId) { + if (confirm('Are you sure you want to delete this entire import batch? This action cannot be undone.')) { + const form = document.createElement('form'); + form.method = 'POST'; + form.action = `/time-attendance/import/batch/${batchId}/delete`; + + const csrfToken = document.querySelector('meta[name="csrf-token"]'); + if (csrfToken) { + const input = document.createElement('input'); + input.type = 'hidden'; + input.name = 'csrf_token'; + input.value = csrfToken.content; + form.appendChild(input); + } + + document.body.appendChild(form); + form.submit(); + } +} {% endblock %} \ No newline at end of file diff --git a/templates/time_attendance_import.html b/templates/time_attendance_import.html index 1b1c3a6..2b677ca 100644 --- a/templates/time_attendance_import.html +++ b/templates/time_attendance_import.html @@ -3,6 +3,115 @@ {% block extra_head %} + {% endblock %} {% block page_title %}Import Time Attendance{% endblock %} @@ -24,9 +133,16 @@ Import Time Attendance Data

- Upload Excel files containing employee time attendance records + Upload Excel files containing employee time attendance records with enhanced validation

+ +
+ + + Download Template + +
@@ -59,8 +175,16 @@
-

Data Validation

-

Automatic validation and error reporting for invalid data

+

Smart Validation

+

Automatic duplicate detection and data validation

+ + +
+
+ +
+

Duplicate Handling

+

Automatically skip duplicate records during import

@@ -89,7 +213,7 @@ 12345 John Doe iPhone - iOS - 2025-09-12 + 2025-10-06 09:00:00 HQ Suite 210 Check In @@ -100,7 +224,7 @@ 67890 Jane Smith Android - 2025-09-12 + 2025-10-06 17:30:00 HQ Suite 210 Check Out @@ -147,6 +271,31 @@ +
+

+ + Import Options +

+
+
+ + +
+ +
+ + +
+
+
+ +
+ +
+ +

Processing your file, please wait...

+
+
- + {% if validation_result %}
@@ -197,29 +348,31 @@
-
-
+
+
{{ validation_result.total_rows }}
Total Rows
-
+
+
{{ validation_result.valid_rows }}
+
Valid Rows
+
+ {% if validation_result.invalid_rows > 0 %} +
+
{{ validation_result.invalid_rows }}
+
Invalid Rows
+
+ {% endif %} +
{{ validation_result.columns|length }}
Columns Found
- {% if validation_result.errors %} -
-
{{ validation_result.errors|length }}
-
Errors
-
- {% endif %} - {% if validation_result.warnings %} -
-
{{ validation_result.warnings|length }}
-
Warnings
-
- {% endif %}
+ {% if validation_result.file_info %} +

File Size: {{ validation_result.file_info.size_mb }} MB

+ {% endif %} + {% if validation_result.errors %}

@@ -252,7 +405,7 @@

- Sample Data Preview + Sample Data Preview (First 3 Rows)

@@ -280,7 +433,7 @@ {% endif %} - + {% if import_result %}
@@ -295,17 +448,23 @@
-
-
+
+
{{ import_result.total_records }}
Total Records
-
+
{{ import_result.imported_records }}
Successfully Imported
+ {% if import_result.duplicate_records > 0 %} +
+
{{ import_result.duplicate_records }}
+
Duplicates Skipped
+
+ {% endif %} {% if import_result.failed_records > 0 %} -
+
{{ import_result.failed_records }}
Failed Records
@@ -335,12 +494,32 @@

- Import Errors + Import Errors (Showing first 10)

    - {% for error in import_result.errors %} + {% for error in import_result.errors[:10] %}
  • {{ error }}
    • {% endfor %} + {% if import_result.errors|length > 10 %} +
  • ...and {{ import_result.errors|length - 10 }} more errors
    • + {% endif %} +
+
+ {% endif %} + + {% if import_result.warnings %} +
+

+ + Import Warnings (Showing first 10) +

+
    + {% for warning in import_result.warnings[:10] %} +
  • {{ warning }}
    • + {% endfor %} + {% if import_result.warnings|length > 10 %} +
  • ...and {{ import_result.warnings|length - 10 }} more warnings
    • + {% endif %}
{% endif %} @@ -349,344 +528,87 @@ {% endif %}
- - {% endblock %} \ No newline at end of file diff --git a/time_attendance_import_service.py b/time_attendance_import_service.py index cc554da..b673316 100644 --- a/time_attendance_import_service.py +++ b/time_attendance_import_service.py @@ -1,20 +1,21 @@ """ -Time Attendance Import Service -============================= +Enhanced Time Attendance Import Service +======================================== -Service to handle importing time attendance data from Excel files. -Provides functionality to parse Excel files and import data into the time_attendance table. +Improved service with duplicate detection, advanced validation, +and better error handling for Excel imports. """ import pandas as pd import uuid -from datetime import datetime +from datetime import datetime, time from sqlalchemy.exc import SQLAlchemyError from typing import Dict, List, Any, Optional, Tuple import traceback +import hashlib class TimeAttendanceImportService: - """Service to handle time attendance data import from Excel files""" + """Enhanced service to handle time attendance data import from Excel files""" def __init__(self, db, logger_handler=None): """Initialize the import service with database and logger""" @@ -22,14 +23,15 @@ class TimeAttendanceImportService: self.logger = logger_handler def import_from_excel(self, file_path: str, created_by: int = None, - import_source: str = None) -> Dict[str, Any]: + import_source: str = None, skip_duplicates: bool = True) -> Dict[str, Any]: """ - Import time attendance data from Excel file + Import time attendance data from Excel file with enhanced validation Args: file_path: Path to the Excel file created_by: User ID who initiated the import import_source: Description of import source + skip_duplicates: Whether to skip duplicate records Returns: Dictionary containing import results @@ -40,7 +42,10 @@ class TimeAttendanceImportService: 'total_records': 0, 'imported_records': 0, 'failed_records': 0, + 'duplicate_records': 0, + 'skipped_records': 0, 'errors': [], + 'warnings': [], 'success': False, 'import_date': datetime.utcnow() } @@ -48,10 +53,22 @@ class TimeAttendanceImportService: try: # Log import start if self.logger: - self.logger.logger.info(f"Starting time attendance import from {file_path} by user {created_by}") + self.logger.logger.info(f"Starting enhanced time attendance import from {file_path} by user {created_by}") - # Read Excel file - df = pd.read_excel(file_path, sheet_name=0) # Read first sheet + # Read Excel file with multiple sheet support + try: + excel_file = pd.ExcelFile(file_path) + sheet_name = excel_file.sheet_names[0] # Use first sheet + df = pd.read_excel(file_path, sheet_name=sheet_name) + + if self.logger: + self.logger.logger.info(f"Reading sheet: {sheet_name} with {len(df)} rows") + except Exception as e: + error_msg = f"Failed to read Excel file: {str(e)}" + import_results['errors'].append(error_msg) + if self.logger: + self.logger.logger.error(error_msg) + return import_results # Validate required columns required_columns = ['ID', 'Name', 'Date', 'Time', 'Location Name', 'Action Description'] @@ -61,38 +78,78 @@ class TimeAttendanceImportService: error_msg = f"Missing required columns: {', '.join(missing_columns)}" import_results['errors'].append(error_msg) if self.logger: - self.logger.logger.error(f"Import failed - {error_msg}") + self.logger.logger.error(error_msg) return import_results + # Remove completely empty rows + df = df.dropna(how='all') import_results['total_records'] = len(df) - # Process each row + if import_results['total_records'] == 0: + error_msg = "No valid data rows found in Excel file" + import_results['errors'].append(error_msg) + return import_results + + # Track duplicates using hash + duplicate_hashes = set() + if skip_duplicates: + duplicate_hashes = self._get_existing_record_hashes() + + # Process each row with enhanced validation for index, row in df.iterrows(): try: - # Parse date and time - attendance_date = pd.to_datetime(row['Date']).date() + # Skip empty rows + if pd.isna(row['ID']) or pd.isna(row['Name']): + import_results['skipped_records'] += 1 + import_results['warnings'].append(f"Row {index + 2}: Skipped due to missing ID or Name") + continue - # Handle time parsing - could be string or time object - time_str = str(row['Time']) - if ':' in time_str: - attendance_time = datetime.strptime(time_str, '%H:%M:%S').time() - else: - # Handle Excel time format - attendance_time = pd.to_datetime(row['Time']).time() + # Validate and parse date + try: + attendance_date = pd.to_datetime(row['Date']).date() + except Exception as date_error: + import_results['failed_records'] += 1 + import_results['errors'].append(f"Row {index + 2}: Invalid date format - {str(date_error)}") + continue + + # Validate and parse time with multiple format support + try: + attendance_time = self._parse_time_field(row['Time']) + except Exception as time_error: + import_results['failed_records'] += 1 + import_results['errors'].append(f"Row {index + 2}: Invalid time format - {str(time_error)}") + continue + + # Prepare record data + record_data = { + 'employee_id': str(row['ID']).strip(), + 'employee_name': str(row['Name']).strip(), + 'platform': str(row.get('Platform', '')).strip() if pd.notna(row.get('Platform')) else None, + 'attendance_date': attendance_date, + 'attendance_time': attendance_time, + 'location_name': str(row['Location Name']).strip(), + 'action_description': str(row['Action Description']).strip(), + 'event_description': str(row.get('Event Description', '')).strip() if pd.notna(row.get('Event Description')) else None, + 'recorded_address': str(row.get('Recorded Address', '')).strip() if pd.notna(row.get('Recorded Address')) else None, + } + + # Check for duplicates + if skip_duplicates: + record_hash = self._generate_record_hash(record_data) + if record_hash in duplicate_hashes: + import_results['duplicate_records'] += 1 + import_results['warnings'].append( + f"Row {index + 2}: Duplicate record for {record_data['employee_name']} " + f"on {attendance_date} at {attendance_time} - Skipped" + ) + continue + duplicate_hashes.add(record_hash) # Create TimeAttendance record from models.time_attendance import TimeAttendance time_attendance_record = TimeAttendance( - employee_id=str(row['ID']).strip(), - employee_name=str(row['Name']).strip(), - platform=str(row.get('Platform', '')).strip() if pd.notna(row.get('Platform')) else None, - attendance_date=attendance_date, - attendance_time=attendance_time, - location_name=str(row['Location Name']).strip(), - action_description=str(row['Action Description']).strip(), - event_description=str(row.get('Event Description', '')).strip() if pd.notna(row.get('Event Description')) else None, - recorded_address=str(row.get('Recorded Address', '')).strip() if pd.notna(row.get('Recorded Address')) else None, + **record_data, import_batch_id=batch_id, import_source=import_source or f"Excel Import - {file_path}", created_by=created_by @@ -101,6 +158,10 @@ class TimeAttendanceImportService: self.db.session.add(time_attendance_record) import_results['imported_records'] += 1 + # Commit in batches for better performance + if import_results['imported_records'] % 100 == 0: + self.db.session.flush() + except Exception as e: import_results['failed_records'] += 1 error_msg = f"Row {index + 2}: {str(e)}" @@ -111,9 +172,9 @@ class TimeAttendanceImportService: continue - # Commit all records + # Final commit self.db.session.commit() - import_results['success'] = True + import_results['success'] = import_results['imported_records'] > 0 # Log successful import if self.logger: @@ -121,7 +182,9 @@ class TimeAttendanceImportService: f"Time attendance import completed - Batch: {batch_id}, " f"Total: {import_results['total_records']}, " f"Imported: {import_results['imported_records']}, " - f"Failed: {import_results['failed_records']}" + f"Failed: {import_results['failed_records']}, " + f"Duplicates: {import_results['duplicate_records']}, " + f"Skipped: {import_results['skipped_records']}" ) except SQLAlchemyError as e: @@ -144,9 +207,99 @@ class TimeAttendanceImportService: return import_results + def _parse_time_field(self, time_value) -> time: + """ + Parse time field with multiple format support + + Args: + time_value: Time value from Excel (string, datetime, or time object) + + Returns: + time object + """ + if pd.isna(time_value): + raise ValueError("Time value is empty") + + # If already a time object + if isinstance(time_value, time): + return time_value + + # Convert to string and try parsing + time_str = str(time_value).strip() + + # Try common time formats + time_formats = [ + '%H:%M:%S', + '%H:%M', + '%I:%M:%S %p', + '%I:%M %p', + ] + + for fmt in time_formats: + try: + return datetime.strptime(time_str, fmt).time() + except ValueError: + continue + + # Try pandas datetime parsing as fallback + try: + return pd.to_datetime(time_value).time() + except: + pass + + raise ValueError(f"Unable to parse time value: {time_value}") + + def _generate_record_hash(self, record_data: Dict) -> str: + """ + Generate unique hash for a record to detect duplicates + + Args: + record_data: Dictionary containing record information + + Returns: + Hash string + """ + hash_string = ( + f"{record_data['employee_id']}_" + f"{record_data['attendance_date']}_" + f"{record_data['attendance_time']}_" + f"{record_data['location_name']}_" + f"{record_data['action_description']}" + ) + return hashlib.md5(hash_string.encode()).hexdigest() + + def _get_existing_record_hashes(self) -> set: + """ + Get hashes of existing records to detect duplicates + + Returns: + Set of record hashes + """ + try: + from models.time_attendance import TimeAttendance + + existing_records = TimeAttendance.query.all() + hashes = set() + + for record in existing_records: + record_data = { + 'employee_id': record.employee_id, + 'attendance_date': record.attendance_date, + 'attendance_time': record.attendance_time, + 'location_name': record.location_name, + 'action_description': record.action_description + } + hashes.add(self._generate_record_hash(record_data)) + + return hashes + except Exception as e: + if self.logger: + self.logger.logger.warning(f"Failed to get existing record hashes: {e}") + return set() + def validate_excel_file(self, file_path: str) -> Dict[str, Any]: """ - Validate Excel file structure before import + Enhanced Excel file validation with detailed analysis Args: file_path: Path to the Excel file @@ -157,19 +310,41 @@ class TimeAttendanceImportService: validation_results = { 'valid': False, 'total_rows': 0, + 'valid_rows': 0, + 'invalid_rows': 0, 'columns': [], 'sample_data': [], 'errors': [], - 'warnings': [] + 'warnings': [], + 'file_info': {} } try: + # Get file information + import os + file_stats = os.stat(file_path) + validation_results['file_info'] = { + 'size': file_stats.st_size, + 'size_mb': round(file_stats.st_size / (1024 * 1024), 2) + } + # Read Excel file - df = pd.read_excel(file_path, sheet_name=0) + excel_file = pd.ExcelFile(file_path) + sheet_name = excel_file.sheet_names[0] + df = pd.read_excel(file_path, sheet_name=sheet_name) + + # Remove empty rows + original_row_count = len(df) + df = df.dropna(how='all') validation_results['total_rows'] = len(df) validation_results['columns'] = df.columns.tolist() + if original_row_count > len(df): + validation_results['warnings'].append( + f"Removed {original_row_count - len(df)} completely empty rows" + ) + # Get sample data (first 5 rows) sample_rows = df.head(5).to_dict('records') validation_results['sample_data'] = sample_rows @@ -179,35 +354,86 @@ class TimeAttendanceImportService: missing_columns = [col for col in required_columns if col not in df.columns] if missing_columns: - validation_results['errors'].append(f"Missing required columns: {', '.join(missing_columns)}") + validation_results['errors'].append( + f"Missing required columns: {', '.join(missing_columns)}" + ) # Check for empty required fields - for col in required_columns: - if col in df.columns: - empty_count = df[col].isna().sum() - if empty_count > 0: - validation_results['warnings'].append( - f"Column '{col}' has {empty_count} empty values" - ) + valid_row_count = 0 + for index, row in df.iterrows(): + is_valid = True + for col in required_columns: + if col in df.columns and pd.isna(row[col]): + is_valid = False + break + + if is_valid: + valid_row_count += 1 + + validation_results['valid_rows'] = valid_row_count + validation_results['invalid_rows'] = len(df) - valid_row_count + + if validation_results['invalid_rows'] > 0: + validation_results['warnings'].append( + f"{validation_results['invalid_rows']} rows have missing required data" + ) # Validate date format if 'Date' in df.columns: - try: - pd.to_datetime(df['Date'], errors='coerce') - except: - validation_results['errors'].append("Invalid date format in 'Date' column") + invalid_dates = 0 + for idx, date_val in df['Date'].items(): + if pd.notna(date_val): + try: + pd.to_datetime(date_val) + except: + invalid_dates += 1 + + if invalid_dates > 0: + validation_results['warnings'].append( + f"{invalid_dates} rows have invalid date format" + ) + + # Validate time format + if 'Time' in df.columns: + invalid_times = 0 + for idx, time_val in df['Time'].items(): + if pd.notna(time_val): + try: + self._parse_time_field(time_val) + except: + invalid_times += 1 + + if invalid_times > 0: + validation_results['warnings'].append( + f"{invalid_times} rows have invalid time format" + ) + + # Check for potential duplicates + if all(col in df.columns for col in ['ID', 'Date', 'Time', 'Location Name']): + duplicate_check = df[['ID', 'Date', 'Time', 'Location Name']].duplicated() + duplicate_count = duplicate_check.sum() + + if duplicate_count > 0: + validation_results['warnings'].append( + f"{duplicate_count} potential duplicate records detected" + ) # Set valid flag - validation_results['valid'] = len(validation_results['errors']) == 0 + validation_results['valid'] = ( + len(validation_results['errors']) == 0 and + validation_results['valid_rows'] > 0 + ) except Exception as e: - validation_results['errors'].append(f"Failed to read Excel file: {str(e)}") + validation_results['errors'].append(f"Failed to validate Excel file: {str(e)}") + if self.logger: + self.logger.logger.error(f"Validation error: {e}") return validation_results def get_import_summary(self, batch_id: str) -> Optional[Dict[str, Any]]: """ - Get summary of imported data by batch ID + Get detailed summary of imported data by batch ID Args: batch_id: Import batch identifier @@ -238,6 +464,17 @@ class TimeAttendanceImportService: action = record.action_description actions[action] = actions.get(action, 0) + 1 + # Group by employee + employee_summary = {} + for record in records: + emp_id = record.employee_id + if emp_id not in employee_summary: + employee_summary[emp_id] = { + 'name': record.employee_name, + 'count': 0 + } + employee_summary[emp_id]['count'] += 1 + return { 'batch_id': batch_id, 'total_records': total_records, @@ -245,6 +482,7 @@ class TimeAttendanceImportService: 'unique_locations': unique_locations, 'date_range': date_range, 'actions': actions, + 'employee_summary': employee_summary, 'import_date': records[0].import_date if records else None, 'import_source': records[0].import_source if records else None } @@ -252,4 +490,58 @@ class TimeAttendanceImportService: except Exception as e: if self.logger: self.logger.logger.error(f"Failed to get import summary for batch {batch_id}: {e}") - return None \ No newline at end of file + return None + + def delete_import_batch(self, batch_id: str, deleted_by: int = None) -> Dict[str, Any]: + """ + Delete all records from a specific import batch + + Args: + batch_id: Import batch identifier + deleted_by: User ID who initiated the deletion + + Returns: + Dictionary containing deletion results + """ + result = { + 'success': False, + 'deleted_count': 0, + 'message': '' + } + + try: + from models.time_attendance import TimeAttendance + + records = TimeAttendance.query.filter_by(import_batch_id=batch_id).all() + deleted_count = len(records) + + if deleted_count == 0: + result['message'] = 'No records found for this batch' + return result + + # Delete records + for record in records: + self.db.session.delete(record) + + self.db.session.commit() + + # Log deletion + if self.logger: + self.logger.logger.info( + f"User {deleted_by} deleted import batch {batch_id} - " + f"Removed {deleted_count} records" + ) + + result['success'] = True + result['deleted_count'] = deleted_count + result['message'] = f'Successfully deleted {deleted_count} records' + + except Exception as e: + self.db.session.rollback() + result['message'] = f'Error deleting batch: {str(e)}' + + if self.logger: + self.logger.logger.error(f"Failed to delete batch {batch_id}: {e}") + + return result + \ No newline at end of file