Updated import duplication review

app.py

@@ -6544,7 +6544,7 @@ def time_attendance_dashboard():
 @admin_required
 @log_database_operations('time_attendance_import')
 def import_time_attendance():
-    """Enhanced import time attendance data from Excel file"""
+    """Enhanced import with duplicate review"""
     if request.method == 'POST':
         try:
             # Check if file is uploaded
@@ -6567,19 +6567,37 @@ def import_time_attendance():
             temp_path = os.path.join(app.config.get('UPLOAD_FOLDER', '/tmp'), 
                                    f"temp_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{filename}")
             
-            # Ensure upload directory exists
             os.makedirs(os.path.dirname(temp_path), exist_ok=True)
             file.save(temp_path)
             
+            # Store file path in session for duplicate review
+            session['pending_import_file'] = temp_path
+            session['pending_import_filename'] = filename
+            
             try:
-                # Initialize enhanced import service
                 import_service = TimeAttendanceImportService(db, logger_handler)
                 
                 # Get import options
                 skip_duplicates = request.form.get('skip_duplicates', 'true').lower() == 'true'
                 validate_only = request.form.get('validate_only', 'false').lower() == 'true'
+                analyze_duplicates = request.form.get('analyze_duplicates', 'false').lower() == 'true'
                 
-                # Validate file first
+                # Check if this is coming from duplicate review
+                force_import_hashes = request.form.getlist('force_import_hashes[]')
+                
+                # If analyzing for duplicates, show review page
+                if analyze_duplicates and not force_import_hashes:
+                    duplicate_analysis = import_service.analyze_for_duplicates(temp_path)
+                    
+                    if duplicate_analysis['duplicate_records'] > 0:
+                        # Show duplicate review page
+                        return render_template('time_attendance_duplicate_review.html',
+                                             analysis=duplicate_analysis,
+                                             filename=filename)
+                    else:
+                        flash('No duplicates found. Proceeding with import.', 'info')
+                
+                # Validate file
                 validation_result = import_service.validate_excel_file(temp_path)
                 
                 if not validation_result['valid']:
@@ -6587,12 +6605,10 @@ def import_time_attendance():
                     return render_template('time_attendance_import.html', 
                                          validation_result=validation_result)
                 
-                # Show warnings if any
                 if validation_result['warnings']:
                     for warning in validation_result['warnings']:
                         flash(warning, 'warning')
                 
-                # If validate only, return validation results
                 if validate_only:
                     flash(f"File validation successful! Found {validation_result['valid_rows']} valid records.", 'success')
                     return render_template('time_attendance_import.html', 
@@ -6604,16 +6620,17 @@ def import_time_attendance():
                     temp_path,
                     created_by=session['user_id'],
                     import_source=import_source,
-                    skip_duplicates=skip_duplicates
+                    skip_duplicates=skip_duplicates,
+                    force_import_hashes=force_import_hashes
                 )
                 
                 if import_result['success']:
-                    # Log successful import
                     logger_handler.logger.info(
                         f"User {session['username']} successfully imported time attendance data - "
                         f"Batch: {import_result['batch_id']}, "
                         f"Records: {import_result['imported_records']}/{import_result['total_records']}, "
                         f"Duplicates: {import_result['duplicate_records']}, "
+                        f"Forced: {import_result['forced_duplicates']}, "
                         f"Failed: {import_result['failed_records']}"
                     )
                     
@@ -6623,6 +6640,9 @@ def import_time_attendance():
                     if import_result['duplicate_records'] > 0:
                         flash(f"Skipped {import_result['duplicate_records']} duplicate records.", 'info')
                     
+                    if import_result['forced_duplicates'] > 0:
+                        flash(f"Imported {import_result['forced_duplicates']} duplicate records as requested.", 'info')
+                    
                     if import_result['failed_records'] > 0:
                         flash(f"Note: {import_result['failed_records']} records failed to import. "
                               f"Check the error details below.", 'warning')
@@ -6637,12 +6657,17 @@ def import_time_attendance():
                                          import_result=import_result)
                 
             finally:
-                # Clean up temporary file
+                # Clean up if import completed or failed (not if showing duplicate review)
-                if os.path.exists(temp_path):
+                if not analyze_duplicates or force_import_hashes:
-                    try:
+                    if os.path.exists(temp_path):
-                        os.remove(temp_path)
+                        try:
-                    except Exception as cleanup_error:
+                            os.remove(temp_path)
-                        logger_handler.logger.warning(f"Failed to cleanup temp file: {cleanup_error}")
+                            if 'pending_import_file' in session:
+                                session.pop('pending_import_file')
+                            if 'pending_import_filename' in session:
+                                session.pop('pending_import_filename')
+                        except Exception as cleanup_error:
+                            logger_handler.logger.warning(f"Failed to cleanup temp file: {cleanup_error}")
             
         except Exception as e:
             logger_handler.log_database_error('time_attendance_import', e)
@@ -6651,6 +6676,95 @@ def import_time_attendance():
     
     return render_template('time_attendance_import.html')
 
+
+@app.route('/time-attendance/import/analyze-duplicates', methods=['POST'])
+@admin_required
+def analyze_import_duplicates():
+    """AJAX endpoint to analyze file for duplicates"""
+    try:
+        if 'file' not in request.files:
+            return jsonify({'success': False, 'message': 'No file provided'}), 400
+        
+        file = request.files['file']
+        if file.filename == '':
+            return jsonify({'success': False, 'message': 'No file selected'}), 400
+        
+        if not file.filename.lower().endswith(('.xlsx', '.xls')):
+            return jsonify({'success': False, 'message': 'Invalid file format'}), 400
+        
+        # Save temporarily
+        filename = secure_filename(file.filename)
+        temp_path = os.path.join(app.config.get('UPLOAD_FOLDER', '/tmp'), 
+                               f"analyze_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{filename}")
+        
+        os.makedirs(os.path.dirname(temp_path), exist_ok=True)
+        file.save(temp_path)
+        
+        # Store in session
+        session['pending_import_file'] = temp_path
+        session['pending_import_filename'] = filename
+        
+        try:
+            import_service = TimeAttendanceImportService(db, logger_handler)
+            analysis = import_service.analyze_for_duplicates(temp_path)
+            
+            # Convert datetime objects to strings for JSON
+            for duplicate in analysis.get('duplicates', []):
+                if 'new_record' in duplicate:
+                    if duplicate['new_record'].get('attendance_date'):
+                        duplicate['new_record']['attendance_date'] = str(duplicate['new_record']['attendance_date'])
+                    if duplicate['new_record'].get('attendance_time'):
+                        duplicate['new_record']['attendance_time'] = str(duplicate['new_record']['attendance_time'])
+                
+                if 'existing_record' in duplicate:
+                    if duplicate['existing_record'].get('attendance_date'):
+                        duplicate['existing_record']['attendance_date'] = str(duplicate['existing_record']['attendance_date'])
+                    if duplicate['existing_record'].get('attendance_time'):
+                        duplicate['existing_record']['attendance_time'] = str(duplicate['existing_record']['attendance_time'])
+                    if duplicate['existing_record'].get('import_date'):
+                        duplicate['existing_record']['import_date'] = str(duplicate['existing_record']['import_date'])
+            
+            return jsonify({
+                'success': True,
+                'analysis': analysis
+            })
+        
+        except Exception as e:
+            # Cleanup on error
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+            raise e
+    
+    except Exception as e:
+        logger_handler.logger.error(f"Duplicate analysis error: {e}")
+        return jsonify({
+            'success': False,
+            'message': f'Analysis failed: {str(e)}'
+        }), 500
+
+
+@app.route('/time-attendance/import/cancel-pending')
+@admin_required
+def cancel_pending_import():
+    """Cancel pending import and cleanup temp file"""
+    try:
+        if 'pending_import_file' in session:
+            temp_path = session['pending_import_file']
+            if os.path.exists(temp_path):
+                os.remove(temp_path)
+            session.pop('pending_import_file')
+        
+        if 'pending_import_filename' in session:
+            session.pop('pending_import_filename')
+        
+        flash('Import cancelled.', 'info')
+    except Exception as e:
+        logger_handler.logger.error(f"Error cancelling import: {e}")
+    
+    return redirect(url_for('import_time_attendance'))
+
+
+
 @app.route('/time-attendance/import/validate', methods=['POST'])
 @admin_required
 def validate_import_file():

templates/time_attendance_duplicate_review.html

@@ -0,0 +1,641 @@
+{% extends "base_authenticated.html" %}
+{% block title %}Review Duplicate Records - {{ COMPANY_NAME }}{% endblock %}
+
+{% block extra_head %}
+<link rel="stylesheet" href="{{ url_for('static', filename='css/time_attendance.css') }}">
+<style>
+.duplicate-review-page {
+  max-width: 1400px;
+  margin: 0 auto;
+  padding: 2rem;
+}
+
+.review-summary {
+  background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
+  color: white;
+  padding: 2rem;
+  border-radius: 12px;
+  margin-bottom: 2rem;
+  box-shadow: 0 8px 24px rgba(0,0,0,0.15);
+}
+
+.summary-stats {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
+  gap: 1.5rem;
+  margin-top: 1.5rem;
+}
+
+.stat-box {
+  background: rgba(255,255,255,0.2);
+  padding: 1.5rem;
+  border-radius: 8px;
+  text-align: center;
+  backdrop-filter: blur(10px);
+}
+
+.stat-box .number {
+  font-size: 2.5rem;
+  font-weight: 700;
+  margin-bottom: 0.5rem;
+}
+
+.stat-box .label {
+  font-size: 0.875rem;
+  opacity: 0.9;
+}
+
+.duplicates-container {
+  display: flex;
+  flex-direction: column;
+  gap: 1.5rem;
+}
+
+.duplicate-item {
+  background: white;
+  border-radius: 12px;
+  box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+  overflow: hidden;
+  border: 2px solid #e2e8f0;
+  transition: all 0.3s ease;
+}
+
+.duplicate-item.selected {
+  border-color: #667eea;
+  box-shadow: 0 4px 12px rgba(102, 126, 234, 0.2);
+}
+
+.duplicate-header {
+  background: #f7fafc;
+  padding: 1rem 1.5rem;
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  border-bottom: 1px solid #e2e8f0;
+}
+
+.duplicate-title {
+  display: flex;
+  align-items: center;
+  gap: 1rem;
+}
+
+.duplicate-title h3 {
+  margin: 0;
+  color: #2d3748;
+  font-size: 1.125rem;
+}
+
+.row-badge {
+  background: #ed8936;
+  color: white;
+  padding: 0.25rem 0.75rem;
+  border-radius: 20px;
+  font-size: 0.75rem;
+  font-weight: 600;
+}
+
+.duplicate-actions {
+  display: flex;
+  gap: 0.5rem;
+}
+
+.duplicate-body {
+  padding: 1.5rem;
+}
+
+.comparison-grid {
+  display: grid;
+  grid-template-columns: 1fr 1fr;
+  gap: 2rem;
+}
+
+.record-card {
+  background: #f8fafc;
+  padding: 1.5rem;
+  border-radius: 8px;
+  border: 2px solid #e2e8f0;
+}
+
+.record-card h4 {
+  margin: 0 0 1rem 0;
+  color: #2d3748;
+  font-size: 1rem;
+  display: flex;
+  align-items: center;
+  gap: 0.5rem;
+}
+
+.record-card.new-record h4 {
+  color: #3182ce;
+}
+
+.record-card.existing-record h4 {
+  color: #805ad5;
+}
+
+.record-details {
+  display: flex;
+  flex-direction: column;
+  gap: 0.75rem;
+}
+
+.detail-row {
+  display: grid;
+  grid-template-columns: 140px 1fr;
+  gap: 0.5rem;
+  font-size: 0.875rem;
+}
+
+.detail-label {
+  color: #718096;
+  font-weight: 600;
+}
+
+.detail-value {
+  color: #2d3748;
+  word-break: break-word;
+}
+
+.detail-value.highlight {
+  background: #fef5e7;
+  padding: 0.25rem 0.5rem;
+  border-radius: 4px;
+  font-weight: 600;
+}
+
+.checkbox-container {
+  display: flex;
+  align-items: center;
+  gap: 0.75rem;
+  padding: 1rem;
+  background: #ebf8ff;
+  border-radius: 8px;
+  margin-top: 1rem;
+}
+
+.checkbox-container input[type="checkbox"] {
+  width: 20px;
+  height: 20px;
+  cursor: pointer;
+}
+
+.checkbox-container label {
+  font-weight: 600;
+  color: #2c5282;
+  cursor: pointer;
+  margin: 0;
+}
+
+.action-buttons {
+  display: flex;
+  justify-content: space-between;
+  gap: 1rem;
+  margin-top: 2rem;
+  padding: 1.5rem;
+  background: white;
+  border-radius: 12px;
+  box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+  position: sticky;
+  bottom: 1rem;
+  z-index: 100;
+}
+
+.action-buttons .left-actions,
+.action-buttons .right-actions {
+  display: flex;
+  gap: 1rem;
+}
+
+.btn {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.5rem;
+  padding: 0.75rem 1.5rem;
+  border: none;
+  border-radius: 8px;
+  font-weight: 600;
+  cursor: pointer;
+  transition: all 0.3s ease;
+  text-decoration: none;
+  font-size: 1rem;
+}
+
+.btn-primary {
+  background: linear-gradient(135deg, #667eea, #764ba2);
+  color: white;
+}
+
+.btn-primary:hover {
+  transform: translateY(-2px);
+  box-shadow: 0 4px 12px rgba(102, 126, 234, 0.4);
+}
+
+.btn-secondary {
+  background: #e2e8f0;
+  color: #2d3748;
+}
+
+.btn-secondary:hover {
+  background: #cbd5e0;
+}
+
+.btn-danger {
+  background: #fc8181;
+  color: white;
+}
+
+.btn-danger:hover {
+  background: #f56565;
+}
+
+.selection-summary {
+  background: #f0fff4;
+  border: 2px solid #9ae6b4;
+  padding: 1rem;
+  border-radius: 8px;
+  margin-bottom: 1rem;
+  display: none;
+}
+
+.selection-summary.visible {
+  display: block;
+}
+
+.selection-summary .count {
+  font-weight: 700;
+  color: #22543d;
+  font-size: 1.125rem;
+}
+
+.empty-state {
+  text-align: center;
+  padding: 4rem 2rem;
+  background: white;
+  border-radius: 12px;
+  box-shadow: 0 2px 8px rgba(0,0,0,0.1);
+}
+
+.empty-state i {
+  font-size: 4rem;
+  color: #48bb78;
+  margin-bottom: 1rem;
+}
+
+.empty-state h3 {
+  color: #2d3748;
+  margin-bottom: 0.5rem;
+}
+
+.empty-state p {
+  color: #718096;
+}
+
+@media (max-width: 768px) {
+  .comparison-grid {
+    grid-template-columns: 1fr;
+  }
+  
+  .action-buttons {
+    flex-direction: column;
+  }
+  
+  .action-buttons .left-actions,
+  .action-buttons .right-actions {
+    width: 100%;
+  }
+}
+</style>
+{% endblock %}
+
+{% block page_title %}Review Duplicate Records{% endblock %}
+
+{% block content %}
+<div class="duplicate-review-page">
+  <!-- Page Header -->
+  <div class="time-attendance-header">
+    <div class="header-navigation">
+      <a href="{{ url_for('import_time_attendance') }}" class="back-button">
+        <i class="fas fa-arrow-left"></i>
+        Back to Import
+      </a>
+    </div>
+    
+    <div class="header-content">
+      <h1>
+        <i class="fas fa-exclamation-triangle"></i>
+        Review Duplicate Records
+      </h1>
+      <p class="header-description">
+        The following records already exist in the system. Please review and select which ones you want to import anyway.
+      </p>
+    </div>
+  </div>
+
+  <!-- Summary Section -->
+  <div class="review-summary">
+    <h2 style="margin: 0 0 0.5rem 0;">
+      <i class="fas fa-file-excel"></i>
+      File: {{ filename }}
+    </h2>
+    <p style="margin: 0; opacity: 0.9;">Review duplicate records before importing</p>
+    
+    <div class="summary-stats">
+      <div class="stat-box">
+        <div class="number">{{ analysis.total_records }}</div>
+        <div class="label">Total Records in File</div>
+      </div>
+      
+      <div class="stat-box">
+        <div class="number">{{ analysis.new_records }}</div>
+        <div class="label">New Records</div>
+      </div>
+      
+      <div class="stat-box">
+        <div class="number">{{ analysis.duplicate_records }}</div>
+        <div class="label">Duplicate Records</div>
+      </div>
+    </div>
+  </div>
+
+  <!-- Selection Summary -->
+  <div id="selectionSummary" class="selection-summary">
+    <i class="fas fa-check-circle"></i>
+    <span class="count">0</span> duplicate record(s) selected to import
+  </div>
+
+  <!-- Duplicates List -->
+  {% if analysis.duplicate_records > 0 %}
+  <form id="duplicateReviewForm" method="POST" action="{{ url_for('import_time_attendance') }}">
+    <input type="hidden" name="analyze_duplicates" value="false">
+    <input type="hidden" name="skip_duplicates" value="true">
+    <input type="hidden" name="import_source" value="Import with Duplicates - {{ filename }}">
+    
+    <div class="duplicates-container">
+      {% for duplicate in analysis.duplicates %}
+      <div class="duplicate-item" data-hash="{{ duplicate.hash }}">
+        <div class="duplicate-header">
+          <div class="duplicate-title">
+            <span class="row-badge">Row {{ duplicate.row_number }}</span>
+            <h3>{{ duplicate.new_record.employee_name }} (ID: {{ duplicate.new_record.employee_id }})</h3>
+          </div>
+        </div>
+        
+        <div class="duplicate-body">
+          <div class="comparison-grid">
+            <!-- New Record (from file) -->
+            <div class="record-card new-record">
+              <h4>
+                <i class="fas fa-file-import"></i>
+                New Record (From File)
+              </h4>
+              <div class="record-details">
+                <div class="detail-row">
+                  <span class="detail-label">Employee ID:</span>
+                  <span class="detail-value highlight">{{ duplicate.new_record.employee_id }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Name:</span>
+                  <span class="detail-value">{{ duplicate.new_record.employee_name }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Date:</span>
+                  <span class="detail-value highlight">{{ duplicate.new_record.attendance_date }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Time:</span>
+                  <span class="detail-value highlight">{{ duplicate.new_record.attendance_time }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Location:</span>
+                  <span class="detail-value highlight">{{ duplicate.new_record.location_name }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Action:</span>
+                  <span class="detail-value highlight">{{ duplicate.new_record.action_description }}</span>
+                </div>
+                {% if duplicate.new_record.platform %}
+                <div class="detail-row">
+                  <span class="detail-label">Platform:</span>
+                  <span class="detail-value">{{ duplicate.new_record.platform }}</span>
+                </div>
+                {% endif %}
+                {% if duplicate.new_record.event_description %}
+                <div class="detail-row">
+                  <span class="detail-label">Event:</span>
+                  <span class="detail-value">{{ duplicate.new_record.event_description }}</span>
+                </div>
+                {% endif %}
+                {% if duplicate.new_record.recorded_address %}
+                <div class="detail-row">
+                  <span class="detail-label">Address:</span>
+                  <span class="detail-value">{{ duplicate.new_record.recorded_address }}</span>
+                </div>
+                {% endif %}
+              </div>
+            </div>
+            
+            <!-- Existing Record (in database) -->
+            <div class="record-card existing-record">
+              <h4>
+                <i class="fas fa-database"></i>
+                Existing Record (In System)
+              </h4>
+              <div class="record-details">
+                <div class="detail-row">
+                  <span class="detail-label">Employee ID:</span>
+                  <span class="detail-value highlight">{{ duplicate.existing_record.employee_id }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Name:</span>
+                  <span class="detail-value">{{ duplicate.existing_record.employee_name }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Date:</span>
+                  <span class="detail-value highlight">{{ duplicate.existing_record.attendance_date }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Time:</span>
+                  <span class="detail-value highlight">{{ duplicate.existing_record.attendance_time }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Location:</span>
+                  <span class="detail-value highlight">{{ duplicate.existing_record.location_name }}</span>
+                </div>
+                <div class="detail-row">
+                  <span class="detail-label">Action:</span>
+                  <span class="detail-value highlight">{{ duplicate.existing_record.action_description }}</span>
+                </div>
+                {% if duplicate.existing_record.platform %}
+                <div class="detail-row">
+                  <span class="detail-label">Platform:</span>
+                  <span class="detail-value">{{ duplicate.existing_record.platform }}</span>
+                </div>
+                {% endif %}
+                {% if duplicate.existing_record.event_description %}
+                <div class="detail-row">
+                  <span class="detail-label">Event:</span>
+                  <span class="detail-value">{{ duplicate.existing_record.event_description }}</span>
+                </div>
+                {% endif %}
+                {% if duplicate.existing_record.recorded_address %}
+                <div class="detail-row">
+                  <span class="detail-label">Address:</span>
+                  <span class="detail-value">{{ duplicate.existing_record.recorded_address }}</span>
+                </div>
+                {% endif %}
+                <div class="detail-row">
+                  <span class="detail-label">Import Date:</span>
+                  <span class="detail-value">
+                    <i class="fas fa-clock"></i>
+                    {{ duplicate.existing_record.import_date.strftime('%Y-%m-%d %H:%M') if duplicate.existing_record.import_date else 'Unknown' }}
+                  </span>
+                </div>
+                {% if duplicate.existing_record.import_source %}
+                <div class="detail-row">
+                  <span class="detail-label">Import Source:</span>
+                  <span class="detail-value">{{ duplicate.existing_record.import_source }}</span>
+                </div>
+                {% endif %}
+              </div>
+            </div>
+          </div>
+          
+          <!-- Selection Checkbox -->
+          <div class="checkbox-container">
+            <input type="checkbox" 
+                   id="duplicate_{{ loop.index }}" 
+                   name="force_import_hashes[]" 
+                   value="{{ duplicate.hash }}"
+                   class="duplicate-checkbox">
+            <label for="duplicate_{{ loop.index }}">
+              Import this duplicate record anyway
+            </label>
+          </div>
+        </div>
+      </div>
+      {% endfor %}
+    </div>
+
+    <!-- Action Buttons -->
+    <div class="action-buttons">
+      <div class="left-actions">
+        <a href="{{ url_for('cancel_pending_import') }}" class="btn btn-secondary">
+          <i class="fas fa-times"></i>
+          Cancel Import
+        </a>
+      </div>
+      
+      <div class="right-actions">
+        <button type="button" id="selectAllBtn" class="btn btn-secondary">
+          <i class="fas fa-check-square"></i>
+          Select All
+        </button>
+        <button type="button" id="skipAllBtn" class="btn btn-secondary">
+          <i class="fas fa-forward"></i>
+          Skip All Duplicates
+        </button>
+        <button type="submit" class="btn btn-primary" id="proceedBtn">
+          <i class="fas fa-upload"></i>
+          Proceed with Import
+        </button>
+      </div>
+    </div>
+  </form>
+  {% else %}
+  <div class="empty-state">
+    <i class="fas fa-check-circle"></i>
+    <h3>No Duplicates Found</h3>
+    <p>All records in the file are unique. You can proceed with the import.</p>
+    <a href="{{ url_for('import_time_attendance') }}" class="btn btn-primary" style="margin-top: 1rem;">
+      <i class="fas fa-arrow-left"></i>
+      Back to Import
+    </a>
+  </div>
+  {% endif %}
+</div>
+
+<script>
+// Track selected duplicates
+const checkboxes = document.querySelectorAll('.duplicate-checkbox');
+const selectionSummary = document.getElementById('selectionSummary');
+const selectAllBtn = document.getElementById('selectAllBtn');
+const skipAllBtn = document.getElementById('skipAllBtn');
+const proceedBtn = document.getElementById('proceedBtn');
+
+function updateSelectionSummary() {
+  const selectedCount = document.querySelectorAll('.duplicate-checkbox:checked').length;
+  const countSpan = selectionSummary.querySelector('.count');
+  countSpan.textContent = selectedCount;
+  
+  if (selectedCount > 0) {
+    selectionSummary.classList.add('visible');
+  } else {
+    selectionSummary.classList.remove('visible');
+  }
+  
+  // Update duplicate items visual state
+  document.querySelectorAll('.duplicate-item').forEach(item => {
+    const checkbox = item.querySelector('.duplicate-checkbox');
+    if (checkbox && checkbox.checked) {
+      item.classList.add('selected');
+    } else {
+      item.classList.remove('selected');
+    }
+  });
+}
+
+// Listen to checkbox changes
+checkboxes.forEach(checkbox => {
+  checkbox.addEventListener('change', updateSelectionSummary);
+});
+
+// Select all button
+if (selectAllBtn) {
+  selectAllBtn.addEventListener('click', () => {
+    checkboxes.forEach(cb => cb.checked = true);
+    updateSelectionSummary();
+  });
+}
+
+// Skip all button (uncheck all and submit)
+if (skipAllBtn) {
+  skipAllBtn.addEventListener('click', () => {
+    if (confirm('This will skip all duplicate records and import only new records. Continue?')) {
+      checkboxes.forEach(cb => cb.checked = false);
+      document.getElementById('duplicateReviewForm').submit();
+    }
+  });
+}
+
+// Form submission confirmation
+if (proceedBtn) {
+  proceedBtn.addEventListener('click', (e) => {
+    const selectedCount = document.querySelectorAll('.duplicate-checkbox:checked').length;
+    const totalDuplicates = checkboxes.length;
+    const newRecordsCount = {{ analysis.new_records }};
+    
+    let message = '';
+    if (selectedCount > 0) {
+      message = `You are about to import:\n\n`;
+      message += `- ${newRecordsCount} new record(s)\n`;
+      message += `- ${selectedCount} duplicate record(s)\n`;
+      message += `- Skipping ${totalDuplicates - selectedCount} duplicate(s)\n\n`;
+      message += `Total: ${newRecordsCount + selectedCount} records will be imported.\n\n`;
+      message += `Continue with import?`;
+    } else {
+      message = `You are about to import ${newRecordsCount} new record(s) only.\n\n`;
+      message += `All ${totalDuplicates} duplicate record(s) will be skipped.\n\n`;
+      message += `Continue with import?`;
+    }
+    
+    if (!confirm(message)) {
+      e.preventDefault();
+    }
+  });
+}
+
+// Initial update
+updateSelectionSummary();
+</script>
+{% endblock %}

templates/time_attendance_import.html

@@ -284,6 +284,14 @@
                 <span>Automatically detect and skip records that already exist in the system</span>
               </label>
             </div>
+
+            <div class="checkbox-item">
+              <input type="checkbox" id="analyze_duplicates" name="analyze_duplicates" value="true">
+              <label for="analyze_duplicates" class="checkbox-label">
+                <strong>Review Duplicates Before Import</strong>
+                <span>Show potential duplicate records for review and allow selective import</span>
+              </label>
+            </div>
             
             <div class="checkbox-item">
               <input type="checkbox" id="validate_only" name="validate_only" value="true">

time_attendance_import_service.py

@@ -1,9 +1,8 @@
 """
-Enhanced Time Attendance Import Service
+Enhanced Time Attendance Import Service with Duplicate Review
-========================================
+============================================================
 
-Improved service with duplicate detection, advanced validation, 
+Added functionality to detect and present duplicates for user review.
-and better error handling for Excel imports.
 """
 
 import pandas as pd
@@ -15,23 +14,130 @@ import traceback
 import hashlib
 
 class TimeAttendanceImportService:
-    """Enhanced service to handle time attendance data import from Excel files"""
+    """Enhanced service with duplicate detection and review"""
     
     def __init__(self, db, logger_handler=None):
         """Initialize the import service with database and logger"""
         self.db = db
         self.logger = logger_handler
-        
+    
-    def import_from_excel(self, file_path: str, created_by: int = None, 
+    def analyze_for_duplicates(self, file_path: str) -> Dict[str, Any]:
-                         import_source: str = None, skip_duplicates: bool = True) -> Dict[str, Any]:
         """
-        Import time attendance data from Excel file with enhanced validation
+        Analyze file for potential duplicates WITHOUT importing
+        
+        Args:
+            file_path: Path to the Excel file
+            
+        Returns:
+            Dictionary containing duplicate analysis
+        """
+        analysis_result = {
+            'success': False,
+            'total_records': 0,
+            'new_records': 0,
+            'duplicate_records': 0,
+            'duplicates': [],
+            'errors': []
+        }
+        
+        try:
+            # Read Excel file
+            excel_file = pd.ExcelFile(file_path)
+            sheet_name = excel_file.sheet_names[0]
+            df = pd.read_excel(file_path, sheet_name=sheet_name)
+            
+            # Validate required columns
+            required_columns = ['ID', 'Name', 'Date', 'Time', 'Location Name', 'Action Description']
+            missing_columns = [col for col in required_columns if col not in df.columns]
+            
+            if missing_columns:
+                analysis_result['errors'].append(f"Missing columns: {', '.join(missing_columns)}")
+                return analysis_result
+            
+            # Remove empty rows
+            df = df.dropna(how='all')
+            analysis_result['total_records'] = len(df)
+            
+            # Get existing record hashes
+            existing_hashes = self._get_existing_record_hashes_with_data()
+            
+            # Process each row
+            duplicates_list = []
+            new_records_count = 0
+            
+            for index, row in df.iterrows():
+                try:
+                    # Skip empty rows
+                    if pd.isna(row['ID']) or pd.isna(row['Name']):
+                        continue
+                    
+                    # Parse date and time
+                    attendance_date = pd.to_datetime(row['Date']).date()
+                    attendance_time = self._parse_time_field(row['Time'])
+                    
+                    # Prepare record data
+                    record_data = {
+                        'employee_id': str(row['ID']).strip(),
+                        'employee_name': str(row['Name']).strip(),
+                        'platform': str(row.get('Platform', '')).strip() if pd.notna(row.get('Platform')) else None,
+                        'attendance_date': attendance_date,
+                        'attendance_time': attendance_time,
+                        'location_name': str(row['Location Name']).strip(),
+                        'action_description': str(row['Action Description']).strip(),
+                        'event_description': str(row.get('Event Description', '')).strip() if pd.notna(row.get('Event Description')) else None,
+                        'recorded_address': str(row.get('Recorded Address', '')).strip() if pd.notna(row.get('Recorded Address')) else None,
+                    }
+                    
+                    # Check for duplicates
+                    record_hash = self._generate_record_hash(record_data)
+                    
+                    if record_hash in existing_hashes:
+                        # Found duplicate - get existing record details
+                        existing_record = existing_hashes[record_hash]
+                        
+                        duplicates_list.append({
+                            'row_number': index + 2,
+                            'new_record': record_data,
+                            'existing_record': existing_record,
+                            'hash': record_hash
+                        })
+                    else:
+                        new_records_count += 1
+                
+                except Exception as e:
+                    analysis_result['errors'].append(f"Row {index + 2}: {str(e)}")
+                    continue
+            
+            analysis_result['success'] = True
+            analysis_result['new_records'] = new_records_count
+            analysis_result['duplicate_records'] = len(duplicates_list)
+            analysis_result['duplicates'] = duplicates_list
+            
+            if self.logger:
+                self.logger.logger.info(
+                    f"Duplicate analysis complete - Total: {analysis_result['total_records']}, "
+                    f"New: {new_records_count}, Duplicates: {len(duplicates_list)}"
+                )
+        
+        except Exception as e:
+            analysis_result['errors'].append(f"Analysis failed: {str(e)}")
+            if self.logger:
+                self.logger.logger.error(f"Duplicate analysis error: {e}")
+        
+        return analysis_result
+    
+    def import_from_excel(self, file_path: str, created_by: int = None, 
+                         import_source: str = None, skip_duplicates: bool = True,
+                         force_import_hashes: List[str] = None) -> Dict[str, Any]:
+        """
+        Import time attendance data from Excel file with enhanced duplicate handling
         
         Args:
             file_path: Path to the Excel file
             created_by: User ID who initiated the import
             import_source: Description of import source
             skip_duplicates: Whether to skip duplicate records
+            force_import_hashes: List of hashes to force import (user confirmed duplicates)
             
         Returns:
             Dictionary containing import results
@@ -44,21 +150,27 @@ class TimeAttendanceImportService:
             'failed_records': 0,
             'duplicate_records': 0,
             'skipped_records': 0,
+            'forced_duplicates': 0,
             'errors': [],
             'warnings': [],
             'success': False,
             'import_date': datetime.utcnow()
         }
         
+        force_import_hashes = force_import_hashes or []
+        
         try:
             # Log import start
             if self.logger:
-                self.logger.logger.info(f"Starting enhanced time attendance import from {file_path} by user {created_by}")
+                self.logger.logger.info(
+                    f"Starting enhanced time attendance import from {file_path} by user {created_by} "
+                    f"(skip_duplicates={skip_duplicates}, force_import={len(force_import_hashes)})"
+                )
             
-            # Read Excel file with multiple sheet support
+            # Read Excel file
             try:
                 excel_file = pd.ExcelFile(file_path)
-                sheet_name = excel_file.sheet_names[0]  # Use first sheet
+                sheet_name = excel_file.sheet_names[0]
                 df = pd.read_excel(file_path, sheet_name=sheet_name)
                 
                 if self.logger:
@@ -112,7 +224,7 @@ class TimeAttendanceImportService:
                         import_results['errors'].append(f"Row {index + 2}: Invalid date format - {str(date_error)}")
                         continue
                     
-                    # Validate and parse time with multiple format support
+                    # Validate and parse time
                     try:
                         attendance_time = self._parse_time_field(row['Time'])
                     except Exception as time_error:
@@ -136,13 +248,20 @@ class TimeAttendanceImportService:
                     # Check for duplicates
                     if skip_duplicates:
                         record_hash = self._generate_record_hash(record_data)
-                        if record_hash in duplicate_hashes:
+                        
+                        # If duplicate and NOT in force import list, skip it
+                        if record_hash in duplicate_hashes and record_hash not in force_import_hashes:
                             import_results['duplicate_records'] += 1
                             import_results['warnings'].append(
                                 f"Row {index + 2}: Duplicate record for {record_data['employee_name']} "
                                 f"on {attendance_date} at {attendance_time} - Skipped"
                             )
                             continue
+                        
+                        # If in force import list, track it
+                        if record_hash in force_import_hashes:
+                            import_results['forced_duplicates'] += 1
+                        
                         duplicate_hashes.add(record_hash)
                     
                     # Create TimeAttendance record
@@ -184,6 +303,7 @@ class TimeAttendanceImportService:
                     f"Imported: {import_results['imported_records']}, "
                     f"Failed: {import_results['failed_records']}, "
                     f"Duplicates: {import_results['duplicate_records']}, "
+                    f"Forced: {import_results['forced_duplicates']}, "
                     f"Skipped: {import_results['skipped_records']}"
                 )
             
@@ -208,26 +328,15 @@ class TimeAttendanceImportService:
         return import_results
     
     def _parse_time_field(self, time_value) -> time:
-        """
+        """Parse time field with multiple format support"""
-        Parse time field with multiple format support
-        
-        Args:
-            time_value: Time value from Excel (string, datetime, or time object)
-            
-        Returns:
-            time object
-        """
         if pd.isna(time_value):
             raise ValueError("Time value is empty")
         
-        # If already a time object
         if isinstance(time_value, time):
             return time_value
         
-        # Convert to string and try parsing
         time_str = str(time_value).strip()
         
-        # Try common time formats
         time_formats = [
             '%H:%M:%S',
             '%H:%M',
@@ -241,7 +350,6 @@ class TimeAttendanceImportService:
             except ValueError:
                 continue
         
-        # Try pandas datetime parsing as fallback
         try:
             return pd.to_datetime(time_value).time()
         except:
@@ -250,15 +358,7 @@ class TimeAttendanceImportService:
         raise ValueError(f"Unable to parse time value: {time_value}")
     
     def _generate_record_hash(self, record_data: Dict) -> str:
-        """
+        """Generate unique hash for a record to detect duplicates"""
-        Generate unique hash for a record to detect duplicates
-        
-        Args:
-            record_data: Dictionary containing record information
-            
-        Returns:
-            Hash string
-        """
         hash_string = (
             f"{record_data['employee_id']}_"
             f"{record_data['attendance_date']}_"
@@ -269,12 +369,7 @@ class TimeAttendanceImportService:
         return hashlib.md5(hash_string.encode()).hexdigest()
     
     def _get_existing_record_hashes(self) -> set:
-        """
+        """Get hashes of existing records (hash only)"""
-        Get hashes of existing records to detect duplicates
-        
-        Returns:
-            Set of record hashes
-        """
         try:
             from models.time_attendance import TimeAttendance
             
@@ -297,16 +392,48 @@ class TimeAttendanceImportService:
                 self.logger.logger.warning(f"Failed to get existing record hashes: {e}")
             return set()
     
-    def validate_excel_file(self, file_path: str) -> Dict[str, Any]:
+    def _get_existing_record_hashes_with_data(self) -> Dict[str, Dict]:
-        """
+        """Get hashes with full existing record data for comparison"""
-        Enhanced Excel file validation with detailed analysis
+        try:
-        
+            from models.time_attendance import TimeAttendance
-        Args:
-            file_path: Path to the Excel file
             
-        Returns:
+            existing_records = TimeAttendance.query.all()
-            Dictionary containing validation results
+            hash_map = {}
-        """
+            
+            for record in existing_records:
+                record_data = {
+                    'employee_id': record.employee_id,
+                    'attendance_date': record.attendance_date,
+                    'attendance_time': record.attendance_time,
+                    'location_name': record.location_name,
+                    'action_description': record.action_description
+                }
+                record_hash = self._generate_record_hash(record_data)
+                
+                hash_map[record_hash] = {
+                    'id': record.id,
+                    'employee_id': record.employee_id,
+                    'employee_name': record.employee_name,
+                    'platform': record.platform,
+                    'attendance_date': record.attendance_date,
+                    'attendance_time': record.attendance_time,
+                    'location_name': record.location_name,
+                    'action_description': record.action_description,
+                    'event_description': record.event_description,
+                    'recorded_address': record.recorded_address,
+                    'import_batch_id': record.import_batch_id,
+                    'import_date': record.import_date,
+                    'import_source': record.import_source
+                }
+            
+            return hash_map
+        except Exception as e:
+            if self.logger:
+                self.logger.logger.warning(f"Failed to get existing records with data: {e}")
+            return {}
+    
+    def validate_excel_file(self, file_path: str) -> Dict[str, Any]:
+        """Enhanced Excel file validation with detailed analysis"""
         validation_results = {
             'valid': False,
             'total_rows': 0,
@@ -320,7 +447,6 @@ class TimeAttendanceImportService:
         }
         
         try:
-            # Get file information
             import os
             file_stats = os.stat(file_path)
             validation_results['file_info'] = {
@@ -328,12 +454,10 @@ class TimeAttendanceImportService:
                 'size_mb': round(file_stats.st_size / (1024 * 1024), 2)
             }
             
-            # Read Excel file
             excel_file = pd.ExcelFile(file_path)
             sheet_name = excel_file.sheet_names[0]
             df = pd.read_excel(file_path, sheet_name=sheet_name)
             
-            # Remove empty rows
             original_row_count = len(df)
             df = df.dropna(how='all')
             
@@ -345,11 +469,9 @@ class TimeAttendanceImportService:
                     f"Removed {original_row_count - len(df)} completely empty rows"
                 )
             
-            # Get sample data (first 5 rows)
             sample_rows = df.head(5).to_dict('records')
             validation_results['sample_data'] = sample_rows
             
-            # Validate required columns
             required_columns = ['ID', 'Name', 'Date', 'Time', 'Location Name', 'Action Description']
             missing_columns = [col for col in required_columns if col not in df.columns]
             
@@ -358,7 +480,6 @@ class TimeAttendanceImportService:
                     f"Missing required columns: {', '.join(missing_columns)}"
                 )
             
-            # Check for empty required fields
             valid_row_count = 0
             for index, row in df.iterrows():
                 is_valid = True
@@ -378,7 +499,6 @@ class TimeAttendanceImportService:
                     f"{validation_results['invalid_rows']} rows have missing required data"
                 )
             
-            # Validate date format
             if 'Date' in df.columns:
                 invalid_dates = 0
                 for idx, date_val in df['Date'].items():
@@ -393,7 +513,6 @@ class TimeAttendanceImportService:
                         f"{invalid_dates} rows have invalid date format"
                     )
             
-            # Validate time format
             if 'Time' in df.columns:
                 invalid_times = 0
                 for idx, time_val in df['Time'].items():
@@ -408,7 +527,6 @@ class TimeAttendanceImportService:
                         f"{invalid_times} rows have invalid time format"
                     )
             
-            # Check for potential duplicates
             if all(col in df.columns for col in ['ID', 'Date', 'Time', 'Location Name']):
                 duplicate_check = df[['ID', 'Date', 'Time', 'Location Name']].duplicated()
                 duplicate_count = duplicate_check.sum()
@@ -418,7 +536,6 @@ class TimeAttendanceImportService:
                         f"{duplicate_count} potential duplicate records detected"
                     )
             
-            # Set valid flag
             validation_results['valid'] = (
                 len(validation_results['errors']) == 0 and 
                 validation_results['valid_rows'] > 0