Updated import duplication review
This commit is contained in:
@@ -6544,7 +6544,7 @@ def time_attendance_dashboard():
|
||||
@admin_required
|
||||
@log_database_operations('time_attendance_import')
|
||||
def import_time_attendance():
|
||||
"""Enhanced import time attendance data from Excel file"""
|
||||
"""Enhanced import with duplicate review"""
|
||||
if request.method == 'POST':
|
||||
try:
|
||||
# Check if file is uploaded
|
||||
@@ -6567,19 +6567,37 @@ def import_time_attendance():
|
||||
temp_path = os.path.join(app.config.get('UPLOAD_FOLDER', '/tmp'),
|
||||
f"temp_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{filename}")
|
||||
|
||||
# Ensure upload directory exists
|
||||
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
||||
file.save(temp_path)
|
||||
|
||||
# Store file path in session for duplicate review
|
||||
session['pending_import_file'] = temp_path
|
||||
session['pending_import_filename'] = filename
|
||||
|
||||
try:
|
||||
# Initialize enhanced import service
|
||||
import_service = TimeAttendanceImportService(db, logger_handler)
|
||||
|
||||
# Get import options
|
||||
skip_duplicates = request.form.get('skip_duplicates', 'true').lower() == 'true'
|
||||
validate_only = request.form.get('validate_only', 'false').lower() == 'true'
|
||||
analyze_duplicates = request.form.get('analyze_duplicates', 'false').lower() == 'true'
|
||||
|
||||
# Validate file first
|
||||
# Check if this is coming from duplicate review
|
||||
force_import_hashes = request.form.getlist('force_import_hashes[]')
|
||||
|
||||
# If analyzing for duplicates, show review page
|
||||
if analyze_duplicates and not force_import_hashes:
|
||||
duplicate_analysis = import_service.analyze_for_duplicates(temp_path)
|
||||
|
||||
if duplicate_analysis['duplicate_records'] > 0:
|
||||
# Show duplicate review page
|
||||
return render_template('time_attendance_duplicate_review.html',
|
||||
analysis=duplicate_analysis,
|
||||
filename=filename)
|
||||
else:
|
||||
flash('No duplicates found. Proceeding with import.', 'info')
|
||||
|
||||
# Validate file
|
||||
validation_result = import_service.validate_excel_file(temp_path)
|
||||
|
||||
if not validation_result['valid']:
|
||||
@@ -6587,12 +6605,10 @@ def import_time_attendance():
|
||||
return render_template('time_attendance_import.html',
|
||||
validation_result=validation_result)
|
||||
|
||||
# Show warnings if any
|
||||
if validation_result['warnings']:
|
||||
for warning in validation_result['warnings']:
|
||||
flash(warning, 'warning')
|
||||
|
||||
# If validate only, return validation results
|
||||
if validate_only:
|
||||
flash(f"File validation successful! Found {validation_result['valid_rows']} valid records.", 'success')
|
||||
return render_template('time_attendance_import.html',
|
||||
@@ -6604,16 +6620,17 @@ def import_time_attendance():
|
||||
temp_path,
|
||||
created_by=session['user_id'],
|
||||
import_source=import_source,
|
||||
skip_duplicates=skip_duplicates
|
||||
skip_duplicates=skip_duplicates,
|
||||
force_import_hashes=force_import_hashes
|
||||
)
|
||||
|
||||
if import_result['success']:
|
||||
# Log successful import
|
||||
logger_handler.logger.info(
|
||||
f"User {session['username']} successfully imported time attendance data - "
|
||||
f"Batch: {import_result['batch_id']}, "
|
||||
f"Records: {import_result['imported_records']}/{import_result['total_records']}, "
|
||||
f"Duplicates: {import_result['duplicate_records']}, "
|
||||
f"Forced: {import_result['forced_duplicates']}, "
|
||||
f"Failed: {import_result['failed_records']}"
|
||||
)
|
||||
|
||||
@@ -6623,6 +6640,9 @@ def import_time_attendance():
|
||||
if import_result['duplicate_records'] > 0:
|
||||
flash(f"Skipped {import_result['duplicate_records']} duplicate records.", 'info')
|
||||
|
||||
if import_result['forced_duplicates'] > 0:
|
||||
flash(f"Imported {import_result['forced_duplicates']} duplicate records as requested.", 'info')
|
||||
|
||||
if import_result['failed_records'] > 0:
|
||||
flash(f"Note: {import_result['failed_records']} records failed to import. "
|
||||
f"Check the error details below.", 'warning')
|
||||
@@ -6637,12 +6657,17 @@ def import_time_attendance():
|
||||
import_result=import_result)
|
||||
|
||||
finally:
|
||||
# Clean up temporary file
|
||||
if os.path.exists(temp_path):
|
||||
try:
|
||||
os.remove(temp_path)
|
||||
except Exception as cleanup_error:
|
||||
logger_handler.logger.warning(f"Failed to cleanup temp file: {cleanup_error}")
|
||||
# Clean up if import completed or failed (not if showing duplicate review)
|
||||
if not analyze_duplicates or force_import_hashes:
|
||||
if os.path.exists(temp_path):
|
||||
try:
|
||||
os.remove(temp_path)
|
||||
if 'pending_import_file' in session:
|
||||
session.pop('pending_import_file')
|
||||
if 'pending_import_filename' in session:
|
||||
session.pop('pending_import_filename')
|
||||
except Exception as cleanup_error:
|
||||
logger_handler.logger.warning(f"Failed to cleanup temp file: {cleanup_error}")
|
||||
|
||||
except Exception as e:
|
||||
logger_handler.log_database_error('time_attendance_import', e)
|
||||
@@ -6651,6 +6676,95 @@ def import_time_attendance():
|
||||
|
||||
return render_template('time_attendance_import.html')
|
||||
|
||||
|
||||
@app.route('/time-attendance/import/analyze-duplicates', methods=['POST'])
|
||||
@admin_required
|
||||
def analyze_import_duplicates():
|
||||
"""AJAX endpoint to analyze file for duplicates"""
|
||||
try:
|
||||
if 'file' not in request.files:
|
||||
return jsonify({'success': False, 'message': 'No file provided'}), 400
|
||||
|
||||
file = request.files['file']
|
||||
if file.filename == '':
|
||||
return jsonify({'success': False, 'message': 'No file selected'}), 400
|
||||
|
||||
if not file.filename.lower().endswith(('.xlsx', '.xls')):
|
||||
return jsonify({'success': False, 'message': 'Invalid file format'}), 400
|
||||
|
||||
# Save temporarily
|
||||
filename = secure_filename(file.filename)
|
||||
temp_path = os.path.join(app.config.get('UPLOAD_FOLDER', '/tmp'),
|
||||
f"analyze_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{filename}")
|
||||
|
||||
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
||||
file.save(temp_path)
|
||||
|
||||
# Store in session
|
||||
session['pending_import_file'] = temp_path
|
||||
session['pending_import_filename'] = filename
|
||||
|
||||
try:
|
||||
import_service = TimeAttendanceImportService(db, logger_handler)
|
||||
analysis = import_service.analyze_for_duplicates(temp_path)
|
||||
|
||||
# Convert datetime objects to strings for JSON
|
||||
for duplicate in analysis.get('duplicates', []):
|
||||
if 'new_record' in duplicate:
|
||||
if duplicate['new_record'].get('attendance_date'):
|
||||
duplicate['new_record']['attendance_date'] = str(duplicate['new_record']['attendance_date'])
|
||||
if duplicate['new_record'].get('attendance_time'):
|
||||
duplicate['new_record']['attendance_time'] = str(duplicate['new_record']['attendance_time'])
|
||||
|
||||
if 'existing_record' in duplicate:
|
||||
if duplicate['existing_record'].get('attendance_date'):
|
||||
duplicate['existing_record']['attendance_date'] = str(duplicate['existing_record']['attendance_date'])
|
||||
if duplicate['existing_record'].get('attendance_time'):
|
||||
duplicate['existing_record']['attendance_time'] = str(duplicate['existing_record']['attendance_time'])
|
||||
if duplicate['existing_record'].get('import_date'):
|
||||
duplicate['existing_record']['import_date'] = str(duplicate['existing_record']['import_date'])
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'analysis': analysis
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
# Cleanup on error
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
raise e
|
||||
|
||||
except Exception as e:
|
||||
logger_handler.logger.error(f"Duplicate analysis error: {e}")
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'message': f'Analysis failed: {str(e)}'
|
||||
}), 500
|
||||
|
||||
|
||||
@app.route('/time-attendance/import/cancel-pending')
|
||||
@admin_required
|
||||
def cancel_pending_import():
|
||||
"""Cancel pending import and cleanup temp file"""
|
||||
try:
|
||||
if 'pending_import_file' in session:
|
||||
temp_path = session['pending_import_file']
|
||||
if os.path.exists(temp_path):
|
||||
os.remove(temp_path)
|
||||
session.pop('pending_import_file')
|
||||
|
||||
if 'pending_import_filename' in session:
|
||||
session.pop('pending_import_filename')
|
||||
|
||||
flash('Import cancelled.', 'info')
|
||||
except Exception as e:
|
||||
logger_handler.logger.error(f"Error cancelling import: {e}")
|
||||
|
||||
return redirect(url_for('import_time_attendance'))
|
||||
|
||||
|
||||
|
||||
@app.route('/time-attendance/import/validate', methods=['POST'])
|
||||
@admin_required
|
||||
def validate_import_file():
|
||||
|
||||
@@ -0,0 +1,641 @@
|
||||
{% extends "base_authenticated.html" %}
|
||||
{% block title %}Review Duplicate Records - {{ COMPANY_NAME }}{% endblock %}
|
||||
|
||||
{% block extra_head %}
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/time_attendance.css') }}">
|
||||
<style>
|
||||
.duplicate-review-page {
|
||||
max-width: 1400px;
|
||||
margin: 0 auto;
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
.review-summary {
|
||||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||||
color: white;
|
||||
padding: 2rem;
|
||||
border-radius: 12px;
|
||||
margin-bottom: 2rem;
|
||||
box-shadow: 0 8px 24px rgba(0,0,0,0.15);
|
||||
}
|
||||
|
||||
.summary-stats {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 1.5rem;
|
||||
margin-top: 1.5rem;
|
||||
}
|
||||
|
||||
.stat-box {
|
||||
background: rgba(255,255,255,0.2);
|
||||
padding: 1.5rem;
|
||||
border-radius: 8px;
|
||||
text-align: center;
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
|
||||
.stat-box .number {
|
||||
font-size: 2.5rem;
|
||||
font-weight: 700;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.stat-box .label {
|
||||
font-size: 0.875rem;
|
||||
opacity: 0.9;
|
||||
}
|
||||
|
||||
.duplicates-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.duplicate-item {
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
overflow: hidden;
|
||||
border: 2px solid #e2e8f0;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
|
||||
.duplicate-item.selected {
|
||||
border-color: #667eea;
|
||||
box-shadow: 0 4px 12px rgba(102, 126, 234, 0.2);
|
||||
}
|
||||
|
||||
.duplicate-header {
|
||||
background: #f7fafc;
|
||||
padding: 1rem 1.5rem;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.duplicate-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.duplicate-title h3 {
|
||||
margin: 0;
|
||||
color: #2d3748;
|
||||
font-size: 1.125rem;
|
||||
}
|
||||
|
||||
.row-badge {
|
||||
background: #ed8936;
|
||||
color: white;
|
||||
padding: 0.25rem 0.75rem;
|
||||
border-radius: 20px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.duplicate-actions {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.duplicate-body {
|
||||
padding: 1.5rem;
|
||||
}
|
||||
|
||||
.comparison-grid {
|
||||
display: grid;
|
||||
grid-template-columns: 1fr 1fr;
|
||||
gap: 2rem;
|
||||
}
|
||||
|
||||
.record-card {
|
||||
background: #f8fafc;
|
||||
padding: 1.5rem;
|
||||
border-radius: 8px;
|
||||
border: 2px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.record-card h4 {
|
||||
margin: 0 0 1rem 0;
|
||||
color: #2d3748;
|
||||
font-size: 1rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.record-card.new-record h4 {
|
||||
color: #3182ce;
|
||||
}
|
||||
|
||||
.record-card.existing-record h4 {
|
||||
color: #805ad5;
|
||||
}
|
||||
|
||||
.record-details {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.detail-row {
|
||||
display: grid;
|
||||
grid-template-columns: 140px 1fr;
|
||||
gap: 0.5rem;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.detail-label {
|
||||
color: #718096;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.detail-value {
|
||||
color: #2d3748;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.detail-value.highlight {
|
||||
background: #fef5e7;
|
||||
padding: 0.25rem 0.5rem;
|
||||
border-radius: 4px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.checkbox-container {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.75rem;
|
||||
padding: 1rem;
|
||||
background: #ebf8ff;
|
||||
border-radius: 8px;
|
||||
margin-top: 1rem;
|
||||
}
|
||||
|
||||
.checkbox-container input[type="checkbox"] {
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.checkbox-container label {
|
||||
font-weight: 600;
|
||||
color: #2c5282;
|
||||
cursor: pointer;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.action-buttons {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
margin-top: 2rem;
|
||||
padding: 1.5rem;
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
position: sticky;
|
||||
bottom: 1rem;
|
||||
z-index: 100;
|
||||
}
|
||||
|
||||
.action-buttons .left-actions,
|
||||
.action-buttons .right-actions {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.btn {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
padding: 0.75rem 1.5rem;
|
||||
border: none;
|
||||
border-radius: 8px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s ease;
|
||||
text-decoration: none;
|
||||
font-size: 1rem;
|
||||
}
|
||||
|
||||
.btn-primary {
|
||||
background: linear-gradient(135deg, #667eea, #764ba2);
|
||||
color: white;
|
||||
}
|
||||
|
||||
.btn-primary:hover {
|
||||
transform: translateY(-2px);
|
||||
box-shadow: 0 4px 12px rgba(102, 126, 234, 0.4);
|
||||
}
|
||||
|
||||
.btn-secondary {
|
||||
background: #e2e8f0;
|
||||
color: #2d3748;
|
||||
}
|
||||
|
||||
.btn-secondary:hover {
|
||||
background: #cbd5e0;
|
||||
}
|
||||
|
||||
.btn-danger {
|
||||
background: #fc8181;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.btn-danger:hover {
|
||||
background: #f56565;
|
||||
}
|
||||
|
||||
.selection-summary {
|
||||
background: #f0fff4;
|
||||
border: 2px solid #9ae6b4;
|
||||
padding: 1rem;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 1rem;
|
||||
display: none;
|
||||
}
|
||||
|
||||
.selection-summary.visible {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.selection-summary .count {
|
||||
font-weight: 700;
|
||||
color: #22543d;
|
||||
font-size: 1.125rem;
|
||||
}
|
||||
|
||||
.empty-state {
|
||||
text-align: center;
|
||||
padding: 4rem 2rem;
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
}
|
||||
|
||||
.empty-state i {
|
||||
font-size: 4rem;
|
||||
color: #48bb78;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.empty-state h3 {
|
||||
color: #2d3748;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.empty-state p {
|
||||
color: #718096;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.comparison-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.action-buttons {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.action-buttons .left-actions,
|
||||
.action-buttons .right-actions {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block page_title %}Review Duplicate Records{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="duplicate-review-page">
|
||||
<!-- Page Header -->
|
||||
<div class="time-attendance-header">
|
||||
<div class="header-navigation">
|
||||
<a href="{{ url_for('import_time_attendance') }}" class="back-button">
|
||||
<i class="fas fa-arrow-left"></i>
|
||||
Back to Import
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="header-content">
|
||||
<h1>
|
||||
<i class="fas fa-exclamation-triangle"></i>
|
||||
Review Duplicate Records
|
||||
</h1>
|
||||
<p class="header-description">
|
||||
The following records already exist in the system. Please review and select which ones you want to import anyway.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Summary Section -->
|
||||
<div class="review-summary">
|
||||
<h2 style="margin: 0 0 0.5rem 0;">
|
||||
<i class="fas fa-file-excel"></i>
|
||||
File: {{ filename }}
|
||||
</h2>
|
||||
<p style="margin: 0; opacity: 0.9;">Review duplicate records before importing</p>
|
||||
|
||||
<div class="summary-stats">
|
||||
<div class="stat-box">
|
||||
<div class="number">{{ analysis.total_records }}</div>
|
||||
<div class="label">Total Records in File</div>
|
||||
</div>
|
||||
|
||||
<div class="stat-box">
|
||||
<div class="number">{{ analysis.new_records }}</div>
|
||||
<div class="label">New Records</div>
|
||||
</div>
|
||||
|
||||
<div class="stat-box">
|
||||
<div class="number">{{ analysis.duplicate_records }}</div>
|
||||
<div class="label">Duplicate Records</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Selection Summary -->
|
||||
<div id="selectionSummary" class="selection-summary">
|
||||
<i class="fas fa-check-circle"></i>
|
||||
<span class="count">0</span> duplicate record(s) selected to import
|
||||
</div>
|
||||
|
||||
<!-- Duplicates List -->
|
||||
{% if analysis.duplicate_records > 0 %}
|
||||
<form id="duplicateReviewForm" method="POST" action="{{ url_for('import_time_attendance') }}">
|
||||
<input type="hidden" name="analyze_duplicates" value="false">
|
||||
<input type="hidden" name="skip_duplicates" value="true">
|
||||
<input type="hidden" name="import_source" value="Import with Duplicates - {{ filename }}">
|
||||
|
||||
<div class="duplicates-container">
|
||||
{% for duplicate in analysis.duplicates %}
|
||||
<div class="duplicate-item" data-hash="{{ duplicate.hash }}">
|
||||
<div class="duplicate-header">
|
||||
<div class="duplicate-title">
|
||||
<span class="row-badge">Row {{ duplicate.row_number }}</span>
|
||||
<h3>{{ duplicate.new_record.employee_name }} (ID: {{ duplicate.new_record.employee_id }})</h3>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="duplicate-body">
|
||||
<div class="comparison-grid">
|
||||
<!-- New Record (from file) -->
|
||||
<div class="record-card new-record">
|
||||
<h4>
|
||||
<i class="fas fa-file-import"></i>
|
||||
New Record (From File)
|
||||
</h4>
|
||||
<div class="record-details">
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Employee ID:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.new_record.employee_id }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Name:</span>
|
||||
<span class="detail-value">{{ duplicate.new_record.employee_name }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Date:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.new_record.attendance_date }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Time:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.new_record.attendance_time }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Location:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.new_record.location_name }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Action:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.new_record.action_description }}</span>
|
||||
</div>
|
||||
{% if duplicate.new_record.platform %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Platform:</span>
|
||||
<span class="detail-value">{{ duplicate.new_record.platform }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if duplicate.new_record.event_description %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Event:</span>
|
||||
<span class="detail-value">{{ duplicate.new_record.event_description }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if duplicate.new_record.recorded_address %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Address:</span>
|
||||
<span class="detail-value">{{ duplicate.new_record.recorded_address }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Existing Record (in database) -->
|
||||
<div class="record-card existing-record">
|
||||
<h4>
|
||||
<i class="fas fa-database"></i>
|
||||
Existing Record (In System)
|
||||
</h4>
|
||||
<div class="record-details">
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Employee ID:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.existing_record.employee_id }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Name:</span>
|
||||
<span class="detail-value">{{ duplicate.existing_record.employee_name }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Date:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.existing_record.attendance_date }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Time:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.existing_record.attendance_time }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Location:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.existing_record.location_name }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Action:</span>
|
||||
<span class="detail-value highlight">{{ duplicate.existing_record.action_description }}</span>
|
||||
</div>
|
||||
{% if duplicate.existing_record.platform %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Platform:</span>
|
||||
<span class="detail-value">{{ duplicate.existing_record.platform }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if duplicate.existing_record.event_description %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Event:</span>
|
||||
<span class="detail-value">{{ duplicate.existing_record.event_description }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if duplicate.existing_record.recorded_address %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Address:</span>
|
||||
<span class="detail-value">{{ duplicate.existing_record.recorded_address }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Import Date:</span>
|
||||
<span class="detail-value">
|
||||
<i class="fas fa-clock"></i>
|
||||
{{ duplicate.existing_record.import_date.strftime('%Y-%m-%d %H:%M') if duplicate.existing_record.import_date else 'Unknown' }}
|
||||
</span>
|
||||
</div>
|
||||
{% if duplicate.existing_record.import_source %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Import Source:</span>
|
||||
<span class="detail-value">{{ duplicate.existing_record.import_source }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Selection Checkbox -->
|
||||
<div class="checkbox-container">
|
||||
<input type="checkbox"
|
||||
id="duplicate_{{ loop.index }}"
|
||||
name="force_import_hashes[]"
|
||||
value="{{ duplicate.hash }}"
|
||||
class="duplicate-checkbox">
|
||||
<label for="duplicate_{{ loop.index }}">
|
||||
Import this duplicate record anyway
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<!-- Action Buttons -->
|
||||
<div class="action-buttons">
|
||||
<div class="left-actions">
|
||||
<a href="{{ url_for('cancel_pending_import') }}" class="btn btn-secondary">
|
||||
<i class="fas fa-times"></i>
|
||||
Cancel Import
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="right-actions">
|
||||
<button type="button" id="selectAllBtn" class="btn btn-secondary">
|
||||
<i class="fas fa-check-square"></i>
|
||||
Select All
|
||||
</button>
|
||||
<button type="button" id="skipAllBtn" class="btn btn-secondary">
|
||||
<i class="fas fa-forward"></i>
|
||||
Skip All Duplicates
|
||||
</button>
|
||||
<button type="submit" class="btn btn-primary" id="proceedBtn">
|
||||
<i class="fas fa-upload"></i>
|
||||
Proceed with Import
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
{% else %}
|
||||
<div class="empty-state">
|
||||
<i class="fas fa-check-circle"></i>
|
||||
<h3>No Duplicates Found</h3>
|
||||
<p>All records in the file are unique. You can proceed with the import.</p>
|
||||
<a href="{{ url_for('import_time_attendance') }}" class="btn btn-primary" style="margin-top: 1rem;">
|
||||
<i class="fas fa-arrow-left"></i>
|
||||
Back to Import
|
||||
</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// Track selected duplicates
|
||||
const checkboxes = document.querySelectorAll('.duplicate-checkbox');
|
||||
const selectionSummary = document.getElementById('selectionSummary');
|
||||
const selectAllBtn = document.getElementById('selectAllBtn');
|
||||
const skipAllBtn = document.getElementById('skipAllBtn');
|
||||
const proceedBtn = document.getElementById('proceedBtn');
|
||||
|
||||
function updateSelectionSummary() {
|
||||
const selectedCount = document.querySelectorAll('.duplicate-checkbox:checked').length;
|
||||
const countSpan = selectionSummary.querySelector('.count');
|
||||
countSpan.textContent = selectedCount;
|
||||
|
||||
if (selectedCount > 0) {
|
||||
selectionSummary.classList.add('visible');
|
||||
} else {
|
||||
selectionSummary.classList.remove('visible');
|
||||
}
|
||||
|
||||
// Update duplicate items visual state
|
||||
document.querySelectorAll('.duplicate-item').forEach(item => {
|
||||
const checkbox = item.querySelector('.duplicate-checkbox');
|
||||
if (checkbox && checkbox.checked) {
|
||||
item.classList.add('selected');
|
||||
} else {
|
||||
item.classList.remove('selected');
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Listen to checkbox changes
|
||||
checkboxes.forEach(checkbox => {
|
||||
checkbox.addEventListener('change', updateSelectionSummary);
|
||||
});
|
||||
|
||||
// Select all button
|
||||
if (selectAllBtn) {
|
||||
selectAllBtn.addEventListener('click', () => {
|
||||
checkboxes.forEach(cb => cb.checked = true);
|
||||
updateSelectionSummary();
|
||||
});
|
||||
}
|
||||
|
||||
// Skip all button (uncheck all and submit)
|
||||
if (skipAllBtn) {
|
||||
skipAllBtn.addEventListener('click', () => {
|
||||
if (confirm('This will skip all duplicate records and import only new records. Continue?')) {
|
||||
checkboxes.forEach(cb => cb.checked = false);
|
||||
document.getElementById('duplicateReviewForm').submit();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Form submission confirmation
|
||||
if (proceedBtn) {
|
||||
proceedBtn.addEventListener('click', (e) => {
|
||||
const selectedCount = document.querySelectorAll('.duplicate-checkbox:checked').length;
|
||||
const totalDuplicates = checkboxes.length;
|
||||
const newRecordsCount = {{ analysis.new_records }};
|
||||
|
||||
let message = '';
|
||||
if (selectedCount > 0) {
|
||||
message = `You are about to import:\n\n`;
|
||||
message += `- ${newRecordsCount} new record(s)\n`;
|
||||
message += `- ${selectedCount} duplicate record(s)\n`;
|
||||
message += `- Skipping ${totalDuplicates - selectedCount} duplicate(s)\n\n`;
|
||||
message += `Total: ${newRecordsCount + selectedCount} records will be imported.\n\n`;
|
||||
message += `Continue with import?`;
|
||||
} else {
|
||||
message = `You are about to import ${newRecordsCount} new record(s) only.\n\n`;
|
||||
message += `All ${totalDuplicates} duplicate record(s) will be skipped.\n\n`;
|
||||
message += `Continue with import?`;
|
||||
}
|
||||
|
||||
if (!confirm(message)) {
|
||||
e.preventDefault();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Initial update
|
||||
updateSelectionSummary();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -284,6 +284,14 @@
|
||||
<span>Automatically detect and skip records that already exist in the system</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="checkbox-item">
|
||||
<input type="checkbox" id="analyze_duplicates" name="analyze_duplicates" value="true">
|
||||
<label for="analyze_duplicates" class="checkbox-label">
|
||||
<strong>Review Duplicates Before Import</strong>
|
||||
<span>Show potential duplicate records for review and allow selective import</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="checkbox-item">
|
||||
<input type="checkbox" id="validate_only" name="validate_only" value="true">
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
"""
|
||||
Enhanced Time Attendance Import Service
|
||||
========================================
|
||||
Enhanced Time Attendance Import Service with Duplicate Review
|
||||
============================================================
|
||||
|
||||
Improved service with duplicate detection, advanced validation,
|
||||
and better error handling for Excel imports.
|
||||
Added functionality to detect and present duplicates for user review.
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
@@ -15,23 +14,130 @@ import traceback
|
||||
import hashlib
|
||||
|
||||
class TimeAttendanceImportService:
|
||||
"""Enhanced service to handle time attendance data import from Excel files"""
|
||||
"""Enhanced service with duplicate detection and review"""
|
||||
|
||||
def __init__(self, db, logger_handler=None):
|
||||
"""Initialize the import service with database and logger"""
|
||||
self.db = db
|
||||
self.logger = logger_handler
|
||||
|
||||
def import_from_excel(self, file_path: str, created_by: int = None,
|
||||
import_source: str = None, skip_duplicates: bool = True) -> Dict[str, Any]:
|
||||
|
||||
def analyze_for_duplicates(self, file_path: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Import time attendance data from Excel file with enhanced validation
|
||||
Analyze file for potential duplicates WITHOUT importing
|
||||
|
||||
Args:
|
||||
file_path: Path to the Excel file
|
||||
|
||||
Returns:
|
||||
Dictionary containing duplicate analysis
|
||||
"""
|
||||
analysis_result = {
|
||||
'success': False,
|
||||
'total_records': 0,
|
||||
'new_records': 0,
|
||||
'duplicate_records': 0,
|
||||
'duplicates': [],
|
||||
'errors': []
|
||||
}
|
||||
|
||||
try:
|
||||
# Read Excel file
|
||||
excel_file = pd.ExcelFile(file_path)
|
||||
sheet_name = excel_file.sheet_names[0]
|
||||
df = pd.read_excel(file_path, sheet_name=sheet_name)
|
||||
|
||||
# Validate required columns
|
||||
required_columns = ['ID', 'Name', 'Date', 'Time', 'Location Name', 'Action Description']
|
||||
missing_columns = [col for col in required_columns if col not in df.columns]
|
||||
|
||||
if missing_columns:
|
||||
analysis_result['errors'].append(f"Missing columns: {', '.join(missing_columns)}")
|
||||
return analysis_result
|
||||
|
||||
# Remove empty rows
|
||||
df = df.dropna(how='all')
|
||||
analysis_result['total_records'] = len(df)
|
||||
|
||||
# Get existing record hashes
|
||||
existing_hashes = self._get_existing_record_hashes_with_data()
|
||||
|
||||
# Process each row
|
||||
duplicates_list = []
|
||||
new_records_count = 0
|
||||
|
||||
for index, row in df.iterrows():
|
||||
try:
|
||||
# Skip empty rows
|
||||
if pd.isna(row['ID']) or pd.isna(row['Name']):
|
||||
continue
|
||||
|
||||
# Parse date and time
|
||||
attendance_date = pd.to_datetime(row['Date']).date()
|
||||
attendance_time = self._parse_time_field(row['Time'])
|
||||
|
||||
# Prepare record data
|
||||
record_data = {
|
||||
'employee_id': str(row['ID']).strip(),
|
||||
'employee_name': str(row['Name']).strip(),
|
||||
'platform': str(row.get('Platform', '')).strip() if pd.notna(row.get('Platform')) else None,
|
||||
'attendance_date': attendance_date,
|
||||
'attendance_time': attendance_time,
|
||||
'location_name': str(row['Location Name']).strip(),
|
||||
'action_description': str(row['Action Description']).strip(),
|
||||
'event_description': str(row.get('Event Description', '')).strip() if pd.notna(row.get('Event Description')) else None,
|
||||
'recorded_address': str(row.get('Recorded Address', '')).strip() if pd.notna(row.get('Recorded Address')) else None,
|
||||
}
|
||||
|
||||
# Check for duplicates
|
||||
record_hash = self._generate_record_hash(record_data)
|
||||
|
||||
if record_hash in existing_hashes:
|
||||
# Found duplicate - get existing record details
|
||||
existing_record = existing_hashes[record_hash]
|
||||
|
||||
duplicates_list.append({
|
||||
'row_number': index + 2,
|
||||
'new_record': record_data,
|
||||
'existing_record': existing_record,
|
||||
'hash': record_hash
|
||||
})
|
||||
else:
|
||||
new_records_count += 1
|
||||
|
||||
except Exception as e:
|
||||
analysis_result['errors'].append(f"Row {index + 2}: {str(e)}")
|
||||
continue
|
||||
|
||||
analysis_result['success'] = True
|
||||
analysis_result['new_records'] = new_records_count
|
||||
analysis_result['duplicate_records'] = len(duplicates_list)
|
||||
analysis_result['duplicates'] = duplicates_list
|
||||
|
||||
if self.logger:
|
||||
self.logger.logger.info(
|
||||
f"Duplicate analysis complete - Total: {analysis_result['total_records']}, "
|
||||
f"New: {new_records_count}, Duplicates: {len(duplicates_list)}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
analysis_result['errors'].append(f"Analysis failed: {str(e)}")
|
||||
if self.logger:
|
||||
self.logger.logger.error(f"Duplicate analysis error: {e}")
|
||||
|
||||
return analysis_result
|
||||
|
||||
def import_from_excel(self, file_path: str, created_by: int = None,
|
||||
import_source: str = None, skip_duplicates: bool = True,
|
||||
force_import_hashes: List[str] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
Import time attendance data from Excel file with enhanced duplicate handling
|
||||
|
||||
Args:
|
||||
file_path: Path to the Excel file
|
||||
created_by: User ID who initiated the import
|
||||
import_source: Description of import source
|
||||
skip_duplicates: Whether to skip duplicate records
|
||||
force_import_hashes: List of hashes to force import (user confirmed duplicates)
|
||||
|
||||
Returns:
|
||||
Dictionary containing import results
|
||||
@@ -44,21 +150,27 @@ class TimeAttendanceImportService:
|
||||
'failed_records': 0,
|
||||
'duplicate_records': 0,
|
||||
'skipped_records': 0,
|
||||
'forced_duplicates': 0,
|
||||
'errors': [],
|
||||
'warnings': [],
|
||||
'success': False,
|
||||
'import_date': datetime.utcnow()
|
||||
}
|
||||
|
||||
force_import_hashes = force_import_hashes or []
|
||||
|
||||
try:
|
||||
# Log import start
|
||||
if self.logger:
|
||||
self.logger.logger.info(f"Starting enhanced time attendance import from {file_path} by user {created_by}")
|
||||
self.logger.logger.info(
|
||||
f"Starting enhanced time attendance import from {file_path} by user {created_by} "
|
||||
f"(skip_duplicates={skip_duplicates}, force_import={len(force_import_hashes)})"
|
||||
)
|
||||
|
||||
# Read Excel file with multiple sheet support
|
||||
# Read Excel file
|
||||
try:
|
||||
excel_file = pd.ExcelFile(file_path)
|
||||
sheet_name = excel_file.sheet_names[0] # Use first sheet
|
||||
sheet_name = excel_file.sheet_names[0]
|
||||
df = pd.read_excel(file_path, sheet_name=sheet_name)
|
||||
|
||||
if self.logger:
|
||||
@@ -112,7 +224,7 @@ class TimeAttendanceImportService:
|
||||
import_results['errors'].append(f"Row {index + 2}: Invalid date format - {str(date_error)}")
|
||||
continue
|
||||
|
||||
# Validate and parse time with multiple format support
|
||||
# Validate and parse time
|
||||
try:
|
||||
attendance_time = self._parse_time_field(row['Time'])
|
||||
except Exception as time_error:
|
||||
@@ -136,13 +248,20 @@ class TimeAttendanceImportService:
|
||||
# Check for duplicates
|
||||
if skip_duplicates:
|
||||
record_hash = self._generate_record_hash(record_data)
|
||||
if record_hash in duplicate_hashes:
|
||||
|
||||
# If duplicate and NOT in force import list, skip it
|
||||
if record_hash in duplicate_hashes and record_hash not in force_import_hashes:
|
||||
import_results['duplicate_records'] += 1
|
||||
import_results['warnings'].append(
|
||||
f"Row {index + 2}: Duplicate record for {record_data['employee_name']} "
|
||||
f"on {attendance_date} at {attendance_time} - Skipped"
|
||||
)
|
||||
continue
|
||||
|
||||
# If in force import list, track it
|
||||
if record_hash in force_import_hashes:
|
||||
import_results['forced_duplicates'] += 1
|
||||
|
||||
duplicate_hashes.add(record_hash)
|
||||
|
||||
# Create TimeAttendance record
|
||||
@@ -184,6 +303,7 @@ class TimeAttendanceImportService:
|
||||
f"Imported: {import_results['imported_records']}, "
|
||||
f"Failed: {import_results['failed_records']}, "
|
||||
f"Duplicates: {import_results['duplicate_records']}, "
|
||||
f"Forced: {import_results['forced_duplicates']}, "
|
||||
f"Skipped: {import_results['skipped_records']}"
|
||||
)
|
||||
|
||||
@@ -208,26 +328,15 @@ class TimeAttendanceImportService:
|
||||
return import_results
|
||||
|
||||
def _parse_time_field(self, time_value) -> time:
|
||||
"""
|
||||
Parse time field with multiple format support
|
||||
|
||||
Args:
|
||||
time_value: Time value from Excel (string, datetime, or time object)
|
||||
|
||||
Returns:
|
||||
time object
|
||||
"""
|
||||
"""Parse time field with multiple format support"""
|
||||
if pd.isna(time_value):
|
||||
raise ValueError("Time value is empty")
|
||||
|
||||
# If already a time object
|
||||
if isinstance(time_value, time):
|
||||
return time_value
|
||||
|
||||
# Convert to string and try parsing
|
||||
time_str = str(time_value).strip()
|
||||
|
||||
# Try common time formats
|
||||
time_formats = [
|
||||
'%H:%M:%S',
|
||||
'%H:%M',
|
||||
@@ -241,7 +350,6 @@ class TimeAttendanceImportService:
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
# Try pandas datetime parsing as fallback
|
||||
try:
|
||||
return pd.to_datetime(time_value).time()
|
||||
except:
|
||||
@@ -250,15 +358,7 @@ class TimeAttendanceImportService:
|
||||
raise ValueError(f"Unable to parse time value: {time_value}")
|
||||
|
||||
def _generate_record_hash(self, record_data: Dict) -> str:
|
||||
"""
|
||||
Generate unique hash for a record to detect duplicates
|
||||
|
||||
Args:
|
||||
record_data: Dictionary containing record information
|
||||
|
||||
Returns:
|
||||
Hash string
|
||||
"""
|
||||
"""Generate unique hash for a record to detect duplicates"""
|
||||
hash_string = (
|
||||
f"{record_data['employee_id']}_"
|
||||
f"{record_data['attendance_date']}_"
|
||||
@@ -269,12 +369,7 @@ class TimeAttendanceImportService:
|
||||
return hashlib.md5(hash_string.encode()).hexdigest()
|
||||
|
||||
def _get_existing_record_hashes(self) -> set:
|
||||
"""
|
||||
Get hashes of existing records to detect duplicates
|
||||
|
||||
Returns:
|
||||
Set of record hashes
|
||||
"""
|
||||
"""Get hashes of existing records (hash only)"""
|
||||
try:
|
||||
from models.time_attendance import TimeAttendance
|
||||
|
||||
@@ -297,16 +392,48 @@ class TimeAttendanceImportService:
|
||||
self.logger.logger.warning(f"Failed to get existing record hashes: {e}")
|
||||
return set()
|
||||
|
||||
def validate_excel_file(self, file_path: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Enhanced Excel file validation with detailed analysis
|
||||
|
||||
Args:
|
||||
file_path: Path to the Excel file
|
||||
def _get_existing_record_hashes_with_data(self) -> Dict[str, Dict]:
|
||||
"""Get hashes with full existing record data for comparison"""
|
||||
try:
|
||||
from models.time_attendance import TimeAttendance
|
||||
|
||||
Returns:
|
||||
Dictionary containing validation results
|
||||
"""
|
||||
existing_records = TimeAttendance.query.all()
|
||||
hash_map = {}
|
||||
|
||||
for record in existing_records:
|
||||
record_data = {
|
||||
'employee_id': record.employee_id,
|
||||
'attendance_date': record.attendance_date,
|
||||
'attendance_time': record.attendance_time,
|
||||
'location_name': record.location_name,
|
||||
'action_description': record.action_description
|
||||
}
|
||||
record_hash = self._generate_record_hash(record_data)
|
||||
|
||||
hash_map[record_hash] = {
|
||||
'id': record.id,
|
||||
'employee_id': record.employee_id,
|
||||
'employee_name': record.employee_name,
|
||||
'platform': record.platform,
|
||||
'attendance_date': record.attendance_date,
|
||||
'attendance_time': record.attendance_time,
|
||||
'location_name': record.location_name,
|
||||
'action_description': record.action_description,
|
||||
'event_description': record.event_description,
|
||||
'recorded_address': record.recorded_address,
|
||||
'import_batch_id': record.import_batch_id,
|
||||
'import_date': record.import_date,
|
||||
'import_source': record.import_source
|
||||
}
|
||||
|
||||
return hash_map
|
||||
except Exception as e:
|
||||
if self.logger:
|
||||
self.logger.logger.warning(f"Failed to get existing records with data: {e}")
|
||||
return {}
|
||||
|
||||
def validate_excel_file(self, file_path: str) -> Dict[str, Any]:
|
||||
"""Enhanced Excel file validation with detailed analysis"""
|
||||
validation_results = {
|
||||
'valid': False,
|
||||
'total_rows': 0,
|
||||
@@ -320,7 +447,6 @@ class TimeAttendanceImportService:
|
||||
}
|
||||
|
||||
try:
|
||||
# Get file information
|
||||
import os
|
||||
file_stats = os.stat(file_path)
|
||||
validation_results['file_info'] = {
|
||||
@@ -328,12 +454,10 @@ class TimeAttendanceImportService:
|
||||
'size_mb': round(file_stats.st_size / (1024 * 1024), 2)
|
||||
}
|
||||
|
||||
# Read Excel file
|
||||
excel_file = pd.ExcelFile(file_path)
|
||||
sheet_name = excel_file.sheet_names[0]
|
||||
df = pd.read_excel(file_path, sheet_name=sheet_name)
|
||||
|
||||
# Remove empty rows
|
||||
original_row_count = len(df)
|
||||
df = df.dropna(how='all')
|
||||
|
||||
@@ -345,11 +469,9 @@ class TimeAttendanceImportService:
|
||||
f"Removed {original_row_count - len(df)} completely empty rows"
|
||||
)
|
||||
|
||||
# Get sample data (first 5 rows)
|
||||
sample_rows = df.head(5).to_dict('records')
|
||||
validation_results['sample_data'] = sample_rows
|
||||
|
||||
# Validate required columns
|
||||
required_columns = ['ID', 'Name', 'Date', 'Time', 'Location Name', 'Action Description']
|
||||
missing_columns = [col for col in required_columns if col not in df.columns]
|
||||
|
||||
@@ -358,7 +480,6 @@ class TimeAttendanceImportService:
|
||||
f"Missing required columns: {', '.join(missing_columns)}"
|
||||
)
|
||||
|
||||
# Check for empty required fields
|
||||
valid_row_count = 0
|
||||
for index, row in df.iterrows():
|
||||
is_valid = True
|
||||
@@ -378,7 +499,6 @@ class TimeAttendanceImportService:
|
||||
f"{validation_results['invalid_rows']} rows have missing required data"
|
||||
)
|
||||
|
||||
# Validate date format
|
||||
if 'Date' in df.columns:
|
||||
invalid_dates = 0
|
||||
for idx, date_val in df['Date'].items():
|
||||
@@ -393,7 +513,6 @@ class TimeAttendanceImportService:
|
||||
f"{invalid_dates} rows have invalid date format"
|
||||
)
|
||||
|
||||
# Validate time format
|
||||
if 'Time' in df.columns:
|
||||
invalid_times = 0
|
||||
for idx, time_val in df['Time'].items():
|
||||
@@ -408,7 +527,6 @@ class TimeAttendanceImportService:
|
||||
f"{invalid_times} rows have invalid time format"
|
||||
)
|
||||
|
||||
# Check for potential duplicates
|
||||
if all(col in df.columns for col in ['ID', 'Date', 'Time', 'Location Name']):
|
||||
duplicate_check = df[['ID', 'Date', 'Time', 'Location Name']].duplicated()
|
||||
duplicate_count = duplicate_check.sum()
|
||||
@@ -418,7 +536,6 @@ class TimeAttendanceImportService:
|
||||
f"{duplicate_count} potential duplicate records detected"
|
||||
)
|
||||
|
||||
# Set valid flag
|
||||
validation_results['valid'] = (
|
||||
len(validation_results['errors']) == 0 and
|
||||
validation_results['valid_rows'] > 0
|
||||
|
||||
Reference in New Issue
Block a user