Updated import functions
This commit is contained in:
@@ -6597,6 +6597,20 @@ def import_time_attendance():
|
||||
else:
|
||||
flash('No duplicates found. Proceeding with import.', 'info')
|
||||
|
||||
# Check for invalid rows and show review if any
|
||||
analyze_invalid = request.form.get('analyze_invalid', 'false').lower() == 'true'
|
||||
|
||||
if analyze_invalid:
|
||||
invalid_analysis = import_service.analyze_for_invalid_rows(temp_path)
|
||||
|
||||
if invalid_analysis['invalid_rows'] > 0:
|
||||
# Show invalid row review page
|
||||
return render_template('time_attendance_invalid_review.html',
|
||||
analysis=invalid_analysis,
|
||||
filename=filename)
|
||||
else:
|
||||
flash('All rows are valid. Proceeding with import.', 'info')
|
||||
|
||||
# Validate file
|
||||
validation_result = import_service.validate_excel_file(temp_path)
|
||||
|
||||
@@ -6741,7 +6755,64 @@ def analyze_import_duplicates():
|
||||
'success': False,
|
||||
'message': f'Analysis failed: {str(e)}'
|
||||
}), 500
|
||||
|
||||
|
||||
@app.route('/time-attendance/import/analyze-invalid', methods=['POST'])
|
||||
@login_required
|
||||
def analyze_import_invalid():
|
||||
"""AJAX endpoint to analyze file for invalid rows"""
|
||||
try:
|
||||
if 'file' not in request.files:
|
||||
return jsonify({'success': False, 'message': 'No file provided'}), 400
|
||||
|
||||
file = request.files['file']
|
||||
if file.filename == '':
|
||||
return jsonify({'success': False, 'message': 'No file selected'}), 400
|
||||
|
||||
if not file.filename.lower().endswith(('.xlsx', '.xls')):
|
||||
return jsonify({'success': False, 'message': 'Invalid file format'}), 400
|
||||
|
||||
# Save temporarily
|
||||
filename = secure_filename(file.filename)
|
||||
temp_path = os.path.join(app.config.get('UPLOAD_FOLDER', '/tmp'),
|
||||
f"analyze_invalid_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{filename}")
|
||||
|
||||
os.makedirs(os.path.dirname(temp_path), exist_ok=True)
|
||||
file.save(temp_path)
|
||||
|
||||
# Store in session
|
||||
session['pending_import_file'] = temp_path
|
||||
session['pending_import_filename'] = filename
|
||||
|
||||
try:
|
||||
import_service = TimeAttendanceImportService(db, logger_handler)
|
||||
analysis = import_service.analyze_for_invalid_rows(temp_path)
|
||||
|
||||
# Convert datetime objects to strings for JSON
|
||||
for invalid in analysis.get('invalid_details', []):
|
||||
if 'row_data' in invalid:
|
||||
if invalid['row_data'].get('attendance_date'):
|
||||
invalid['row_data']['attendance_date'] = str(invalid['row_data']['attendance_date'])
|
||||
if invalid['row_data'].get('attendance_time'):
|
||||
invalid['row_data']['attendance_time'] = str(invalid['row_data']['attendance_time'])
|
||||
|
||||
return jsonify({
|
||||
'success': True,
|
||||
'analysis': analysis
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger_handler.logger.error(f"Invalid row analysis error: {e}")
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'message': f'Analysis failed: {str(e)}'
|
||||
}), 500
|
||||
|
||||
except Exception as e:
|
||||
logger_handler.logger.error(f"Invalid row analysis error: {e}")
|
||||
return jsonify({
|
||||
'success': False,
|
||||
'message': f'Analysis failed: {str(e)}'
|
||||
}), 500
|
||||
|
||||
@app.route('/time-attendance/import/cancel-pending')
|
||||
@login_required
|
||||
|
||||
@@ -377,15 +377,23 @@
|
||||
</h3>
|
||||
<div class="checkbox-group">
|
||||
<div class="checkbox-item">
|
||||
<input type="checkbox" id="analyze_duplicates" name="analyze_duplicates" value="true">
|
||||
<input type="checkbox" id="analyze_duplicates" name="analyze_duplicates" value="true" checked>
|
||||
<label for="analyze_duplicates" class="checkbox-label">
|
||||
<strong>Review Duplicates Before Import</strong>
|
||||
<span>Show potential duplicate records for review and allow selective import</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="checkbox-item">
|
||||
<input type="checkbox" id="analyze_invalid" name="analyze_invalid" value="true" checked>
|
||||
<label for="analyze_invalid" class="checkbox-label">
|
||||
<strong>Review Invalid Rows Before Import</strong>
|
||||
<span>Show all invalid rows with error details for review before importing valid records</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="checkbox-item">
|
||||
<input type="checkbox" id="skip_duplicates" name="skip_duplicates" value="true" checked>
|
||||
<input type="checkbox" id="skip_duplicates" name="skip_duplicates" value="true">
|
||||
<label for="skip_duplicates" class="checkbox-label">
|
||||
<strong>Skip Duplicate Records</strong>
|
||||
<span>Automatically detect and skip records that already exist in the system</span>
|
||||
@@ -393,7 +401,7 @@
|
||||
</div>
|
||||
|
||||
<div class="checkbox-item">
|
||||
<input type="checkbox" id="validate_only" name="validate_only" value="true">
|
||||
<input type="checkbox" id="validate_only" name="validate_only" value="true" checked>
|
||||
<label for="validate_only" class="checkbox-label">
|
||||
<strong>Validate Only (Don't Import)</strong>
|
||||
<span>Check file for errors without actually importing the data</span>
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
{% extends "base_authenticated.html" %}
|
||||
{% block title %}Importing Data - {{ COMPANY_NAME }}{% endblock %}
|
||||
|
||||
{% block extra_head %}
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/time_attendance.css') }}">
|
||||
<style>
|
||||
.import-progress-page {
|
||||
max-width: 800px;
|
||||
margin: 2rem auto;
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
.progress-container {
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 4px 12px rgba(0,0,0,0.1);
|
||||
padding: 3rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.progress-header {
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
.progress-header h1 {
|
||||
color: #2d3748;
|
||||
margin-bottom: 0.5rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.progress-header p {
|
||||
color: #718096;
|
||||
font-size: 1.125rem;
|
||||
}
|
||||
|
||||
.progress-circle {
|
||||
width: 200px;
|
||||
height: 200px;
|
||||
margin: 2rem auto;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.progress-ring {
|
||||
transform: rotate(-90deg);
|
||||
}
|
||||
|
||||
.progress-ring-circle {
|
||||
transition: stroke-dashoffset 0.35s;
|
||||
transform-origin: 50% 50%;
|
||||
}
|
||||
|
||||
.progress-percentage {
|
||||
position: absolute;
|
||||
top: 50%;
|
||||
left: 50%;
|
||||
transform: translate(-50%, -50%);
|
||||
font-size: 3rem;
|
||||
font-weight: 700;
|
||||
color: #667eea;
|
||||
}
|
||||
|
||||
.progress-status {
|
||||
font-size: 1.125rem;
|
||||
color: #4a5568;
|
||||
margin: 2rem 0;
|
||||
min-height: 1.5rem;
|
||||
}
|
||||
|
||||
.progress-details {
|
||||
background: #f7fafc;
|
||||
border-radius: 8px;
|
||||
padding: 1.5rem;
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
.detail-row {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
padding: 0.75rem 0;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.detail-row:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.detail-label {
|
||||
color: #718096;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.detail-value {
|
||||
color: #2d3748;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.completion-message {
|
||||
display: none;
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
.completion-message.show {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.completion-actions {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
justify-content: center;
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
.spinner {
|
||||
display: inline-block;
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
border: 3px solid #e2e8f0;
|
||||
border-top-color: #667eea;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block page_title %}Importing Time Attendance Data{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="import-progress-page">
|
||||
<div class="progress-container">
|
||||
<div class="progress-header">
|
||||
<h1>
|
||||
<i class="fas fa-cloud-upload-alt"></i>
|
||||
Importing Data
|
||||
</h1>
|
||||
<p>File: <strong>{{ filename }}</strong></p>
|
||||
</div>
|
||||
|
||||
<!-- Progress Circle -->
|
||||
<div class="progress-circle">
|
||||
<svg class="progress-ring" width="200" height="200">
|
||||
<circle
|
||||
class="progress-ring-circle-bg"
|
||||
stroke="#e2e8f0"
|
||||
stroke-width="12"
|
||||
fill="transparent"
|
||||
r="90"
|
||||
cx="100"
|
||||
cy="100"
|
||||
/>
|
||||
<circle
|
||||
class="progress-ring-circle"
|
||||
stroke="#667eea"
|
||||
stroke-width="12"
|
||||
fill="transparent"
|
||||
r="90"
|
||||
cx="100"
|
||||
cy="100"
|
||||
stroke-dasharray="565.48"
|
||||
stroke-dashoffset="565.48"
|
||||
stroke-linecap="round"
|
||||
/>
|
||||
</svg>
|
||||
<div class="progress-percentage" id="progressPercentage">0%</div>
|
||||
</div>
|
||||
|
||||
<!-- Status Message -->
|
||||
<div class="progress-status" id="progressStatus">
|
||||
<span class="spinner"></span>
|
||||
Initializing import...
|
||||
</div>
|
||||
|
||||
<!-- Progress Details -->
|
||||
<div class="progress-details">
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Total Records:</span>
|
||||
<span class="detail-value">{{ total_rows }}</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Processed:</span>
|
||||
<span class="detail-value" id="processedCount">0</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Remaining:</span>
|
||||
<span class="detail-value" id="remainingCount">{{ total_rows }}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Completion Message -->
|
||||
<div class="completion-message" id="completionMessage">
|
||||
<h2 style="color: #48bb78; margin-bottom: 1rem;">
|
||||
<i class="fas fa-check-circle"></i>
|
||||
Import Completed Successfully!
|
||||
</h2>
|
||||
<div class="completion-actions">
|
||||
<a href="{{ url_for('time_attendance_dashboard') }}" class="btn btn-primary">
|
||||
<i class="fas fa-chart-line"></i>
|
||||
View Dashboard
|
||||
</a>
|
||||
<a href="{{ url_for('time_attendance_records') }}" class="btn btn-secondary">
|
||||
<i class="fas fa-list"></i>
|
||||
View Records
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
const batchId = "{{ batch_id }}";
|
||||
const totalRows = {{ total_rows }};
|
||||
const eventSource = new EventSource(`/time-attendance/import/progress/${batchId}`);
|
||||
const progressCircle = document.querySelector('.progress-ring-circle');
|
||||
const progressPercentage = document.getElementById('progressPercentage');
|
||||
const progressStatus = document.getElementById('progressStatus');
|
||||
const processedCount = document.getElementById('processedCount');
|
||||
const remainingCount = document.getElementById('remainingCount');
|
||||
const completionMessage = document.getElementById('completionMessage');
|
||||
const circumference = 2 * Math.PI * 90;
|
||||
|
||||
// Initialize circle
|
||||
progressCircle.style.strokeDasharray = `${circumference} ${circumference}`;
|
||||
progressCircle.style.strokeDashoffset = circumference;
|
||||
|
||||
function setProgress(percent) {
|
||||
const offset = circumference - (percent / 100) * circumference;
|
||||
progressCircle.style.strokeDashoffset = offset;
|
||||
}
|
||||
|
||||
// Start the import
|
||||
fetch('/time-attendance/import/execute', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
batch_id: batchId,
|
||||
skip_duplicates: {{ 'true' if skip_duplicates else 'false' }},
|
||||
force_import_hashes: {{ force_import_hashes | tojson }},
|
||||
import_source: "{{ import_source }}"
|
||||
})
|
||||
})
|
||||
.then(response => response.json())
|
||||
.then(data => {
|
||||
console.log('Import execution started:', data);
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Error starting import:', error);
|
||||
progressStatus.innerHTML = '<span style="color: #f56565;">Error starting import. Please try again.</span>';
|
||||
});
|
||||
|
||||
// Listen for progress updates
|
||||
eventSource.onmessage = function(event) {
|
||||
const data = JSON.parse(event.data);
|
||||
|
||||
if (data.completed) {
|
||||
eventSource.close();
|
||||
|
||||
// Show 100% completion
|
||||
setProgress(100);
|
||||
progressPercentage.textContent = '100%';
|
||||
processedCount.textContent = totalRows;
|
||||
remainingCount.textContent = '0';
|
||||
progressStatus.innerHTML = '<i class="fas fa-check-circle" style="color: #48bb78;"></i> Import completed successfully!';
|
||||
|
||||
// Show completion message
|
||||
setTimeout(() => {
|
||||
completionMessage.classList.add('show');
|
||||
}, 500);
|
||||
} else {
|
||||
// Update progress
|
||||
const percentage = data.percentage || 0;
|
||||
const current = data.current || 0;
|
||||
const remaining = totalRows - current;
|
||||
|
||||
setProgress(percentage);
|
||||
progressPercentage.textContent = percentage + '%';
|
||||
processedCount.textContent = current;
|
||||
remainingCount.textContent = remaining;
|
||||
progressStatus.innerHTML = `<span class="spinner"></span> ${data.status || 'Processing...'}`;
|
||||
}
|
||||
};
|
||||
|
||||
eventSource.onerror = function(error) {
|
||||
console.error('EventSource error:', error);
|
||||
eventSource.close();
|
||||
progressStatus.innerHTML = '<span style="color: #f56565;">Connection lost. Checking final status...</span>';
|
||||
|
||||
// Try to check if import completed
|
||||
setTimeout(() => {
|
||||
window.location.href = "{{ url_for('time_attendance_dashboard') }}";
|
||||
}, 2000);
|
||||
};
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -0,0 +1,485 @@
|
||||
{% extends "base_authenticated.html" %}
|
||||
{% block title %}Review Invalid Rows - {{ COMPANY_NAME }}{% endblock %}
|
||||
|
||||
{% block extra_head %}
|
||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/time_attendance.css') }}">
|
||||
<style>
|
||||
.invalid-review-page {
|
||||
max-width: 1400px;
|
||||
margin: 0 auto;
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
.review-summary {
|
||||
background: linear-gradient(135deg, #fc8181 0%, #f56565 100%);
|
||||
color: white;
|
||||
padding: 2rem;
|
||||
border-radius: 12px;
|
||||
margin-bottom: 2rem;
|
||||
box-shadow: 0 8px 24px rgba(0,0,0,0.15);
|
||||
}
|
||||
|
||||
.summary-stats {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 1.5rem;
|
||||
margin-top: 1.5rem;
|
||||
}
|
||||
|
||||
.stat-box {
|
||||
background: rgba(255,255,255,0.2);
|
||||
padding: 1.5rem;
|
||||
border-radius: 8px;
|
||||
text-align: center;
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
|
||||
.stat-box .number {
|
||||
font-size: 2.5rem;
|
||||
font-weight: 700;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.stat-box .label {
|
||||
font-size: 0.875rem;
|
||||
opacity: 0.9;
|
||||
}
|
||||
|
||||
.invalid-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 1.5rem;
|
||||
}
|
||||
|
||||
.invalid-item {
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
overflow: hidden;
|
||||
border: 2px solid #fca5a5;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
|
||||
.invalid-item:hover {
|
||||
box-shadow: 0 4px 16px rgba(252, 165, 165, 0.3);
|
||||
transform: translateY(-2px);
|
||||
}
|
||||
|
||||
.invalid-header {
|
||||
background: #fee2e2;
|
||||
padding: 1rem 1.5rem;
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
border-bottom: 1px solid #fca5a5;
|
||||
}
|
||||
|
||||
.invalid-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.invalid-title h3 {
|
||||
margin: 0;
|
||||
color: #7f1d1d;
|
||||
font-size: 1.125rem;
|
||||
}
|
||||
|
||||
.row-badge {
|
||||
background: #dc2626;
|
||||
color: white;
|
||||
padding: 0.25rem 0.75rem;
|
||||
border-radius: 20px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.error-badge {
|
||||
background: #dc2626;
|
||||
color: white;
|
||||
padding: 0.25rem 0.75rem;
|
||||
border-radius: 20px;
|
||||
font-size: 0.75rem;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.invalid-body {
|
||||
padding: 1.5rem;
|
||||
}
|
||||
|
||||
.error-list {
|
||||
background: #fef2f2;
|
||||
border: 2px solid #fca5a5;
|
||||
border-radius: 8px;
|
||||
padding: 1rem 1.25rem;
|
||||
margin-bottom: 1.5rem;
|
||||
}
|
||||
|
||||
.error-list h4 {
|
||||
margin: 0 0 0.75rem 0;
|
||||
color: #991b1b;
|
||||
font-size: 1rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.error-list ul {
|
||||
margin: 0;
|
||||
padding-left: 0;
|
||||
list-style: none;
|
||||
}
|
||||
|
||||
.error-list li {
|
||||
color: #991b1b;
|
||||
font-weight: 600;
|
||||
margin-bottom: 0.5rem;
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.error-list li:last-child {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.error-list li:before {
|
||||
content: "✖";
|
||||
color: #dc2626;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.record-card {
|
||||
background: #f8fafc;
|
||||
padding: 1.5rem;
|
||||
border-radius: 8px;
|
||||
border: 2px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.record-card h4 {
|
||||
margin: 0 0 1rem 0;
|
||||
color: #991b1b;
|
||||
font-size: 1rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.record-details {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 0.75rem;
|
||||
}
|
||||
|
||||
.detail-row {
|
||||
display: grid;
|
||||
grid-template-columns: 140px 1fr;
|
||||
gap: 0.5rem;
|
||||
font-size: 0.875rem;
|
||||
}
|
||||
|
||||
.detail-label {
|
||||
color: #718096;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.detail-value {
|
||||
color: #2d3748;
|
||||
word-break: break-word;
|
||||
}
|
||||
|
||||
.detail-value.missing {
|
||||
color: #9ca3af;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.action-buttons {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
gap: 1rem;
|
||||
margin-top: 2rem;
|
||||
padding: 1.5rem;
|
||||
background: white;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 2px 8px rgba(0,0,0,0.1);
|
||||
position: sticky;
|
||||
bottom: 1rem;
|
||||
z-index: 100;
|
||||
}
|
||||
|
||||
.action-buttons .left-actions,
|
||||
.action-buttons .right-actions {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.info-message {
|
||||
background: #dbeafe;
|
||||
border: 2px solid #60a5fa;
|
||||
padding: 1.25rem;
|
||||
border-radius: 8px;
|
||||
margin-bottom: 2rem;
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
gap: 1rem;
|
||||
}
|
||||
|
||||
.info-message i {
|
||||
color: #1e40af;
|
||||
font-size: 1.5rem;
|
||||
margin-top: 0.125rem;
|
||||
}
|
||||
|
||||
.info-message-content h4 {
|
||||
margin: 0 0 0.5rem 0;
|
||||
color: #1e3a8a;
|
||||
font-size: 1.125rem;
|
||||
}
|
||||
|
||||
.info-message-content p {
|
||||
margin: 0;
|
||||
color: #1e40af;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
@media (max-width: 768px) {
|
||||
.action-buttons {
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.action-buttons .left-actions,
|
||||
.action-buttons .right-actions {
|
||||
width: 100%;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block page_title %}Review Invalid Rows{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="invalid-review-page">
|
||||
<!-- Page Header -->
|
||||
<div class="time-attendance-header">
|
||||
<div class="header-navigation">
|
||||
<a href="{{ url_for('import_time_attendance') }}" class="back-button">
|
||||
<i class="fas fa-arrow-left"></i>
|
||||
Back to Import
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="header-content">
|
||||
<h1>
|
||||
<i class="fas fa-exclamation-circle"></i>
|
||||
Review Invalid Rows
|
||||
</h1>
|
||||
<p class="header-description">
|
||||
The following rows contain errors and cannot be imported. Please review the issues below.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Summary Section -->
|
||||
<div class="review-summary">
|
||||
<h2 style="margin: 0 0 0.5rem 0;">
|
||||
<i class="fas fa-file-excel"></i>
|
||||
File: {{ filename }}
|
||||
</h2>
|
||||
<p style="margin: 0; opacity: 0.9;">Review invalid rows before importing valid records</p>
|
||||
|
||||
<div class="summary-stats">
|
||||
<div class="stat-box">
|
||||
<div class="number">{{ analysis.total_rows }}</div>
|
||||
<div class="label">Total Rows in File</div>
|
||||
</div>
|
||||
|
||||
<div class="stat-box">
|
||||
<div class="number">{{ analysis.valid_rows }}</div>
|
||||
<div class="label">Valid Rows</div>
|
||||
</div>
|
||||
|
||||
<div class="stat-box">
|
||||
<div class="number">{{ analysis.invalid_rows }}</div>
|
||||
<div class="label">Invalid Rows</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Info Message -->
|
||||
<div class="info-message">
|
||||
<i class="fas fa-info-circle"></i>
|
||||
<div class="info-message-content">
|
||||
<h4>What happens next?</h4>
|
||||
<p>
|
||||
The invalid rows shown below will be skipped during import. Only the <strong>{{ analysis.valid_rows }} valid rows</strong>
|
||||
will be imported into the system. You can proceed with importing the valid records, or cancel and fix the issues in your Excel file.
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Invalid Rows List -->
|
||||
{% if analysis.invalid_rows > 0 %}
|
||||
<form id="invalidReviewForm" method="POST" action="{{ url_for('import_time_attendance') }}">
|
||||
<input type="hidden" name="analyze_invalid" value="false">
|
||||
<input type="hidden" name="skip_duplicates" value="true">
|
||||
<input type="hidden" name="import_source" value="Import (Skipped Invalid Rows) - {{ filename }}">
|
||||
|
||||
<div class="invalid-container">
|
||||
{% for invalid in analysis.invalid_details %}
|
||||
<div class="invalid-item">
|
||||
<div class="invalid-header">
|
||||
<div class="invalid-title">
|
||||
<span class="row-badge">Row {{ invalid.row_number }}</span>
|
||||
<h3>
|
||||
{% if invalid.row_data.employee_name %}
|
||||
{{ invalid.row_data.employee_name }}
|
||||
{% if invalid.row_data.employee_id %}(ID: {{ invalid.row_data.employee_id }}){% endif %}
|
||||
{% else %}
|
||||
Invalid Record
|
||||
{% endif %}
|
||||
</h3>
|
||||
</div>
|
||||
<span class="error-badge">
|
||||
{{ invalid.errors|length }} Error{% if invalid.errors|length != 1 %}s{% endif %}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div class="invalid-body">
|
||||
<!-- Error List -->
|
||||
<div class="error-list">
|
||||
<h4>
|
||||
<i class="fas fa-exclamation-triangle"></i>
|
||||
Validation Errors
|
||||
</h4>
|
||||
<ul>
|
||||
{% for error in invalid.errors %}
|
||||
<li>{{ error }}</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<!-- Row Data -->
|
||||
<div class="record-card">
|
||||
<h4>
|
||||
<i class="fas fa-table"></i>
|
||||
Row Data
|
||||
</h4>
|
||||
<div class="record-details">
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Employee ID:</span>
|
||||
<span class="detail-value {% if not invalid.row_data.employee_id %}missing{% endif %}">
|
||||
{{ invalid.row_data.employee_id or 'Missing' }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Name:</span>
|
||||
<span class="detail-value {% if not invalid.row_data.employee_name %}missing{% endif %}">
|
||||
{{ invalid.row_data.employee_name or 'Missing' }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Date:</span>
|
||||
<span class="detail-value {% if not invalid.row_data.attendance_date %}missing{% endif %}">
|
||||
{{ invalid.row_data.attendance_date or 'Missing' }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Time:</span>
|
||||
<span class="detail-value {% if not invalid.row_data.attendance_time %}missing{% endif %}">
|
||||
{{ invalid.row_data.attendance_time or 'Missing' }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Location:</span>
|
||||
<span class="detail-value {% if not invalid.row_data.location_name %}missing{% endif %}">
|
||||
{{ invalid.row_data.location_name or 'Missing' }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Action:</span>
|
||||
<span class="detail-value {% if not invalid.row_data.action_description %}missing{% endif %}">
|
||||
{{ invalid.row_data.action_description or 'Missing' }}
|
||||
</span>
|
||||
</div>
|
||||
{% if invalid.row_data.platform %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Platform:</span>
|
||||
<span class="detail-value">{{ invalid.row_data.platform }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if invalid.row_data.event_description %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Event:</span>
|
||||
<span class="detail-value">{{ invalid.row_data.event_description }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
{% if invalid.row_data.recorded_address %}
|
||||
<div class="detail-row">
|
||||
<span class="detail-label">Address:</span>
|
||||
<span class="detail-value">{{ invalid.row_data.recorded_address }}</span>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</div>
|
||||
|
||||
<!-- Action Buttons -->
|
||||
<div class="action-buttons">
|
||||
<div class="left-actions">
|
||||
<a href="{{ url_for('cancel_pending_import') }}" class="btn btn-secondary">
|
||||
<i class="fas fa-times"></i>
|
||||
Cancel Import
|
||||
</a>
|
||||
</div>
|
||||
<div class="right-actions">
|
||||
<button type="submit" class="btn btn-primary" id="proceedBtn">
|
||||
<i class="fas fa-check-circle"></i>
|
||||
Import {{ analysis.valid_rows }} Valid Records
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</form>
|
||||
{% else %}
|
||||
<div class="empty-state">
|
||||
<i class="fas fa-check-circle"></i>
|
||||
<h3>All rows are valid!</h3>
|
||||
<p>No invalid rows found. You can proceed with the import.</p>
|
||||
<a href="{{ url_for('import_time_attendance') }}" class="btn btn-primary" style="margin-top: 1rem;">
|
||||
<i class="fas fa-arrow-left"></i>
|
||||
Back to Import
|
||||
</a>
|
||||
</div>
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
<script>
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
const form = document.getElementById('invalidReviewForm');
|
||||
const proceedBtn = document.getElementById('proceedBtn');
|
||||
|
||||
if (form && proceedBtn) {
|
||||
proceedBtn.addEventListener('click', function(e) {
|
||||
const validCount = {{ analysis.valid_rows }};
|
||||
const invalidCount = {{ analysis.invalid_rows }};
|
||||
|
||||
const message = `You are about to import ${validCount} valid record(s).\n\n` +
|
||||
`${invalidCount} invalid row(s) will be skipped.\n\n` +
|
||||
`Continue with import?`;
|
||||
|
||||
if (!confirm(message)) {
|
||||
e.preventDefault();
|
||||
return;
|
||||
}
|
||||
|
||||
// Show loading state
|
||||
proceedBtn.disabled = true;
|
||||
proceedBtn.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Processing Import...';
|
||||
});
|
||||
}
|
||||
});
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -21,6 +21,24 @@ class TimeAttendanceImportService:
|
||||
self.db = db
|
||||
self.logger = logger_handler
|
||||
|
||||
def _update_progress(self, current: int, total: int, status: str = "Processing"):
|
||||
"""
|
||||
Update progress information for real-time tracking
|
||||
|
||||
Args:
|
||||
current: Current record number being processed
|
||||
total: Total number of records
|
||||
status: Status message
|
||||
"""
|
||||
if hasattr(self, 'progress_callback') and self.progress_callback:
|
||||
percentage = int((current / total) * 100) if total > 0 else 0
|
||||
self.progress_callback({
|
||||
'current': current,
|
||||
'total': total,
|
||||
'percentage': percentage,
|
||||
'status': status
|
||||
})
|
||||
|
||||
def _parse_excel_hyperlink(self, cell_value: str) -> str:
|
||||
"""
|
||||
Parse Excel HYPERLINK formula to extract the display text (address)
|
||||
@@ -139,7 +157,16 @@ class TimeAttendanceImportService:
|
||||
duplicates_list = []
|
||||
new_records_count = 0
|
||||
|
||||
# Process each row with enhanced validation
|
||||
for index, row in df.iterrows():
|
||||
# Update progress every 10 records or on last record
|
||||
if (index + 1) % 10 == 0 or (index + 1) == len(df):
|
||||
self._update_progress(
|
||||
index + 1,
|
||||
len(df),
|
||||
f"Processing row {index + 2} of {len(df) + 1}"
|
||||
)
|
||||
|
||||
try:
|
||||
# Skip empty rows
|
||||
if pd.isna(row['ID']) or pd.isna(row['Name']):
|
||||
@@ -200,6 +227,142 @@ class TimeAttendanceImportService:
|
||||
|
||||
return analysis_result
|
||||
|
||||
def analyze_for_invalid_rows(self, file_path: str) -> Dict[str, Any]:
|
||||
"""
|
||||
Analyze file for invalid rows with detailed error information
|
||||
|
||||
Args:
|
||||
file_path: Path to the Excel file
|
||||
|
||||
Returns:
|
||||
Dictionary containing invalid row analysis
|
||||
"""
|
||||
analysis_result = {
|
||||
'success': False,
|
||||
'total_rows': 0,
|
||||
'valid_rows': 0,
|
||||
'invalid_rows': 0,
|
||||
'invalid_details': [],
|
||||
'errors': []
|
||||
}
|
||||
|
||||
try:
|
||||
# Read Excel file
|
||||
excel_file = pd.ExcelFile(file_path)
|
||||
sheet_name = excel_file.sheet_names[0]
|
||||
df = pd.read_excel(file_path, sheet_name=sheet_name)
|
||||
|
||||
# Validate required columns
|
||||
required_columns = ['ID', 'Name', 'Date', 'Time', 'Location Name', 'Action Description']
|
||||
missing_columns = [col for col in required_columns if col not in df.columns]
|
||||
|
||||
if missing_columns:
|
||||
analysis_result['errors'].append(f"Missing columns: {', '.join(missing_columns)}")
|
||||
return analysis_result
|
||||
|
||||
# Remove empty rows
|
||||
df = df.dropna(how='all')
|
||||
analysis_result['total_rows'] = len(df)
|
||||
|
||||
# Process each row and collect invalid ones
|
||||
invalid_list = []
|
||||
valid_count = 0
|
||||
|
||||
for index, row in df.iterrows():
|
||||
row_errors = []
|
||||
row_data = {
|
||||
'employee_id': None,
|
||||
'employee_name': None,
|
||||
'platform': None,
|
||||
'attendance_date': None,
|
||||
'attendance_time': None,
|
||||
'location_name': None,
|
||||
'action_description': None,
|
||||
'event_description': None,
|
||||
'recorded_address': None
|
||||
}
|
||||
|
||||
# Check ID
|
||||
if pd.isna(row['ID']):
|
||||
row_errors.append("Missing Employee ID")
|
||||
else:
|
||||
row_data['employee_id'] = str(row['ID']).strip()
|
||||
|
||||
# Check Name
|
||||
if pd.isna(row['Name']):
|
||||
row_errors.append("Missing Employee Name")
|
||||
else:
|
||||
row_data['employee_name'] = str(row['Name']).strip()
|
||||
|
||||
# Check and parse Date
|
||||
if pd.isna(row['Date']):
|
||||
row_errors.append("Missing Date")
|
||||
else:
|
||||
try:
|
||||
attendance_date = pd.to_datetime(row['Date']).date()
|
||||
row_data['attendance_date'] = attendance_date
|
||||
except Exception:
|
||||
row_errors.append(f"Invalid date format: {row['Date']}")
|
||||
|
||||
# Check and parse Time
|
||||
if pd.isna(row['Time']):
|
||||
row_errors.append("Missing Time")
|
||||
else:
|
||||
try:
|
||||
attendance_time = self._parse_time_field(row['Time'])
|
||||
row_data['attendance_time'] = attendance_time
|
||||
except Exception as e:
|
||||
row_errors.append(f"Invalid time format: {row['Time']}")
|
||||
|
||||
# Check Location Name
|
||||
if pd.isna(row['Location Name']):
|
||||
row_errors.append("Missing Location Name")
|
||||
else:
|
||||
row_data['location_name'] = str(row['Location Name']).strip()
|
||||
|
||||
# Check Action Description
|
||||
if pd.isna(row['Action Description']):
|
||||
row_errors.append("Missing Action Description")
|
||||
else:
|
||||
row_data['action_description'] = str(row['Action Description']).strip()
|
||||
|
||||
# Optional fields
|
||||
if pd.notna(row.get('Platform')):
|
||||
row_data['platform'] = str(row['Platform']).strip()
|
||||
|
||||
if pd.notna(row.get('Event Description')):
|
||||
row_data['event_description'] = str(row['Event Description']).strip()
|
||||
|
||||
row_data['recorded_address'] = self._process_recorded_address(row)
|
||||
|
||||
# If row has errors, add to invalid list
|
||||
if row_errors:
|
||||
invalid_list.append({
|
||||
'row_number': index + 2, # +2 for header and 0-based index
|
||||
'row_data': row_data,
|
||||
'errors': row_errors
|
||||
})
|
||||
else:
|
||||
valid_count += 1
|
||||
|
||||
analysis_result['success'] = True
|
||||
analysis_result['valid_rows'] = valid_count
|
||||
analysis_result['invalid_rows'] = len(invalid_list)
|
||||
analysis_result['invalid_details'] = invalid_list
|
||||
|
||||
if self.logger:
|
||||
self.logger.logger.info(
|
||||
f"Invalid row analysis complete - Total: {analysis_result['total_rows']}, "
|
||||
f"Valid: {valid_count}, Invalid: {len(invalid_list)}"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
analysis_result['errors'].append(f"Analysis failed: {str(e)}")
|
||||
if self.logger:
|
||||
self.logger.logger.error(f"Invalid row analysis error: {e}")
|
||||
|
||||
return analysis_result
|
||||
|
||||
def import_from_excel(self, file_path: str, created_by: int = None,
|
||||
import_source: str = None, skip_duplicates: bool = True,
|
||||
force_import_hashes: List[str] = None) -> Dict[str, Any]:
|
||||
@@ -546,7 +709,7 @@ class TimeAttendanceImportService:
|
||||
sample_rows = df.head(5).to_dict('records')
|
||||
validation_results['sample_data'] = sample_rows
|
||||
|
||||
required_columns = ['ID', 'Name', 'Date', 'Time', 'Location Name', 'Action Description']
|
||||
required_columns = ['ID', 'Date', 'Time', 'Location Name', 'Action Description', 'Event Description', 'Recorded Address']
|
||||
missing_columns = [col for col in required_columns if col not in df.columns]
|
||||
|
||||
if missing_columns:
|
||||
|
||||
Reference in New Issue
Block a user