Feb25 2026: updated import progress bar and fixed big file import fail
This commit is contained in:
@@ -8716,6 +8716,244 @@ def analyze_import_invalid():
|
||||
'message': f'Analysis failed: {str(e)}'
|
||||
}), 500
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Time Attendance Import — SSE progress streaming (disk-based, multi-worker safe)
|
||||
#
|
||||
# Design: progress state is written to a small JSON file on disk so that any
|
||||
# gunicorn worker process can read it. No shared in-memory state is required.
|
||||
# The /stream endpoint runs the import itself (synchronously inside the SSE
|
||||
# generator) while writing progress to the file and yielding events to the
|
||||
# browser — compatible with gunicorn gevent workers.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _progress_file_path(job_id: str) -> str:
|
||||
"""Return the path for the on-disk progress file for a given job_id."""
|
||||
upload_dir = app.config.get('UPLOAD_FOLDER', '/tmp')
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
return os.path.join(upload_dir, f"import_progress_{job_id}.json")
|
||||
|
||||
|
||||
def _write_progress(job_id: str, event: dict) -> None:
|
||||
"""Atomically write the latest progress event to disk."""
|
||||
path = _progress_file_path(job_id)
|
||||
try:
|
||||
tmp = path + '.tmp'
|
||||
with open(tmp, 'w') as f:
|
||||
json.dump(event, f)
|
||||
os.replace(tmp, path)
|
||||
except Exception:
|
||||
pass # Best-effort; import will continue regardless
|
||||
|
||||
|
||||
@app.route('/time-attendance/import/start', methods=['POST'])
|
||||
@login_required
|
||||
def start_import_job():
|
||||
"""
|
||||
Validates the uploaded file, saves it to disk, stores import options in a
|
||||
progress file, then returns a job_id. The actual import runs inside the
|
||||
SSE stream endpoint so no background thread or shared memory is needed.
|
||||
"""
|
||||
try:
|
||||
if 'files' not in request.files:
|
||||
return jsonify({'success': False, 'error': 'No file uploaded.'}), 400
|
||||
|
||||
files = request.files.getlist('files')
|
||||
if not files or files[0].filename == '':
|
||||
return jsonify({'success': False, 'error': 'No file selected.'}), 400
|
||||
|
||||
file = files[0]
|
||||
if not file.filename.lower().endswith(('.xlsx', '.xls')):
|
||||
return jsonify({'success': False, 'error': 'Invalid file format.'}), 400
|
||||
|
||||
filename = secure_filename(file.filename)
|
||||
upload_dir = app.config.get('UPLOAD_FOLDER', '/tmp')
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
job_id = str(uuid.uuid4())
|
||||
temp_path = os.path.join(upload_dir,
|
||||
f"stream_{job_id}_{filename}")
|
||||
file.save(temp_path)
|
||||
|
||||
# Store import options alongside the file so the stream endpoint can
|
||||
# read them without depending on session or shared memory.
|
||||
job_meta = {
|
||||
'type': 'pending',
|
||||
'temp_path': temp_path,
|
||||
'filename': filename,
|
||||
'skip_duplicates': request.form.get('skip_duplicates', 'true').lower() == 'true',
|
||||
'project_id': int(request.form.get('project_id')) if request.form.get('project_id') else None,
|
||||
'import_source': request.form.get('import_source', f"Manual Import - {filename}"),
|
||||
'created_by': session['user_id'],
|
||||
'username': session.get('username', 'unknown'),
|
||||
}
|
||||
_write_progress(job_id, job_meta)
|
||||
|
||||
logger_handler.logger.info(
|
||||
f"User {job_meta['username']} queued time attendance import job {job_id} for file {filename}"
|
||||
)
|
||||
return jsonify({'success': True, 'job_id': job_id})
|
||||
|
||||
except Exception as e:
|
||||
logger_handler.logger.error(f"Error queuing import job: {e}")
|
||||
return jsonify({'success': False, 'error': str(e)}), 500
|
||||
|
||||
|
||||
@app.route('/time-attendance/import/stream/<job_id>')
|
||||
@login_required
|
||||
def stream_import_progress(job_id):
|
||||
"""
|
||||
SSE endpoint — runs the import synchronously while streaming progress to
|
||||
the browser. Works across multiple gunicorn workers because all state is
|
||||
stored on disk (no in-memory job store).
|
||||
"""
|
||||
progress_path = _progress_file_path(job_id)
|
||||
|
||||
def generate():
|
||||
import time as _time
|
||||
|
||||
# ── Read the job metadata written by /start ────────────────────────
|
||||
deadline = _time.time() + 15 # Wait up to 15 s for the file to appear
|
||||
meta = None
|
||||
while _time.time() < deadline:
|
||||
if os.path.exists(progress_path):
|
||||
try:
|
||||
with open(progress_path) as f:
|
||||
meta = json.load(f)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
yield "data: " + json.dumps({'type': 'heartbeat'}) + "\n\n"
|
||||
_time.sleep(0.3)
|
||||
|
||||
if not meta or meta.get('type') != 'pending':
|
||||
yield "data: " + json.dumps({
|
||||
'type': 'error',
|
||||
'message': 'Job metadata not found. Please try importing again.'
|
||||
}) + "\n\n"
|
||||
return
|
||||
|
||||
temp_path = meta['temp_path']
|
||||
skip_dupes = meta['skip_duplicates']
|
||||
project_id = meta['project_id']
|
||||
import_source= meta['import_source']
|
||||
created_by = meta['created_by']
|
||||
username = meta['username']
|
||||
|
||||
if not os.path.exists(temp_path):
|
||||
yield "data: " + json.dumps({
|
||||
'type': 'error',
|
||||
'message': 'Uploaded file not found. Please try importing again.'
|
||||
}) + "\n\n"
|
||||
return
|
||||
|
||||
yield "data: " + json.dumps({'type': 'status', 'message': 'Reading and validating file...'}) + "\n\n"
|
||||
|
||||
# ── Run the import with a progress callback ────────────────────────
|
||||
try:
|
||||
svc = TimeAttendanceImportService(db, logger_handler)
|
||||
|
||||
# progress_callback writes to disk AND yields an SSE event.
|
||||
# We collect events in a list so the generator can yield them.
|
||||
_pending_events = []
|
||||
|
||||
def on_progress(current, total, message):
|
||||
pct = int(current / total * 100) if total else 0
|
||||
event = {
|
||||
'type': 'progress',
|
||||
'current': current,
|
||||
'total': total,
|
||||
'percent': pct,
|
||||
'message': message,
|
||||
}
|
||||
_write_progress(job_id, event)
|
||||
_pending_events.append(event)
|
||||
|
||||
# We need to interleave yielding with the synchronous import loop.
|
||||
# Strategy: run import_from_excel; the callback appends to
|
||||
# _pending_events; after every DB commit batch (50 records) we
|
||||
# flush pending events to the SSE stream.
|
||||
import threading as _threading
|
||||
result_holder = [None]
|
||||
error_holder = [None]
|
||||
done_event = _threading.Event()
|
||||
|
||||
def _run():
|
||||
# Push an application context so the thread can access
|
||||
# Flask-SQLAlchemy, Employee.query, etc.
|
||||
with app.app_context():
|
||||
try:
|
||||
result_holder[0] = svc.import_from_excel(
|
||||
temp_path,
|
||||
created_by=created_by,
|
||||
import_source=import_source,
|
||||
skip_duplicates=skip_dupes,
|
||||
force_import_hashes=[],
|
||||
project_id=project_id,
|
||||
progress_callback=on_progress,
|
||||
)
|
||||
except Exception as exc:
|
||||
error_holder[0] = exc
|
||||
finally:
|
||||
done_event.set()
|
||||
|
||||
t = _threading.Thread(target=_run, daemon=True)
|
||||
t.start()
|
||||
|
||||
# Yield progress events as they arrive while the import thread runs
|
||||
while not done_event.is_set():
|
||||
while _pending_events:
|
||||
yield "data: " + json.dumps(_pending_events.pop(0)) + "\n\n"
|
||||
yield "data: " + json.dumps({'type': 'heartbeat'}) + "\n\n"
|
||||
_time.sleep(0.4)
|
||||
|
||||
# Drain any remaining events after the thread finishes
|
||||
while _pending_events:
|
||||
yield "data: " + json.dumps(_pending_events.pop(0)) + "\n\n"
|
||||
|
||||
if error_holder[0]:
|
||||
raise error_holder[0]
|
||||
|
||||
result = result_holder[0]
|
||||
|
||||
if result and result['success']:
|
||||
logger_handler.logger.info(
|
||||
f"User {username} imported {result['imported_records']} time attendance records "
|
||||
f"via stream (batch: {result['batch_id']})"
|
||||
)
|
||||
|
||||
# Sanitize result dict for JSON serialization — convert any
|
||||
# datetime objects (e.g. import_date) to ISO-format strings.
|
||||
if result and isinstance(result.get('import_date'), datetime):
|
||||
result['import_date'] = result['import_date'].isoformat()
|
||||
done_event_data = {'type': 'done', 'result': result}
|
||||
_write_progress(job_id, done_event_data)
|
||||
yield "data: " + json.dumps(done_event_data) + "\n\n"
|
||||
|
||||
except Exception as e:
|
||||
logger_handler.logger.error(f"Import stream error for job {job_id}: {e}")
|
||||
error_event = {'type': 'error', 'message': str(e)}
|
||||
_write_progress(job_id, error_event)
|
||||
yield "data: " + json.dumps(error_event) + "\n\n"
|
||||
|
||||
finally:
|
||||
# Clean up temp files
|
||||
for path in (temp_path, progress_path):
|
||||
try:
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return Response(
|
||||
generate(),
|
||||
mimetype='text/event-stream',
|
||||
headers={
|
||||
'Cache-Control': 'no-cache',
|
||||
'X-Accel-Buffering': 'no', # Disable nginx buffering for SSE
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@app.route('/time-attendance/import/cancel-pending')
|
||||
@login_required
|
||||
def cancel_pending_import():
|
||||
|
||||
@@ -96,26 +96,107 @@
|
||||
|
||||
.progress-indicator {
|
||||
display: none;
|
||||
margin-top: 1rem;
|
||||
padding: 1rem;
|
||||
margin-top: 1.25rem;
|
||||
padding: 1.25rem 1.5rem;
|
||||
background: #ebf8ff;
|
||||
border: 1px solid #90cdf4;
|
||||
border-radius: 8px;
|
||||
text-align: center;
|
||||
border-radius: 10px;
|
||||
}
|
||||
|
||||
.progress-indicator.active {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.progress-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.6rem;
|
||||
margin-bottom: 0.75rem;
|
||||
font-weight: 600;
|
||||
color: #2b6cb0;
|
||||
}
|
||||
|
||||
.progress-bar-track {
|
||||
background: #bee3f8;
|
||||
border-radius: 999px;
|
||||
height: 18px;
|
||||
overflow: hidden;
|
||||
margin-bottom: 0.5rem;
|
||||
box-shadow: inset 0 1px 3px rgba(0,0,0,0.1);
|
||||
}
|
||||
|
||||
.progress-bar-fill {
|
||||
height: 100%;
|
||||
width: 0%;
|
||||
background: linear-gradient(90deg, #4299e1, #3182ce);
|
||||
border-radius: 999px;
|
||||
transition: width 0.3s ease;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: flex-end;
|
||||
padding-right: 6px;
|
||||
}
|
||||
|
||||
.progress-bar-fill .pct-label {
|
||||
font-size: 0.7rem;
|
||||
font-weight: 700;
|
||||
color: white;
|
||||
white-space: nowrap;
|
||||
opacity: 0;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
|
||||
.progress-bar-fill.has-width .pct-label {
|
||||
opacity: 1;
|
||||
}
|
||||
|
||||
.progress-counters {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
font-size: 0.8rem;
|
||||
color: #4a5568;
|
||||
}
|
||||
|
||||
.progress-message {
|
||||
margin-top: 0.4rem;
|
||||
font-size: 0.8rem;
|
||||
color: #718096;
|
||||
font-style: italic;
|
||||
min-height: 1.1em;
|
||||
}
|
||||
|
||||
.progress-done-message {
|
||||
display: none;
|
||||
margin-top: 0.75rem;
|
||||
padding: 0.75rem 1rem;
|
||||
border-radius: 8px;
|
||||
font-weight: 600;
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.progress-done-message.success {
|
||||
background: #f0fff4;
|
||||
border: 1px solid #68d391;
|
||||
color: #276749;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.progress-done-message.error {
|
||||
background: #fff5f5;
|
||||
border: 1px solid #fc8181;
|
||||
color: #c53030;
|
||||
display: block;
|
||||
}
|
||||
|
||||
.spinner {
|
||||
display: inline-block;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border: 3px solid #e2e8f0;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border: 2.5px solid #bee3f8;
|
||||
border-top-color: #4299e1;
|
||||
border-radius: 50%;
|
||||
animation: spin 0.8s linear infinite;
|
||||
flex-shrink: 0;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
@@ -472,8 +553,21 @@
|
||||
|
||||
<!-- Progress Indicator -->
|
||||
<div id="progressIndicator" class="progress-indicator">
|
||||
<div class="spinner"></div>
|
||||
<p style="margin-top: 0.5rem;">Processing your file, please wait...</p>
|
||||
<div class="progress-header">
|
||||
<div class="spinner" id="progressSpinner"></div>
|
||||
<span id="progressTitle">Uploading & preparing import...</span>
|
||||
</div>
|
||||
<div class="progress-bar-track">
|
||||
<div class="progress-bar-fill" id="progressBarFill">
|
||||
<span class="pct-label" id="progressPctLabel">0%</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="progress-counters">
|
||||
<span id="progressCounter">0 of 0 records processed</span>
|
||||
<span id="progressPct">0%</span>
|
||||
</div>
|
||||
<div class="progress-message" id="progressMessage">Starting...</div>
|
||||
<div class="progress-done-message" id="progressDoneMsg"></div>
|
||||
</div>
|
||||
|
||||
<!-- Form Actions -->
|
||||
@@ -687,118 +781,210 @@
|
||||
</div>
|
||||
|
||||
<script>
|
||||
// File upload handling
|
||||
const fileInput = document.getElementById('fileInput');
|
||||
const fileUploadArea = document.getElementById('fileUploadArea');
|
||||
const fileInfo = document.getElementById('fileInfo');
|
||||
const submitBtn = document.getElementById('submitBtn');
|
||||
const importForm = document.getElementById('importForm');
|
||||
const progressIndicator = document.getElementById('progressIndicator');
|
||||
// ─── DOM references ────────────────────────────────────────────────────────
|
||||
const fileInput = document.getElementById('fileInput');
|
||||
const fileUploadArea = document.getElementById('fileUploadArea');
|
||||
const fileInfo = document.getElementById('fileInfo');
|
||||
const submitBtn = document.getElementById('submitBtn');
|
||||
const importForm = document.getElementById('importForm');
|
||||
const progressIndicator= document.getElementById('progressIndicator');
|
||||
const progressBarFill = document.getElementById('progressBarFill');
|
||||
const progressPctLabel = document.getElementById('progressPctLabel');
|
||||
const progressCounter = document.getElementById('progressCounter');
|
||||
const progressPct = document.getElementById('progressPct');
|
||||
const progressMessage = document.getElementById('progressMessage');
|
||||
const progressTitle = document.getElementById('progressTitle');
|
||||
const progressDoneMsg = document.getElementById('progressDoneMsg');
|
||||
|
||||
// Drag and drop handlers
|
||||
// ─── Drag & drop ───────────────────────────────────────────────────────────
|
||||
fileUploadArea.addEventListener('dragover', (e) => {
|
||||
e.preventDefault();
|
||||
fileUploadArea.classList.add('dragover');
|
||||
});
|
||||
|
||||
fileUploadArea.addEventListener('dragleave', () => {
|
||||
fileUploadArea.classList.remove('dragover');
|
||||
});
|
||||
|
||||
fileUploadArea.addEventListener('drop', (e) => {
|
||||
e.preventDefault();
|
||||
fileUploadArea.classList.remove('dragover');
|
||||
|
||||
const files = e.dataTransfer.files;
|
||||
if (files.length > 0) {
|
||||
fileInput.files = files;
|
||||
handleFileSelect();
|
||||
}
|
||||
if (files.length > 0) { fileInput.files = files; handleFileSelect(); }
|
||||
});
|
||||
|
||||
fileUploadArea.addEventListener('click', () => {
|
||||
fileInput.click();
|
||||
});
|
||||
|
||||
fileUploadArea.addEventListener('click', () => fileInput.click());
|
||||
fileInput.addEventListener('change', handleFileSelect);
|
||||
|
||||
function handleFileSelect() {
|
||||
const files = fileInput.files;
|
||||
if (files.length > 0) {
|
||||
// Validate all files
|
||||
let invalidFiles = [];
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const fileName = files[i].name;
|
||||
if (!fileName.toLowerCase().endsWith('.xlsx') && !fileName.toLowerCase().endsWith('.xls')) {
|
||||
invalidFiles.push(fileName);
|
||||
}
|
||||
}
|
||||
if (!files.length) return;
|
||||
|
||||
if (invalidFiles.length > 0) {
|
||||
alert('Invalid file format detected:\n' + invalidFiles.join('\n') + '\n\nPlease select only Excel files (.xlsx or .xls)');
|
||||
resetForm();
|
||||
return;
|
||||
}
|
||||
|
||||
// Calculate total size
|
||||
let totalSize = 0;
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
totalSize += files[i].size;
|
||||
}
|
||||
|
||||
// Display file info
|
||||
if (files.length === 1) {
|
||||
fileInfo.querySelector('.file-name').textContent = files[0].name;
|
||||
fileInfo.querySelector('.file-size').textContent = (totalSize / 1024 / 1024).toFixed(2) + ' MB';
|
||||
fileInfo.querySelector('.file-count').textContent = '';
|
||||
} else {
|
||||
fileInfo.querySelector('.file-name').textContent = files.length + ' files selected';
|
||||
fileInfo.querySelector('.file-size').textContent = (totalSize / 1024 / 1024).toFixed(2) + ' MB total';
|
||||
const fileNames = Array.from(files).map(f => f.name).join(', ');
|
||||
fileInfo.querySelector('.file-count').textContent = '(' + fileNames + ')';
|
||||
fileInfo.querySelector('.file-count').style.fontSize = '0.75rem';
|
||||
fileInfo.querySelector('.file-count').style.color = '#718096';
|
||||
fileInfo.querySelector('.file-count').style.display = 'block';
|
||||
fileInfo.querySelector('.file-count').style.marginTop = '0.5rem';
|
||||
}
|
||||
|
||||
fileInfo.style.display = 'flex';
|
||||
fileInfo.style.flexDirection = 'column';
|
||||
submitBtn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
// Form submission
|
||||
importForm.addEventListener('submit', (e) => {
|
||||
if (fileInput.files.length === 0) {
|
||||
e.preventDefault();
|
||||
alert('Please select at least one file to import');
|
||||
const invalid = Array.from(files).filter(
|
||||
f => !f.name.toLowerCase().endsWith('.xlsx') && !f.name.toLowerCase().endsWith('.xls')
|
||||
);
|
||||
if (invalid.length) {
|
||||
alert('Invalid file format:\n' + invalid.map(f => f.name).join('\n') +
|
||||
'\n\nPlease select only Excel files (.xlsx or .xls)');
|
||||
resetForm();
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate project selection
|
||||
const totalSize = Array.from(files).reduce((s, f) => s + f.size, 0);
|
||||
if (files.length === 1) {
|
||||
fileInfo.querySelector('.file-name').textContent = files[0].name;
|
||||
fileInfo.querySelector('.file-size').textContent = (totalSize / 1024 / 1024).toFixed(2) + ' MB';
|
||||
fileInfo.querySelector('.file-count').textContent = '';
|
||||
} else {
|
||||
fileInfo.querySelector('.file-name').textContent = files.length + ' files selected';
|
||||
fileInfo.querySelector('.file-size').textContent = (totalSize / 1024 / 1024).toFixed(2) + ' MB total';
|
||||
const names = Array.from(files).map(f => f.name).join(', ');
|
||||
const countEl = fileInfo.querySelector('.file-count');
|
||||
countEl.textContent = '(' + names + ')';
|
||||
Object.assign(countEl.style, { fontSize: '0.75rem', color: '#718096', display: 'block', marginTop: '0.5rem' });
|
||||
}
|
||||
fileInfo.style.display = 'flex';
|
||||
fileInfo.style.flexDirection = 'column';
|
||||
submitBtn.disabled = false;
|
||||
}
|
||||
|
||||
// ─── Progress bar helpers ──────────────────────────────────────────────────
|
||||
function setProgress(current, total, message) {
|
||||
const pct = total > 0 ? Math.min(100, Math.round(current / total * 100)) : 0;
|
||||
progressBarFill.style.width = pct + '%';
|
||||
if (pct > 8) {
|
||||
progressBarFill.classList.add('has-width');
|
||||
progressPctLabel.textContent = pct + '%';
|
||||
}
|
||||
progressPct.textContent = pct + '%';
|
||||
progressCounter.textContent = current + ' of ' + total + ' records processed';
|
||||
if (message) progressMessage.textContent = message;
|
||||
}
|
||||
|
||||
function showDone(success, message) {
|
||||
document.getElementById('progressSpinner').style.display = 'none';
|
||||
progressDoneMsg.className = 'progress-done-message ' + (success ? 'success' : 'error');
|
||||
progressDoneMsg.textContent = message;
|
||||
progressTitle.textContent = success ? 'Import complete!' : 'Import failed';
|
||||
}
|
||||
|
||||
// ─── Form submit → SSE import flow ────────────────────────────────────────
|
||||
importForm.addEventListener('submit', async (e) => {
|
||||
e.preventDefault(); // Always intercept; we use fetch + SSE instead
|
||||
|
||||
if (!fileInput.files.length) {
|
||||
alert('Please select at least one file to import.');
|
||||
return;
|
||||
}
|
||||
const projectSelect = document.getElementById('project_id');
|
||||
if (!projectSelect.value || projectSelect.value === '') {
|
||||
e.preventDefault();
|
||||
alert('Please select a project. Project selection is required for attendance imports.');
|
||||
if (!projectSelect.value) {
|
||||
alert('Please select a project. Project selection is required.');
|
||||
projectSelect.focus();
|
||||
projectSelect.style.borderColor = '#ef4444';
|
||||
return;
|
||||
}
|
||||
|
||||
// Show progress indicator
|
||||
// Show progress UI
|
||||
progressIndicator.classList.add('active');
|
||||
progressDoneMsg.className = 'progress-done-message'; // hide
|
||||
progressDoneMsg.textContent = '';
|
||||
setProgress(0, 0, 'Uploading file...');
|
||||
progressTitle.textContent = 'Uploading & preparing import...';
|
||||
document.getElementById('progressSpinner').style.display = '';
|
||||
submitBtn.disabled = true;
|
||||
const fileCount = fileInput.files.length;
|
||||
const fileText = fileCount === 1 ? 'file' : 'files';
|
||||
submitBtn.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Processing ' + fileCount + ' ' + fileText + '...';
|
||||
submitBtn.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Importing...';
|
||||
|
||||
// Build FormData from the existing form (includes all checkboxes, selects, etc.)
|
||||
const formData = new FormData(importForm);
|
||||
|
||||
// ── Step 1: POST the file to /start → get job_id ──────────────────────
|
||||
let jobId;
|
||||
try {
|
||||
const resp = await fetch('{{ url_for("start_import_job") }}', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (!data.success) {
|
||||
showDone(false, 'Upload failed: ' + (data.error || 'Unknown error'));
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
|
||||
return;
|
||||
}
|
||||
jobId = data.job_id;
|
||||
} catch (err) {
|
||||
showDone(false, 'Network error during upload: ' + err.message);
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
|
||||
return;
|
||||
}
|
||||
|
||||
progressMessage.textContent = 'File received. Starting import...';
|
||||
|
||||
// ── Step 2: Open SSE stream for this job ──────────────────────────────
|
||||
const streamUrl = '{{ url_for("stream_import_progress", job_id="__JOB_ID__") }}'.replace('__JOB_ID__', jobId);
|
||||
const evtSource = new EventSource(streamUrl);
|
||||
|
||||
evtSource.onmessage = (event) => {
|
||||
let msg;
|
||||
try { msg = JSON.parse(event.data); } catch { return; }
|
||||
|
||||
if (msg.type === 'status') {
|
||||
progressMessage.textContent = msg.message;
|
||||
progressTitle.textContent = 'Importing records...';
|
||||
|
||||
} else if (msg.type === 'progress') {
|
||||
setProgress(msg.current, msg.total, msg.message);
|
||||
if (msg.percent >= 5) {
|
||||
progressTitle.textContent = 'Importing records — ' + msg.percent + '% complete';
|
||||
}
|
||||
|
||||
} else if (msg.type === 'done') {
|
||||
evtSource.close();
|
||||
const result = msg.result;
|
||||
setProgress(result.total_records, result.total_records, 'Import finished.');
|
||||
|
||||
let summary = '✅ Imported ' + result.imported_records + ' of ' + result.total_records + ' records.';
|
||||
if (result.duplicate_records > 0)
|
||||
summary += ' Skipped ' + result.duplicate_records + ' duplicates.';
|
||||
if (result.failed_records > 0)
|
||||
summary += ' ⚠️ ' + result.failed_records + ' records failed.';
|
||||
|
||||
showDone(result.success, summary);
|
||||
submitBtn.innerHTML = '<i class="fas fa-check"></i> Done';
|
||||
|
||||
// Redirect to records view after a short delay
|
||||
if (result.success && result.batch_id) {
|
||||
setTimeout(() => {
|
||||
window.location.href = '{{ url_for("time_attendance_records") }}?import_batch=' + result.batch_id;
|
||||
}, 2500);
|
||||
} else {
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
|
||||
}
|
||||
|
||||
} else if (msg.type === 'error') {
|
||||
evtSource.close();
|
||||
showDone(false, '❌ Import error: ' + msg.message);
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
|
||||
|
||||
} else if (msg.type === 'heartbeat') {
|
||||
// Keep-alive — no action needed
|
||||
}
|
||||
};
|
||||
|
||||
evtSource.onerror = () => {
|
||||
evtSource.close();
|
||||
// Only show error if import hasn't completed yet
|
||||
if (!progressDoneMsg.classList.contains('success')) {
|
||||
showDone(false, 'Connection lost. Please check the records page to verify import status.');
|
||||
}
|
||||
submitBtn.disabled = false;
|
||||
submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
|
||||
};
|
||||
});
|
||||
|
||||
// Reset border color when project is selected
|
||||
// ─── Misc ─────────────────────────────────────────────────────────────────
|
||||
document.getElementById('project_id').addEventListener('change', function() {
|
||||
if (this.value) {
|
||||
this.style.borderColor = '#e2e8f0';
|
||||
}
|
||||
if (this.value) this.style.borderColor = '#e2e8f0';
|
||||
});
|
||||
|
||||
function resetForm() {
|
||||
@@ -807,9 +993,14 @@ function resetForm() {
|
||||
submitBtn.disabled = true;
|
||||
progressIndicator.classList.remove('active');
|
||||
submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
|
||||
progressBarFill.style.width = '0%';
|
||||
progressBarFill.classList.remove('has-width');
|
||||
progressDoneMsg.className = 'progress-done-message';
|
||||
progressMessage.textContent = '';
|
||||
progressCounter.textContent = '0 of 0 records processed';
|
||||
progressPct.textContent = '0%';
|
||||
}
|
||||
|
||||
// Prevent form resubmission
|
||||
if (window.history.replaceState) {
|
||||
window.history.replaceState(null, null, window.location.href);
|
||||
}
|
||||
|
||||
@@ -573,9 +573,10 @@ class TimeAttendanceImportService:
|
||||
|
||||
def import_from_excel(self, file_path: str, created_by: int = None,
|
||||
import_source: str = None, skip_duplicates: bool = True,
|
||||
force_import_hashes: List[str] = None, project_id: int = None) -> Dict[str, Any]:
|
||||
force_import_hashes: List[str] = None, project_id: int = None,
|
||||
progress_callback=None) -> Dict[str, Any]:
|
||||
"""
|
||||
Import time attendance data from Excel file with enhanced duplicate handling
|
||||
Import time attendance data from Excel file with enhanced duplicate handling.
|
||||
|
||||
Args:
|
||||
file_path: Path to the Excel file
|
||||
@@ -583,6 +584,7 @@ class TimeAttendanceImportService:
|
||||
import_source: Description of import source
|
||||
skip_duplicates: Whether to skip duplicate records
|
||||
force_import_hashes: List of hashes to force import (user confirmed duplicates)
|
||||
progress_callback: Optional callable(current, total, message) for real-time progress
|
||||
|
||||
Returns:
|
||||
Dictionary containing import results
|
||||
@@ -733,9 +735,21 @@ class TimeAttendanceImportService:
|
||||
self.db.session.add(time_attendance_record)
|
||||
import_results['imported_records'] += 1
|
||||
|
||||
# Commit in batches for better performance
|
||||
if import_results['imported_records'] % 100 == 0:
|
||||
self.db.session.flush()
|
||||
# Commit in batches of 50 to prevent memory buildup on large files
|
||||
if import_results['imported_records'] % 50 == 0:
|
||||
self.db.session.commit()
|
||||
|
||||
# Report progress via callback if provided
|
||||
if progress_callback:
|
||||
processed = (import_results['imported_records'] +
|
||||
import_results['failed_records'] +
|
||||
import_results['duplicate_records'] +
|
||||
import_results['skipped_records'])
|
||||
progress_callback(
|
||||
processed,
|
||||
import_results['total_records'],
|
||||
f"Importing record {processed} of {import_results['total_records']}..."
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
import_results['failed_records'] += 1
|
||||
|
||||
Reference in New Issue
Block a user