Feb25 2026: updated import progress bar and fixed big file import fail

app.py

@@ -8716,6 +8716,244 @@ def analyze_import_invalid():
             'message': f'Analysis failed: {str(e)}'
         }), 500
 
+
+# ---------------------------------------------------------------------------
+# Time Attendance Import — SSE progress streaming (disk-based, multi-worker safe)
+#
+# Design: progress state is written to a small JSON file on disk so that any
+# gunicorn worker process can read it.  No shared in-memory state is required.
+# The /stream endpoint runs the import itself (synchronously inside the SSE
+# generator) while writing progress to the file and yielding events to the
+# browser — compatible with gunicorn gevent workers.
+# ---------------------------------------------------------------------------
+
+def _progress_file_path(job_id: str) -> str:
+    """Return the path for the on-disk progress file for a given job_id."""
+    upload_dir = app.config.get('UPLOAD_FOLDER', '/tmp')
+    os.makedirs(upload_dir, exist_ok=True)
+    return os.path.join(upload_dir, f"import_progress_{job_id}.json")
+
+
+def _write_progress(job_id: str, event: dict) -> None:
+    """Atomically write the latest progress event to disk."""
+    path = _progress_file_path(job_id)
+    try:
+        tmp = path + '.tmp'
+        with open(tmp, 'w') as f:
+            json.dump(event, f)
+        os.replace(tmp, path)
+    except Exception:
+        pass  # Best-effort; import will continue regardless
+
+
+@app.route('/time-attendance/import/start', methods=['POST'])
+@login_required
+def start_import_job():
+    """
+    Validates the uploaded file, saves it to disk, stores import options in a
+    progress file, then returns a job_id.  The actual import runs inside the
+    SSE stream endpoint so no background thread or shared memory is needed.
+    """
+    try:
+        if 'files' not in request.files:
+            return jsonify({'success': False, 'error': 'No file uploaded.'}), 400
+
+        files = request.files.getlist('files')
+        if not files or files[0].filename == '':
+            return jsonify({'success': False, 'error': 'No file selected.'}), 400
+
+        file = files[0]
+        if not file.filename.lower().endswith(('.xlsx', '.xls')):
+            return jsonify({'success': False, 'error': 'Invalid file format.'}), 400
+
+        filename = secure_filename(file.filename)
+        upload_dir = app.config.get('UPLOAD_FOLDER', '/tmp')
+        os.makedirs(upload_dir, exist_ok=True)
+        job_id = str(uuid.uuid4())
+        temp_path = os.path.join(upload_dir,
+                                 f"stream_{job_id}_{filename}")
+        file.save(temp_path)
+
+        # Store import options alongside the file so the stream endpoint can
+        # read them without depending on session or shared memory.
+        job_meta = {
+            'type': 'pending',
+            'temp_path': temp_path,
+            'filename': filename,
+            'skip_duplicates': request.form.get('skip_duplicates', 'true').lower() == 'true',
+            'project_id': int(request.form.get('project_id')) if request.form.get('project_id') else None,
+            'import_source': request.form.get('import_source', f"Manual Import - {filename}"),
+            'created_by': session['user_id'],
+            'username': session.get('username', 'unknown'),
+        }
+        _write_progress(job_id, job_meta)
+
+        logger_handler.logger.info(
+            f"User {job_meta['username']} queued time attendance import job {job_id} for file {filename}"
+        )
+        return jsonify({'success': True, 'job_id': job_id})
+
+    except Exception as e:
+        logger_handler.logger.error(f"Error queuing import job: {e}")
+        return jsonify({'success': False, 'error': str(e)}), 500
+
+
+@app.route('/time-attendance/import/stream/<job_id>')
+@login_required
+def stream_import_progress(job_id):
+    """
+    SSE endpoint — runs the import synchronously while streaming progress to
+    the browser.  Works across multiple gunicorn workers because all state is
+    stored on disk (no in-memory job store).
+    """
+    progress_path = _progress_file_path(job_id)
+
+    def generate():
+        import time as _time
+
+        # ── Read the job metadata written by /start ────────────────────────
+        deadline = _time.time() + 15  # Wait up to 15 s for the file to appear
+        meta = None
+        while _time.time() < deadline:
+            if os.path.exists(progress_path):
+                try:
+                    with open(progress_path) as f:
+                        meta = json.load(f)
+                    break
+                except Exception:
+                    pass
+            yield "data: " + json.dumps({'type': 'heartbeat'}) + "\n\n"
+            _time.sleep(0.3)
+
+        if not meta or meta.get('type') != 'pending':
+            yield "data: " + json.dumps({
+                'type': 'error',
+                'message': 'Job metadata not found. Please try importing again.'
+            }) + "\n\n"
+            return
+
+        temp_path    = meta['temp_path']
+        skip_dupes   = meta['skip_duplicates']
+        project_id   = meta['project_id']
+        import_source= meta['import_source']
+        created_by   = meta['created_by']
+        username     = meta['username']
+
+        if not os.path.exists(temp_path):
+            yield "data: " + json.dumps({
+                'type': 'error',
+                'message': 'Uploaded file not found. Please try importing again.'
+            }) + "\n\n"
+            return
+
+        yield "data: " + json.dumps({'type': 'status', 'message': 'Reading and validating file...'}) + "\n\n"
+
+        # ── Run the import with a progress callback ────────────────────────
+        try:
+            svc = TimeAttendanceImportService(db, logger_handler)
+
+            # progress_callback writes to disk AND yields an SSE event.
+            # We collect events in a list so the generator can yield them.
+            _pending_events = []
+
+            def on_progress(current, total, message):
+                pct = int(current / total * 100) if total else 0
+                event = {
+                    'type': 'progress',
+                    'current': current,
+                    'total': total,
+                    'percent': pct,
+                    'message': message,
+                }
+                _write_progress(job_id, event)
+                _pending_events.append(event)
+
+            # We need to interleave yielding with the synchronous import loop.
+            # Strategy: run import_from_excel; the callback appends to
+            # _pending_events; after every DB commit batch (50 records) we
+            # flush pending events to the SSE stream.
+            import threading as _threading
+            result_holder = [None]
+            error_holder  = [None]
+            done_event    = _threading.Event()
+
+            def _run():
+                # Push an application context so the thread can access
+                # Flask-SQLAlchemy, Employee.query, etc.
+                with app.app_context():
+                    try:
+                        result_holder[0] = svc.import_from_excel(
+                            temp_path,
+                            created_by=created_by,
+                            import_source=import_source,
+                            skip_duplicates=skip_dupes,
+                            force_import_hashes=[],
+                            project_id=project_id,
+                            progress_callback=on_progress,
+                        )
+                    except Exception as exc:
+                        error_holder[0] = exc
+                    finally:
+                        done_event.set()
+
+            t = _threading.Thread(target=_run, daemon=True)
+            t.start()
+
+            # Yield progress events as they arrive while the import thread runs
+            while not done_event.is_set():
+                while _pending_events:
+                    yield "data: " + json.dumps(_pending_events.pop(0)) + "\n\n"
+                yield "data: " + json.dumps({'type': 'heartbeat'}) + "\n\n"
+                _time.sleep(0.4)
+
+            # Drain any remaining events after the thread finishes
+            while _pending_events:
+                yield "data: " + json.dumps(_pending_events.pop(0)) + "\n\n"
+
+            if error_holder[0]:
+                raise error_holder[0]
+
+            result = result_holder[0]
+
+            if result and result['success']:
+                logger_handler.logger.info(
+                    f"User {username} imported {result['imported_records']} time attendance records "
+                    f"via stream (batch: {result['batch_id']})"
+                )
+
+            # Sanitize result dict for JSON serialization — convert any
+            # datetime objects (e.g. import_date) to ISO-format strings.
+            if result and isinstance(result.get('import_date'), datetime):
+                result['import_date'] = result['import_date'].isoformat()
+            done_event_data = {'type': 'done', 'result': result}
+            _write_progress(job_id, done_event_data)
+            yield "data: " + json.dumps(done_event_data) + "\n\n"
+
+        except Exception as e:
+            logger_handler.logger.error(f"Import stream error for job {job_id}: {e}")
+            error_event = {'type': 'error', 'message': str(e)}
+            _write_progress(job_id, error_event)
+            yield "data: " + json.dumps(error_event) + "\n\n"
+
+        finally:
+            # Clean up temp files
+            for path in (temp_path, progress_path):
+                try:
+                    if os.path.exists(path):
+                        os.remove(path)
+                except Exception:
+                    pass
+
+    return Response(
+        generate(),
+        mimetype='text/event-stream',
+        headers={
+            'Cache-Control': 'no-cache',
+            'X-Accel-Buffering': 'no',   # Disable nginx buffering for SSE
+        }
+    )
+
+
 @app.route('/time-attendance/import/cancel-pending')
 @login_required
 def cancel_pending_import():

templates/time_attendance_import.html

@@ -96,26 +96,107 @@
 
 .progress-indicator {
   display: none;
-  margin-top: 1rem;
+  margin-top: 1.25rem;
-  padding: 1rem;
+  padding: 1.25rem 1.5rem;
   background: #ebf8ff;
   border: 1px solid #90cdf4;
-  border-radius: 8px;
+  border-radius: 10px;
-  text-align: center;
 }
 
 .progress-indicator.active {
   display: block;
 }
 
+.progress-header {
+  display: flex;
+  align-items: center;
+  gap: 0.6rem;
+  margin-bottom: 0.75rem;
+  font-weight: 600;
+  color: #2b6cb0;
+}
+
+.progress-bar-track {
+  background: #bee3f8;
+  border-radius: 999px;
+  height: 18px;
+  overflow: hidden;
+  margin-bottom: 0.5rem;
+  box-shadow: inset 0 1px 3px rgba(0,0,0,0.1);
+}
+
+.progress-bar-fill {
+  height: 100%;
+  width: 0%;
+  background: linear-gradient(90deg, #4299e1, #3182ce);
+  border-radius: 999px;
+  transition: width 0.3s ease;
+  display: flex;
+  align-items: center;
+  justify-content: flex-end;
+  padding-right: 6px;
+}
+
+.progress-bar-fill .pct-label {
+  font-size: 0.7rem;
+  font-weight: 700;
+  color: white;
+  white-space: nowrap;
+  opacity: 0;
+  transition: opacity 0.2s;
+}
+
+.progress-bar-fill.has-width .pct-label {
+  opacity: 1;
+}
+
+.progress-counters {
+  display: flex;
+  justify-content: space-between;
+  font-size: 0.8rem;
+  color: #4a5568;
+}
+
+.progress-message {
+  margin-top: 0.4rem;
+  font-size: 0.8rem;
+  color: #718096;
+  font-style: italic;
+  min-height: 1.1em;
+}
+
+.progress-done-message {
+  display: none;
+  margin-top: 0.75rem;
+  padding: 0.75rem 1rem;
+  border-radius: 8px;
+  font-weight: 600;
+  font-size: 0.9rem;
+}
+
+.progress-done-message.success {
+  background: #f0fff4;
+  border: 1px solid #68d391;
+  color: #276749;
+  display: block;
+}
+
+.progress-done-message.error {
+  background: #fff5f5;
+  border: 1px solid #fc8181;
+  color: #c53030;
+  display: block;
+}
+
 .spinner {
   display: inline-block;
-  width: 20px;
+  width: 16px;
-  height: 20px;
+  height: 16px;
-  border: 3px solid #e2e8f0;
+  border: 2.5px solid #bee3f8;
   border-top-color: #4299e1;
   border-radius: 50%;
   animation: spin 0.8s linear infinite;
+  flex-shrink: 0;
 }
 
 @keyframes spin {
@@ -472,8 +553,21 @@
 
         <!-- Progress Indicator -->
         <div id="progressIndicator" class="progress-indicator">
-          <div class="spinner"></div>
+          <div class="progress-header">
-          <p style="margin-top: 0.5rem;">Processing your file, please wait...</p>
+            <div class="spinner" id="progressSpinner"></div>
+            <span id="progressTitle">Uploading &amp; preparing import...</span>
+          </div>
+          <div class="progress-bar-track">
+            <div class="progress-bar-fill" id="progressBarFill">
+              <span class="pct-label" id="progressPctLabel">0%</span>
+            </div>
+          </div>
+          <div class="progress-counters">
+            <span id="progressCounter">0 of 0 records processed</span>
+            <span id="progressPct">0%</span>
+          </div>
+          <div class="progress-message" id="progressMessage">Starting...</div>
+          <div class="progress-done-message" id="progressDoneMsg"></div>
         </div>
 
         <!-- Form Actions -->
@@ -687,118 +781,210 @@
 </div>
 
 <script>
-// File upload handling
+// ─── DOM references ────────────────────────────────────────────────────────
-const fileInput = document.getElementById('fileInput');
+const fileInput        = document.getElementById('fileInput');
-const fileUploadArea = document.getElementById('fileUploadArea');
+const fileUploadArea   = document.getElementById('fileUploadArea');
-const fileInfo = document.getElementById('fileInfo');
+const fileInfo         = document.getElementById('fileInfo');
-const submitBtn = document.getElementById('submitBtn');
+const submitBtn        = document.getElementById('submitBtn');
-const importForm = document.getElementById('importForm');
+const importForm       = document.getElementById('importForm');
-const progressIndicator = document.getElementById('progressIndicator');
+const progressIndicator= document.getElementById('progressIndicator');
+const progressBarFill  = document.getElementById('progressBarFill');
+const progressPctLabel = document.getElementById('progressPctLabel');
+const progressCounter  = document.getElementById('progressCounter');
+const progressPct      = document.getElementById('progressPct');
+const progressMessage  = document.getElementById('progressMessage');
+const progressTitle    = document.getElementById('progressTitle');
+const progressDoneMsg  = document.getElementById('progressDoneMsg');
 
-// Drag and drop handlers
+// ─── Drag & drop ───────────────────────────────────────────────────────────
 fileUploadArea.addEventListener('dragover', (e) => {
   e.preventDefault();
   fileUploadArea.classList.add('dragover');
 });
-
 fileUploadArea.addEventListener('dragleave', () => {
   fileUploadArea.classList.remove('dragover');
 });
-
 fileUploadArea.addEventListener('drop', (e) => {
   e.preventDefault();
   fileUploadArea.classList.remove('dragover');
-  
   const files = e.dataTransfer.files;
-  if (files.length > 0) {
+  if (files.length > 0) { fileInput.files = files; handleFileSelect(); }
-    fileInput.files = files;
-    handleFileSelect();
-  }
 });
-
+fileUploadArea.addEventListener('click', () => fileInput.click());
-fileUploadArea.addEventListener('click', () => {
-  fileInput.click();
-});
-
 fileInput.addEventListener('change', handleFileSelect);
 
 function handleFileSelect() {
   const files = fileInput.files;
-  if (files.length > 0) {
+  if (!files.length) return;
-    // Validate all files
-    let invalidFiles = [];
-    for (let i = 0; i < files.length; i++) {
-      const fileName = files[i].name;
-      if (!fileName.toLowerCase().endsWith('.xlsx') && !fileName.toLowerCase().endsWith('.xls')) {
-        invalidFiles.push(fileName);
-      }
-    }
-    
-    if (invalidFiles.length > 0) {
-      alert('Invalid file format detected:\n' + invalidFiles.join('\n') + '\n\nPlease select only Excel files (.xlsx or .xls)');
-      resetForm();
-      return;
-    }
-    
-    // Calculate total size
-    let totalSize = 0;
-    for (let i = 0; i < files.length; i++) {
-      totalSize += files[i].size;
-    }
-    
-    // Display file info
-    if (files.length === 1) {
-      fileInfo.querySelector('.file-name').textContent = files[0].name;
-      fileInfo.querySelector('.file-size').textContent = (totalSize / 1024 / 1024).toFixed(2) + ' MB';
-      fileInfo.querySelector('.file-count').textContent = '';
-    } else {
-      fileInfo.querySelector('.file-name').textContent = files.length + ' files selected';
-      fileInfo.querySelector('.file-size').textContent = (totalSize / 1024 / 1024).toFixed(2) + ' MB total';
-      const fileNames = Array.from(files).map(f => f.name).join(', ');
-      fileInfo.querySelector('.file-count').textContent = '(' + fileNames + ')';
-      fileInfo.querySelector('.file-count').style.fontSize = '0.75rem';
-      fileInfo.querySelector('.file-count').style.color = '#718096';
-      fileInfo.querySelector('.file-count').style.display = 'block';
-      fileInfo.querySelector('.file-count').style.marginTop = '0.5rem';
-    }
-    
-    fileInfo.style.display = 'flex';
-    fileInfo.style.flexDirection = 'column';
-    submitBtn.disabled = false;
-  }
-}
 
-// Form submission
+  const invalid = Array.from(files).filter(
-importForm.addEventListener('submit', (e) => {
+    f => !f.name.toLowerCase().endsWith('.xlsx') && !f.name.toLowerCase().endsWith('.xls')
-  if (fileInput.files.length === 0) {
+  );
-    e.preventDefault();
+  if (invalid.length) {
-    alert('Please select at least one file to import');
+    alert('Invalid file format:\n' + invalid.map(f => f.name).join('\n') +
+          '\n\nPlease select only Excel files (.xlsx or .xls)');
+    resetForm();
     return;
   }
 
-  // Validate project selection
+  const totalSize = Array.from(files).reduce((s, f) => s + f.size, 0);
+  if (files.length === 1) {
+    fileInfo.querySelector('.file-name').textContent = files[0].name;
+    fileInfo.querySelector('.file-size').textContent = (totalSize / 1024 / 1024).toFixed(2) + ' MB';
+    fileInfo.querySelector('.file-count').textContent = '';
+  } else {
+    fileInfo.querySelector('.file-name').textContent = files.length + ' files selected';
+    fileInfo.querySelector('.file-size').textContent = (totalSize / 1024 / 1024).toFixed(2) + ' MB total';
+    const names = Array.from(files).map(f => f.name).join(', ');
+    const countEl = fileInfo.querySelector('.file-count');
+    countEl.textContent = '(' + names + ')';
+    Object.assign(countEl.style, { fontSize: '0.75rem', color: '#718096', display: 'block', marginTop: '0.5rem' });
+  }
+  fileInfo.style.display = 'flex';
+  fileInfo.style.flexDirection = 'column';
+  submitBtn.disabled = false;
+}
+
+// ─── Progress bar helpers ──────────────────────────────────────────────────
+function setProgress(current, total, message) {
+  const pct = total > 0 ? Math.min(100, Math.round(current / total * 100)) : 0;
+  progressBarFill.style.width = pct + '%';
+  if (pct > 8) {
+    progressBarFill.classList.add('has-width');
+    progressPctLabel.textContent = pct + '%';
+  }
+  progressPct.textContent = pct + '%';
+  progressCounter.textContent = current + ' of ' + total + ' records processed';
+  if (message) progressMessage.textContent = message;
+}
+
+function showDone(success, message) {
+  document.getElementById('progressSpinner').style.display = 'none';
+  progressDoneMsg.className = 'progress-done-message ' + (success ? 'success' : 'error');
+  progressDoneMsg.textContent = message;
+  progressTitle.textContent = success ? 'Import complete!' : 'Import failed';
+}
+
+// ─── Form submit → SSE import flow ────────────────────────────────────────
+importForm.addEventListener('submit', async (e) => {
+  e.preventDefault();   // Always intercept; we use fetch + SSE instead
+
+  if (!fileInput.files.length) {
+    alert('Please select at least one file to import.');
+    return;
+  }
   const projectSelect = document.getElementById('project_id');
-  if (!projectSelect.value || projectSelect.value === '') {
+  if (!projectSelect.value) {
-    e.preventDefault();
+    alert('Please select a project. Project selection is required.');
-    alert('Please select a project. Project selection is required for attendance imports.');
     projectSelect.focus();
     projectSelect.style.borderColor = '#ef4444';
     return;
   }
-  
+
-  // Show progress indicator
+  // Show progress UI
   progressIndicator.classList.add('active');
+  progressDoneMsg.className = 'progress-done-message';  // hide
+  progressDoneMsg.textContent = '';
+  setProgress(0, 0, 'Uploading file...');
+  progressTitle.textContent = 'Uploading & preparing import...';
+  document.getElementById('progressSpinner').style.display = '';
   submitBtn.disabled = true;
-  const fileCount = fileInput.files.length;
+  submitBtn.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Importing...';
-  const fileText = fileCount === 1 ? 'file' : 'files';
+
-  submitBtn.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Processing ' + fileCount + ' ' + fileText + '...';
+  // Build FormData from the existing form (includes all checkboxes, selects, etc.)
+  const formData = new FormData(importForm);
+
+  // ── Step 1: POST the file to /start → get job_id ──────────────────────
+  let jobId;
+  try {
+    const resp = await fetch('{{ url_for("start_import_job") }}', {
+      method: 'POST',
+      body: formData
+    });
+    const data = await resp.json();
+    if (!data.success) {
+      showDone(false, 'Upload failed: ' + (data.error || 'Unknown error'));
+      submitBtn.disabled = false;
+      submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
+      return;
+    }
+    jobId = data.job_id;
+  } catch (err) {
+    showDone(false, 'Network error during upload: ' + err.message);
+    submitBtn.disabled = false;
+    submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
+    return;
+  }
+
+  progressMessage.textContent = 'File received. Starting import...';
+
+  // ── Step 2: Open SSE stream for this job ──────────────────────────────
+  const streamUrl = '{{ url_for("stream_import_progress", job_id="__JOB_ID__") }}'.replace('__JOB_ID__', jobId);
+  const evtSource = new EventSource(streamUrl);
+
+  evtSource.onmessage = (event) => {
+    let msg;
+    try { msg = JSON.parse(event.data); } catch { return; }
+
+    if (msg.type === 'status') {
+      progressMessage.textContent = msg.message;
+      progressTitle.textContent = 'Importing records...';
+
+    } else if (msg.type === 'progress') {
+      setProgress(msg.current, msg.total, msg.message);
+      if (msg.percent >= 5) {
+        progressTitle.textContent = 'Importing records — ' + msg.percent + '% complete';
+      }
+
+    } else if (msg.type === 'done') {
+      evtSource.close();
+      const result = msg.result;
+      setProgress(result.total_records, result.total_records, 'Import finished.');
+
+      let summary = '✅ Imported ' + result.imported_records + ' of ' + result.total_records + ' records.';
+      if (result.duplicate_records > 0)
+        summary += '  Skipped ' + result.duplicate_records + ' duplicates.';
+      if (result.failed_records > 0)
+        summary += '  ⚠️ ' + result.failed_records + ' records failed.';
+
+      showDone(result.success, summary);
+      submitBtn.innerHTML = '<i class="fas fa-check"></i> Done';
+
+      // Redirect to records view after a short delay
+      if (result.success && result.batch_id) {
+        setTimeout(() => {
+          window.location.href = '{{ url_for("time_attendance_records") }}?import_batch=' + result.batch_id;
+        }, 2500);
+      } else {
+        submitBtn.disabled = false;
+        submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
+      }
+
+    } else if (msg.type === 'error') {
+      evtSource.close();
+      showDone(false, '❌ Import error: ' + msg.message);
+      submitBtn.disabled = false;
+      submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
+
+    } else if (msg.type === 'heartbeat') {
+      // Keep-alive — no action needed
+    }
+  };
+
+  evtSource.onerror = () => {
+    evtSource.close();
+    // Only show error if import hasn't completed yet
+    if (!progressDoneMsg.classList.contains('success')) {
+      showDone(false, 'Connection lost. Please check the records page to verify import status.');
+    }
+    submitBtn.disabled = false;
+    submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
+  };
 });
 
-// Reset border color when project is selected
+// ─── Misc ─────────────────────────────────────────────────────────────────
 document.getElementById('project_id').addEventListener('change', function() {
-  if (this.value) {
+  if (this.value) this.style.borderColor = '#e2e8f0';
-    this.style.borderColor = '#e2e8f0';
-  }
 });
 
 function resetForm() {
@@ -807,9 +993,14 @@ function resetForm() {
   submitBtn.disabled = true;
   progressIndicator.classList.remove('active');
   submitBtn.innerHTML = '<i class="fas fa-upload"></i> <span class="btn-text">Import Data</span>';
+  progressBarFill.style.width = '0%';
+  progressBarFill.classList.remove('has-width');
+  progressDoneMsg.className = 'progress-done-message';
+  progressMessage.textContent = '';
+  progressCounter.textContent = '0 of 0 records processed';
+  progressPct.textContent = '0%';
 }
 
-// Prevent form resubmission
 if (window.history.replaceState) {
   window.history.replaceState(null, null, window.location.href);
 }

time_attendance_import_service.py

@@ -571,19 +571,21 @@ class TimeAttendanceImportService:
         
         return analysis_result
     
-    def import_from_excel(self, file_path: str, created_by: int = None, 
+    def import_from_excel(self, file_path: str, created_by: int = None,
                          import_source: str = None, skip_duplicates: bool = True,
-                         force_import_hashes: List[str] = None, project_id: int = None) -> Dict[str, Any]:
+                         force_import_hashes: List[str] = None, project_id: int = None,
+                         progress_callback=None) -> Dict[str, Any]:
         """
-        Import time attendance data from Excel file with enhanced duplicate handling
+        Import time attendance data from Excel file with enhanced duplicate handling.
-        
+
         Args:
             file_path: Path to the Excel file
             created_by: User ID who initiated the import
             import_source: Description of import source
             skip_duplicates: Whether to skip duplicate records
             force_import_hashes: List of hashes to force import (user confirmed duplicates)
-            
+            progress_callback: Optional callable(current, total, message) for real-time progress
+
         Returns:
             Dictionary containing import results
         """
@@ -732,10 +734,22 @@ class TimeAttendanceImportService:
                     
                     self.db.session.add(time_attendance_record)
                     import_results['imported_records'] += 1
-                    
+
-                    # Commit in batches for better performance
+                    # Commit in batches of 50 to prevent memory buildup on large files
-                    if import_results['imported_records'] % 100 == 0:
+                    if import_results['imported_records'] % 50 == 0:
-                        self.db.session.flush()
+                        self.db.session.commit()
+
+                    # Report progress via callback if provided
+                    if progress_callback:
+                        processed = (import_results['imported_records'] +
+                                     import_results['failed_records'] +
+                                     import_results['duplicate_records'] +
+                                     import_results['skipped_records'])
+                        progress_callback(
+                            processed,
+                            import_results['total_records'],
+                            f"Importing record {processed} of {import_results['total_records']}..."
+                        )
                     
                 except Exception as e:
                     import_results['failed_records'] += 1