From cacc92636ab3339f249e852fccbf4279df5ccc0c Mon Sep 17 00:00:00 2001 From: NguyenND Date: Mon, 13 Apr 2026 18:18:15 -0400 Subject: [PATCH] 04/13/2026 Fixed email ingres function --- app/__init__.py | 83 +++++++--------- app/routes/admin.py | 120 +++++++++++++++++++++++- app/services/email_ingestion_service.py | 17 +++- app/templates/admin/settings.html | 96 ++++++++++++++++++- 4 files changed, 257 insertions(+), 59 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 3bd4fa1..8ded18d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -182,39 +182,47 @@ def create_app(config_name=None): _seed_settings() _seed_ticket_templates() - # ── Email ingestion background scheduler ──────────────────────────────────── - _start_email_ingestion_scheduler(app) - - # ── SLA background scheduler ────────────────────────────────────────────── - # APScheduler runs inside the gunicorn worker process (single-worker - # eventlet setup), so no cross-process coordination is needed. - # The scheduler is only started in the main process — not during Flask's - # reloader child process — to prevent duplicate job execution. - _start_sla_scheduler(app) + # ── Background schedulers (SLA + Email Ingestion) ─────────────────────────── + # Both jobs share a single BackgroundScheduler instance to avoid duplicate + # thread pools and simplify startup/shutdown. The scheduler is only started + # in the main process — not during Flask's reloader child — to prevent + # duplicate job execution. + _start_background_schedulers(app) return app -def _start_sla_scheduler(app): - """Start the APScheduler background job that checks for SLA breaches. - Safe to call on every app startup: the scheduler is idempotent and - jobstore deduplication prevents double-registration on hot-reloads. - Under gunicorn with preload_app=False each worker calls create_app() - once, so there is exactly one scheduler per worker. +def _start_background_schedulers(app): + """Start a single shared APScheduler instance hosting both the SLA breach + check and the inbound email ingestion job. + + A single BackgroundScheduler is used rather than two separate instances to + avoid creating duplicate thread pools under eventlet. Both jobs are + registered here so they share one daemon thread pool. + + The scheduler is skipped in Flask's reloader subprocess (identified by the + WERKZEUG_RUN_MAIN env var) to prevent double-registration on hot-reloads. + Under gunicorn with preload_app=False, each worker calls create_app() once, + giving exactly one scheduler per worker. """ - # Skip inside Flask's reloader subprocess (identified by the env var it sets). import os as _os if _os.environ.get('WERKZEUG_RUN_MAIN') == 'true': - # Reloader is active — the child process will start its own scheduler. - return + return # skip reloader child process try: from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.triggers.interval import IntervalTrigger from app.services.sla_service import check_sla_breaches + from app.services.email_ingestion_service import check_inbound_email + from app.models import SystemSetting + + with app.app_context(): + email_interval = int(SystemSetting.get('email_ingestion_interval', '5') or '5') scheduler = BackgroundScheduler(daemon=True) + + # SLA breach check — every 30 minutes scheduler.add_job( func = check_sla_breaches, trigger = IntervalTrigger(minutes=30), @@ -223,49 +231,22 @@ def _start_sla_scheduler(app): replace_existing = True, args = [app], ) - scheduler.start() - app.logger.info('[SLA] APScheduler started — SLA breach check every 30 minutes') - except Exception as exc: - app.logger.error(f'[SLA] Failed to start APScheduler: {exc}') - -def _start_email_ingestion_scheduler(app): - """Start the APScheduler job that polls the inbound mailbox for new emails. - - The interval is read from SystemSetting at job creation time (default 5 min). - The job is a no-op when email_ingestion_enabled = '0', so it is safe to - always register it — no credentials are required until the admin enables it. - """ - import os as _os - if _os.environ.get('WERKZEUG_RUN_MAIN') == 'true': - return # skip reloader child process - - try: - from apscheduler.schedulers.background import BackgroundScheduler - from apscheduler.triggers.interval import IntervalTrigger - from app.services.email_ingestion_service import check_inbound_email - from app.models import SystemSetting - - with app.app_context(): - interval = int(SystemSetting.get('email_ingestion_interval', '5') or '5') - - scheduler = BackgroundScheduler(daemon=True) + # Inbound email ingestion — interval from SystemSetting scheduler.add_job( func = check_inbound_email, - trigger = IntervalTrigger(minutes=interval), + trigger = IntervalTrigger(minutes=email_interval), id = 'email_ingest', name = 'Inbound Email Ingestion', replace_existing = True, args = [app], ) + scheduler.start() - app.logger.info( - f'[EMAIL INGEST] APScheduler started — polling every {interval} minute(s)' - ) + app.logger.info('[SCHEDULER] APScheduler started — SLA check every 30 min, ' + f'email ingestion every {email_interval} min') except Exception as exc: - app.logger.error(f'[EMAIL INGEST] Failed to start scheduler: {exc}') - - + app.logger.error(f'[SCHEDULER] Failed to start APScheduler: {exc}') def _seed_ticket_templates(): diff --git a/app/routes/admin.py b/app/routes/admin.py index bd9c6ef..28f9a5c 100644 --- a/app/routes/admin.py +++ b/app/routes/admin.py @@ -1434,4 +1434,122 @@ def settings(): branding_settings=branding_settings, email_settings=email_settings, current_tz=current_tz, - common_timezones=common_timezones) \ No newline at end of file + common_timezones=common_timezones) + +# ── Email Ingestion: Test Connection ───────────────────────────────────────── + +@admin_bp.route('/settings/email-ingestion/test', methods=['POST']) +@login_required +@admin_required +def email_ingestion_test(): + """AJAX endpoint — attempt an IMAP login with the supplied credentials + and return a JSON result. The password field is optional: if left blank + the stored password is used so the admin does not have to re-enter it + just to run a connection test. + """ + from app.models import SystemSetting + import imaplib + + host = request.form.get('host', '').strip() + port_str = request.form.get('port', '993').strip() + user = request.form.get('user', '').strip() + password = request.form.get('password', '').strip() + folder = request.form.get('folder', 'INBOX').strip() + + # Fall back to the stored password if the admin left the field blank + if not password: + password = SystemSetting.get('email_ingestion_password', '') + + if not host or not user or not password: + return jsonify(ok=False, message='Host, username, and password are required.') + + try: + port = int(port_str) + except ValueError: + return jsonify(ok=False, message=f'Invalid port number: {port_str!r}') + + try: + imap = imaplib.IMAP4_SSL(host, port) + imap.login(user, password) + # Verify the watch folder exists + quoted_folder = f'"{folder}"' + status, data = imap.select(quoted_folder, readonly=True) + if status != 'OK': + imap.logout() + return jsonify( + ok=False, + message=( + f'Login succeeded but folder {folder!r} was not found or ' + f'is not accessible. Check the Watch Folder setting.' + ) + ) + # Count unseen messages for informational feedback + _, msg_data = imap.search(None, 'UNSEEN') + unseen = len(msg_data[0].split()) if msg_data and msg_data[0] else 0 + imap.logout() + + log_action( + current_user.id, 'email_ingestion_test', 'system_setting', None, + f'host={host} port={port} user={user} folder={folder} unseen={unseen}' + ) + logger.info( + f'[ADMIN EMAIL INGEST] Connection test OK: host={host} user={user} ' + f'folder={folder} unseen={unseen} by admin_id={current_user.id}' + ) + return jsonify( + ok=True, + message=( + f'Connection successful. Folder {folder!r} is accessible ' + f'({unseen} unseen message(s)).' + ) + ) + + except imaplib.IMAP4.error as exc: + logger.warning( + f'[ADMIN EMAIL INGEST] Connection test failed: host={host} user={user} ' + f'error={exc} by admin_id={current_user.id}' + ) + return jsonify(ok=False, message=f'IMAP error: {exc}') + except OSError as exc: + logger.warning( + f'[ADMIN EMAIL INGEST] Connection test network error: host={host} ' + f'port={port} error={exc} by admin_id={current_user.id}' + ) + return jsonify(ok=False, message=f'Network error: {exc}') + except Exception as exc: + logger.error( + f'[ADMIN EMAIL INGEST] Connection test unexpected error: {exc}', + exc_info=True + ) + return jsonify(ok=False, message=f'Unexpected error: {exc}') + + +# ── Email Ingestion: Run Now ────────────────────────────────────────────────── + +@admin_bp.route('/settings/email-ingestion/run-now', methods=['POST']) +@login_required +@admin_required +def email_ingestion_run_now(): + """Trigger an immediate ingestion run outside the scheduler cycle. + + Runs synchronously in the request (the job is fast for small mailboxes). + Returns JSON so the UI can surface the outcome without a full page reload. + """ + from app.models import SystemSetting + from app.services.email_ingestion_service import _run_ingestion + + if SystemSetting.get('email_ingestion_enabled', '0') != '1': + return jsonify(ok=False, message='Email ingestion is currently disabled. Enable it first.') + + log_action( + current_user.id, 'email_ingestion_run_now', 'system_setting', None, + f'manual run triggered by admin_id={current_user.id}' + ) + logger.info(f'[ADMIN EMAIL INGEST] Manual run triggered by admin_id={current_user.id}') + + try: + _run_ingestion(current_app._get_current_object()) + return jsonify(ok=True, message='Ingestion run completed. Check the activity log for details.') + except Exception as exc: + logger.error(f'[ADMIN EMAIL INGEST] Manual run error: {exc}', exc_info=True) + return jsonify(ok=False, message=f'Run failed: {exc}') diff --git a/app/services/email_ingestion_service.py b/app/services/email_ingestion_service.py index f3b60b6..00bbb6b 100644 --- a/app/services/email_ingestion_service.py +++ b/app/services/email_ingestion_service.py @@ -436,19 +436,28 @@ def _run_ingestion(app): f'from={from_email} subject="{subject[:60]}" ' f'attachments={attach_count}' ) - notify_new_ticket(ticket) + try: + notify_new_ticket(ticket) + except Exception as notify_exc: + logger.error( + f'[EMAIL INGEST] Notification failed for ticket ' + f'{ticket.ticket_number}: {notify_exc}' + ) # Track message ID for deduplication if message_id: new_ids.add(message_id) created += 1 - # Move processed message to done folder + # Move processed message to done folder. + # RFC 3501: folder names with spaces or special chars must + # be double-quoted in IMAP commands. + quoted_move_to = f'"{move_to}"' try: - imap.create(move_to) + imap.create(quoted_move_to) except Exception: pass # folder may already exist - imap.copy(num, move_to) + imap.copy(num, quoted_move_to) imap.store(num, '+FLAGS', '\\Deleted') except Exception as exc: diff --git a/app/templates/admin/settings.html b/app/templates/admin/settings.html index e31f6f4..e405fae 100644 --- a/app/templates/admin/settings.html +++ b/app/templates/admin/settings.html @@ -237,13 +237,103 @@ For other providers, ensure IMAP is enabled and check their specific settings. - +
+ + + +
+ + +