317 lines
12 KiB
Python
317 lines
12 KiB
Python
"""
|
|
Email Ingestion Service — convert inbound emails into tickets.
|
|
|
|
Architecture
|
|
------------
|
|
A scheduled APScheduler job (check_inbound_email) runs every N minutes,
|
|
connects to a configured IMAP mailbox, and converts unread messages into
|
|
tickets. The job is wired into create_app() alongside the SLA scheduler.
|
|
|
|
Sender resolution
|
|
-----------------
|
|
The From address is matched against existing User.email rows.
|
|
- Match found → ticket is created under that user's account.
|
|
- No match found → ticket is created under a configurable fallback user
|
|
(default: the system admin). A comment is prepended noting the external
|
|
sender so IT staff can follow up.
|
|
|
|
Duplicate suppression
|
|
---------------------
|
|
Message-IDs (from the Message-ID header) are stored in the SystemSetting
|
|
key email_ingested_message_ids as a comma-separated list (capped at 500
|
|
entries). Re-delivering an already-processed message is a no-op.
|
|
|
|
Configuration (all stored in SystemSetting, editable from admin/settings)
|
|
----------
|
|
email_ingestion_enabled '1' / '0'
|
|
email_ingestion_host IMAP server hostname
|
|
email_ingestion_port IMAP port (default 993)
|
|
email_ingestion_user Mailbox username / email address
|
|
email_ingestion_password Mailbox password
|
|
email_ingestion_interval Poll interval in minutes (default 5)
|
|
email_ingestion_folder IMAP folder to watch (default INBOX)
|
|
email_ingestion_move_to Folder to move processed mail into (default Processed)
|
|
"""
|
|
|
|
import email
|
|
import imaplib
|
|
import logging
|
|
import re
|
|
from datetime import datetime
|
|
from email.header import decode_header
|
|
from email.utils import parseaddr, getaddresses
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_MAX_STORED_IDS = 500 # cap on the message-ID suppression list
|
|
|
|
|
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
def _decode_header_value(raw):
|
|
"""Decode an RFC-2047 encoded email header value to a plain string."""
|
|
if raw is None:
|
|
return ''
|
|
parts = []
|
|
for chunk, charset in decode_header(raw):
|
|
if isinstance(chunk, bytes):
|
|
try:
|
|
parts.append(chunk.decode(charset or 'utf-8', errors='replace'))
|
|
except (LookupError, UnicodeDecodeError):
|
|
parts.append(chunk.decode('utf-8', errors='replace'))
|
|
else:
|
|
parts.append(chunk)
|
|
return ''.join(parts).strip()
|
|
|
|
|
|
def _extract_plain_text(msg):
|
|
"""Walk a MIME message and return the first text/plain part, or a
|
|
stripped-down version of the first text/html part as a fallback."""
|
|
plain = None
|
|
html = None
|
|
if msg.is_multipart():
|
|
for part in msg.walk():
|
|
ct = part.get_content_type()
|
|
cd = str(part.get('Content-Disposition', ''))
|
|
if 'attachment' in cd:
|
|
continue
|
|
if ct == 'text/plain' and plain is None:
|
|
try:
|
|
plain = part.get_payload(decode=True).decode(
|
|
part.get_content_charset() or 'utf-8', errors='replace'
|
|
)
|
|
except Exception:
|
|
pass
|
|
elif ct == 'text/html' and html is None:
|
|
try:
|
|
html = part.get_payload(decode=True).decode(
|
|
part.get_content_charset() or 'utf-8', errors='replace'
|
|
)
|
|
except Exception:
|
|
pass
|
|
else:
|
|
ct = msg.get_content_type()
|
|
try:
|
|
body = msg.get_payload(decode=True).decode(
|
|
msg.get_content_charset() or 'utf-8', errors='replace'
|
|
)
|
|
except Exception:
|
|
body = ''
|
|
if ct == 'text/plain':
|
|
plain = body
|
|
elif ct == 'text/html':
|
|
html = body
|
|
|
|
if plain:
|
|
return plain.strip()
|
|
|
|
if html:
|
|
# Minimal HTML → plain text strip
|
|
text = re.sub(r'<br\s*/?>', '\n', html, flags=re.IGNORECASE)
|
|
text = re.sub(r'<[^>]+>', '', text)
|
|
import html as html_module
|
|
return html_module.unescape(text).strip()
|
|
|
|
return ''
|
|
|
|
|
|
def _get_setting(key, default=''):
|
|
"""Read a SystemSetting value inside an existing app context."""
|
|
from app.models import SystemSetting
|
|
return SystemSetting.get(key, default)
|
|
|
|
|
|
def _load_seen_ids():
|
|
raw = _get_setting('email_ingested_message_ids', '')
|
|
return set(x.strip() for x in raw.split(',') if x.strip())
|
|
|
|
|
|
def _save_seen_ids(seen: set):
|
|
from app.models import SystemSetting
|
|
from app import db
|
|
# Keep the most recent N IDs to prevent unbounded growth
|
|
trimmed = sorted(seen)[-_MAX_STORED_IDS:]
|
|
SystemSetting.set(
|
|
'email_ingested_message_ids',
|
|
','.join(trimmed),
|
|
'Message-IDs of emails already converted to tickets',
|
|
)
|
|
db.session.commit()
|
|
|
|
|
|
# ── Core ingestion logic ──────────────────────────────────────────────────────
|
|
|
|
def check_inbound_email(app):
|
|
"""Entry point called by APScheduler. Wraps _run_ingestion with
|
|
error isolation so a transient IMAP failure never kills the worker."""
|
|
with app.app_context():
|
|
try:
|
|
if _get_setting('email_ingestion_enabled', '0') != '1':
|
|
return
|
|
_run_ingestion(app)
|
|
except Exception as exc:
|
|
logger.error(f'[EMAIL INGEST] Unhandled error: {exc}', exc_info=True)
|
|
|
|
|
|
def _run_ingestion(app):
|
|
from app import db
|
|
from app.models import (Ticket, TicketStatus, TicketPriority,
|
|
TicketCategory, User, UserRole)
|
|
from app.services.notification_service import notify_new_ticket
|
|
from app.services.sla_service import set_due_date
|
|
|
|
host = _get_setting('email_ingestion_host', '')
|
|
port = int(_get_setting('email_ingestion_port', '993'))
|
|
username = _get_setting('email_ingestion_user', '')
|
|
password = _get_setting('email_ingestion_password', '')
|
|
folder = _get_setting('email_ingestion_folder', 'INBOX')
|
|
move_to = _get_setting('email_ingestion_move_to', 'Processed')
|
|
|
|
if not host or not username or not password:
|
|
logger.warning('[EMAIL INGEST] Missing IMAP credentials — skipping')
|
|
return
|
|
|
|
seen_ids = _load_seen_ids()
|
|
new_ids = set()
|
|
created = 0
|
|
|
|
try:
|
|
imap = imaplib.IMAP4_SSL(host, port)
|
|
imap.login(username, password)
|
|
except Exception as exc:
|
|
logger.error(f'[EMAIL INGEST] IMAP login failed: {exc}')
|
|
return
|
|
|
|
try:
|
|
imap.select(folder)
|
|
# Search for unseen messages only
|
|
status, data = imap.search(None, 'UNSEEN')
|
|
if status != 'OK' or not data[0]:
|
|
return
|
|
|
|
msg_ids = data[0].split()
|
|
logger.info(f'[EMAIL INGEST] Found {len(msg_ids)} unseen message(s) in {folder}')
|
|
|
|
for num in msg_ids:
|
|
try:
|
|
_, raw = imap.fetch(num, '(RFC822)')
|
|
msg = email.message_from_bytes(raw[0][1])
|
|
|
|
message_id = msg.get('Message-ID', '').strip()
|
|
if message_id and message_id in seen_ids:
|
|
logger.debug(f'[EMAIL INGEST] Skipping duplicate {message_id}')
|
|
continue
|
|
|
|
# ── Parse headers ─────────────────────────────────────────────
|
|
subject = _decode_header_value(msg.get('Subject', '(No Subject)'))
|
|
from_raw = msg.get('From', '')
|
|
from_name, from_email = parseaddr(from_raw)
|
|
from_email = from_email.lower().strip()
|
|
body = _extract_plain_text(msg)
|
|
|
|
if not body:
|
|
body = f'[Email received from {from_email} with no readable body]'
|
|
|
|
# Truncate very long bodies to 8000 chars
|
|
if len(body) > 8000:
|
|
body = body[:8000] + '\n\n[…message truncated…]'
|
|
|
|
# ── Resolve sender to a user ──────────────────────────────────
|
|
sender_user = User.query.filter_by(
|
|
email=from_email, is_active=True
|
|
).first()
|
|
|
|
if sender_user:
|
|
created_by_id = sender_user.id
|
|
external_note = None
|
|
else:
|
|
# Fall back to the first active admin
|
|
fallback = User.query.filter(
|
|
User.role == UserRole.ADMIN,
|
|
User.is_active == True,
|
|
).first()
|
|
if not fallback:
|
|
logger.warning(
|
|
f'[EMAIL INGEST] No fallback admin found, skipping: {from_email}'
|
|
)
|
|
continue
|
|
created_by_id = fallback.id
|
|
external_note = (
|
|
f'**[Email received from unknown sender]**\n\n'
|
|
f'From: {from_name} <{from_email}>\n\n'
|
|
f'This ticket was automatically created from an inbound email. '
|
|
f'The sender is not a registered user — please follow up directly.'
|
|
)
|
|
|
|
# ── Create the ticket ─────────────────────────────────────────
|
|
ticket = Ticket(
|
|
title = subject[:200],
|
|
description = body,
|
|
category = TicketCategory.OTHER,
|
|
priority = TicketPriority.MEDIUM,
|
|
status = TicketStatus.OPEN,
|
|
created_by_id = created_by_id,
|
|
ai_generated = False,
|
|
)
|
|
ticket.ticket_number = ticket.generate_ticket_number()
|
|
set_due_date(ticket, app)
|
|
db.session.add(ticket)
|
|
db.session.flush()
|
|
|
|
# Prepend external-sender note as a comment if needed
|
|
if external_note:
|
|
from app.models import Comment
|
|
from app.services.validation_service import render_comment_body
|
|
comment = Comment(
|
|
ticket_id = ticket.id,
|
|
author_id = created_by_id,
|
|
body = render_comment_body(external_note),
|
|
is_internal= True, # IT staff only
|
|
)
|
|
db.session.add(comment)
|
|
|
|
from app.services.log_service import log_action
|
|
log_action(
|
|
created_by_id, 'ticket_create_email', 'ticket', ticket.id,
|
|
f'ticket_number={ticket.ticket_number} '
|
|
f'from_email={from_email} message_id={message_id or "none"}'
|
|
)
|
|
db.session.commit()
|
|
logger.info(
|
|
f'[EMAIL INGEST] Created ticket {ticket.ticket_number} '
|
|
f'from={from_email} subject="{subject[:60]}"'
|
|
)
|
|
notify_new_ticket(ticket)
|
|
|
|
# Track message ID for deduplication
|
|
if message_id:
|
|
new_ids.add(message_id)
|
|
created += 1
|
|
|
|
# Move processed message to done folder
|
|
try:
|
|
imap.create(move_to)
|
|
except Exception:
|
|
pass # folder may already exist
|
|
imap.copy(num, move_to)
|
|
imap.store(num, '+FLAGS', '\\Deleted')
|
|
|
|
except Exception as exc:
|
|
db.session.rollback()
|
|
logger.error(f'[EMAIL INGEST] Failed to process message {num}: {exc}',
|
|
exc_info=True)
|
|
|
|
imap.expunge()
|
|
|
|
finally:
|
|
try:
|
|
imap.logout()
|
|
except Exception:
|
|
pass
|
|
|
|
if new_ids:
|
|
_save_seen_ids(seen_ids | new_ids)
|
|
|
|
if created:
|
|
logger.info(f'[EMAIL INGEST] Run complete — {created} ticket(s) created')
|