04/07 updated timezone, report via email, etc
This commit is contained in:
@@ -0,0 +1,316 @@
|
||||
"""
|
||||
Email Ingestion Service — convert inbound emails into tickets.
|
||||
|
||||
Architecture
|
||||
------------
|
||||
A scheduled APScheduler job (check_inbound_email) runs every N minutes,
|
||||
connects to a configured IMAP mailbox, and converts unread messages into
|
||||
tickets. The job is wired into create_app() alongside the SLA scheduler.
|
||||
|
||||
Sender resolution
|
||||
-----------------
|
||||
The From address is matched against existing User.email rows.
|
||||
- Match found → ticket is created under that user's account.
|
||||
- No match found → ticket is created under a configurable fallback user
|
||||
(default: the system admin). A comment is prepended noting the external
|
||||
sender so IT staff can follow up.
|
||||
|
||||
Duplicate suppression
|
||||
---------------------
|
||||
Message-IDs (from the Message-ID header) are stored in the SystemSetting
|
||||
key email_ingested_message_ids as a comma-separated list (capped at 500
|
||||
entries). Re-delivering an already-processed message is a no-op.
|
||||
|
||||
Configuration (all stored in SystemSetting, editable from admin/settings)
|
||||
----------
|
||||
email_ingestion_enabled '1' / '0'
|
||||
email_ingestion_host IMAP server hostname
|
||||
email_ingestion_port IMAP port (default 993)
|
||||
email_ingestion_user Mailbox username / email address
|
||||
email_ingestion_password Mailbox password
|
||||
email_ingestion_interval Poll interval in minutes (default 5)
|
||||
email_ingestion_folder IMAP folder to watch (default INBOX)
|
||||
email_ingestion_move_to Folder to move processed mail into (default Processed)
|
||||
"""
|
||||
|
||||
import email
|
||||
import imaplib
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime
|
||||
from email.header import decode_header
|
||||
from email.utils import parseaddr, getaddresses
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_STORED_IDS = 500 # cap on the message-ID suppression list
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
def _decode_header_value(raw):
|
||||
"""Decode an RFC-2047 encoded email header value to a plain string."""
|
||||
if raw is None:
|
||||
return ''
|
||||
parts = []
|
||||
for chunk, charset in decode_header(raw):
|
||||
if isinstance(chunk, bytes):
|
||||
try:
|
||||
parts.append(chunk.decode(charset or 'utf-8', errors='replace'))
|
||||
except (LookupError, UnicodeDecodeError):
|
||||
parts.append(chunk.decode('utf-8', errors='replace'))
|
||||
else:
|
||||
parts.append(chunk)
|
||||
return ''.join(parts).strip()
|
||||
|
||||
|
||||
def _extract_plain_text(msg):
|
||||
"""Walk a MIME message and return the first text/plain part, or a
|
||||
stripped-down version of the first text/html part as a fallback."""
|
||||
plain = None
|
||||
html = None
|
||||
if msg.is_multipart():
|
||||
for part in msg.walk():
|
||||
ct = part.get_content_type()
|
||||
cd = str(part.get('Content-Disposition', ''))
|
||||
if 'attachment' in cd:
|
||||
continue
|
||||
if ct == 'text/plain' and plain is None:
|
||||
try:
|
||||
plain = part.get_payload(decode=True).decode(
|
||||
part.get_content_charset() or 'utf-8', errors='replace'
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
elif ct == 'text/html' and html is None:
|
||||
try:
|
||||
html = part.get_payload(decode=True).decode(
|
||||
part.get_content_charset() or 'utf-8', errors='replace'
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
ct = msg.get_content_type()
|
||||
try:
|
||||
body = msg.get_payload(decode=True).decode(
|
||||
msg.get_content_charset() or 'utf-8', errors='replace'
|
||||
)
|
||||
except Exception:
|
||||
body = ''
|
||||
if ct == 'text/plain':
|
||||
plain = body
|
||||
elif ct == 'text/html':
|
||||
html = body
|
||||
|
||||
if plain:
|
||||
return plain.strip()
|
||||
|
||||
if html:
|
||||
# Minimal HTML → plain text strip
|
||||
text = re.sub(r'<br\s*/?>', '\n', html, flags=re.IGNORECASE)
|
||||
text = re.sub(r'<[^>]+>', '', text)
|
||||
import html as html_module
|
||||
return html_module.unescape(text).strip()
|
||||
|
||||
return ''
|
||||
|
||||
|
||||
def _get_setting(key, default=''):
|
||||
"""Read a SystemSetting value inside an existing app context."""
|
||||
from app.models import SystemSetting
|
||||
return SystemSetting.get(key, default)
|
||||
|
||||
|
||||
def _load_seen_ids():
|
||||
raw = _get_setting('email_ingested_message_ids', '')
|
||||
return set(x.strip() for x in raw.split(',') if x.strip())
|
||||
|
||||
|
||||
def _save_seen_ids(seen: set):
|
||||
from app.models import SystemSetting
|
||||
from app import db
|
||||
# Keep the most recent N IDs to prevent unbounded growth
|
||||
trimmed = sorted(seen)[-_MAX_STORED_IDS:]
|
||||
SystemSetting.set(
|
||||
'email_ingested_message_ids',
|
||||
','.join(trimmed),
|
||||
'Message-IDs of emails already converted to tickets',
|
||||
)
|
||||
db.session.commit()
|
||||
|
||||
|
||||
# ── Core ingestion logic ──────────────────────────────────────────────────────
|
||||
|
||||
def check_inbound_email(app):
|
||||
"""Entry point called by APScheduler. Wraps _run_ingestion with
|
||||
error isolation so a transient IMAP failure never kills the worker."""
|
||||
with app.app_context():
|
||||
try:
|
||||
if _get_setting('email_ingestion_enabled', '0') != '1':
|
||||
return
|
||||
_run_ingestion(app)
|
||||
except Exception as exc:
|
||||
logger.error(f'[EMAIL INGEST] Unhandled error: {exc}', exc_info=True)
|
||||
|
||||
|
||||
def _run_ingestion(app):
|
||||
from app import db
|
||||
from app.models import (Ticket, TicketStatus, TicketPriority,
|
||||
TicketCategory, User, UserRole)
|
||||
from app.services.notification_service import notify_new_ticket
|
||||
from app.services.sla_service import set_due_date
|
||||
|
||||
host = _get_setting('email_ingestion_host', '')
|
||||
port = int(_get_setting('email_ingestion_port', '993'))
|
||||
username = _get_setting('email_ingestion_user', '')
|
||||
password = _get_setting('email_ingestion_password', '')
|
||||
folder = _get_setting('email_ingestion_folder', 'INBOX')
|
||||
move_to = _get_setting('email_ingestion_move_to', 'Processed')
|
||||
|
||||
if not host or not username or not password:
|
||||
logger.warning('[EMAIL INGEST] Missing IMAP credentials — skipping')
|
||||
return
|
||||
|
||||
seen_ids = _load_seen_ids()
|
||||
new_ids = set()
|
||||
created = 0
|
||||
|
||||
try:
|
||||
imap = imaplib.IMAP4_SSL(host, port)
|
||||
imap.login(username, password)
|
||||
except Exception as exc:
|
||||
logger.error(f'[EMAIL INGEST] IMAP login failed: {exc}')
|
||||
return
|
||||
|
||||
try:
|
||||
imap.select(folder)
|
||||
# Search for unseen messages only
|
||||
status, data = imap.search(None, 'UNSEEN')
|
||||
if status != 'OK' or not data[0]:
|
||||
return
|
||||
|
||||
msg_ids = data[0].split()
|
||||
logger.info(f'[EMAIL INGEST] Found {len(msg_ids)} unseen message(s) in {folder}')
|
||||
|
||||
for num in msg_ids:
|
||||
try:
|
||||
_, raw = imap.fetch(num, '(RFC822)')
|
||||
msg = email.message_from_bytes(raw[0][1])
|
||||
|
||||
message_id = msg.get('Message-ID', '').strip()
|
||||
if message_id and message_id in seen_ids:
|
||||
logger.debug(f'[EMAIL INGEST] Skipping duplicate {message_id}')
|
||||
continue
|
||||
|
||||
# ── Parse headers ─────────────────────────────────────────────
|
||||
subject = _decode_header_value(msg.get('Subject', '(No Subject)'))
|
||||
from_raw = msg.get('From', '')
|
||||
from_name, from_email = parseaddr(from_raw)
|
||||
from_email = from_email.lower().strip()
|
||||
body = _extract_plain_text(msg)
|
||||
|
||||
if not body:
|
||||
body = f'[Email received from {from_email} with no readable body]'
|
||||
|
||||
# Truncate very long bodies to 8000 chars
|
||||
if len(body) > 8000:
|
||||
body = body[:8000] + '\n\n[…message truncated…]'
|
||||
|
||||
# ── Resolve sender to a user ──────────────────────────────────
|
||||
sender_user = User.query.filter_by(
|
||||
email=from_email, is_active=True
|
||||
).first()
|
||||
|
||||
if sender_user:
|
||||
created_by_id = sender_user.id
|
||||
external_note = None
|
||||
else:
|
||||
# Fall back to the first active admin
|
||||
fallback = User.query.filter(
|
||||
User.role == UserRole.ADMIN,
|
||||
User.is_active == True,
|
||||
).first()
|
||||
if not fallback:
|
||||
logger.warning(
|
||||
f'[EMAIL INGEST] No fallback admin found, skipping: {from_email}'
|
||||
)
|
||||
continue
|
||||
created_by_id = fallback.id
|
||||
external_note = (
|
||||
f'**[Email received from unknown sender]**\n\n'
|
||||
f'From: {from_name} <{from_email}>\n\n'
|
||||
f'This ticket was automatically created from an inbound email. '
|
||||
f'The sender is not a registered user — please follow up directly.'
|
||||
)
|
||||
|
||||
# ── Create the ticket ─────────────────────────────────────────
|
||||
ticket = Ticket(
|
||||
title = subject[:200],
|
||||
description = body,
|
||||
category = TicketCategory.OTHER,
|
||||
priority = TicketPriority.MEDIUM,
|
||||
status = TicketStatus.OPEN,
|
||||
created_by_id = created_by_id,
|
||||
ai_generated = False,
|
||||
)
|
||||
ticket.ticket_number = ticket.generate_ticket_number()
|
||||
set_due_date(ticket, app)
|
||||
db.session.add(ticket)
|
||||
db.session.flush()
|
||||
|
||||
# Prepend external-sender note as a comment if needed
|
||||
if external_note:
|
||||
from app.models import Comment
|
||||
from app.services.validation_service import render_comment_body
|
||||
comment = Comment(
|
||||
ticket_id = ticket.id,
|
||||
author_id = created_by_id,
|
||||
body = render_comment_body(external_note),
|
||||
is_internal= True, # IT staff only
|
||||
)
|
||||
db.session.add(comment)
|
||||
|
||||
from app.services.log_service import log_action
|
||||
log_action(
|
||||
created_by_id, 'ticket_create_email', 'ticket', ticket.id,
|
||||
f'ticket_number={ticket.ticket_number} '
|
||||
f'from_email={from_email} message_id={message_id or "none"}'
|
||||
)
|
||||
db.session.commit()
|
||||
logger.info(
|
||||
f'[EMAIL INGEST] Created ticket {ticket.ticket_number} '
|
||||
f'from={from_email} subject="{subject[:60]}"'
|
||||
)
|
||||
notify_new_ticket(ticket)
|
||||
|
||||
# Track message ID for deduplication
|
||||
if message_id:
|
||||
new_ids.add(message_id)
|
||||
created += 1
|
||||
|
||||
# Move processed message to done folder
|
||||
try:
|
||||
imap.create(move_to)
|
||||
except Exception:
|
||||
pass # folder may already exist
|
||||
imap.copy(num, move_to)
|
||||
imap.store(num, '+FLAGS', '\\Deleted')
|
||||
|
||||
except Exception as exc:
|
||||
db.session.rollback()
|
||||
logger.error(f'[EMAIL INGEST] Failed to process message {num}: {exc}',
|
||||
exc_info=True)
|
||||
|
||||
imap.expunge()
|
||||
|
||||
finally:
|
||||
try:
|
||||
imap.logout()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if new_ids:
|
||||
_save_seen_ids(seen_ids | new_ids)
|
||||
|
||||
if created:
|
||||
logger.info(f'[EMAIL INGEST] Run complete — {created} ticket(s) created')
|
||||
Reference in New Issue
Block a user