From 36945f59767666e122472d30e8534b7f8b3d2cd5 Mon Sep 17 00:00:00 2001 From: NguyenND Date: Tue, 7 Apr 2026 16:01:26 -0400 Subject: [PATCH] 04/07 updated report via email including attachments --- app/services/email_ingestion_service.py | 163 +++++++++++++++++++++++- 1 file changed, 159 insertions(+), 4 deletions(-) diff --git a/app/services/email_ingestion_service.py b/app/services/email_ingestion_service.py index 4ab5752..f3b60b6 100644 --- a/app/services/email_ingestion_service.py +++ b/app/services/email_ingestion_service.py @@ -36,14 +36,23 @@ email_ingestion_move_to Folder to move processed mail into (default Processed import email import imaplib import logging +import os import re +import uuid from datetime import datetime from email.header import decode_header from email.utils import parseaddr, getaddresses logger = logging.getLogger(__name__) -_MAX_STORED_IDS = 500 # cap on the message-ID suppression list +_MAX_STORED_IDS = 500 # cap on the message-ID suppression list +_MAX_ATTACH_BYTES = 16 * 1024 * 1024 # 16 MB per attachment — matches manual upload limit +_MAX_ATTACH_COUNT = 10 # max attachments per email +_ALLOWED_EXT = { # mirrors ALLOWED_EXT in tickets.py + 'png', 'jpg', 'jpeg', 'gif', + 'pdf', 'doc', 'docx', + 'txt', 'zip', 'log', +} # ── Helpers ─────────────────────────────────────────────────────────────────── @@ -115,6 +124,129 @@ def _extract_plain_text(msg): return '' +def _extract_attachments(msg): + """Walk a MIME message and collect all file attachments. + + Returns a list of (filename, content_type, data_bytes) tuples. + + Selection criteria + ------------------ + A MIME part is treated as an attachment when it meets ANY of these: + - Content-Disposition is 'attachment' (standard), OR + - Content-Disposition is 'inline' and the part has a filename parameter + (common for images pasted inline, e.g. screenshots). + + Parts without a usable filename, parts that are the text body + (text/plain or text/html without a filename), and parts with + Content-Disposition: inline but no filename are all skipped. + + Limits enforced here (caller enforces the count cap separately): + - Files larger than _MAX_ATTACH_BYTES are skipped with a warning. + - Files whose extension is not in _ALLOWED_EXT are skipped with a warning. + """ + attachments = [] + + for part in msg.walk(): + cd = part.get('Content-Disposition', '') or '' + ct = part.get_content_type() or 'application/octet-stream' + + # Derive filename from Content-Disposition or Content-Type params + filename = ( + part.get_filename() + or part.get_param('name') # some clients use Content-Type: name= + or part.get_param('filename') + ) + + is_attachment = 'attachment' in cd.lower() + is_inline_file = 'inline' in cd.lower() and filename + + if not (is_attachment or is_inline_file): + continue + + if not filename: + continue + + # Decode RFC-2047 encoded filenames + filename = _decode_header_value(filename) + if not filename: + continue + + # Extension check + ext = filename.rsplit('.', 1)[-1].lower() if '.' in filename else '' + if ext not in _ALLOWED_EXT: + logger.warning( + f'[EMAIL INGEST] Skipping attachment "{filename}" ' + f'— extension ".{ext}" not in allowed list' + ) + continue + + # Read the payload + try: + data = part.get_payload(decode=True) + except Exception as exc: + logger.warning(f'[EMAIL INGEST] Could not decode attachment "{filename}": {exc}') + continue + + if not data: + continue + + # Size check + if len(data) > _MAX_ATTACH_BYTES: + size_mb = len(data) / (1024 * 1024) + logger.warning( + f'[EMAIL INGEST] Skipping attachment "{filename}" ' + f'— size {size_mb:.1f} MB exceeds {_MAX_ATTACH_BYTES // (1024*1024)} MB limit' + ) + continue + + attachments.append((filename, ct, data)) + + return attachments + + +def _save_email_attachments(attachments, ticket_id, uploader_id, upload_dir): + """Persist attachment files to disk and create Attachment DB rows. + + This mirrors the logic in tickets.py:save_attachment() exactly, + using UUID-based stored names so files never collide. + + Returns the number of attachments successfully saved. + Caller is responsible for committing the session afterwards. + """ + from app.models import Attachment + from app import db + + saved = 0 + for filename, content_type, data in attachments: + ext = filename.rsplit('.', 1)[-1].lower() if '.' in filename else 'bin' + stored_name = f"{uuid.uuid4().hex}.{ext}" + filepath = os.path.join(upload_dir, stored_name) + try: + with open(filepath, 'wb') as fh: + fh.write(data) + except OSError as exc: + logger.error(f'[EMAIL INGEST] Failed to write attachment "{filename}": {exc}') + continue + + att = Attachment( + ticket_id = ticket_id, + comment_id = None, + filename = filename, + stored_name = stored_name, + file_size = len(data), + mime_type = content_type, + uploaded_by = uploader_id, + ) + db.session.add(att) + saved += 1 + logger.info( + f'[EMAIL INGEST] Saved attachment "{filename}" ' + f'({len(data)} bytes) → {stored_name} for ticket_id={ticket_id}' + ) + + return saved + + def _get_setting(key, default=''): """Read a SystemSetting value inside an existing app context.""" from app.models import SystemSetting @@ -216,6 +348,16 @@ def _run_ingestion(app): if len(body) > 8000: body = body[:8000] + '\n\n[…message truncated…]' + # ── Extract attachments from the MIME message ────────────── + # Done before any DB work so we can log counts accurately. + raw_attachments = _extract_attachments(msg) + if len(raw_attachments) > _MAX_ATTACH_COUNT: + logger.warning( + f'[EMAIL INGEST] Email has {len(raw_attachments)} attachments ' + f'— capping at {_MAX_ATTACH_COUNT}' + ) + raw_attachments = raw_attachments[:_MAX_ATTACH_COUNT] + # ── Resolve sender to a user ────────────────────────────────── sender_user = User.query.filter_by( email=from_email, is_active=True @@ -270,16 +412,29 @@ def _run_ingestion(app): ) db.session.add(comment) + # ── Save email attachments ──────────────────────────────── + # ticket.id is available after flush(), so Attachment rows + # can reference it. Files are written to UPLOAD_FOLDER and + # flushed into the session here; the single commit() below + # persists ticket + comment + all attachments atomically. + upload_dir = app.config['UPLOAD_FOLDER'] + os.makedirs(upload_dir, exist_ok=True) + attach_count = _save_email_attachments( + raw_attachments, ticket.id, created_by_id, upload_dir + ) + from app.services.log_service import log_action log_action( created_by_id, 'ticket_create_email', 'ticket', ticket.id, f'ticket_number={ticket.ticket_number} ' - f'from_email={from_email} message_id={message_id or "none"}' + f'from_email={from_email} message_id={message_id or "none"} ' + f'attachments={attach_count}' ) db.session.commit() logger.info( f'[EMAIL INGEST] Created ticket {ticket.ticket_number} ' - f'from={from_email} subject="{subject[:60]}"' + f'from={from_email} subject="{subject[:60]}" ' + f'attachments={attach_count}' ) notify_new_ticket(ticket) @@ -313,4 +468,4 @@ def _run_ingestion(app): _save_seen_ids(seen_ids | new_ids) if created: - logger.info(f'[EMAIL INGEST] Run complete — {created} ticket(s) created') + logger.info(f'[EMAIL INGEST] Run complete — {created} ticket(s) created') \ No newline at end of file