04/07 updated report via email including attachments
This commit is contained in:
@@ -36,14 +36,23 @@ email_ingestion_move_to Folder to move processed mail into (default Processed
|
||||
import email
|
||||
import imaplib
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from email.header import decode_header
|
||||
from email.utils import parseaddr, getaddresses
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_STORED_IDS = 500 # cap on the message-ID suppression list
|
||||
_MAX_STORED_IDS = 500 # cap on the message-ID suppression list
|
||||
_MAX_ATTACH_BYTES = 16 * 1024 * 1024 # 16 MB per attachment — matches manual upload limit
|
||||
_MAX_ATTACH_COUNT = 10 # max attachments per email
|
||||
_ALLOWED_EXT = { # mirrors ALLOWED_EXT in tickets.py
|
||||
'png', 'jpg', 'jpeg', 'gif',
|
||||
'pdf', 'doc', 'docx',
|
||||
'txt', 'zip', 'log',
|
||||
}
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
@@ -115,6 +124,129 @@ def _extract_plain_text(msg):
|
||||
return ''
|
||||
|
||||
|
||||
def _extract_attachments(msg):
|
||||
"""Walk a MIME message and collect all file attachments.
|
||||
|
||||
Returns a list of (filename, content_type, data_bytes) tuples.
|
||||
|
||||
Selection criteria
|
||||
------------------
|
||||
A MIME part is treated as an attachment when it meets ANY of these:
|
||||
- Content-Disposition is 'attachment' (standard), OR
|
||||
- Content-Disposition is 'inline' and the part has a filename parameter
|
||||
(common for images pasted inline, e.g. screenshots).
|
||||
|
||||
Parts without a usable filename, parts that are the text body
|
||||
(text/plain or text/html without a filename), and parts with
|
||||
Content-Disposition: inline but no filename are all skipped.
|
||||
|
||||
Limits enforced here (caller enforces the count cap separately):
|
||||
- Files larger than _MAX_ATTACH_BYTES are skipped with a warning.
|
||||
- Files whose extension is not in _ALLOWED_EXT are skipped with a warning.
|
||||
"""
|
||||
attachments = []
|
||||
|
||||
for part in msg.walk():
|
||||
cd = part.get('Content-Disposition', '') or ''
|
||||
ct = part.get_content_type() or 'application/octet-stream'
|
||||
|
||||
# Derive filename from Content-Disposition or Content-Type params
|
||||
filename = (
|
||||
part.get_filename()
|
||||
or part.get_param('name') # some clients use Content-Type: name=
|
||||
or part.get_param('filename')
|
||||
)
|
||||
|
||||
is_attachment = 'attachment' in cd.lower()
|
||||
is_inline_file = 'inline' in cd.lower() and filename
|
||||
|
||||
if not (is_attachment or is_inline_file):
|
||||
continue
|
||||
|
||||
if not filename:
|
||||
continue
|
||||
|
||||
# Decode RFC-2047 encoded filenames
|
||||
filename = _decode_header_value(filename)
|
||||
if not filename:
|
||||
continue
|
||||
|
||||
# Extension check
|
||||
ext = filename.rsplit('.', 1)[-1].lower() if '.' in filename else ''
|
||||
if ext not in _ALLOWED_EXT:
|
||||
logger.warning(
|
||||
f'[EMAIL INGEST] Skipping attachment "{filename}" '
|
||||
f'— extension ".{ext}" not in allowed list'
|
||||
)
|
||||
continue
|
||||
|
||||
# Read the payload
|
||||
try:
|
||||
data = part.get_payload(decode=True)
|
||||
except Exception as exc:
|
||||
logger.warning(f'[EMAIL INGEST] Could not decode attachment "{filename}": {exc}')
|
||||
continue
|
||||
|
||||
if not data:
|
||||
continue
|
||||
|
||||
# Size check
|
||||
if len(data) > _MAX_ATTACH_BYTES:
|
||||
size_mb = len(data) / (1024 * 1024)
|
||||
logger.warning(
|
||||
f'[EMAIL INGEST] Skipping attachment "{filename}" '
|
||||
f'— size {size_mb:.1f} MB exceeds {_MAX_ATTACH_BYTES // (1024*1024)} MB limit'
|
||||
)
|
||||
continue
|
||||
|
||||
attachments.append((filename, ct, data))
|
||||
|
||||
return attachments
|
||||
|
||||
|
||||
def _save_email_attachments(attachments, ticket_id, uploader_id, upload_dir):
|
||||
"""Persist attachment files to disk and create Attachment DB rows.
|
||||
|
||||
This mirrors the logic in tickets.py:save_attachment() exactly,
|
||||
using UUID-based stored names so files never collide.
|
||||
|
||||
Returns the number of attachments successfully saved.
|
||||
Caller is responsible for committing the session afterwards.
|
||||
"""
|
||||
from app.models import Attachment
|
||||
from app import db
|
||||
|
||||
saved = 0
|
||||
for filename, content_type, data in attachments:
|
||||
ext = filename.rsplit('.', 1)[-1].lower() if '.' in filename else 'bin'
|
||||
stored_name = f"{uuid.uuid4().hex}.{ext}"
|
||||
filepath = os.path.join(upload_dir, stored_name)
|
||||
try:
|
||||
with open(filepath, 'wb') as fh:
|
||||
fh.write(data)
|
||||
except OSError as exc:
|
||||
logger.error(f'[EMAIL INGEST] Failed to write attachment "{filename}": {exc}')
|
||||
continue
|
||||
|
||||
att = Attachment(
|
||||
ticket_id = ticket_id,
|
||||
comment_id = None,
|
||||
filename = filename,
|
||||
stored_name = stored_name,
|
||||
file_size = len(data),
|
||||
mime_type = content_type,
|
||||
uploaded_by = uploader_id,
|
||||
)
|
||||
db.session.add(att)
|
||||
saved += 1
|
||||
logger.info(
|
||||
f'[EMAIL INGEST] Saved attachment "{filename}" '
|
||||
f'({len(data)} bytes) → {stored_name} for ticket_id={ticket_id}'
|
||||
)
|
||||
|
||||
return saved
|
||||
|
||||
|
||||
def _get_setting(key, default=''):
|
||||
"""Read a SystemSetting value inside an existing app context."""
|
||||
from app.models import SystemSetting
|
||||
@@ -216,6 +348,16 @@ def _run_ingestion(app):
|
||||
if len(body) > 8000:
|
||||
body = body[:8000] + '\n\n[…message truncated…]'
|
||||
|
||||
# ── Extract attachments from the MIME message ──────────────
|
||||
# Done before any DB work so we can log counts accurately.
|
||||
raw_attachments = _extract_attachments(msg)
|
||||
if len(raw_attachments) > _MAX_ATTACH_COUNT:
|
||||
logger.warning(
|
||||
f'[EMAIL INGEST] Email has {len(raw_attachments)} attachments '
|
||||
f'— capping at {_MAX_ATTACH_COUNT}'
|
||||
)
|
||||
raw_attachments = raw_attachments[:_MAX_ATTACH_COUNT]
|
||||
|
||||
# ── Resolve sender to a user ──────────────────────────────────
|
||||
sender_user = User.query.filter_by(
|
||||
email=from_email, is_active=True
|
||||
@@ -270,16 +412,29 @@ def _run_ingestion(app):
|
||||
)
|
||||
db.session.add(comment)
|
||||
|
||||
# ── Save email attachments ────────────────────────────────
|
||||
# ticket.id is available after flush(), so Attachment rows
|
||||
# can reference it. Files are written to UPLOAD_FOLDER and
|
||||
# flushed into the session here; the single commit() below
|
||||
# persists ticket + comment + all attachments atomically.
|
||||
upload_dir = app.config['UPLOAD_FOLDER']
|
||||
os.makedirs(upload_dir, exist_ok=True)
|
||||
attach_count = _save_email_attachments(
|
||||
raw_attachments, ticket.id, created_by_id, upload_dir
|
||||
)
|
||||
|
||||
from app.services.log_service import log_action
|
||||
log_action(
|
||||
created_by_id, 'ticket_create_email', 'ticket', ticket.id,
|
||||
f'ticket_number={ticket.ticket_number} '
|
||||
f'from_email={from_email} message_id={message_id or "none"}'
|
||||
f'from_email={from_email} message_id={message_id or "none"} '
|
||||
f'attachments={attach_count}'
|
||||
)
|
||||
db.session.commit()
|
||||
logger.info(
|
||||
f'[EMAIL INGEST] Created ticket {ticket.ticket_number} '
|
||||
f'from={from_email} subject="{subject[:60]}"'
|
||||
f'from={from_email} subject="{subject[:60]}" '
|
||||
f'attachments={attach_count}'
|
||||
)
|
||||
notify_new_ticket(ticket)
|
||||
|
||||
@@ -313,4 +468,4 @@ def _run_ingestion(app):
|
||||
_save_seen_ids(seen_ids | new_ids)
|
||||
|
||||
if created:
|
||||
logger.info(f'[EMAIL INGEST] Run complete — {created} ticket(s) created')
|
||||
logger.info(f'[EMAIL INGEST] Run complete — {created} ticket(s) created')
|
||||
Reference in New Issue
Block a user