06/05 Optimize app: logs will be saved to db

2026-06-05 12:48:55 -04:00
parent 8f8b0348c6
commit f8114e7997
8 changed files with 355 additions and 94 deletions
@@ -78,6 +78,14 @@ def create_app(config_name=None):
    csrf.init_app(app)
    limiter.init_app(app)

+    # Add DB log handler after db is ready
+    from app.utils.db_log_handler import DBLogHandler
+    _db_handler = DBLogHandler()
+    _db_handler.setLevel(logging.INFO)
+    app_log = logging.getLogger('app')
+    if not any(isinstance(h, DBLogHandler) for h in app_log.handlers):
+        app_log.addHandler(_db_handler)
+
    # ── Sentry (optional) ────────────────────────────────────────────────────
    sentry_dsn = app.config.get('SENTRY_DSN', '')
    if sentry_dsn:
@@ -135,6 +143,8 @@ def create_app(config_name=None):
        from app.models.teller_enrollment import TellerEnrollment, TellerAccount
        from app.models.schwab_connection import SchwabConnection, SchwabAccount
        from app.models.plaid_item import PlaidItem, PlaidAccount, PlaidSyncPreview
+        from app.models.app_log import AppLog
+        from app.models.audit_log import AuditLog

    # ── Session idle timeout ──────────────────────────────────────────────────
    from flask import session as _session, request as _request
@@ -0,0 +1,15 @@
+from app.extensions import db
+from datetime import datetime
+
+
+class AppLog(db.Model):
+    __tablename__ = 'app_logs'
+
+    id        = db.Column(db.Integer, primary_key=True)
+    timestamp = db.Column(db.DateTime, default=datetime.utcnow, index=True)
+    level     = db.Column(db.String(16), nullable=False, index=True)
+    module    = db.Column(db.String(128))
+    message   = db.Column(db.Text)
+
+    def __repr__(self):
+        return f'<AppLog {self.level} {self.timestamp}>'
@@ -1,5 +1,7 @@
 import os
 import logging
+from datetime import datetime, timedelta
+
 from flask import (Blueprint, render_template, request, jsonify,
                   current_app, send_file, abort)
 from flask_login import login_required
@@ -8,115 +10,108 @@ logs_bp = Blueprint('logs', __name__, url_prefix='/logs')
 log = logging.getLogger(__name__)


-def _tail_lines(filepath, n=2000):
-    """Return the last n lines from a file without loading it all into memory."""
-    try:
-        with open(filepath, 'rb') as f:
-            f.seek(0, 2)
-            size = f.tell()
-            if size == 0:
-                return []
-            buf = bytearray()
-            pos = size
-            while len(buf) < 256 * 1024 and pos > 0:   # max 256 KB scan
-                chunk = min(65536, pos)
-                pos -= chunk
-                f.seek(pos)
-                buf = bytearray(f.read(chunk)) + buf
-            lines = buf.decode('utf-8', errors='replace').splitlines()
-            return lines[-n:]
-    except FileNotFoundError:
-        return []
-    except Exception as exc:
-        log.error('log viewer _tail_lines error: %s', exc)
-        return []
-
-
-def _parse_line(raw):
-    """Parse pipe-delimited log line: timestamp|level|name|message"""
-    parts = raw.split('|', 3)
-    if len(parts) == 4:
-        return {
-            'ts':      parts[0],
-            'level':   parts[1],
-            'name':    parts[2],
-            'message': parts[3],
-        }
-    # Fallback for lines that don't match the format (e.g. tracebacks)
-    return {'ts': '', 'level': 'RAW', 'name': '', 'message': raw}
-
+# ── Index ─────────────────────────────────────────────────────────────────────

@logs_bp.route('/')
@login_required
 def index():
-    import datetime
    log_file = current_app.config.get('LOG_FILE_PATH', '')
    file_exists = bool(log_file) and os.path.isfile(log_file)
    file_size = os.path.getsize(log_file) if file_exists else 0
    file_mtime = None
    if file_exists:
        ts = os.path.getmtime(log_file)
-        file_mtime = datetime.datetime.fromtimestamp(ts).strftime('%b %d, %H:%M')
+        file_mtime = datetime.fromtimestamp(ts).strftime('%b %d, %H:%M')

    def fmt_size(b):
        if b < 1024: return f'{b} B'
        if b < 1024**2: return f'{b/1024:.1f} KB'
        return f'{b/1024**2:.1f} MB'

+    # DB row count
+    db_count = 0
+    try:
+        from app.models.app_log import AppLog
+        db_count = AppLog.query.count()
+    except Exception:
+        pass
+
    return render_template('logs/index.html',
                           log_file=log_file,
                           file_exists=file_exists,
                           file_size=file_size,
                           file_size_fmt=fmt_size(file_size),
-                           file_mtime=file_mtime)
+                           file_mtime=file_mtime,
+                           db_count=db_count)


+# ── API (reads from DB) ───────────────────────────────────────────────────────
+
@logs_bp.route('/api')
@login_required
 def api():
-    """JSON endpoint used by the log viewer to fetch and filter entries."""
-    log_file = current_app.config.get('LOG_FILE_PATH', '')
-    if not log_file:
-        return jsonify({'entries': [], 'error': 'LOG_FILE_PATH not configured'})
+    try:
+        from app.models.app_log import AppLog
+        from app.extensions import db
+    except Exception as e:
+        return jsonify({'entries': [], 'error': str(e)})

    level_filter = request.args.get('level', 'ALL').upper()
    search       = request.args.get('search', '').lower()
    limit        = min(int(request.args.get('limit', 200)), 2000)
    module       = request.args.get('module', '').lower()

-    raw_lines = _tail_lines(log_file, n=5000)
-    entries = [_parse_line(l) for l in raw_lines if l.strip()]
+    try:
+        query = AppLog.query

-    if level_filter not in ('ALL', ''):
-        entries = [e for e in entries if e['level'] == level_filter]
+        if level_filter not in ('ALL', ''):
+            query = query.filter(AppLog.level == level_filter)

-    if module:
-        entries = [e for e in entries if module in e['name'].lower()]
+        if module:
+            query = query.filter(AppLog.module.ilike(f'%{module}%'))

-    if search:
-        entries = [e for e in entries
-                   if search in e['message'].lower() or search in e['name'].lower()]
+        if search:
+            query = query.filter(
+                db.or_(
+                    AppLog.message.ilike(f'%{search}%'),
+                    AppLog.module.ilike(f'%{search}%'),
+                )
+            )

-    # Most-recent first, capped at limit
-    entries = list(reversed(entries))[:limit]
+        total = query.count()
+        rows = query.order_by(AppLog.timestamp.desc()).limit(limit).all()

-    # Count per level + unique modules across ALL unfiltered lines
-    all_entries = [_parse_line(l) for l in raw_lines if l.strip()]
-    counts = {}
-    modules_seen = set()
-    for e in all_entries:
-        counts[e['level']] = counts.get(e['level'], 0) + 1
-        if e['name']:
-            modules_seen.add(e['name'])
+        entries = [
+            {
+                'ts':      r.timestamp.strftime('%Y-%m-%d %H:%M:%S') if r.timestamp else '',
+                'level':   r.level or 'INFO',
+                'name':    r.module or '',
+                'message': r.message or '',
+            }
+            for r in rows
+        ]

-    return jsonify({
-        'entries':    entries,
-        'counts':     counts,
-        'total_raw':  len(raw_lines),
-        'log_file':   log_file,
-        'modules':    sorted(modules_seen),
-    })
+        # Counts and modules across ALL unfiltered rows (capped for performance)
+        all_rows = AppLog.query.with_entities(AppLog.level, AppLog.module).all()
+        counts = {}
+        modules_seen = set()
+        for lvl, mod in all_rows:
+            counts[lvl] = counts.get(lvl, 0) + 1
+            if mod:
+                modules_seen.add(mod)

+        return jsonify({
+            'entries':   entries,
+            'counts':    counts,
+            'total_raw': total,
+            'modules':   sorted(modules_seen),
+        })
+    except Exception as e:
+        log.error('logs api error: %s', e)
+        return jsonify({'entries': [], 'error': str(e), 'counts': {}, 'total_raw': 0, 'modules': []})
+
+
+# ── Download (still from file) ────────────────────────────────────────────────

@logs_bp.route('/download')
@login_required
@@ -127,16 +122,57 @@ def download():
    return send_file(log_file, as_attachment=True, download_name='pfm-app.log')


+# ── Clear (truncate DB table + file) ─────────────────────────────────────────
+
@logs_bp.route('/clear', methods=['POST'])
@login_required
 def clear():
-    log_file = current_app.config.get('LOG_FILE_PATH', '')
-    if not log_file:
-        return jsonify({'error': 'LOG_FILE_PATH not configured'}), 400
+    errors = []
+
+    # Clear DB table
    try:
-        with open(log_file, 'w', encoding='utf-8'):
-            pass
-        log.info('Log file cleared by user')
-        return jsonify({'status': 'ok'})
+        from app.models.app_log import AppLog
+        from app.extensions import db
+        AppLog.query.delete()
+        db.session.commit()
    except Exception as exc:
+        errors.append(f'DB: {exc}')
+
+    # Clear log file
+    log_file = current_app.config.get('LOG_FILE_PATH', '')
+    if log_file:
+        try:
+            with open(log_file, 'w', encoding='utf-8'):
+                pass
+        except Exception as exc:
+            errors.append(f'file: {exc}')
+
+    if errors:
+        return jsonify({'error': '; '.join(errors)}), 500
+    log.info('App logs cleared by user')
+    return jsonify({'status': 'ok'})
+
+
+# ── Purge (delete entries older than N days) ──────────────────────────────────
+
+@logs_bp.route('/purge', methods=['POST'])
+@login_required
+def purge():
+    try:
+        days = int(request.form.get('days', 30))
+        if days not in (7, 30, 90):
+            return jsonify({'error': 'days must be 7, 30, or 90'}), 400
+    except (ValueError, TypeError):
+        return jsonify({'error': 'Invalid days parameter'}), 400
+
+    cutoff = datetime.utcnow() - timedelta(days=days)
+    try:
+        from app.models.app_log import AppLog
+        from app.extensions import db
+        deleted = AppLog.query.filter(AppLog.timestamp < cutoff).delete()
+        db.session.commit()
+        log.info('App logs purged: %d entries older than %d days deleted', deleted, days)
+        return jsonify({'status': 'ok', 'deleted': deleted})
+    except Exception as exc:
+        log.error('App log purge failed: %s', exc)
        return jsonify({'error': str(exc)}), 500
@@ -470,3 +470,24 @@ def audit_log():
                           logs=pagination.items,
                           actions=actions,
                           action=action)
+
+
+@settings_bp.route('/audit/purge', methods=['POST'])
+@login_required
+def audit_purge():
+    from app.models.audit_log import AuditLog
+    from datetime import timedelta, datetime
+    try:
+        days = int(request.form.get('days', 30))
+        if days not in (7, 30, 90):
+            flash('Invalid purge period.', 'danger')
+            return redirect(url_for('settings.audit_log'))
+    except (ValueError, TypeError):
+        flash('Invalid purge period.', 'danger')
+        return redirect(url_for('settings.audit_log'))
+
+    cutoff = datetime.utcnow() - timedelta(days=days)
+    deleted = AuditLog.query.filter(AuditLog.timestamp < cutoff).delete()
+    db.session.commit()
+    flash(f'Deleted {deleted} audit log entries older than {days} days.', 'success')
+    return redirect(url_for('settings.audit_log'))
@@ -168,12 +168,10 @@ tr.expanded-row > td {
        {% if file_exists %}
        <span class="log-meta-chip"><i class="bi bi-hdd"></i>{{ file_size_fmt }}</span>
        <span class="log-meta-chip"><i class="bi bi-clock"></i>{{ file_mtime }}</span>
-        <span class="log-meta-chip" style="max-width:360px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap;" title="{{ log_file }}">
-            <i class="bi bi-file-text"></i>{{ log_file }}
-        </span>
        {% else %}
        <span class="log-meta-chip" style="color:#ef4444;"><i class="bi bi-exclamation-triangle"></i>Log file not found</span>
        {% endif %}
+        <span class="log-meta-chip"><i class="bi bi-database"></i>{{ db_count }} entries in DB</span>
    </div>
    <div class="ms-auto">
        <span id="spinner" class="spinner-border spinner-border-sm text-secondary"></span>
@@ -226,8 +224,21 @@ tr.expanded-row > td {
        <i class="bi bi-download me-1"></i><span class="d-none d-sm-inline">Download</span>
    </a>

-    <button class="btn btn-sm btn-outline-danger" id="clear-btn" title="Clear log file">
-        <i class="bi bi-trash me-1"></i><span class="d-none d-sm-inline">Clear</span>
+    <div class="dropdown">
+        <button class="btn btn-sm btn-outline-warning dropdown-toggle" type="button"
+                data-bs-toggle="dropdown" title="Purge old log entries from database">
+            <i class="bi bi-clock-history me-1"></i><span class="d-none d-sm-inline">Purge</span>
+        </button>
+        <ul class="dropdown-menu dropdown-menu-end" style="font-size:13px;">
+            <li><h6 class="dropdown-header">Delete entries older than…</h6></li>
+            <li><a class="dropdown-item purge-item" href="#" data-days="7">7 days</a></li>
+            <li><a class="dropdown-item purge-item" href="#" data-days="30">30 days</a></li>
+            <li><a class="dropdown-item purge-item" href="#" data-days="90">90 days</a></li>
+        </ul>
+    </div>
+
+    <button class="btn btn-sm btn-outline-danger" id="clear-btn" title="Clear all logs (DB + file)">
+        <i class="bi bi-trash me-1"></i><span class="d-none d-sm-inline">Clear All</span>
    </button>
 </div>

@@ -470,18 +481,45 @@ document.getElementById('ar-toggle').addEventListener('click', () => {

 // ── Clear ───────────────────────────────────────────────────────────────────
 document.getElementById('clear-btn').addEventListener('click', function() {
-    if (!confirm('Clear all log entries from the file?\nThis cannot be undone.')) return;
+    if (!confirm('Clear ALL log entries from the database and log file?\nThis cannot be undone.')) return;
    fetch('{{ url_for("logs.clear") }}', {
        method: 'POST',
        headers: { 'X-CSRFToken': CSRF, 'Content-Type': 'application/json' },
    })
    .then(r => r.json())
    .then(d => {
-        if (d.status === 'ok') load();
+        if (d.status === 'ok') { load(); location.reload(); }
        else alert('Clear failed: ' + d.error);
    });
 });

+// ── Purge ───────────────────────────────────────────────────────────────────
+document.querySelectorAll('.purge-item').forEach(function(el) {
+    el.addEventListener('click', function(e) {
+        e.preventDefault();
+        const days = this.dataset.days;
+        if (!confirm(`Delete all log entries older than ${days} days?\nThis cannot be undone.`)) return;
+        const fd = new FormData();
+        fd.append('days', days);
+        fetch('{{ url_for("logs.purge") }}', {
+            method: 'POST',
+            headers: { 'X-CSRFToken': CSRF },
+            body: fd,
+        })
+        .then(r => r.json())
+        .then(d => {
+            if (d.status === 'ok') {
+                load();
+                // Update DB count chip
+                const chip = document.querySelector('.log-meta-chip .bi-database');
+                if (chip) chip.parentElement.innerHTML = `<i class="bi bi-database"></i> refreshed`;
+            } else {
+                alert('Purge failed: ' + d.error);
+            }
+        });
+    });
+});
+
 // ── Keyboard shortcuts ──────────────────────────────────────────────────────
 document.addEventListener('keydown', function(e) {
    if (e.target.tagName === 'INPUT' || e.target.tagName === 'SELECT') return;
@@ -4,16 +4,54 @@

 {% block content %}
 <div class="pcard pcard-sm mb-3">
-    <form method="GET" class="d-flex gap-2 align-items-center flex-wrap">
-        <select name="action" class="form-select form-select-sm" style="max-width:220px;">
-            <option value="">All events</option>
-            {% for a in actions %}
-            <option value="{{ a }}" {% if a == action %}selected{% endif %}>{{ a | replace('_',' ') | title }}</option>
-            {% endfor %}
-        </select>
-        <button type="submit" class="btn btn-sm btn-outline-secondary"><i class="bi bi-filter me-1"></i>Filter</button>
-        {% if action %}<a href="{{ url_for('settings.audit_log') }}" class="btn btn-sm btn-outline-secondary"><i class="bi bi-x-lg"></i></a>{% endif %}
-    </form>
+    <div class="d-flex gap-2 align-items-center flex-wrap">
+        <form method="GET" class="d-flex gap-2 align-items-center flex-wrap">
+            <select name="action" class="form-select form-select-sm" style="max-width:220px;">
+                <option value="">All events</option>
+                {% for a in actions %}
+                <option value="{{ a }}" {% if a == action %}selected{% endif %}>{{ a | replace('_',' ') | title }}</option>
+                {% endfor %}
+            </select>
+            <button type="submit" class="btn btn-sm btn-outline-secondary"><i class="bi bi-filter me-1"></i>Filter</button>
+            {% if action %}<a href="{{ url_for('settings.audit_log') }}" class="btn btn-sm btn-outline-secondary"><i class="bi bi-x-lg"></i></a>{% endif %}
+        </form>
+
+        <div class="ms-auto">
+            <div class="dropdown">
+                <button class="btn btn-sm btn-outline-warning dropdown-toggle" type="button"
+                        data-bs-toggle="dropdown" title="Purge old audit log entries">
+                    <i class="bi bi-clock-history me-1"></i>Purge
+                </button>
+                <ul class="dropdown-menu dropdown-menu-end" style="font-size:13px;">
+                    <li><h6 class="dropdown-header">Delete entries older than…</h6></li>
+                    <li>
+                        <form method="POST" action="{{ url_for('settings.audit_purge') }}"
+                              onsubmit="return confirm('Delete audit log entries older than 7 days?')">
+                            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+                            <input type="hidden" name="days" value="7">
+                            <button type="submit" class="dropdown-item">7 days</button>
+                        </form>
+                    </li>
+                    <li>
+                        <form method="POST" action="{{ url_for('settings.audit_purge') }}"
+                              onsubmit="return confirm('Delete audit log entries older than 30 days?')">
+                            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+                            <input type="hidden" name="days" value="30">
+                            <button type="submit" class="dropdown-item">30 days</button>
+                        </form>
+                    </li>
+                    <li>
+                        <form method="POST" action="{{ url_for('settings.audit_purge') }}"
+                              onsubmit="return confirm('Delete audit log entries older than 90 days?')">
+                            <input type="hidden" name="csrf_token" value="{{ csrf_token() }}">
+                            <input type="hidden" name="days" value="90">
+                            <button type="submit" class="dropdown-item">90 days</button>
+                        </form>
+                    </li>
+                </ul>
+            </div>
+        </div>
+    </div>
 </div>

 <div class="pcard p-0">
@@ -0,0 +1,49 @@
+"""
+SQLAlchemy-backed logging handler.
+
+Writes app.* log records to the app_logs table so the web viewer
+can query, filter, and purge them without touching the log file.
+
+Safety rules:
+  - Never raises — all errors are silently swallowed to avoid crashing the app
+  - Reentrancy guard prevents infinite recursion (SQLAlchemy emits its own logs)
+  - Skips sqlalchemy.* and werkzeug loggers to avoid noise / recursion
+"""
+
+import logging
+import traceback
+from datetime import datetime
+
+
+_SKIP_PREFIXES = ('sqlalchemy', 'werkzeug', 'urllib3', 'plaid_handler')
+
+
+class DBLogHandler(logging.Handler):
+    _inside_emit = False
+
+    def emit(self, record):
+        if DBLogHandler._inside_emit:
+            return
+        if any(record.name.startswith(p) for p in _SKIP_PREFIXES):
+            return
+        DBLogHandler._inside_emit = True
+        try:
+            from app.extensions import db
+            from app.models.app_log import AppLog
+
+            msg = record.getMessage()
+            if record.exc_info:
+                msg += '\n' + ''.join(traceback.format_exception(*record.exc_info)).rstrip()
+
+            entry = AppLog(
+                timestamp=datetime.utcfromtimestamp(record.created),
+                level=record.levelname,
+                module=record.name,
+                message=msg,
+            )
+            db.session.add(entry)
+            db.session.commit()
+        except Exception:
+            pass
+        finally:
+            DBLogHandler._inside_emit = False