diff --git a/app/__init__.py b/app/__init__.py index 7dfbb8e..4016df7 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -78,6 +78,14 @@ def create_app(config_name=None): csrf.init_app(app) limiter.init_app(app) + # Add DB log handler after db is ready + from app.utils.db_log_handler import DBLogHandler + _db_handler = DBLogHandler() + _db_handler.setLevel(logging.INFO) + app_log = logging.getLogger('app') + if not any(isinstance(h, DBLogHandler) for h in app_log.handlers): + app_log.addHandler(_db_handler) + # ── Sentry (optional) ──────────────────────────────────────────────────── sentry_dsn = app.config.get('SENTRY_DSN', '') if sentry_dsn: @@ -135,6 +143,8 @@ def create_app(config_name=None): from app.models.teller_enrollment import TellerEnrollment, TellerAccount from app.models.schwab_connection import SchwabConnection, SchwabAccount from app.models.plaid_item import PlaidItem, PlaidAccount, PlaidSyncPreview + from app.models.app_log import AppLog + from app.models.audit_log import AuditLog # ── Session idle timeout ────────────────────────────────────────────────── from flask import session as _session, request as _request diff --git a/app/models/app_log.py b/app/models/app_log.py new file mode 100644 index 0000000..b0c8238 --- /dev/null +++ b/app/models/app_log.py @@ -0,0 +1,15 @@ +from app.extensions import db +from datetime import datetime + + +class AppLog(db.Model): + __tablename__ = 'app_logs' + + id = db.Column(db.Integer, primary_key=True) + timestamp = db.Column(db.DateTime, default=datetime.utcnow, index=True) + level = db.Column(db.String(16), nullable=False, index=True) + module = db.Column(db.String(128)) + message = db.Column(db.Text) + + def __repr__(self): + return f'' diff --git a/app/routes/logs.py b/app/routes/logs.py index 0c97470..585274b 100644 --- a/app/routes/logs.py +++ b/app/routes/logs.py @@ -1,5 +1,7 @@ import os import logging +from datetime import datetime, timedelta + from flask import (Blueprint, render_template, request, jsonify, current_app, send_file, abort) from flask_login import login_required @@ -8,115 +10,108 @@ logs_bp = Blueprint('logs', __name__, url_prefix='/logs') log = logging.getLogger(__name__) -def _tail_lines(filepath, n=2000): - """Return the last n lines from a file without loading it all into memory.""" - try: - with open(filepath, 'rb') as f: - f.seek(0, 2) - size = f.tell() - if size == 0: - return [] - buf = bytearray() - pos = size - while len(buf) < 256 * 1024 and pos > 0: # max 256 KB scan - chunk = min(65536, pos) - pos -= chunk - f.seek(pos) - buf = bytearray(f.read(chunk)) + buf - lines = buf.decode('utf-8', errors='replace').splitlines() - return lines[-n:] - except FileNotFoundError: - return [] - except Exception as exc: - log.error('log viewer _tail_lines error: %s', exc) - return [] - - -def _parse_line(raw): - """Parse pipe-delimited log line: timestamp|level|name|message""" - parts = raw.split('|', 3) - if len(parts) == 4: - return { - 'ts': parts[0], - 'level': parts[1], - 'name': parts[2], - 'message': parts[3], - } - # Fallback for lines that don't match the format (e.g. tracebacks) - return {'ts': '', 'level': 'RAW', 'name': '', 'message': raw} - +# ── Index ───────────────────────────────────────────────────────────────────── @logs_bp.route('/') @login_required def index(): - import datetime log_file = current_app.config.get('LOG_FILE_PATH', '') file_exists = bool(log_file) and os.path.isfile(log_file) file_size = os.path.getsize(log_file) if file_exists else 0 file_mtime = None if file_exists: ts = os.path.getmtime(log_file) - file_mtime = datetime.datetime.fromtimestamp(ts).strftime('%b %d, %H:%M') + file_mtime = datetime.fromtimestamp(ts).strftime('%b %d, %H:%M') def fmt_size(b): if b < 1024: return f'{b} B' if b < 1024**2: return f'{b/1024:.1f} KB' return f'{b/1024**2:.1f} MB' + # DB row count + db_count = 0 + try: + from app.models.app_log import AppLog + db_count = AppLog.query.count() + except Exception: + pass + return render_template('logs/index.html', log_file=log_file, file_exists=file_exists, file_size=file_size, file_size_fmt=fmt_size(file_size), - file_mtime=file_mtime) + file_mtime=file_mtime, + db_count=db_count) +# ── API (reads from DB) ─────────────────────────────────────────────────────── + @logs_bp.route('/api') @login_required def api(): - """JSON endpoint used by the log viewer to fetch and filter entries.""" - log_file = current_app.config.get('LOG_FILE_PATH', '') - if not log_file: - return jsonify({'entries': [], 'error': 'LOG_FILE_PATH not configured'}) + try: + from app.models.app_log import AppLog + from app.extensions import db + except Exception as e: + return jsonify({'entries': [], 'error': str(e)}) level_filter = request.args.get('level', 'ALL').upper() search = request.args.get('search', '').lower() limit = min(int(request.args.get('limit', 200)), 2000) module = request.args.get('module', '').lower() - raw_lines = _tail_lines(log_file, n=5000) - entries = [_parse_line(l) for l in raw_lines if l.strip()] + try: + query = AppLog.query - if level_filter not in ('ALL', ''): - entries = [e for e in entries if e['level'] == level_filter] + if level_filter not in ('ALL', ''): + query = query.filter(AppLog.level == level_filter) - if module: - entries = [e for e in entries if module in e['name'].lower()] + if module: + query = query.filter(AppLog.module.ilike(f'%{module}%')) - if search: - entries = [e for e in entries - if search in e['message'].lower() or search in e['name'].lower()] + if search: + query = query.filter( + db.or_( + AppLog.message.ilike(f'%{search}%'), + AppLog.module.ilike(f'%{search}%'), + ) + ) - # Most-recent first, capped at limit - entries = list(reversed(entries))[:limit] + total = query.count() + rows = query.order_by(AppLog.timestamp.desc()).limit(limit).all() - # Count per level + unique modules across ALL unfiltered lines - all_entries = [_parse_line(l) for l in raw_lines if l.strip()] - counts = {} - modules_seen = set() - for e in all_entries: - counts[e['level']] = counts.get(e['level'], 0) + 1 - if e['name']: - modules_seen.add(e['name']) + entries = [ + { + 'ts': r.timestamp.strftime('%Y-%m-%d %H:%M:%S') if r.timestamp else '', + 'level': r.level or 'INFO', + 'name': r.module or '', + 'message': r.message or '', + } + for r in rows + ] - return jsonify({ - 'entries': entries, - 'counts': counts, - 'total_raw': len(raw_lines), - 'log_file': log_file, - 'modules': sorted(modules_seen), - }) + # Counts and modules across ALL unfiltered rows (capped for performance) + all_rows = AppLog.query.with_entities(AppLog.level, AppLog.module).all() + counts = {} + modules_seen = set() + for lvl, mod in all_rows: + counts[lvl] = counts.get(lvl, 0) + 1 + if mod: + modules_seen.add(mod) + return jsonify({ + 'entries': entries, + 'counts': counts, + 'total_raw': total, + 'modules': sorted(modules_seen), + }) + except Exception as e: + log.error('logs api error: %s', e) + return jsonify({'entries': [], 'error': str(e), 'counts': {}, 'total_raw': 0, 'modules': []}) + + +# ── Download (still from file) ──────────────────────────────────────────────── @logs_bp.route('/download') @login_required @@ -127,16 +122,57 @@ def download(): return send_file(log_file, as_attachment=True, download_name='pfm-app.log') +# ── Clear (truncate DB table + file) ───────────────────────────────────────── + @logs_bp.route('/clear', methods=['POST']) @login_required def clear(): - log_file = current_app.config.get('LOG_FILE_PATH', '') - if not log_file: - return jsonify({'error': 'LOG_FILE_PATH not configured'}), 400 + errors = [] + + # Clear DB table try: - with open(log_file, 'w', encoding='utf-8'): - pass - log.info('Log file cleared by user') - return jsonify({'status': 'ok'}) + from app.models.app_log import AppLog + from app.extensions import db + AppLog.query.delete() + db.session.commit() except Exception as exc: + errors.append(f'DB: {exc}') + + # Clear log file + log_file = current_app.config.get('LOG_FILE_PATH', '') + if log_file: + try: + with open(log_file, 'w', encoding='utf-8'): + pass + except Exception as exc: + errors.append(f'file: {exc}') + + if errors: + return jsonify({'error': '; '.join(errors)}), 500 + log.info('App logs cleared by user') + return jsonify({'status': 'ok'}) + + +# ── Purge (delete entries older than N days) ────────────────────────────────── + +@logs_bp.route('/purge', methods=['POST']) +@login_required +def purge(): + try: + days = int(request.form.get('days', 30)) + if days not in (7, 30, 90): + return jsonify({'error': 'days must be 7, 30, or 90'}), 400 + except (ValueError, TypeError): + return jsonify({'error': 'Invalid days parameter'}), 400 + + cutoff = datetime.utcnow() - timedelta(days=days) + try: + from app.models.app_log import AppLog + from app.extensions import db + deleted = AppLog.query.filter(AppLog.timestamp < cutoff).delete() + db.session.commit() + log.info('App logs purged: %d entries older than %d days deleted', deleted, days) + return jsonify({'status': 'ok', 'deleted': deleted}) + except Exception as exc: + log.error('App log purge failed: %s', exc) return jsonify({'error': str(exc)}), 500 diff --git a/app/routes/settings.py b/app/routes/settings.py index eb1d0d0..9988c3b 100644 --- a/app/routes/settings.py +++ b/app/routes/settings.py @@ -470,3 +470,24 @@ def audit_log(): logs=pagination.items, actions=actions, action=action) + + +@settings_bp.route('/audit/purge', methods=['POST']) +@login_required +def audit_purge(): + from app.models.audit_log import AuditLog + from datetime import timedelta, datetime + try: + days = int(request.form.get('days', 30)) + if days not in (7, 30, 90): + flash('Invalid purge period.', 'danger') + return redirect(url_for('settings.audit_log')) + except (ValueError, TypeError): + flash('Invalid purge period.', 'danger') + return redirect(url_for('settings.audit_log')) + + cutoff = datetime.utcnow() - timedelta(days=days) + deleted = AuditLog.query.filter(AuditLog.timestamp < cutoff).delete() + db.session.commit() + flash(f'Deleted {deleted} audit log entries older than {days} days.', 'success') + return redirect(url_for('settings.audit_log')) diff --git a/app/templates/logs/index.html b/app/templates/logs/index.html index 98a1537..212a10f 100644 --- a/app/templates/logs/index.html +++ b/app/templates/logs/index.html @@ -168,12 +168,10 @@ tr.expanded-row > td { {% if file_exists %} {{ file_size_fmt }} {{ file_mtime }} - - {{ log_file }} - {% else %} Log file not found {% endif %} + {{ db_count }} entries in DB
@@ -226,8 +224,21 @@ tr.expanded-row > td { Download - + +
+ + @@ -470,18 +481,45 @@ document.getElementById('ar-toggle').addEventListener('click', () => { // ── Clear ─────────────────────────────────────────────────────────────────── document.getElementById('clear-btn').addEventListener('click', function() { - if (!confirm('Clear all log entries from the file?\nThis cannot be undone.')) return; + if (!confirm('Clear ALL log entries from the database and log file?\nThis cannot be undone.')) return; fetch('{{ url_for("logs.clear") }}', { method: 'POST', headers: { 'X-CSRFToken': CSRF, 'Content-Type': 'application/json' }, }) .then(r => r.json()) .then(d => { - if (d.status === 'ok') load(); + if (d.status === 'ok') { load(); location.reload(); } else alert('Clear failed: ' + d.error); }); }); +// ── Purge ─────────────────────────────────────────────────────────────────── +document.querySelectorAll('.purge-item').forEach(function(el) { + el.addEventListener('click', function(e) { + e.preventDefault(); + const days = this.dataset.days; + if (!confirm(`Delete all log entries older than ${days} days?\nThis cannot be undone.`)) return; + const fd = new FormData(); + fd.append('days', days); + fetch('{{ url_for("logs.purge") }}', { + method: 'POST', + headers: { 'X-CSRFToken': CSRF }, + body: fd, + }) + .then(r => r.json()) + .then(d => { + if (d.status === 'ok') { + load(); + // Update DB count chip + const chip = document.querySelector('.log-meta-chip .bi-database'); + if (chip) chip.parentElement.innerHTML = ` refreshed`; + } else { + alert('Purge failed: ' + d.error); + } + }); + }); +}); + // ── Keyboard shortcuts ────────────────────────────────────────────────────── document.addEventListener('keydown', function(e) { if (e.target.tagName === 'INPUT' || e.target.tagName === 'SELECT') return; diff --git a/app/templates/settings/audit.html b/app/templates/settings/audit.html index d4c1e2d..02c03b6 100644 --- a/app/templates/settings/audit.html +++ b/app/templates/settings/audit.html @@ -4,16 +4,54 @@ {% block content %}
-
- - - {% if action %}{% endif %} -
+
+
+ + + {% if action %}{% endif %} +
+ +
+
+ +
    +
  • Delete entries older than…
    • +
  • +
    + + + +
    +
    • +
  • +
    + + + +
    +
    • +
  • +
    + + + +
    +
    • +
+
+
+
diff --git a/app/utils/db_log_handler.py b/app/utils/db_log_handler.py new file mode 100644 index 0000000..f7fbf0b --- /dev/null +++ b/app/utils/db_log_handler.py @@ -0,0 +1,49 @@ +""" +SQLAlchemy-backed logging handler. + +Writes app.* log records to the app_logs table so the web viewer +can query, filter, and purge them without touching the log file. + +Safety rules: + - Never raises — all errors are silently swallowed to avoid crashing the app + - Reentrancy guard prevents infinite recursion (SQLAlchemy emits its own logs) + - Skips sqlalchemy.* and werkzeug loggers to avoid noise / recursion +""" + +import logging +import traceback +from datetime import datetime + + +_SKIP_PREFIXES = ('sqlalchemy', 'werkzeug', 'urllib3', 'plaid_handler') + + +class DBLogHandler(logging.Handler): + _inside_emit = False + + def emit(self, record): + if DBLogHandler._inside_emit: + return + if any(record.name.startswith(p) for p in _SKIP_PREFIXES): + return + DBLogHandler._inside_emit = True + try: + from app.extensions import db + from app.models.app_log import AppLog + + msg = record.getMessage() + if record.exc_info: + msg += '\n' + ''.join(traceback.format_exception(*record.exc_info)).rstrip() + + entry = AppLog( + timestamp=datetime.utcfromtimestamp(record.created), + level=record.levelname, + module=record.name, + message=msg, + ) + db.session.add(entry) + db.session.commit() + except Exception: + pass + finally: + DBLogHandler._inside_emit = False diff --git a/scripts/add_log_tables.py b/scripts/add_log_tables.py new file mode 100644 index 0000000..b20b93b --- /dev/null +++ b/scripts/add_log_tables.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +""" +Migration: create audit_logs and app_logs tables. + +Run once: python scripts/add_log_tables.py +Safe to re-run — uses CREATE TABLE IF NOT EXISTS. +""" +import sys, os +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from app import create_app +from app.extensions import db + +app = create_app() + +TABLES = [ + ( + 'audit_logs', + """ + CREATE TABLE IF NOT EXISTS audit_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + action VARCHAR(64) NOT NULL, + description VARCHAR(255), + ip_address VARCHAR(45), + INDEX ix_audit_logs_timestamp (timestamp), + INDEX ix_audit_logs_action (action) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """ + ), + ( + 'app_logs', + """ + CREATE TABLE IF NOT EXISTS app_logs ( + id INT AUTO_INCREMENT PRIMARY KEY, + timestamp DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + level VARCHAR(16) NOT NULL, + module VARCHAR(128), + message TEXT, + INDEX ix_app_logs_timestamp (timestamp), + INDEX ix_app_logs_level (level) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 + """ + ), +] + +with app.app_context(): + with db.engine.connect() as conn: + for name, sql in TABLES: + conn.execute(db.text(sql)) + conn.commit() + print(f' {name} — created (or already exists).') + +print('Done.')