06/05 Optimize app: logs will be saved to db
This commit is contained in:
+113
-77
@@ -1,5 +1,7 @@
|
||||
import os
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from flask import (Blueprint, render_template, request, jsonify,
|
||||
current_app, send_file, abort)
|
||||
from flask_login import login_required
|
||||
@@ -8,115 +10,108 @@ logs_bp = Blueprint('logs', __name__, url_prefix='/logs')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _tail_lines(filepath, n=2000):
|
||||
"""Return the last n lines from a file without loading it all into memory."""
|
||||
try:
|
||||
with open(filepath, 'rb') as f:
|
||||
f.seek(0, 2)
|
||||
size = f.tell()
|
||||
if size == 0:
|
||||
return []
|
||||
buf = bytearray()
|
||||
pos = size
|
||||
while len(buf) < 256 * 1024 and pos > 0: # max 256 KB scan
|
||||
chunk = min(65536, pos)
|
||||
pos -= chunk
|
||||
f.seek(pos)
|
||||
buf = bytearray(f.read(chunk)) + buf
|
||||
lines = buf.decode('utf-8', errors='replace').splitlines()
|
||||
return lines[-n:]
|
||||
except FileNotFoundError:
|
||||
return []
|
||||
except Exception as exc:
|
||||
log.error('log viewer _tail_lines error: %s', exc)
|
||||
return []
|
||||
|
||||
|
||||
def _parse_line(raw):
|
||||
"""Parse pipe-delimited log line: timestamp|level|name|message"""
|
||||
parts = raw.split('|', 3)
|
||||
if len(parts) == 4:
|
||||
return {
|
||||
'ts': parts[0],
|
||||
'level': parts[1],
|
||||
'name': parts[2],
|
||||
'message': parts[3],
|
||||
}
|
||||
# Fallback for lines that don't match the format (e.g. tracebacks)
|
||||
return {'ts': '', 'level': 'RAW', 'name': '', 'message': raw}
|
||||
|
||||
# ── Index ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
@logs_bp.route('/')
|
||||
@login_required
|
||||
def index():
|
||||
import datetime
|
||||
log_file = current_app.config.get('LOG_FILE_PATH', '')
|
||||
file_exists = bool(log_file) and os.path.isfile(log_file)
|
||||
file_size = os.path.getsize(log_file) if file_exists else 0
|
||||
file_mtime = None
|
||||
if file_exists:
|
||||
ts = os.path.getmtime(log_file)
|
||||
file_mtime = datetime.datetime.fromtimestamp(ts).strftime('%b %d, %H:%M')
|
||||
file_mtime = datetime.fromtimestamp(ts).strftime('%b %d, %H:%M')
|
||||
|
||||
def fmt_size(b):
|
||||
if b < 1024: return f'{b} B'
|
||||
if b < 1024**2: return f'{b/1024:.1f} KB'
|
||||
return f'{b/1024**2:.1f} MB'
|
||||
|
||||
# DB row count
|
||||
db_count = 0
|
||||
try:
|
||||
from app.models.app_log import AppLog
|
||||
db_count = AppLog.query.count()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return render_template('logs/index.html',
|
||||
log_file=log_file,
|
||||
file_exists=file_exists,
|
||||
file_size=file_size,
|
||||
file_size_fmt=fmt_size(file_size),
|
||||
file_mtime=file_mtime)
|
||||
file_mtime=file_mtime,
|
||||
db_count=db_count)
|
||||
|
||||
|
||||
# ── API (reads from DB) ───────────────────────────────────────────────────────
|
||||
|
||||
@logs_bp.route('/api')
|
||||
@login_required
|
||||
def api():
|
||||
"""JSON endpoint used by the log viewer to fetch and filter entries."""
|
||||
log_file = current_app.config.get('LOG_FILE_PATH', '')
|
||||
if not log_file:
|
||||
return jsonify({'entries': [], 'error': 'LOG_FILE_PATH not configured'})
|
||||
try:
|
||||
from app.models.app_log import AppLog
|
||||
from app.extensions import db
|
||||
except Exception as e:
|
||||
return jsonify({'entries': [], 'error': str(e)})
|
||||
|
||||
level_filter = request.args.get('level', 'ALL').upper()
|
||||
search = request.args.get('search', '').lower()
|
||||
limit = min(int(request.args.get('limit', 200)), 2000)
|
||||
module = request.args.get('module', '').lower()
|
||||
|
||||
raw_lines = _tail_lines(log_file, n=5000)
|
||||
entries = [_parse_line(l) for l in raw_lines if l.strip()]
|
||||
try:
|
||||
query = AppLog.query
|
||||
|
||||
if level_filter not in ('ALL', ''):
|
||||
entries = [e for e in entries if e['level'] == level_filter]
|
||||
if level_filter not in ('ALL', ''):
|
||||
query = query.filter(AppLog.level == level_filter)
|
||||
|
||||
if module:
|
||||
entries = [e for e in entries if module in e['name'].lower()]
|
||||
if module:
|
||||
query = query.filter(AppLog.module.ilike(f'%{module}%'))
|
||||
|
||||
if search:
|
||||
entries = [e for e in entries
|
||||
if search in e['message'].lower() or search in e['name'].lower()]
|
||||
if search:
|
||||
query = query.filter(
|
||||
db.or_(
|
||||
AppLog.message.ilike(f'%{search}%'),
|
||||
AppLog.module.ilike(f'%{search}%'),
|
||||
)
|
||||
)
|
||||
|
||||
# Most-recent first, capped at limit
|
||||
entries = list(reversed(entries))[:limit]
|
||||
total = query.count()
|
||||
rows = query.order_by(AppLog.timestamp.desc()).limit(limit).all()
|
||||
|
||||
# Count per level + unique modules across ALL unfiltered lines
|
||||
all_entries = [_parse_line(l) for l in raw_lines if l.strip()]
|
||||
counts = {}
|
||||
modules_seen = set()
|
||||
for e in all_entries:
|
||||
counts[e['level']] = counts.get(e['level'], 0) + 1
|
||||
if e['name']:
|
||||
modules_seen.add(e['name'])
|
||||
entries = [
|
||||
{
|
||||
'ts': r.timestamp.strftime('%Y-%m-%d %H:%M:%S') if r.timestamp else '',
|
||||
'level': r.level or 'INFO',
|
||||
'name': r.module or '',
|
||||
'message': r.message or '',
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
return jsonify({
|
||||
'entries': entries,
|
||||
'counts': counts,
|
||||
'total_raw': len(raw_lines),
|
||||
'log_file': log_file,
|
||||
'modules': sorted(modules_seen),
|
||||
})
|
||||
# Counts and modules across ALL unfiltered rows (capped for performance)
|
||||
all_rows = AppLog.query.with_entities(AppLog.level, AppLog.module).all()
|
||||
counts = {}
|
||||
modules_seen = set()
|
||||
for lvl, mod in all_rows:
|
||||
counts[lvl] = counts.get(lvl, 0) + 1
|
||||
if mod:
|
||||
modules_seen.add(mod)
|
||||
|
||||
return jsonify({
|
||||
'entries': entries,
|
||||
'counts': counts,
|
||||
'total_raw': total,
|
||||
'modules': sorted(modules_seen),
|
||||
})
|
||||
except Exception as e:
|
||||
log.error('logs api error: %s', e)
|
||||
return jsonify({'entries': [], 'error': str(e), 'counts': {}, 'total_raw': 0, 'modules': []})
|
||||
|
||||
|
||||
# ── Download (still from file) ────────────────────────────────────────────────
|
||||
|
||||
@logs_bp.route('/download')
|
||||
@login_required
|
||||
@@ -127,16 +122,57 @@ def download():
|
||||
return send_file(log_file, as_attachment=True, download_name='pfm-app.log')
|
||||
|
||||
|
||||
# ── Clear (truncate DB table + file) ─────────────────────────────────────────
|
||||
|
||||
@logs_bp.route('/clear', methods=['POST'])
|
||||
@login_required
|
||||
def clear():
|
||||
log_file = current_app.config.get('LOG_FILE_PATH', '')
|
||||
if not log_file:
|
||||
return jsonify({'error': 'LOG_FILE_PATH not configured'}), 400
|
||||
errors = []
|
||||
|
||||
# Clear DB table
|
||||
try:
|
||||
with open(log_file, 'w', encoding='utf-8'):
|
||||
pass
|
||||
log.info('Log file cleared by user')
|
||||
return jsonify({'status': 'ok'})
|
||||
from app.models.app_log import AppLog
|
||||
from app.extensions import db
|
||||
AppLog.query.delete()
|
||||
db.session.commit()
|
||||
except Exception as exc:
|
||||
errors.append(f'DB: {exc}')
|
||||
|
||||
# Clear log file
|
||||
log_file = current_app.config.get('LOG_FILE_PATH', '')
|
||||
if log_file:
|
||||
try:
|
||||
with open(log_file, 'w', encoding='utf-8'):
|
||||
pass
|
||||
except Exception as exc:
|
||||
errors.append(f'file: {exc}')
|
||||
|
||||
if errors:
|
||||
return jsonify({'error': '; '.join(errors)}), 500
|
||||
log.info('App logs cleared by user')
|
||||
return jsonify({'status': 'ok'})
|
||||
|
||||
|
||||
# ── Purge (delete entries older than N days) ──────────────────────────────────
|
||||
|
||||
@logs_bp.route('/purge', methods=['POST'])
|
||||
@login_required
|
||||
def purge():
|
||||
try:
|
||||
days = int(request.form.get('days', 30))
|
||||
if days not in (7, 30, 90):
|
||||
return jsonify({'error': 'days must be 7, 30, or 90'}), 400
|
||||
except (ValueError, TypeError):
|
||||
return jsonify({'error': 'Invalid days parameter'}), 400
|
||||
|
||||
cutoff = datetime.utcnow() - timedelta(days=days)
|
||||
try:
|
||||
from app.models.app_log import AppLog
|
||||
from app.extensions import db
|
||||
deleted = AppLog.query.filter(AppLog.timestamp < cutoff).delete()
|
||||
db.session.commit()
|
||||
log.info('App logs purged: %d entries older than %d days deleted', deleted, days)
|
||||
return jsonify({'status': 'ok', 'deleted': deleted})
|
||||
except Exception as exc:
|
||||
log.error('App log purge failed: %s', exc)
|
||||
return jsonify({'error': str(exc)}), 500
|
||||
|
||||
Reference in New Issue
Block a user