06/01 Implement PDF statement file import
This commit is contained in:
@@ -90,6 +90,7 @@ def create_app(config_name=None):
|
||||
from app.routes.settings import settings_bp
|
||||
from app.routes.teller import teller_bp
|
||||
from app.routes.logs import logs_bp
|
||||
from app.routes.bank_import import bank_import_bp
|
||||
|
||||
app.register_blueprint(auth_bp)
|
||||
app.register_blueprint(dashboard_bp)
|
||||
@@ -104,6 +105,7 @@ def create_app(config_name=None):
|
||||
app.register_blueprint(settings_bp)
|
||||
app.register_blueprint(teller_bp)
|
||||
app.register_blueprint(logs_bp)
|
||||
app.register_blueprint(bank_import_bp)
|
||||
|
||||
with app.app_context():
|
||||
from app.models import (
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
import json
|
||||
import logging
|
||||
from flask import Blueprint, render_template, request, jsonify
|
||||
from flask_login import login_required
|
||||
from app.models.account import Account
|
||||
from app.models.category import Category
|
||||
from app.services.bank_import_service import parse_file, do_import
|
||||
|
||||
bank_import_bp = Blueprint('bank_import', __name__, url_prefix='/bank-import')
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ALLOWED_EXTENSIONS = {'csv', 'ofx', 'qfx', 'txt', 'pdf'}
|
||||
|
||||
|
||||
@bank_import_bp.route('/')
|
||||
@login_required
|
||||
def index():
|
||||
accounts = Account.query.filter_by(is_active=True).order_by(Account.name).all()
|
||||
categories = Category.query.filter_by(is_active=True).order_by(Category.name).all()
|
||||
return render_template('bank_import/index.html',
|
||||
accounts=accounts,
|
||||
categories=categories)
|
||||
|
||||
|
||||
@bank_import_bp.route('/parse', methods=['POST'])
|
||||
@login_required
|
||||
def parse():
|
||||
"""
|
||||
Accepts a multipart/form-data POST with:
|
||||
file — the bank statement file
|
||||
col_map — JSON string (optional); user-supplied column mapping
|
||||
|
||||
Returns JSON with parsed rows, detected format, errors, or needs_mapping flag.
|
||||
"""
|
||||
f = request.files.get('file')
|
||||
if not f or not f.filename:
|
||||
return jsonify({'error': 'No file uploaded'}), 400
|
||||
|
||||
ext = f.filename.rsplit('.', 1)[-1].lower() if '.' in f.filename else ''
|
||||
if ext not in ALLOWED_EXTENSIONS:
|
||||
return jsonify({'error': f'Unsupported file type: .{ext}. Use CSV, OFX, or QFX.'}), 400
|
||||
|
||||
col_map = None
|
||||
raw_map = request.form.get('col_map', '')
|
||||
if raw_map:
|
||||
try:
|
||||
col_map = json.loads(raw_map)
|
||||
except (ValueError, TypeError):
|
||||
return jsonify({'error': 'Invalid col_map JSON'}), 400
|
||||
|
||||
file_bytes = f.read()
|
||||
log.info('[bank_import] parse request: filename=%s size=%d bytes', f.filename, len(file_bytes))
|
||||
|
||||
try:
|
||||
result = parse_file(file_bytes, f.filename, col_map=col_map)
|
||||
except Exception as e:
|
||||
log.error('[bank_import] parse failed: %s', e, exc_info=True)
|
||||
return jsonify({'error': f'Failed to parse file: {e}'}), 500
|
||||
|
||||
return jsonify(result)
|
||||
|
||||
|
||||
@bank_import_bp.route('/import', methods=['POST'])
|
||||
@login_required
|
||||
def confirm_import():
|
||||
"""
|
||||
Accepts JSON:
|
||||
{
|
||||
"account_id": int,
|
||||
"skip_dupes": bool,
|
||||
"rows": [
|
||||
{"date": "YYYY-MM-DD", "transaction_type": "expense|income",
|
||||
"amount": float, "description": str, "category_id": int|null,
|
||||
"notes": str, "source_id": str|null},
|
||||
...
|
||||
]
|
||||
}
|
||||
Returns {"imported": N, "skipped": N}.
|
||||
"""
|
||||
data = request.get_json()
|
||||
if not data:
|
||||
return jsonify({'error': 'No JSON payload'}), 400
|
||||
|
||||
rows = data.get('rows', [])
|
||||
account_id = data.get('account_id')
|
||||
skip_dupes = data.get('skip_dupes', True)
|
||||
|
||||
if not rows:
|
||||
return jsonify({'error': 'No rows provided'}), 400
|
||||
if not account_id:
|
||||
return jsonify({'error': 'account_id is required'}), 400
|
||||
|
||||
# Validate account exists
|
||||
acct = Account.query.get(int(account_id))
|
||||
if not acct:
|
||||
return jsonify({'error': f'Account {account_id} not found'}), 400
|
||||
|
||||
log.info('[bank_import] importing %d rows into account_id=%s skip_dupes=%s',
|
||||
len(rows), account_id, skip_dupes)
|
||||
try:
|
||||
imported, skipped = do_import(rows, int(account_id), skip_dupes=skip_dupes)
|
||||
except Exception as e:
|
||||
log.error('[bank_import] import failed: %s', e, exc_info=True)
|
||||
return jsonify({'error': f'Import failed: {e}'}), 500
|
||||
|
||||
return jsonify({'imported': imported, 'skipped': skipped})
|
||||
@@ -0,0 +1,787 @@
|
||||
"""
|
||||
Bank Statement Import Service
|
||||
|
||||
Supported formats:
|
||||
PDF — Digital bank statement PDFs; text extracted via pdfplumber then
|
||||
parsed by Groq LLM (requires GROQ_API_KEY).
|
||||
OFX / QFX — Open Financial Exchange (exported by most US banks & Quicken)
|
||||
CSV — Auto-detects Chase, Bank of America, Citi, Capital One, Discover,
|
||||
USAA, Wells Fargo; falls back to a generic heuristic; asks user
|
||||
to map columns when detection fails.
|
||||
|
||||
Public API:
|
||||
parse_file(file_bytes, filename, col_map=None) → dict
|
||||
do_import(rows, default_account_id, skip_dupes=True) → (imported, skipped)
|
||||
auto_categorize(description) → category_name_str
|
||||
"""
|
||||
|
||||
import csv
|
||||
import io
|
||||
import re
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# ── Keyword → PFM category name map ──────────────────────────────────────────
|
||||
|
||||
KEYWORD_CATEGORIES = {
|
||||
'Food & Dining': [
|
||||
'restaurant', 'dining', 'cafe', 'coffee', 'starbucks', 'mcdonald',
|
||||
'burger', 'pizza', 'sushi', 'taco', 'chipotle', 'subway', 'doordash',
|
||||
'ubereats', 'grubhub', 'seamless', 'instacart', 'grocery', 'safeway',
|
||||
'whole foods', 'trader joe', 'kroger', 'wegmans', 'publix', 'aldi',
|
||||
'panera', 'dunkin', 'chick-fil-a', 'wendy', 'taco bell', 'kfc',
|
||||
'popeyes', 'five guys', 'domino', 'papa john', 'little caesar',
|
||||
'denny', 'ihop', 'olive garden', 'applebee', 'cheesecake factory',
|
||||
],
|
||||
'Transport': [
|
||||
'uber', 'lyft', 'taxi', 'gas station', 'shell', 'chevron', 'exxon',
|
||||
'mobil', 'citgo', 'speedway', 'sunoco', 'circle k', 'wawa fuel',
|
||||
'parking', 'toll', 'transit', 'metro', 'bart', 'mta ', 'wmata',
|
||||
'amtrak', 'greyhound', 'zipcar', 'enterprise rent', 'hertz', 'avis',
|
||||
'budget rent', 'national car', 'jiffy lube', 'valvoline', 'firestone',
|
||||
'oil change', 'autozone', 'advance auto', 'napa auto', 'pep boys',
|
||||
'airline', 'delta ', 'united air', 'southwest', 'american air',
|
||||
'jetblue', 'spirit air',
|
||||
],
|
||||
'Shopping': [
|
||||
'amazon', 'ebay', 'walmart', 'target', 'costco', 'best buy',
|
||||
'apple store', 'nike', 'adidas', 'gap', 'old navy', 'h&m', 'zara',
|
||||
'nordstrom', 'macy', 'tj maxx', 'marshalls', 'ross store',
|
||||
'dollar tree', 'dollar general', 'home depot', 'lowes', 'ikea',
|
||||
'bed bath', 'wayfair', 'etsy', 'paypal ', 'venmo ',
|
||||
],
|
||||
'Entertainment': [
|
||||
'netflix', 'spotify', 'hulu', 'disney+', 'hbo max', 'amazon prime',
|
||||
'apple tv', 'youtube premium', 'twitch', 'steam ', 'playstation',
|
||||
'xbox', 'nintendo', 'movie', 'cinema', 'theater', 'concert',
|
||||
'ticketmaster', 'stubhub', 'eventbrite', 'live nation', 'amc ',
|
||||
'regal cinema', 'fandango', 'bowling', 'arcade', 'museum', 'zoo',
|
||||
],
|
||||
'Utilities': [
|
||||
'electric', 'gas utility', 'natural gas', 'water bill', 'sewage',
|
||||
'trash', 'waste mgmt', 'internet', 'comcast', 'xfinity', 'verizon',
|
||||
'at&t', 'att bill', 't-mobile', 'sprint', 'charter', 'spectrum',
|
||||
'cox comm', 'centurylink', 'utility bill', 'pge ', 'con edison',
|
||||
],
|
||||
'Health': [
|
||||
'pharmacy', 'cvs ', 'walgreens', 'rite aid', 'hospital', 'clinic',
|
||||
'doctor', 'dental', 'dentist', 'vision center', 'optometrist',
|
||||
'labcorp', 'quest diag', 'urgent care', 'medical', 'health care',
|
||||
'gym', 'planet fitness', 'la fitness', 'equinox', 'peloton',
|
||||
],
|
||||
'Housing': [
|
||||
'rent payment', 'mortgage', 'hoa ', 'property tax', 'maintenance',
|
||||
'plumber', 'electrician', 'hvac', 'pest control', 'lawn care',
|
||||
'landscap', 'cleaning service', 'airbnb', 'vrbo', 'hotel', 'motel',
|
||||
'hilton', 'marriott', 'hyatt', 'sheraton', 'holiday inn',
|
||||
],
|
||||
'Subscriptions': [
|
||||
'subscription', 'membership fee', 'annual fee', 'monthly fee',
|
||||
'adobe ', 'microsoft 365', 'office 365', 'dropbox', 'icloud',
|
||||
'google one', '1password', 'nordvpn', 'expressvpn', 'duolingo',
|
||||
'headspace', 'calm app', 'audible', 'kindle unlimited',
|
||||
'new york times', 'wall street journal', 'nytimes.com',
|
||||
],
|
||||
'Education': [
|
||||
'tuition', 'university', 'college', 'coursera', 'udemy',
|
||||
'linkedin learning', 'pluralsight', 'skillshare', 'textbook', 'chegg',
|
||||
'grammarly', 'turnitin', 'canvas ', 'blackboard',
|
||||
],
|
||||
'Insurance': [
|
||||
'insurance', 'geico', 'state farm', 'allstate', 'progressive ins',
|
||||
'liberty mutual', 'farmers ins', 'aaa ins', 'aflac', 'metlife',
|
||||
'cigna', 'aetna', 'humana', 'blue cross', 'united health',
|
||||
],
|
||||
'Gifts': [
|
||||
'gift card', 'donation', 'charity', 'church tithe', 'gofundme',
|
||||
'red cross', 'goodwill', 'salvation army', 'habitat humanity',
|
||||
],
|
||||
'Investment': [
|
||||
'robinhood', 'fidelity inv', 'schwab', 'vanguard', 'e*trade',
|
||||
'td ameritrade', 'coinbase', 'binance', 'crypto purchase',
|
||||
],
|
||||
'Salary': [
|
||||
'payroll', 'direct deposit', 'salary', 'wages deposit', 'paycheck',
|
||||
'adp llc', 'paychex', 'gusto payroll', 'paylocity', 'intuit payroll',
|
||||
],
|
||||
'Freelance': [
|
||||
'freelance', 'consulting fee', 'contract payment', 'upwork',
|
||||
'fiverr', 'toptal',
|
||||
],
|
||||
'Other Income': [
|
||||
'interest paid', 'dividend', 'refund', 'cashback', 'reward',
|
||||
'reimburs', 'tax refund', 'irs treas',
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
# ── Known bank CSV format profiles ───────────────────────────────────────────
|
||||
|
||||
BANK_FORMATS = [
|
||||
{
|
||||
'name': 'Chase',
|
||||
'required': {'transaction date', 'description', 'amount'},
|
||||
'map': {
|
||||
'date': 'transaction date', 'description': 'description',
|
||||
'amount': 'amount', 'positive_is_expense': False,
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'Bank of America',
|
||||
'required': {'posted date', 'payee', 'amount'},
|
||||
'map': {
|
||||
'date': 'posted date', 'description': 'payee',
|
||||
'amount': 'amount', 'positive_is_expense': False,
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'Citi',
|
||||
'required': {'date', 'description', 'debit', 'credit'},
|
||||
'map': {
|
||||
'date': 'date', 'description': 'description',
|
||||
'debit': 'debit', 'credit': 'credit',
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'Capital One',
|
||||
'required': {'transaction date', 'posted date', 'card no.', 'description', 'debit', 'credit'},
|
||||
'map': {
|
||||
'date': 'transaction date', 'description': 'description',
|
||||
'debit': 'debit', 'credit': 'credit',
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'Discover',
|
||||
'required': {'trans. date', 'post date', 'description', 'amount', 'category'},
|
||||
'map': {
|
||||
'date': 'trans. date', 'description': 'description',
|
||||
'amount': 'amount', 'positive_is_expense': True,
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'American Express',
|
||||
'required': {'date', 'description', 'amount', 'extended details', 'appears on your statement as'},
|
||||
'map': {
|
||||
'date': 'date', 'description': 'description',
|
||||
'amount': 'amount', 'positive_is_expense': False,
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'USAA',
|
||||
'required': {'date', 'description', 'original description', 'category', 'amount', 'status'},
|
||||
'map': {
|
||||
'date': 'date', 'description': 'description',
|
||||
'amount': 'amount', 'positive_is_expense': False,
|
||||
},
|
||||
},
|
||||
{
|
||||
'name': 'Wells Fargo',
|
||||
'required': {'date', 'amount', 'description'},
|
||||
'map': {
|
||||
'date': 'date', 'description': 'description',
|
||||
'amount': 'amount', 'positive_is_expense': False,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
_DATE_FMTS = [
|
||||
'%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d',
|
||||
'%d-%m-%Y', '%m-%d-%Y', '%b %d, %Y', '%d %b %Y',
|
||||
'%B %d, %Y', '%d %B %Y', '%m/%d/%y',
|
||||
]
|
||||
|
||||
|
||||
# ── Internal helpers ──────────────────────────────────────────────────────────
|
||||
|
||||
def _parse_date(s):
|
||||
s = str(s).strip()
|
||||
# OFX: 20250115120000[-5:EST] — take first 8 digits
|
||||
if re.match(r'^\d{8}', s):
|
||||
return datetime.strptime(s[:8], '%Y%m%d').date()
|
||||
for fmt in _DATE_FMTS:
|
||||
try:
|
||||
return datetime.strptime(s, fmt).date()
|
||||
except ValueError:
|
||||
pass
|
||||
raise ValueError(f'Unrecognised date: {s!r}')
|
||||
|
||||
|
||||
def _clean_amount(s):
|
||||
if s is None:
|
||||
return 0.0
|
||||
cleaned = re.sub(r'[^\d.\-+]', '', str(s).strip())
|
||||
try:
|
||||
return float(cleaned) if cleaned else 0.0
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _extract_tag(text, tag):
|
||||
"""Extract value of an OFX SGML/XML tag (stops at newline or next tag)."""
|
||||
m = re.search(rf'<{tag}>\s*([^\r\n<]+)', text, re.IGNORECASE)
|
||||
return m.group(1).strip() if m else ''
|
||||
|
||||
|
||||
# ── OFX / QFX parser ─────────────────────────────────────────────────────────
|
||||
|
||||
def _parse_ofx(content):
|
||||
"""Parse OFX/QFX (handles both SGML and XML variants)."""
|
||||
# Strip header block above <OFX>
|
||||
ofx_start = re.search(r'<OFX>', content, re.IGNORECASE)
|
||||
if ofx_start:
|
||||
content = content[ofx_start.start():]
|
||||
|
||||
# Try XML-style first (closing tags present)
|
||||
xml_blocks = re.findall(r'<STMTTRN>(.*?)</STMTTRN>', content,
|
||||
re.IGNORECASE | re.DOTALL)
|
||||
if not xml_blocks:
|
||||
# SGML-style: split on opening tags
|
||||
xml_blocks = re.split(r'<STMTTRN>', content, flags=re.IGNORECASE)[1:]
|
||||
|
||||
raw = []
|
||||
for block in xml_blocks:
|
||||
dtposted = _extract_tag(block, 'DTPOSTED')
|
||||
trnamt = _extract_tag(block, 'TRNAMT')
|
||||
trntype = _extract_tag(block, 'TRNTYPE').upper()
|
||||
fitid = _extract_tag(block, 'FITID')
|
||||
name = _extract_tag(block, 'NAME')
|
||||
memo = _extract_tag(block, 'MEMO')
|
||||
|
||||
if not dtposted or not trnamt:
|
||||
continue
|
||||
try:
|
||||
txn_date = _parse_date(dtposted)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
amount = _clean_amount(trnamt)
|
||||
if amount == 0.0:
|
||||
continue
|
||||
|
||||
# OFX sign: negative = debit (expense), positive = credit (income)
|
||||
if amount < 0:
|
||||
txn_type = 'expense'
|
||||
else:
|
||||
txn_type = 'income'
|
||||
amount = abs(amount)
|
||||
|
||||
# TRNTYPE tag can override when sign is ambiguous
|
||||
if trntype in ('DEBIT', 'CHECK', 'PAYMENT', 'REPEATPMT', 'ATM', 'POS', 'XFER'):
|
||||
txn_type = 'expense'
|
||||
elif trntype in ('CREDIT', 'INT', 'DIV', 'DIRECTDEP', 'DEPOSIT', 'REFUND'):
|
||||
txn_type = 'income'
|
||||
|
||||
description = name or memo or f'OFX {trntype}'
|
||||
raw.append({
|
||||
'date': txn_date,
|
||||
'transaction_type': txn_type,
|
||||
'amount': amount,
|
||||
'description': description,
|
||||
'notes': memo if memo and memo != name else '',
|
||||
'source_id': fitid,
|
||||
})
|
||||
|
||||
log.info('[bank_import] OFX: parsed %d transactions', len(raw))
|
||||
return raw
|
||||
|
||||
|
||||
# ── CSV helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
def _detect_csv_format(headers_set):
|
||||
"""Return BANK_FORMATS entry whose required columns are all present, or None."""
|
||||
for fmt in BANK_FORMATS:
|
||||
if fmt['required'].issubset(headers_set):
|
||||
return fmt
|
||||
return None
|
||||
|
||||
|
||||
def _build_generic_map(headers_set):
|
||||
"""
|
||||
Attempt to build a column map from an unknown bank CSV.
|
||||
Looks for common column name patterns.
|
||||
Returns col_map dict or None if key columns cannot be found.
|
||||
"""
|
||||
date_names = ['date', 'posted date', 'transaction date', 'trans. date',
|
||||
'post date', 'settlement date', 'value date', 'effective date']
|
||||
desc_names = ['description', 'payee', 'name', 'merchant', 'memo',
|
||||
'transaction description', 'details', 'narrative', 'remarks']
|
||||
amt_names = ['amount', 'transaction amount', 'net amount']
|
||||
debit_names = ['debit', 'withdrawal', 'withdrawals', 'charge', 'charges',
|
||||
'debit amount', 'out', 'outflow']
|
||||
cred_names = ['credit', 'deposit', 'deposits', 'payment', 'in',
|
||||
'credit amount', 'inflow']
|
||||
|
||||
date_col = next((c for c in date_names if c in headers_set), None)
|
||||
desc_col = next((c for c in desc_names if c in headers_set), None)
|
||||
amt_col = next((c for c in amt_names if c in headers_set), None)
|
||||
debit_col = next((c for c in debit_names if c in headers_set), None)
|
||||
cred_col = next((c for c in cred_names if c in headers_set), None)
|
||||
|
||||
if not date_col or not desc_col:
|
||||
return None
|
||||
if not amt_col and not debit_col and not cred_col:
|
||||
return None
|
||||
|
||||
return {
|
||||
'date': date_col,
|
||||
'description': desc_col,
|
||||
'amount': amt_col,
|
||||
'debit': debit_col,
|
||||
'credit': cred_col,
|
||||
'positive_is_expense': False,
|
||||
}
|
||||
|
||||
|
||||
def _parse_csv_with_map(content, col_map):
|
||||
"""
|
||||
Parse CSV content using the provided column map.
|
||||
Returns (raw_rows, errors).
|
||||
raw_rows: list of dicts with date (date obj), transaction_type, amount, description, notes, source_id.
|
||||
"""
|
||||
reader = csv.DictReader(io.StringIO(content))
|
||||
# Build {lowercase: original_case} lookup for header matching
|
||||
header_lookup = {h.strip().lower(): h.strip() for h in (reader.fieldnames or [])}
|
||||
|
||||
def get_col(key):
|
||||
mapped = col_map.get(key)
|
||||
if not mapped:
|
||||
return None
|
||||
return header_lookup.get(mapped.lower())
|
||||
|
||||
date_col = get_col('date')
|
||||
desc_col = get_col('description')
|
||||
amt_col = get_col('amount')
|
||||
debit_col = get_col('debit')
|
||||
cred_col = get_col('credit')
|
||||
positive_is_expense = col_map.get('positive_is_expense', False)
|
||||
|
||||
raw = []
|
||||
errors = []
|
||||
|
||||
for i, row in enumerate(reader, start=2):
|
||||
raw_date = row.get(date_col, '').strip() if date_col else ''
|
||||
if not raw_date:
|
||||
continue # skip blank rows silently
|
||||
|
||||
try:
|
||||
txn_date = _parse_date(raw_date)
|
||||
except ValueError as e:
|
||||
errors.append(f'Row {i}: {e}')
|
||||
continue
|
||||
|
||||
description = (row.get(desc_col, '') if desc_col else '').strip()
|
||||
if not description:
|
||||
description = f'Transaction on {raw_date}'
|
||||
|
||||
# Amount resolution: debit/credit columns take priority
|
||||
if debit_col or cred_col:
|
||||
debit = abs(_clean_amount(row.get(debit_col, '') if debit_col else ''))
|
||||
credit = abs(_clean_amount(row.get(cred_col, '') if cred_col else ''))
|
||||
if debit > 0 and credit > 0:
|
||||
# Both filled — treat as expense (unusual, but be safe)
|
||||
amount = debit
|
||||
txn_type = 'expense'
|
||||
elif debit > 0:
|
||||
amount = debit
|
||||
txn_type = 'expense'
|
||||
elif credit > 0:
|
||||
amount = credit
|
||||
txn_type = 'income'
|
||||
else:
|
||||
continue # skip zero / blank rows
|
||||
elif amt_col:
|
||||
amount_raw = _clean_amount(row.get(amt_col, ''))
|
||||
if amount_raw == 0.0:
|
||||
continue
|
||||
if positive_is_expense:
|
||||
txn_type = 'expense' if amount_raw > 0 else 'income'
|
||||
else:
|
||||
txn_type = 'expense' if amount_raw < 0 else 'income'
|
||||
amount = abs(amount_raw)
|
||||
else:
|
||||
errors.append(f'Row {i}: no amount column')
|
||||
continue
|
||||
|
||||
raw.append({
|
||||
'date': txn_date,
|
||||
'transaction_type': txn_type,
|
||||
'amount': amount,
|
||||
'description': description,
|
||||
'notes': '',
|
||||
'source_id': None,
|
||||
})
|
||||
|
||||
return raw, errors
|
||||
|
||||
|
||||
# ── Auto-categorizer ──────────────────────────────────────────────────────────
|
||||
|
||||
def auto_categorize(description):
|
||||
"""
|
||||
Return a PFM category name based on keywords in the description.
|
||||
Returns empty string when no match is found.
|
||||
"""
|
||||
lower = description.lower()
|
||||
for cat_name, keywords in KEYWORD_CATEGORIES.items():
|
||||
if any(kw in lower for kw in keywords):
|
||||
return cat_name
|
||||
return ''
|
||||
|
||||
|
||||
def _build_cat_id_map():
|
||||
from app.models.category import Category
|
||||
return {c.name: c.id for c in Category.query.filter_by(is_active=True).all()}
|
||||
|
||||
|
||||
# ── Enrichment ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _enrich(raw_rows):
|
||||
"""Add category_id + category_name to raw rows; convert date to ISO string."""
|
||||
cat_map = _build_cat_id_map()
|
||||
enriched = []
|
||||
for r in raw_rows:
|
||||
d = r['date']
|
||||
cat_name = auto_categorize(r['description'])
|
||||
enriched.append({
|
||||
**r,
|
||||
'date': d.isoformat() if hasattr(d, 'isoformat') else str(d),
|
||||
'category_name': cat_name,
|
||||
'category_id': cat_map.get(cat_name),
|
||||
})
|
||||
return enriched
|
||||
|
||||
|
||||
# ── PDF parser ────────────────────────────────────────────────────────────────
|
||||
|
||||
# Maximum characters sent to Groq (~8 K tokens; covers ~6 months of transactions)
|
||||
_PDF_MAX_CHARS = 30_000
|
||||
|
||||
|
||||
def _groq_parse_statement(text):
|
||||
"""
|
||||
Send extracted PDF text to Groq and return a list of raw transaction dicts.
|
||||
Raises RuntimeError on API / parse failure.
|
||||
"""
|
||||
import json
|
||||
import requests as _requests
|
||||
from flask import current_app
|
||||
|
||||
api_key = current_app.config.get('GROQ_API_KEY', '')
|
||||
if not api_key:
|
||||
raise RuntimeError('GROQ_API_KEY is not configured — cannot AI-parse PDFs.')
|
||||
|
||||
model = current_app.config.get('GROQ_MODEL', 'llama-3.3-70b-versatile')
|
||||
|
||||
prompt = (
|
||||
'Extract ALL transactions from the bank statement text below.\n'
|
||||
'Return ONLY a JSON array — no explanation, no markdown fences.\n\n'
|
||||
'Each element must have exactly these keys:\n'
|
||||
' "date" – YYYY-MM-DD\n'
|
||||
' "description" – clean merchant or payee name\n'
|
||||
' "amount" – positive number (never negative)\n'
|
||||
' "transaction_type" – "expense" for debits/withdrawals/charges;\n'
|
||||
' "income" for credits/deposits/refunds/interest\n\n'
|
||||
'Rules:\n'
|
||||
'- Include every transaction; do not omit any.\n'
|
||||
'- Skip opening balance, closing balance, and summary lines.\n'
|
||||
'- If the year is missing, infer it from the statement period header.\n'
|
||||
'- amount is always a positive float.\n\n'
|
||||
f'Bank statement text:\n{text}'
|
||||
)
|
||||
|
||||
try:
|
||||
resp = _requests.post(
|
||||
'https://api.groq.com/openai/v1/chat/completions',
|
||||
headers={
|
||||
'Authorization': f'Bearer {api_key}',
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
json={
|
||||
'model': model,
|
||||
'messages': [
|
||||
{'role': 'system',
|
||||
'content': 'You are a precise financial data extractor. '
|
||||
'Respond with valid JSON only — nothing else.'},
|
||||
{'role': 'user', 'content': prompt},
|
||||
],
|
||||
'max_tokens': 4096,
|
||||
'temperature': 0.1,
|
||||
'stream': False,
|
||||
},
|
||||
timeout=60,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
raw_content = resp.json()['choices'][0]['message']['content']
|
||||
log.info('[bank_import] Groq response length=%d chars', len(raw_content))
|
||||
except Exception as exc:
|
||||
log.error('[bank_import] Groq API call failed: %s', exc, exc_info=True)
|
||||
raise RuntimeError(f'Groq API error: {exc}')
|
||||
|
||||
# Strip markdown code fences if the model wrapped its output
|
||||
content = raw_content.strip()
|
||||
if '```' in content:
|
||||
m = re.search(r'```(?:json)?\s*([\s\S]+?)\s*```', content)
|
||||
content = m.group(1).strip() if m else content
|
||||
|
||||
# Ensure we start at the JSON array
|
||||
if not content.startswith('['):
|
||||
m = re.search(r'\[[\s\S]+\]', content)
|
||||
if m:
|
||||
content = m.group(0)
|
||||
|
||||
try:
|
||||
transactions = json.loads(content)
|
||||
except (ValueError, json.JSONDecodeError) as exc:
|
||||
log.error('[bank_import] Groq JSON parse failed: %s | raw=%r', exc, raw_content[:500])
|
||||
raise RuntimeError(
|
||||
f'Groq returned a response that could not be parsed as JSON: {exc}. '
|
||||
'Try again or use the CSV export from your bank instead.'
|
||||
)
|
||||
|
||||
if not isinstance(transactions, list):
|
||||
raise RuntimeError('Groq returned unexpected structure (expected a JSON array).')
|
||||
|
||||
raw_rows = []
|
||||
for txn in transactions:
|
||||
try:
|
||||
txn_date = _parse_date(str(txn.get('date', '')))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
amount = float(txn.get('amount', 0) or 0)
|
||||
if amount <= 0:
|
||||
continue
|
||||
|
||||
txn_type = str(txn.get('transaction_type', 'expense')).lower().strip()
|
||||
if txn_type not in ('income', 'expense'):
|
||||
txn_type = 'expense'
|
||||
|
||||
raw_rows.append({
|
||||
'date': txn_date,
|
||||
'transaction_type': txn_type,
|
||||
'amount': amount,
|
||||
'description': str(txn.get('description', '')).strip() or 'Bank transaction',
|
||||
'notes': '',
|
||||
'source_id': None,
|
||||
})
|
||||
|
||||
log.info('[bank_import] Groq extracted %d transactions from PDF', len(raw_rows))
|
||||
return raw_rows
|
||||
|
||||
|
||||
def _parse_pdf(file_bytes):
|
||||
"""
|
||||
Extract text from a digital PDF using pdfplumber, then parse with Groq.
|
||||
|
||||
Returns (raw_rows, warnings_list).
|
||||
Raises RuntimeError for unrecoverable problems (scanned PDF, bad file, etc.).
|
||||
"""
|
||||
try:
|
||||
import pdfplumber
|
||||
except ImportError:
|
||||
raise RuntimeError(
|
||||
'pdfplumber is not installed. Run: pip install pdfplumber==0.11.4'
|
||||
)
|
||||
|
||||
warnings = []
|
||||
text_parts = []
|
||||
|
||||
try:
|
||||
with pdfplumber.open(io.BytesIO(file_bytes)) as pdf:
|
||||
num_pages = len(pdf.pages)
|
||||
log.info('[bank_import] PDF has %d page(s)', num_pages)
|
||||
for page in pdf.pages:
|
||||
text = page.extract_text(x_tolerance=2, y_tolerance=2)
|
||||
if text:
|
||||
text_parts.append(text)
|
||||
except Exception as exc:
|
||||
log.error('[bank_import] pdfplumber failed: %s', exc, exc_info=True)
|
||||
raise RuntimeError(
|
||||
f'Could not read the PDF ({exc}). '
|
||||
'Make sure the file is not password-protected and is a valid PDF.'
|
||||
)
|
||||
|
||||
full_text = '\n'.join(text_parts).strip()
|
||||
log.info('[bank_import] PDF text extracted: %d characters', len(full_text))
|
||||
|
||||
if len(full_text) < 150:
|
||||
raise RuntimeError(
|
||||
'This PDF appears to be a scanned image (no text layer found). '
|
||||
'Please download the electronic/digital statement from your bank portal, '
|
||||
'or export your transactions as a CSV or OFX file instead.'
|
||||
)
|
||||
|
||||
truncated = len(full_text) > _PDF_MAX_CHARS
|
||||
if truncated:
|
||||
full_text = full_text[:_PDF_MAX_CHARS]
|
||||
warnings.append(
|
||||
f'The PDF is large — only the first {_PDF_MAX_CHARS:,} characters were '
|
||||
'sent to the AI parser. If transactions are missing, split the statement '
|
||||
'into shorter date ranges and import each part separately.'
|
||||
)
|
||||
|
||||
raw_rows = _groq_parse_statement(full_text)
|
||||
|
||||
if not raw_rows:
|
||||
raise RuntimeError(
|
||||
'The AI could not find any transactions in this PDF. '
|
||||
'Check that the file is a bank statement and not a summary or letter, '
|
||||
'then try again.'
|
||||
)
|
||||
|
||||
return raw_rows, warnings
|
||||
|
||||
|
||||
# ── Public parse entry point ──────────────────────────────────────────────────
|
||||
|
||||
def parse_file(file_bytes, filename, col_map=None):
|
||||
"""
|
||||
Detect format and parse a bank statement file.
|
||||
|
||||
Returns dict:
|
||||
rows — list of enriched transaction dicts (empty on needs_mapping)
|
||||
errors — list of non-fatal parse error strings
|
||||
format_name — human-readable detected format
|
||||
needs_mapping — True when CSV columns could not be auto-detected
|
||||
headers — list of CSV column headers (populated when needs_mapping)
|
||||
"""
|
||||
fname_lower = filename.lower()
|
||||
|
||||
# ── PDF — must be handled before decode attempt ────────────────────────
|
||||
if fname_lower.endswith('.pdf') or (
|
||||
isinstance(file_bytes, bytes) and file_bytes[:4] == b'%PDF'
|
||||
):
|
||||
raw, warnings = _parse_pdf(file_bytes)
|
||||
return {
|
||||
'rows': _enrich(raw), 'errors': warnings,
|
||||
'format_name': 'PDF (AI-parsed)',
|
||||
'needs_mapping': False, 'headers': [],
|
||||
}
|
||||
|
||||
if isinstance(file_bytes, bytes):
|
||||
try:
|
||||
content = file_bytes.decode('utf-8-sig')
|
||||
except UnicodeDecodeError:
|
||||
content = file_bytes.decode('latin-1')
|
||||
else:
|
||||
content = file_bytes
|
||||
|
||||
# ── OFX / QFX ─────────────────────────────────────────────────────────
|
||||
if fname_lower.endswith(('.ofx', '.qfx')) or '<OFX>' in content.upper()[:1000]:
|
||||
raw = _parse_ofx(content)
|
||||
return {
|
||||
'rows': _enrich(raw), 'errors': [],
|
||||
'format_name': 'OFX / QFX',
|
||||
'needs_mapping': False, 'headers': [],
|
||||
}
|
||||
|
||||
# ── CSV ────────────────────────────────────────────────────────────────
|
||||
reader = csv.DictReader(io.StringIO(content))
|
||||
headers = [h.strip() for h in (reader.fieldnames or [])]
|
||||
headers_set = {h.lower() for h in headers}
|
||||
|
||||
# User-supplied mapping (from the mapping UI)
|
||||
if col_map:
|
||||
effective_map = col_map
|
||||
format_name = 'Custom mapping'
|
||||
else:
|
||||
fmt = _detect_csv_format(headers_set)
|
||||
if fmt:
|
||||
effective_map = fmt['map']
|
||||
format_name = fmt['name']
|
||||
else:
|
||||
effective_map = _build_generic_map(headers_set)
|
||||
if effective_map:
|
||||
format_name = 'Generic CSV'
|
||||
else:
|
||||
log.info('[bank_import] CSV column auto-detect failed; headers=%s', headers)
|
||||
return {
|
||||
'rows': [], 'errors': [],
|
||||
'format_name': 'Unknown',
|
||||
'needs_mapping': True, 'headers': headers,
|
||||
}
|
||||
|
||||
raw, errors = _parse_csv_with_map(content, effective_map)
|
||||
log.info('[bank_import] CSV parsed %d rows (format=%s, errors=%d)',
|
||||
len(raw), format_name, len(errors))
|
||||
return {
|
||||
'rows': _enrich(raw), 'errors': errors,
|
||||
'format_name': format_name,
|
||||
'needs_mapping': False, 'headers': [],
|
||||
}
|
||||
|
||||
|
||||
# ── Import ─────────────────────────────────────────────────────────────────────
|
||||
|
||||
def do_import(rows, default_account_id, skip_dupes=True):
|
||||
"""
|
||||
Persist a list of row dicts to the DB.
|
||||
|
||||
Each row must have: date (ISO str), transaction_type, amount, description.
|
||||
Optional: category_id, notes, source_id (bank ref for dupe detection).
|
||||
|
||||
Returns (imported_count, skipped_count).
|
||||
"""
|
||||
from datetime import date as date_cls
|
||||
from app.extensions import db
|
||||
from app.models.transaction import Transaction
|
||||
from app.services.account_service import calc_balance
|
||||
|
||||
imported = 0
|
||||
skipped = 0
|
||||
affected = set()
|
||||
|
||||
for r in rows:
|
||||
txn_date = date_cls.fromisoformat(r['date']) if isinstance(r['date'], str) else r['date']
|
||||
txn_type = r['transaction_type']
|
||||
amount = float(r['amount'])
|
||||
description = (r.get('description') or '').strip() or 'Imported'
|
||||
category_id = r.get('category_id') or None
|
||||
account_id = int(r.get('account_id') or default_account_id)
|
||||
notes = (r.get('notes') or '').strip()
|
||||
source_id = r.get('source_id') or None
|
||||
|
||||
if skip_dupes:
|
||||
if source_id:
|
||||
dup = Transaction.query.filter(
|
||||
Transaction.notes.like(f'%import:{source_id}%')
|
||||
).first()
|
||||
else:
|
||||
dup = Transaction.query.filter_by(
|
||||
date=txn_date,
|
||||
amount=amount,
|
||||
transaction_type=txn_type,
|
||||
description=description,
|
||||
).first()
|
||||
if dup:
|
||||
skipped += 1
|
||||
continue
|
||||
|
||||
note_parts = []
|
||||
if source_id:
|
||||
note_parts.append(f'import:{source_id}')
|
||||
if notes:
|
||||
note_parts.append(notes)
|
||||
|
||||
txn = Transaction(
|
||||
date=txn_date,
|
||||
transaction_type=txn_type,
|
||||
amount=amount,
|
||||
description=description,
|
||||
category_id=category_id,
|
||||
account_id=account_id,
|
||||
notes=' | '.join(note_parts),
|
||||
)
|
||||
db.session.add(txn)
|
||||
affected.add(account_id)
|
||||
imported += 1
|
||||
|
||||
db.session.commit()
|
||||
for aid in affected:
|
||||
calc_balance(aid)
|
||||
|
||||
log.info('[bank_import] do_import: imported=%d skipped=%d', imported, skipped)
|
||||
return imported, skipped
|
||||
@@ -0,0 +1,621 @@
|
||||
{% extends 'base.html' %}
|
||||
{% block title %}Import Bank Statement{% endblock %}
|
||||
{% block page_title %}Import Bank Statement{% endblock %}
|
||||
|
||||
{% block extra_css %}
|
||||
<style>
|
||||
/* ── Drop zone ── */
|
||||
#drop-zone {
|
||||
border: 2px dashed var(--border); border-radius: 12px;
|
||||
padding: 40px 20px; text-align: center; cursor: pointer;
|
||||
transition: all .2s; background: #f8fafc; position: relative;
|
||||
}
|
||||
#drop-zone.drag-over { border-color: var(--accent); background: #eff6ff; }
|
||||
#drop-zone.has-file { border-color: #10b981; background: #f0fdf4; }
|
||||
#drop-zone input[type=file] { position: absolute; inset: 0; opacity: 0; cursor: pointer; }
|
||||
#file-name { font-size: 13px; color: var(--muted); margin-top: 8px; }
|
||||
|
||||
/* ── Steps ── */
|
||||
.step { display: none; }
|
||||
.step.active { display: block; }
|
||||
|
||||
/* ── Preview table ── */
|
||||
#preview-table { font-size: 12.5px; }
|
||||
#preview-table th {
|
||||
font-size: 10px; font-weight: 600; text-transform: uppercase;
|
||||
letter-spacing: .06em; color: var(--muted); padding: 9px 12px;
|
||||
background: #f8fafc; border-bottom: 1px solid var(--border);
|
||||
white-space: nowrap;
|
||||
}
|
||||
#preview-table td { padding: 7px 12px; border-bottom: 1px solid var(--border); vertical-align: middle; }
|
||||
#preview-table tbody tr:hover { background: #f8fafc; }
|
||||
#preview-table tbody tr.row-unchecked { opacity: .45; }
|
||||
#preview-table .cat-select { font-size: 12px; min-width: 150px; }
|
||||
.mono-amt { font-family: 'DM Mono', monospace; font-size: 12px; white-space: nowrap; }
|
||||
|
||||
/* ── Format badge ── */
|
||||
.fmt-badge {
|
||||
display: inline-flex; align-items: center; gap: 6px;
|
||||
background: #dbeafe; color: #1e40af; font-size: 12px;
|
||||
font-weight: 600; padding: 3px 10px; border-radius: 20px;
|
||||
}
|
||||
|
||||
/* ── Column mapping ── */
|
||||
.map-row { display: grid; grid-template-columns: 160px 1fr; align-items: center; gap: 12px; margin-bottom: 12px; }
|
||||
.map-row label { font-size: 13px; font-weight: 500; }
|
||||
|
||||
/* ── Supported formats list ── */
|
||||
.fmt-list { display: flex; flex-wrap: wrap; gap: 6px; }
|
||||
.fmt-chip {
|
||||
font-size: 11px; padding: 3px 10px; border-radius: 20px;
|
||||
background: #f1f5f9; color: #475569; font-weight: 500;
|
||||
}
|
||||
</style>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div class="row g-4">
|
||||
|
||||
{# ── LEFT COLUMN: upload form + supported formats ── #}
|
||||
<div class="col-12 col-lg-4">
|
||||
<div class="pcard">
|
||||
<div class="pcard-title mb-3">Upload Statement</div>
|
||||
|
||||
{# Drop zone #}
|
||||
<div id="drop-zone" class="mb-3">
|
||||
<input type="file" id="file-input" accept=".csv,.ofx,.qfx,.txt,.pdf">
|
||||
<i class="bi bi-cloud-upload fs-2 text-muted d-block mb-2"></i>
|
||||
<div class="fw-medium" style="font-size:13px;">Drag & drop or click to browse</div>
|
||||
<div class="text-muted" style="font-size:12px; margin-top:4px;">CSV, OFX, QFX, PDF</div>
|
||||
</div>
|
||||
<div id="file-name" class="mb-3 d-none"><i class="bi bi-file-earmark-text me-1"></i><span id="fn-text"></span></div>
|
||||
|
||||
<div class="mb-3">
|
||||
<label class="form-label fw-medium" style="font-size:13px;">Account</label>
|
||||
<select id="account-select" class="form-select form-select-sm">
|
||||
<option value="">— Select account —</option>
|
||||
{% for a in accounts %}
|
||||
<option value="{{ a.id }}">{{ a.name }}</option>
|
||||
{% endfor %}
|
||||
</select>
|
||||
<small class="text-muted" style="font-size:11px;">All transactions will be linked to this account.</small>
|
||||
</div>
|
||||
|
||||
<div class="form-check mb-4">
|
||||
<input class="form-check-input" type="checkbox" id="skip-dupes" checked>
|
||||
<label class="form-check-label" for="skip-dupes" style="font-size:13px;">Skip duplicate transactions</label>
|
||||
</div>
|
||||
|
||||
<button id="parse-btn" class="btn btn-primary w-100" disabled>
|
||||
<span id="parse-spinner" class="spinner-border spinner-border-sm me-1 d-none"></span>
|
||||
<i class="bi bi-search me-1" id="parse-icon"></i>Parse Statement
|
||||
</button>
|
||||
|
||||
<div id="parse-error" class="alert alert-danger mt-3 d-none" style="font-size:12px;"></div>
|
||||
</div>
|
||||
|
||||
{# Supported formats #}
|
||||
<div class="pcard mt-3">
|
||||
<div class="pcard-title mb-2">Supported Formats</div>
|
||||
<div class="fmt-list mb-3">
|
||||
<span class="fmt-chip" style="background:#fce7f3;color:#9d174d;">PDF <i class="bi bi-stars" style="font-size:10px;"></i></span>
|
||||
<span class="fmt-chip">OFX / QFX</span>
|
||||
<span class="fmt-chip">Chase</span>
|
||||
<span class="fmt-chip">Bank of America</span>
|
||||
<span class="fmt-chip">Citi</span>
|
||||
<span class="fmt-chip">Capital One</span>
|
||||
<span class="fmt-chip">Discover</span>
|
||||
<span class="fmt-chip">USAA</span>
|
||||
<span class="fmt-chip">Wells Fargo</span>
|
||||
<span class="fmt-chip">Amex</span>
|
||||
<span class="fmt-chip">Generic CSV</span>
|
||||
</div>
|
||||
<div class="text-muted" style="font-size:12px; line-height:1.7;">
|
||||
<div class="d-flex gap-2 mb-2 p-2" style="background:#fdf4ff;border-radius:8px;border:1px solid #f0abfc;">
|
||||
<i class="bi bi-stars text-purple" style="color:#9333ea;flex-shrink:0;margin-top:1px;"></i>
|
||||
<div><strong style="color:#7e22ce;">PDF</strong> — Text is extracted from digital statements and parsed by the AI (Groq).
|
||||
Works with any bank's PDF. Scanned/image PDFs are not supported.</div>
|
||||
</div>
|
||||
<strong>OFX/QFX</strong> is the most reliable structured format — download it from your bank's website.<br>
|
||||
For CSV, the file must have at least a <strong>date</strong>, <strong>description</strong>, and
|
||||
<strong>amount</strong> (or separate <strong>debit/credit</strong>) column.
|
||||
If the format isn't recognised, you'll be asked to map columns manually.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{# ── RIGHT COLUMN: results ── #}
|
||||
<div class="col-12 col-lg-8">
|
||||
|
||||
{# Step: column mapping #}
|
||||
<div id="step-mapping" class="step pcard">
|
||||
<div class="d-flex align-items-center gap-2 mb-3">
|
||||
<i class="bi bi-columns-gap fs-5 text-warning"></i>
|
||||
<span class="fw-semibold">Column Mapping Required</span>
|
||||
</div>
|
||||
<p class="text-muted" style="font-size:13px;">
|
||||
The CSV format wasn't recognised automatically. Please select which column contains each field.
|
||||
</p>
|
||||
<div id="mapping-form">
|
||||
<div class="map-row">
|
||||
<label>Date <span class="text-danger">*</span></label>
|
||||
<select id="map-date" class="form-select form-select-sm"><option value="">— select column —</option></select>
|
||||
</div>
|
||||
<div class="map-row">
|
||||
<label>Description <span class="text-danger">*</span></label>
|
||||
<select id="map-desc" class="form-select form-select-sm"><option value="">— select column —</option></select>
|
||||
</div>
|
||||
<div class="map-row">
|
||||
<label>Amount</label>
|
||||
<select id="map-amount" class="form-select form-select-sm"><option value="">— select column —</option></select>
|
||||
</div>
|
||||
<div class="map-row">
|
||||
<label>Debit (expense)</label>
|
||||
<select id="map-debit" class="form-select form-select-sm"><option value="">— select column —</option></select>
|
||||
</div>
|
||||
<div class="map-row">
|
||||
<label>Credit (income)</label>
|
||||
<select id="map-credit" class="form-select form-select-sm"><option value="">— select column —</option></select>
|
||||
</div>
|
||||
<div class="map-row">
|
||||
<label>Amount sign</label>
|
||||
<select id="map-sign" class="form-select form-select-sm">
|
||||
<option value="0">Negative = expense (most banks)</option>
|
||||
<option value="1">Positive = expense (Discover)</option>
|
||||
</select>
|
||||
</div>
|
||||
</div>
|
||||
<div id="mapping-error" class="alert alert-danger mt-2 d-none" style="font-size:12px;"></div>
|
||||
<button id="apply-mapping-btn" class="btn btn-primary mt-3">
|
||||
<i class="bi bi-arrow-right me-1"></i>Apply Mapping & Preview
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{# Step: preview #}
|
||||
<div id="step-preview" class="step">
|
||||
<div class="d-flex align-items-center flex-wrap gap-2 mb-3">
|
||||
<span class="fmt-badge"><i class="bi bi-check-circle"></i><span id="fmt-label"></span></span>
|
||||
<span class="text-muted" style="font-size:13px;" id="row-count-label"></span>
|
||||
<div class="ms-auto d-flex gap-2 align-items-center">
|
||||
<button id="select-all-btn" class="btn btn-sm btn-outline-secondary">Select All</button>
|
||||
<button id="select-none-btn" class="btn btn-sm btn-outline-secondary">None</button>
|
||||
<button id="import-btn" class="btn btn-sm btn-success px-3">
|
||||
<span id="import-spinner" class="spinner-border spinner-border-sm me-1 d-none"></span>
|
||||
<i class="bi bi-cloud-upload me-1" id="import-icon"></i>
|
||||
Import <span id="import-count">0</span> transactions
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{# Parse errors (non-fatal) #}
|
||||
<div id="parse-warnings" class="alert alert-warning d-none mb-3" style="font-size:12px;">
|
||||
<strong>Warnings</strong> — these rows were skipped:<br>
|
||||
<div id="parse-warnings-list"></div>
|
||||
</div>
|
||||
|
||||
<div class="pcard p-0" style="overflow:hidden;">
|
||||
<div style="overflow-x:auto;">
|
||||
<table id="preview-table" class="w-100" style="border-collapse:collapse;">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="width:36px; padding-left:16px;"><input type="checkbox" id="header-checkbox" checked></th>
|
||||
<th>Date</th>
|
||||
<th>Description</th>
|
||||
<th style="width:80px;">Type</th>
|
||||
<th style="width:110px;">Amount</th>
|
||||
<th>Category</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="preview-body"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="import-error" class="alert alert-danger mt-3 d-none" style="font-size:12px;"></div>
|
||||
</div>
|
||||
|
||||
{# Step: done #}
|
||||
<div id="step-done" class="step pcard text-center py-5">
|
||||
<i class="bi bi-check-circle-fill text-success fs-1 d-block mb-3"></i>
|
||||
<h5 class="fw-semibold mb-1" id="done-title"></h5>
|
||||
<p class="text-muted mb-4" id="done-subtitle"></p>
|
||||
<div class="d-flex justify-content-center gap-3">
|
||||
<a href="{{ url_for('transactions.index') }}" class="btn btn-primary">
|
||||
<i class="bi bi-arrow-right me-1"></i>View Transactions
|
||||
</a>
|
||||
<button id="import-another-btn" class="btn btn-outline-secondary">Import Another File</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{# Placeholder when nothing parsed yet #}
|
||||
<div id="step-empty" class="step pcard text-center py-5 text-muted">
|
||||
<i class="bi bi-bank fs-2 d-block mb-3 opacity-25"></i>
|
||||
<div style="font-size:13px;">Upload a bank statement to get started.</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block extra_js %}
|
||||
<script>
|
||||
(function () {
|
||||
'use strict';
|
||||
|
||||
const CSRF = document.querySelector('meta[name="csrf-token"]')?.content || '';
|
||||
|
||||
// ── Category data for dropdowns ──────────────────────────────────────────
|
||||
const CATEGORIES = [
|
||||
{% for c in categories %}
|
||||
{ id: {{ c.id }}, name: {{ c.name | tojson }}, type: {{ c.category_type | tojson }} },
|
||||
{% endfor %}
|
||||
];
|
||||
|
||||
// ── State ────────────────────────────────────────────────────────────────
|
||||
let parsedRows = []; // enriched rows from /parse
|
||||
let selectedFile = null; // File object
|
||||
|
||||
// ── Element shortcuts ────────────────────────────────────────────────────
|
||||
const $ = id => document.getElementById(id);
|
||||
|
||||
function showStep(name) {
|
||||
['step-mapping','step-preview','step-done','step-empty'].forEach(id => {
|
||||
$(id).classList.remove('active');
|
||||
});
|
||||
$(name).classList.add('active');
|
||||
}
|
||||
|
||||
// ── Drop zone ────────────────────────────────────────────────────────────
|
||||
const dropZone = $('drop-zone');
|
||||
const fileInput = $('file-input');
|
||||
|
||||
dropZone.addEventListener('dragover', e => { e.preventDefault(); dropZone.classList.add('drag-over'); });
|
||||
dropZone.addEventListener('dragleave', () => dropZone.classList.remove('drag-over'));
|
||||
dropZone.addEventListener('drop', e => {
|
||||
e.preventDefault();
|
||||
dropZone.classList.remove('drag-over');
|
||||
const f = e.dataTransfer.files[0];
|
||||
if (f) setFile(f);
|
||||
});
|
||||
fileInput.addEventListener('change', () => {
|
||||
if (fileInput.files[0]) setFile(fileInput.files[0]);
|
||||
});
|
||||
|
||||
function setFile(f) {
|
||||
selectedFile = f;
|
||||
dropZone.classList.add('has-file');
|
||||
$('file-name').classList.remove('d-none');
|
||||
$('fn-text').textContent = f.name + ' (' + (f.size / 1024).toFixed(1) + ' KB)';
|
||||
$('parse-btn').disabled = false;
|
||||
$('parse-error').classList.add('d-none');
|
||||
// Reset preview
|
||||
parsedRows = [];
|
||||
showStep('step-empty');
|
||||
}
|
||||
|
||||
// ── Parse ────────────────────────────────────────────────────────────────
|
||||
$('parse-btn').addEventListener('click', () => doParseRequest(null));
|
||||
|
||||
function doParseRequest(colMap) {
|
||||
if (!selectedFile) return;
|
||||
const accountId = $('account-select').value;
|
||||
if (!accountId) {
|
||||
$('parse-error').textContent = 'Please select an account first.';
|
||||
$('parse-error').classList.remove('d-none');
|
||||
return;
|
||||
}
|
||||
|
||||
setParseLoading(true);
|
||||
$('parse-error').classList.add('d-none');
|
||||
|
||||
const fd = new FormData();
|
||||
fd.append('file', selectedFile);
|
||||
if (colMap) fd.append('col_map', JSON.stringify(colMap));
|
||||
|
||||
fetch('{{ url_for("bank_import.parse") }}', { method: 'POST', body: fd })
|
||||
.then(r => r.json())
|
||||
.then(data => {
|
||||
setParseLoading(false);
|
||||
if (data.error) {
|
||||
$('parse-error').textContent = data.error;
|
||||
$('parse-error').classList.remove('d-none');
|
||||
return;
|
||||
}
|
||||
if (data.needs_mapping) {
|
||||
buildMappingUI(data.headers);
|
||||
showStep('step-mapping');
|
||||
return;
|
||||
}
|
||||
parsedRows = data.rows || [];
|
||||
renderPreview(data);
|
||||
})
|
||||
.catch(err => {
|
||||
setParseLoading(false);
|
||||
$('parse-error').textContent = 'Request failed: ' + err;
|
||||
$('parse-error').classList.remove('d-none');
|
||||
});
|
||||
}
|
||||
|
||||
function setParseLoading(on) {
|
||||
$('parse-spinner').classList.toggle('d-none', !on);
|
||||
$('parse-icon').classList.toggle('d-none', on);
|
||||
$('parse-btn').disabled = on;
|
||||
// PDF parsing calls Groq and can take 15-30 s — show a helpful message
|
||||
if (on && selectedFile && selectedFile.name.toLowerCase().endsWith('.pdf')) {
|
||||
$('parse-btn').textContent = '';
|
||||
$('parse-btn').appendChild($('parse-spinner'));
|
||||
$('parse-btn').append(' Analysing PDF with AI… (may take ~20s)');
|
||||
} else if (!on) {
|
||||
$('parse-btn').innerHTML =
|
||||
'<span id="parse-spinner" class="spinner-border spinner-border-sm me-1 d-none"></span>' +
|
||||
'<i class="bi bi-search me-1" id="parse-icon"></i>Parse Statement';
|
||||
}
|
||||
}
|
||||
|
||||
// ── Column mapping UI ────────────────────────────────────────────────────
|
||||
function buildMappingUI(headers) {
|
||||
const selectors = ['map-date','map-desc','map-amount','map-debit','map-credit'];
|
||||
selectors.forEach(id => {
|
||||
const sel = $(id);
|
||||
while (sel.options.length > 1) sel.remove(1);
|
||||
headers.forEach(h => {
|
||||
const opt = document.createElement('option');
|
||||
opt.value = h.toLowerCase();
|
||||
opt.textContent = h;
|
||||
sel.appendChild(opt);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
$('apply-mapping-btn').addEventListener('click', () => {
|
||||
const dateCol = $('map-date').value;
|
||||
const descCol = $('map-desc').value;
|
||||
const amtCol = $('map-amount').value;
|
||||
const debitCol = $('map-debit').value;
|
||||
const credCol = $('map-credit').value;
|
||||
const sign = $('map-sign').value === '1';
|
||||
|
||||
if (!dateCol || !descCol) {
|
||||
$('mapping-error').textContent = 'Date and Description columns are required.';
|
||||
$('mapping-error').classList.remove('d-none');
|
||||
return;
|
||||
}
|
||||
if (!amtCol && !debitCol && !credCol) {
|
||||
$('mapping-error').textContent = 'At least one of Amount, Debit, or Credit column is required.';
|
||||
$('mapping-error').classList.remove('d-none');
|
||||
return;
|
||||
}
|
||||
$('mapping-error').classList.add('d-none');
|
||||
|
||||
const colMap = {
|
||||
date: dateCol, description: descCol,
|
||||
amount: amtCol || null, debit: debitCol || null, credit: credCol || null,
|
||||
positive_is_expense: sign,
|
||||
};
|
||||
doParseRequest(colMap);
|
||||
});
|
||||
|
||||
// ── Preview table ────────────────────────────────────────────────────────
|
||||
function renderPreview(data) {
|
||||
const rows = data.rows || [];
|
||||
const errors = data.errors || [];
|
||||
|
||||
$('fmt-label').textContent = data.format_name || 'Detected';
|
||||
$('row-count-label').textContent = rows.length + ' transaction' + (rows.length !== 1 ? 's' : '') + ' found';
|
||||
|
||||
// Warnings
|
||||
if (errors.length) {
|
||||
$('parse-warnings').classList.remove('d-none');
|
||||
$('parse-warnings-list').innerHTML = errors.map(e => `<div>${esc(e)}</div>`).join('');
|
||||
} else {
|
||||
$('parse-warnings').classList.add('d-none');
|
||||
}
|
||||
|
||||
const tbody = $('preview-body');
|
||||
tbody.innerHTML = '';
|
||||
|
||||
if (!rows.length) {
|
||||
tbody.innerHTML = '<tr><td colspan="6" class="text-center py-4 text-muted">No transactions found in this file.</td></tr>';
|
||||
showStep('step-preview');
|
||||
updateImportCount();
|
||||
return;
|
||||
}
|
||||
|
||||
rows.forEach((row, idx) => {
|
||||
const tr = document.createElement('tr');
|
||||
tr.dataset.idx = idx;
|
||||
|
||||
// Checkbox
|
||||
const tdChk = document.createElement('td');
|
||||
tdChk.style.paddingLeft = '16px';
|
||||
const chk = document.createElement('input');
|
||||
chk.type = 'checkbox'; chk.checked = true; chk.className = 'row-chk';
|
||||
chk.addEventListener('change', () => {
|
||||
tr.classList.toggle('row-unchecked', !chk.checked);
|
||||
updateImportCount();
|
||||
});
|
||||
tdChk.appendChild(chk);
|
||||
tr.appendChild(tdChk);
|
||||
|
||||
// Date
|
||||
tr.appendChild(tdText(formatDate(row.date), 'color:var(--muted);font-size:12px;white-space:nowrap;'));
|
||||
|
||||
// Description
|
||||
const tdDesc = document.createElement('td');
|
||||
tdDesc.textContent = row.description;
|
||||
tdDesc.style.maxWidth = '220px';
|
||||
tdDesc.style.overflow = 'hidden';
|
||||
tdDesc.style.textOverflow = 'ellipsis';
|
||||
tdDesc.style.whiteSpace = 'nowrap';
|
||||
tr.appendChild(tdDesc);
|
||||
|
||||
// Type badge
|
||||
const tdType = document.createElement('td');
|
||||
const badge = document.createElement('span');
|
||||
badge.className = 'badge ' + (row.transaction_type === 'income' ? 'badge-income' : 'badge-expense');
|
||||
badge.style.fontSize = '11px';
|
||||
badge.textContent = row.transaction_type === 'income' ? 'Income' : 'Expense';
|
||||
tdType.appendChild(badge);
|
||||
tr.appendChild(tdType);
|
||||
|
||||
// Amount
|
||||
const tdAmt = document.createElement('td');
|
||||
tdAmt.className = 'mono-amt ' + (row.transaction_type === 'income' ? 'text-income' : 'text-expense');
|
||||
tdAmt.textContent = (row.transaction_type === 'income' ? '+' : '-') + fmtAmt(row.amount);
|
||||
tr.appendChild(tdAmt);
|
||||
|
||||
// Category dropdown
|
||||
const tdCat = document.createElement('td');
|
||||
const sel = buildCategorySelect(row.category_id, row.transaction_type);
|
||||
sel.dataset.idx = idx;
|
||||
tdCat.appendChild(sel);
|
||||
tr.appendChild(tdCat);
|
||||
|
||||
tbody.appendChild(tr);
|
||||
});
|
||||
|
||||
updateImportCount();
|
||||
showStep('step-preview');
|
||||
}
|
||||
|
||||
function buildCategorySelect(selectedId, txnType) {
|
||||
const sel = document.createElement('select');
|
||||
sel.className = 'form-select form-select-sm cat-select';
|
||||
|
||||
const blank = document.createElement('option');
|
||||
blank.value = ''; blank.textContent = '— Uncategorized —';
|
||||
sel.appendChild(blank);
|
||||
|
||||
CATEGORIES.forEach(cat => {
|
||||
if (cat.type !== 'both' && cat.type !== txnType) return;
|
||||
const opt = document.createElement('option');
|
||||
opt.value = cat.id;
|
||||
opt.textContent = cat.name;
|
||||
if (cat.id == selectedId) opt.selected = true;
|
||||
sel.appendChild(opt);
|
||||
});
|
||||
return sel;
|
||||
}
|
||||
|
||||
function updateImportCount() {
|
||||
const checked = document.querySelectorAll('#preview-body .row-chk:checked').length;
|
||||
$('import-count').textContent = checked;
|
||||
$('import-btn').disabled = checked === 0;
|
||||
}
|
||||
|
||||
// Header checkbox — select/deselect all
|
||||
$('header-checkbox').addEventListener('change', function () {
|
||||
document.querySelectorAll('#preview-body .row-chk').forEach(chk => {
|
||||
chk.checked = this.checked;
|
||||
chk.closest('tr').classList.toggle('row-unchecked', !this.checked);
|
||||
});
|
||||
updateImportCount();
|
||||
});
|
||||
$('select-all-btn').addEventListener('click', () => {
|
||||
document.querySelectorAll('#preview-body .row-chk').forEach(chk => {
|
||||
chk.checked = true; chk.closest('tr').classList.remove('row-unchecked');
|
||||
});
|
||||
$('header-checkbox').checked = true;
|
||||
updateImportCount();
|
||||
});
|
||||
$('select-none-btn').addEventListener('click', () => {
|
||||
document.querySelectorAll('#preview-body .row-chk').forEach(chk => {
|
||||
chk.checked = false; chk.closest('tr').classList.add('row-unchecked');
|
||||
});
|
||||
$('header-checkbox').checked = false;
|
||||
updateImportCount();
|
||||
});
|
||||
|
||||
// ── Import ────────────────────────────────────────────────────────────────
|
||||
$('import-btn').addEventListener('click', () => {
|
||||
const accountId = $('account-select').value;
|
||||
const skipDupes = $('skip-dupes').checked;
|
||||
|
||||
// Collect selected rows with current category selections
|
||||
const rows = [];
|
||||
document.querySelectorAll('#preview-body tr').forEach(tr => {
|
||||
const chk = tr.querySelector('.row-chk');
|
||||
if (!chk || !chk.checked) return;
|
||||
const idx = parseInt(tr.dataset.idx);
|
||||
const row = { ...parsedRows[idx] };
|
||||
const sel = tr.querySelector('.cat-select');
|
||||
row.category_id = sel && sel.value ? parseInt(sel.value) : null;
|
||||
rows.push(row);
|
||||
});
|
||||
|
||||
if (!rows.length) return;
|
||||
|
||||
setImportLoading(true);
|
||||
$('import-error').classList.add('d-none');
|
||||
|
||||
fetch('{{ url_for("bank_import.confirm_import") }}', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'X-CSRFToken': CSRF,
|
||||
},
|
||||
body: JSON.stringify({ account_id: parseInt(accountId), skip_dupes: skipDupes, rows }),
|
||||
})
|
||||
.then(r => r.json())
|
||||
.then(data => {
|
||||
setImportLoading(false);
|
||||
if (data.error) {
|
||||
$('import-error').textContent = data.error;
|
||||
$('import-error').classList.remove('d-none');
|
||||
return;
|
||||
}
|
||||
$('done-title').textContent =
|
||||
data.imported + ' transaction' + (data.imported !== 1 ? 's' : '') + ' imported successfully';
|
||||
$('done-subtitle').textContent =
|
||||
data.skipped > 0 ? data.skipped + ' duplicate(s) skipped.' : 'All transactions were new.';
|
||||
showStep('step-done');
|
||||
})
|
||||
.catch(err => {
|
||||
setImportLoading(false);
|
||||
$('import-error').textContent = 'Request failed: ' + err;
|
||||
$('import-error').classList.remove('d-none');
|
||||
});
|
||||
});
|
||||
|
||||
function setImportLoading(on) {
|
||||
$('import-spinner').classList.toggle('d-none', !on);
|
||||
$('import-icon').classList.toggle('d-none', on);
|
||||
$('import-btn').disabled = on;
|
||||
}
|
||||
|
||||
// ── Import another ────────────────────────────────────────────────────────
|
||||
$('import-another-btn').addEventListener('click', () => {
|
||||
selectedFile = null;
|
||||
parsedRows = [];
|
||||
fileInput.value = '';
|
||||
dropZone.classList.remove('has-file');
|
||||
$('file-name').classList.add('d-none');
|
||||
$('parse-btn').disabled = true;
|
||||
$('parse-error').classList.add('d-none');
|
||||
showStep('step-empty');
|
||||
});
|
||||
|
||||
// ── Utility ───────────────────────────────────────────────────────────────
|
||||
function esc(s) {
|
||||
return String(s)
|
||||
.replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>');
|
||||
}
|
||||
function tdText(text, style) {
|
||||
const td = document.createElement('td');
|
||||
td.textContent = text;
|
||||
if (style) td.style.cssText = style;
|
||||
return td;
|
||||
}
|
||||
function formatDate(iso) {
|
||||
const d = new Date(iso + 'T00:00:00');
|
||||
return d.toLocaleDateString('en-US', { month: 'short', day: 'numeric', year: 'numeric' });
|
||||
}
|
||||
function fmtAmt(n) {
|
||||
return parseFloat(n).toLocaleString('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 });
|
||||
}
|
||||
|
||||
// Init
|
||||
showStep('step-empty');
|
||||
})();
|
||||
</script>
|
||||
{% endblock %}
|
||||
@@ -215,6 +215,12 @@
|
||||
>
|
||||
<i class="bi bi-wallet2"></i><span class="lt">Accounts</span>
|
||||
</a>
|
||||
<a
|
||||
href="{{ url_for('bank_import.index') }}"
|
||||
class="sb-link {% if request.blueprint == 'bank_import' %}active{% endif %}"
|
||||
>
|
||||
<i class="bi bi-bank2"></i><span class="lt">Import Statement</span>
|
||||
</a>
|
||||
|
||||
<div class="sb-section">Planning</div>
|
||||
<a
|
||||
|
||||
Reference in New Issue
Block a user