From b43b42450fe80ff7ded8f8fc87341d12915b65a1 Mon Sep 17 00:00:00 2001 From: NguyenND Date: Mon, 1 Jun 2026 12:53:34 -0400 Subject: [PATCH] 06/01 Implement PDF statement file import --- app/__init__.py | 2 + app/routes/bank_import.py | 106 ++++ app/services/bank_import_service.py | 787 +++++++++++++++++++++++++++ app/templates/bank_import/index.html | 621 +++++++++++++++++++++ app/templates/base.html | 6 + requirements.txt | 1 + 6 files changed, 1523 insertions(+) create mode 100644 app/routes/bank_import.py create mode 100644 app/services/bank_import_service.py create mode 100644 app/templates/bank_import/index.html diff --git a/app/__init__.py b/app/__init__.py index b02c291..e0e4f88 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -90,6 +90,7 @@ def create_app(config_name=None): from app.routes.settings import settings_bp from app.routes.teller import teller_bp from app.routes.logs import logs_bp + from app.routes.bank_import import bank_import_bp app.register_blueprint(auth_bp) app.register_blueprint(dashboard_bp) @@ -104,6 +105,7 @@ def create_app(config_name=None): app.register_blueprint(settings_bp) app.register_blueprint(teller_bp) app.register_blueprint(logs_bp) + app.register_blueprint(bank_import_bp) with app.app_context(): from app.models import ( diff --git a/app/routes/bank_import.py b/app/routes/bank_import.py new file mode 100644 index 0000000..06dc2dd --- /dev/null +++ b/app/routes/bank_import.py @@ -0,0 +1,106 @@ +import json +import logging +from flask import Blueprint, render_template, request, jsonify +from flask_login import login_required +from app.models.account import Account +from app.models.category import Category +from app.services.bank_import_service import parse_file, do_import + +bank_import_bp = Blueprint('bank_import', __name__, url_prefix='/bank-import') +log = logging.getLogger(__name__) + +ALLOWED_EXTENSIONS = {'csv', 'ofx', 'qfx', 'txt', 'pdf'} + + +@bank_import_bp.route('/') +@login_required +def index(): + accounts = Account.query.filter_by(is_active=True).order_by(Account.name).all() + categories = Category.query.filter_by(is_active=True).order_by(Category.name).all() + return render_template('bank_import/index.html', + accounts=accounts, + categories=categories) + + +@bank_import_bp.route('/parse', methods=['POST']) +@login_required +def parse(): + """ + Accepts a multipart/form-data POST with: + file — the bank statement file + col_map — JSON string (optional); user-supplied column mapping + + Returns JSON with parsed rows, detected format, errors, or needs_mapping flag. + """ + f = request.files.get('file') + if not f or not f.filename: + return jsonify({'error': 'No file uploaded'}), 400 + + ext = f.filename.rsplit('.', 1)[-1].lower() if '.' in f.filename else '' + if ext not in ALLOWED_EXTENSIONS: + return jsonify({'error': f'Unsupported file type: .{ext}. Use CSV, OFX, or QFX.'}), 400 + + col_map = None + raw_map = request.form.get('col_map', '') + if raw_map: + try: + col_map = json.loads(raw_map) + except (ValueError, TypeError): + return jsonify({'error': 'Invalid col_map JSON'}), 400 + + file_bytes = f.read() + log.info('[bank_import] parse request: filename=%s size=%d bytes', f.filename, len(file_bytes)) + + try: + result = parse_file(file_bytes, f.filename, col_map=col_map) + except Exception as e: + log.error('[bank_import] parse failed: %s', e, exc_info=True) + return jsonify({'error': f'Failed to parse file: {e}'}), 500 + + return jsonify(result) + + +@bank_import_bp.route('/import', methods=['POST']) +@login_required +def confirm_import(): + """ + Accepts JSON: + { + "account_id": int, + "skip_dupes": bool, + "rows": [ + {"date": "YYYY-MM-DD", "transaction_type": "expense|income", + "amount": float, "description": str, "category_id": int|null, + "notes": str, "source_id": str|null}, + ... + ] + } + Returns {"imported": N, "skipped": N}. + """ + data = request.get_json() + if not data: + return jsonify({'error': 'No JSON payload'}), 400 + + rows = data.get('rows', []) + account_id = data.get('account_id') + skip_dupes = data.get('skip_dupes', True) + + if not rows: + return jsonify({'error': 'No rows provided'}), 400 + if not account_id: + return jsonify({'error': 'account_id is required'}), 400 + + # Validate account exists + acct = Account.query.get(int(account_id)) + if not acct: + return jsonify({'error': f'Account {account_id} not found'}), 400 + + log.info('[bank_import] importing %d rows into account_id=%s skip_dupes=%s', + len(rows), account_id, skip_dupes) + try: + imported, skipped = do_import(rows, int(account_id), skip_dupes=skip_dupes) + except Exception as e: + log.error('[bank_import] import failed: %s', e, exc_info=True) + return jsonify({'error': f'Import failed: {e}'}), 500 + + return jsonify({'imported': imported, 'skipped': skipped}) diff --git a/app/services/bank_import_service.py b/app/services/bank_import_service.py new file mode 100644 index 0000000..b0a1c60 --- /dev/null +++ b/app/services/bank_import_service.py @@ -0,0 +1,787 @@ +""" +Bank Statement Import Service + +Supported formats: + PDF — Digital bank statement PDFs; text extracted via pdfplumber then + parsed by Groq LLM (requires GROQ_API_KEY). + OFX / QFX — Open Financial Exchange (exported by most US banks & Quicken) + CSV — Auto-detects Chase, Bank of America, Citi, Capital One, Discover, + USAA, Wells Fargo; falls back to a generic heuristic; asks user + to map columns when detection fails. + +Public API: + parse_file(file_bytes, filename, col_map=None) → dict + do_import(rows, default_account_id, skip_dupes=True) → (imported, skipped) + auto_categorize(description) → category_name_str +""" + +import csv +import io +import re +import logging +from datetime import datetime + +log = logging.getLogger(__name__) + + +# ── Keyword → PFM category name map ────────────────────────────────────────── + +KEYWORD_CATEGORIES = { + 'Food & Dining': [ + 'restaurant', 'dining', 'cafe', 'coffee', 'starbucks', 'mcdonald', + 'burger', 'pizza', 'sushi', 'taco', 'chipotle', 'subway', 'doordash', + 'ubereats', 'grubhub', 'seamless', 'instacart', 'grocery', 'safeway', + 'whole foods', 'trader joe', 'kroger', 'wegmans', 'publix', 'aldi', + 'panera', 'dunkin', 'chick-fil-a', 'wendy', 'taco bell', 'kfc', + 'popeyes', 'five guys', 'domino', 'papa john', 'little caesar', + 'denny', 'ihop', 'olive garden', 'applebee', 'cheesecake factory', + ], + 'Transport': [ + 'uber', 'lyft', 'taxi', 'gas station', 'shell', 'chevron', 'exxon', + 'mobil', 'citgo', 'speedway', 'sunoco', 'circle k', 'wawa fuel', + 'parking', 'toll', 'transit', 'metro', 'bart', 'mta ', 'wmata', + 'amtrak', 'greyhound', 'zipcar', 'enterprise rent', 'hertz', 'avis', + 'budget rent', 'national car', 'jiffy lube', 'valvoline', 'firestone', + 'oil change', 'autozone', 'advance auto', 'napa auto', 'pep boys', + 'airline', 'delta ', 'united air', 'southwest', 'american air', + 'jetblue', 'spirit air', + ], + 'Shopping': [ + 'amazon', 'ebay', 'walmart', 'target', 'costco', 'best buy', + 'apple store', 'nike', 'adidas', 'gap', 'old navy', 'h&m', 'zara', + 'nordstrom', 'macy', 'tj maxx', 'marshalls', 'ross store', + 'dollar tree', 'dollar general', 'home depot', 'lowes', 'ikea', + 'bed bath', 'wayfair', 'etsy', 'paypal ', 'venmo ', + ], + 'Entertainment': [ + 'netflix', 'spotify', 'hulu', 'disney+', 'hbo max', 'amazon prime', + 'apple tv', 'youtube premium', 'twitch', 'steam ', 'playstation', + 'xbox', 'nintendo', 'movie', 'cinema', 'theater', 'concert', + 'ticketmaster', 'stubhub', 'eventbrite', 'live nation', 'amc ', + 'regal cinema', 'fandango', 'bowling', 'arcade', 'museum', 'zoo', + ], + 'Utilities': [ + 'electric', 'gas utility', 'natural gas', 'water bill', 'sewage', + 'trash', 'waste mgmt', 'internet', 'comcast', 'xfinity', 'verizon', + 'at&t', 'att bill', 't-mobile', 'sprint', 'charter', 'spectrum', + 'cox comm', 'centurylink', 'utility bill', 'pge ', 'con edison', + ], + 'Health': [ + 'pharmacy', 'cvs ', 'walgreens', 'rite aid', 'hospital', 'clinic', + 'doctor', 'dental', 'dentist', 'vision center', 'optometrist', + 'labcorp', 'quest diag', 'urgent care', 'medical', 'health care', + 'gym', 'planet fitness', 'la fitness', 'equinox', 'peloton', + ], + 'Housing': [ + 'rent payment', 'mortgage', 'hoa ', 'property tax', 'maintenance', + 'plumber', 'electrician', 'hvac', 'pest control', 'lawn care', + 'landscap', 'cleaning service', 'airbnb', 'vrbo', 'hotel', 'motel', + 'hilton', 'marriott', 'hyatt', 'sheraton', 'holiday inn', + ], + 'Subscriptions': [ + 'subscription', 'membership fee', 'annual fee', 'monthly fee', + 'adobe ', 'microsoft 365', 'office 365', 'dropbox', 'icloud', + 'google one', '1password', 'nordvpn', 'expressvpn', 'duolingo', + 'headspace', 'calm app', 'audible', 'kindle unlimited', + 'new york times', 'wall street journal', 'nytimes.com', + ], + 'Education': [ + 'tuition', 'university', 'college', 'coursera', 'udemy', + 'linkedin learning', 'pluralsight', 'skillshare', 'textbook', 'chegg', + 'grammarly', 'turnitin', 'canvas ', 'blackboard', + ], + 'Insurance': [ + 'insurance', 'geico', 'state farm', 'allstate', 'progressive ins', + 'liberty mutual', 'farmers ins', 'aaa ins', 'aflac', 'metlife', + 'cigna', 'aetna', 'humana', 'blue cross', 'united health', + ], + 'Gifts': [ + 'gift card', 'donation', 'charity', 'church tithe', 'gofundme', + 'red cross', 'goodwill', 'salvation army', 'habitat humanity', + ], + 'Investment': [ + 'robinhood', 'fidelity inv', 'schwab', 'vanguard', 'e*trade', + 'td ameritrade', 'coinbase', 'binance', 'crypto purchase', + ], + 'Salary': [ + 'payroll', 'direct deposit', 'salary', 'wages deposit', 'paycheck', + 'adp llc', 'paychex', 'gusto payroll', 'paylocity', 'intuit payroll', + ], + 'Freelance': [ + 'freelance', 'consulting fee', 'contract payment', 'upwork', + 'fiverr', 'toptal', + ], + 'Other Income': [ + 'interest paid', 'dividend', 'refund', 'cashback', 'reward', + 'reimburs', 'tax refund', 'irs treas', + ], +} + + +# ── Known bank CSV format profiles ─────────────────────────────────────────── + +BANK_FORMATS = [ + { + 'name': 'Chase', + 'required': {'transaction date', 'description', 'amount'}, + 'map': { + 'date': 'transaction date', 'description': 'description', + 'amount': 'amount', 'positive_is_expense': False, + }, + }, + { + 'name': 'Bank of America', + 'required': {'posted date', 'payee', 'amount'}, + 'map': { + 'date': 'posted date', 'description': 'payee', + 'amount': 'amount', 'positive_is_expense': False, + }, + }, + { + 'name': 'Citi', + 'required': {'date', 'description', 'debit', 'credit'}, + 'map': { + 'date': 'date', 'description': 'description', + 'debit': 'debit', 'credit': 'credit', + }, + }, + { + 'name': 'Capital One', + 'required': {'transaction date', 'posted date', 'card no.', 'description', 'debit', 'credit'}, + 'map': { + 'date': 'transaction date', 'description': 'description', + 'debit': 'debit', 'credit': 'credit', + }, + }, + { + 'name': 'Discover', + 'required': {'trans. date', 'post date', 'description', 'amount', 'category'}, + 'map': { + 'date': 'trans. date', 'description': 'description', + 'amount': 'amount', 'positive_is_expense': True, + }, + }, + { + 'name': 'American Express', + 'required': {'date', 'description', 'amount', 'extended details', 'appears on your statement as'}, + 'map': { + 'date': 'date', 'description': 'description', + 'amount': 'amount', 'positive_is_expense': False, + }, + }, + { + 'name': 'USAA', + 'required': {'date', 'description', 'original description', 'category', 'amount', 'status'}, + 'map': { + 'date': 'date', 'description': 'description', + 'amount': 'amount', 'positive_is_expense': False, + }, + }, + { + 'name': 'Wells Fargo', + 'required': {'date', 'amount', 'description'}, + 'map': { + 'date': 'date', 'description': 'description', + 'amount': 'amount', 'positive_is_expense': False, + }, + }, +] + +_DATE_FMTS = [ + '%Y-%m-%d', '%m/%d/%Y', '%d/%m/%Y', '%Y/%m/%d', + '%d-%m-%Y', '%m-%d-%Y', '%b %d, %Y', '%d %b %Y', + '%B %d, %Y', '%d %B %Y', '%m/%d/%y', +] + + +# ── Internal helpers ────────────────────────────────────────────────────────── + +def _parse_date(s): + s = str(s).strip() + # OFX: 20250115120000[-5:EST] — take first 8 digits + if re.match(r'^\d{8}', s): + return datetime.strptime(s[:8], '%Y%m%d').date() + for fmt in _DATE_FMTS: + try: + return datetime.strptime(s, fmt).date() + except ValueError: + pass + raise ValueError(f'Unrecognised date: {s!r}') + + +def _clean_amount(s): + if s is None: + return 0.0 + cleaned = re.sub(r'[^\d.\-+]', '', str(s).strip()) + try: + return float(cleaned) if cleaned else 0.0 + except ValueError: + return 0.0 + + +def _extract_tag(text, tag): + """Extract value of an OFX SGML/XML tag (stops at newline or next tag).""" + m = re.search(rf'<{tag}>\s*([^\r\n<]+)', text, re.IGNORECASE) + return m.group(1).strip() if m else '' + + +# ── OFX / QFX parser ───────────────────────────────────────────────────────── + +def _parse_ofx(content): + """Parse OFX/QFX (handles both SGML and XML variants).""" + # Strip header block above + ofx_start = re.search(r'', content, re.IGNORECASE) + if ofx_start: + content = content[ofx_start.start():] + + # Try XML-style first (closing tags present) + xml_blocks = re.findall(r'(.*?)', content, + re.IGNORECASE | re.DOTALL) + if not xml_blocks: + # SGML-style: split on opening tags + xml_blocks = re.split(r'', content, flags=re.IGNORECASE)[1:] + + raw = [] + for block in xml_blocks: + dtposted = _extract_tag(block, 'DTPOSTED') + trnamt = _extract_tag(block, 'TRNAMT') + trntype = _extract_tag(block, 'TRNTYPE').upper() + fitid = _extract_tag(block, 'FITID') + name = _extract_tag(block, 'NAME') + memo = _extract_tag(block, 'MEMO') + + if not dtposted or not trnamt: + continue + try: + txn_date = _parse_date(dtposted) + except ValueError: + continue + + amount = _clean_amount(trnamt) + if amount == 0.0: + continue + + # OFX sign: negative = debit (expense), positive = credit (income) + if amount < 0: + txn_type = 'expense' + else: + txn_type = 'income' + amount = abs(amount) + + # TRNTYPE tag can override when sign is ambiguous + if trntype in ('DEBIT', 'CHECK', 'PAYMENT', 'REPEATPMT', 'ATM', 'POS', 'XFER'): + txn_type = 'expense' + elif trntype in ('CREDIT', 'INT', 'DIV', 'DIRECTDEP', 'DEPOSIT', 'REFUND'): + txn_type = 'income' + + description = name or memo or f'OFX {trntype}' + raw.append({ + 'date': txn_date, + 'transaction_type': txn_type, + 'amount': amount, + 'description': description, + 'notes': memo if memo and memo != name else '', + 'source_id': fitid, + }) + + log.info('[bank_import] OFX: parsed %d transactions', len(raw)) + return raw + + +# ── CSV helpers ─────────────────────────────────────────────────────────────── + +def _detect_csv_format(headers_set): + """Return BANK_FORMATS entry whose required columns are all present, or None.""" + for fmt in BANK_FORMATS: + if fmt['required'].issubset(headers_set): + return fmt + return None + + +def _build_generic_map(headers_set): + """ + Attempt to build a column map from an unknown bank CSV. + Looks for common column name patterns. + Returns col_map dict or None if key columns cannot be found. + """ + date_names = ['date', 'posted date', 'transaction date', 'trans. date', + 'post date', 'settlement date', 'value date', 'effective date'] + desc_names = ['description', 'payee', 'name', 'merchant', 'memo', + 'transaction description', 'details', 'narrative', 'remarks'] + amt_names = ['amount', 'transaction amount', 'net amount'] + debit_names = ['debit', 'withdrawal', 'withdrawals', 'charge', 'charges', + 'debit amount', 'out', 'outflow'] + cred_names = ['credit', 'deposit', 'deposits', 'payment', 'in', + 'credit amount', 'inflow'] + + date_col = next((c for c in date_names if c in headers_set), None) + desc_col = next((c for c in desc_names if c in headers_set), None) + amt_col = next((c for c in amt_names if c in headers_set), None) + debit_col = next((c for c in debit_names if c in headers_set), None) + cred_col = next((c for c in cred_names if c in headers_set), None) + + if not date_col or not desc_col: + return None + if not amt_col and not debit_col and not cred_col: + return None + + return { + 'date': date_col, + 'description': desc_col, + 'amount': amt_col, + 'debit': debit_col, + 'credit': cred_col, + 'positive_is_expense': False, + } + + +def _parse_csv_with_map(content, col_map): + """ + Parse CSV content using the provided column map. + Returns (raw_rows, errors). + raw_rows: list of dicts with date (date obj), transaction_type, amount, description, notes, source_id. + """ + reader = csv.DictReader(io.StringIO(content)) + # Build {lowercase: original_case} lookup for header matching + header_lookup = {h.strip().lower(): h.strip() for h in (reader.fieldnames or [])} + + def get_col(key): + mapped = col_map.get(key) + if not mapped: + return None + return header_lookup.get(mapped.lower()) + + date_col = get_col('date') + desc_col = get_col('description') + amt_col = get_col('amount') + debit_col = get_col('debit') + cred_col = get_col('credit') + positive_is_expense = col_map.get('positive_is_expense', False) + + raw = [] + errors = [] + + for i, row in enumerate(reader, start=2): + raw_date = row.get(date_col, '').strip() if date_col else '' + if not raw_date: + continue # skip blank rows silently + + try: + txn_date = _parse_date(raw_date) + except ValueError as e: + errors.append(f'Row {i}: {e}') + continue + + description = (row.get(desc_col, '') if desc_col else '').strip() + if not description: + description = f'Transaction on {raw_date}' + + # Amount resolution: debit/credit columns take priority + if debit_col or cred_col: + debit = abs(_clean_amount(row.get(debit_col, '') if debit_col else '')) + credit = abs(_clean_amount(row.get(cred_col, '') if cred_col else '')) + if debit > 0 and credit > 0: + # Both filled — treat as expense (unusual, but be safe) + amount = debit + txn_type = 'expense' + elif debit > 0: + amount = debit + txn_type = 'expense' + elif credit > 0: + amount = credit + txn_type = 'income' + else: + continue # skip zero / blank rows + elif amt_col: + amount_raw = _clean_amount(row.get(amt_col, '')) + if amount_raw == 0.0: + continue + if positive_is_expense: + txn_type = 'expense' if amount_raw > 0 else 'income' + else: + txn_type = 'expense' if amount_raw < 0 else 'income' + amount = abs(amount_raw) + else: + errors.append(f'Row {i}: no amount column') + continue + + raw.append({ + 'date': txn_date, + 'transaction_type': txn_type, + 'amount': amount, + 'description': description, + 'notes': '', + 'source_id': None, + }) + + return raw, errors + + +# ── Auto-categorizer ────────────────────────────────────────────────────────── + +def auto_categorize(description): + """ + Return a PFM category name based on keywords in the description. + Returns empty string when no match is found. + """ + lower = description.lower() + for cat_name, keywords in KEYWORD_CATEGORIES.items(): + if any(kw in lower for kw in keywords): + return cat_name + return '' + + +def _build_cat_id_map(): + from app.models.category import Category + return {c.name: c.id for c in Category.query.filter_by(is_active=True).all()} + + +# ── Enrichment ──────────────────────────────────────────────────────────────── + +def _enrich(raw_rows): + """Add category_id + category_name to raw rows; convert date to ISO string.""" + cat_map = _build_cat_id_map() + enriched = [] + for r in raw_rows: + d = r['date'] + cat_name = auto_categorize(r['description']) + enriched.append({ + **r, + 'date': d.isoformat() if hasattr(d, 'isoformat') else str(d), + 'category_name': cat_name, + 'category_id': cat_map.get(cat_name), + }) + return enriched + + +# ── PDF parser ──────────────────────────────────────────────────────────────── + +# Maximum characters sent to Groq (~8 K tokens; covers ~6 months of transactions) +_PDF_MAX_CHARS = 30_000 + + +def _groq_parse_statement(text): + """ + Send extracted PDF text to Groq and return a list of raw transaction dicts. + Raises RuntimeError on API / parse failure. + """ + import json + import requests as _requests + from flask import current_app + + api_key = current_app.config.get('GROQ_API_KEY', '') + if not api_key: + raise RuntimeError('GROQ_API_KEY is not configured — cannot AI-parse PDFs.') + + model = current_app.config.get('GROQ_MODEL', 'llama-3.3-70b-versatile') + + prompt = ( + 'Extract ALL transactions from the bank statement text below.\n' + 'Return ONLY a JSON array — no explanation, no markdown fences.\n\n' + 'Each element must have exactly these keys:\n' + ' "date" – YYYY-MM-DD\n' + ' "description" – clean merchant or payee name\n' + ' "amount" – positive number (never negative)\n' + ' "transaction_type" – "expense" for debits/withdrawals/charges;\n' + ' "income" for credits/deposits/refunds/interest\n\n' + 'Rules:\n' + '- Include every transaction; do not omit any.\n' + '- Skip opening balance, closing balance, and summary lines.\n' + '- If the year is missing, infer it from the statement period header.\n' + '- amount is always a positive float.\n\n' + f'Bank statement text:\n{text}' + ) + + try: + resp = _requests.post( + 'https://api.groq.com/openai/v1/chat/completions', + headers={ + 'Authorization': f'Bearer {api_key}', + 'Content-Type': 'application/json', + }, + json={ + 'model': model, + 'messages': [ + {'role': 'system', + 'content': 'You are a precise financial data extractor. ' + 'Respond with valid JSON only — nothing else.'}, + {'role': 'user', 'content': prompt}, + ], + 'max_tokens': 4096, + 'temperature': 0.1, + 'stream': False, + }, + timeout=60, + ) + resp.raise_for_status() + raw_content = resp.json()['choices'][0]['message']['content'] + log.info('[bank_import] Groq response length=%d chars', len(raw_content)) + except Exception as exc: + log.error('[bank_import] Groq API call failed: %s', exc, exc_info=True) + raise RuntimeError(f'Groq API error: {exc}') + + # Strip markdown code fences if the model wrapped its output + content = raw_content.strip() + if '```' in content: + m = re.search(r'```(?:json)?\s*([\s\S]+?)\s*```', content) + content = m.group(1).strip() if m else content + + # Ensure we start at the JSON array + if not content.startswith('['): + m = re.search(r'\[[\s\S]+\]', content) + if m: + content = m.group(0) + + try: + transactions = json.loads(content) + except (ValueError, json.JSONDecodeError) as exc: + log.error('[bank_import] Groq JSON parse failed: %s | raw=%r', exc, raw_content[:500]) + raise RuntimeError( + f'Groq returned a response that could not be parsed as JSON: {exc}. ' + 'Try again or use the CSV export from your bank instead.' + ) + + if not isinstance(transactions, list): + raise RuntimeError('Groq returned unexpected structure (expected a JSON array).') + + raw_rows = [] + for txn in transactions: + try: + txn_date = _parse_date(str(txn.get('date', ''))) + except ValueError: + continue + + amount = float(txn.get('amount', 0) or 0) + if amount <= 0: + continue + + txn_type = str(txn.get('transaction_type', 'expense')).lower().strip() + if txn_type not in ('income', 'expense'): + txn_type = 'expense' + + raw_rows.append({ + 'date': txn_date, + 'transaction_type': txn_type, + 'amount': amount, + 'description': str(txn.get('description', '')).strip() or 'Bank transaction', + 'notes': '', + 'source_id': None, + }) + + log.info('[bank_import] Groq extracted %d transactions from PDF', len(raw_rows)) + return raw_rows + + +def _parse_pdf(file_bytes): + """ + Extract text from a digital PDF using pdfplumber, then parse with Groq. + + Returns (raw_rows, warnings_list). + Raises RuntimeError for unrecoverable problems (scanned PDF, bad file, etc.). + """ + try: + import pdfplumber + except ImportError: + raise RuntimeError( + 'pdfplumber is not installed. Run: pip install pdfplumber==0.11.4' + ) + + warnings = [] + text_parts = [] + + try: + with pdfplumber.open(io.BytesIO(file_bytes)) as pdf: + num_pages = len(pdf.pages) + log.info('[bank_import] PDF has %d page(s)', num_pages) + for page in pdf.pages: + text = page.extract_text(x_tolerance=2, y_tolerance=2) + if text: + text_parts.append(text) + except Exception as exc: + log.error('[bank_import] pdfplumber failed: %s', exc, exc_info=True) + raise RuntimeError( + f'Could not read the PDF ({exc}). ' + 'Make sure the file is not password-protected and is a valid PDF.' + ) + + full_text = '\n'.join(text_parts).strip() + log.info('[bank_import] PDF text extracted: %d characters', len(full_text)) + + if len(full_text) < 150: + raise RuntimeError( + 'This PDF appears to be a scanned image (no text layer found). ' + 'Please download the electronic/digital statement from your bank portal, ' + 'or export your transactions as a CSV or OFX file instead.' + ) + + truncated = len(full_text) > _PDF_MAX_CHARS + if truncated: + full_text = full_text[:_PDF_MAX_CHARS] + warnings.append( + f'The PDF is large — only the first {_PDF_MAX_CHARS:,} characters were ' + 'sent to the AI parser. If transactions are missing, split the statement ' + 'into shorter date ranges and import each part separately.' + ) + + raw_rows = _groq_parse_statement(full_text) + + if not raw_rows: + raise RuntimeError( + 'The AI could not find any transactions in this PDF. ' + 'Check that the file is a bank statement and not a summary or letter, ' + 'then try again.' + ) + + return raw_rows, warnings + + +# ── Public parse entry point ────────────────────────────────────────────────── + +def parse_file(file_bytes, filename, col_map=None): + """ + Detect format and parse a bank statement file. + + Returns dict: + rows — list of enriched transaction dicts (empty on needs_mapping) + errors — list of non-fatal parse error strings + format_name — human-readable detected format + needs_mapping — True when CSV columns could not be auto-detected + headers — list of CSV column headers (populated when needs_mapping) + """ + fname_lower = filename.lower() + + # ── PDF — must be handled before decode attempt ──────────────────────── + if fname_lower.endswith('.pdf') or ( + isinstance(file_bytes, bytes) and file_bytes[:4] == b'%PDF' + ): + raw, warnings = _parse_pdf(file_bytes) + return { + 'rows': _enrich(raw), 'errors': warnings, + 'format_name': 'PDF (AI-parsed)', + 'needs_mapping': False, 'headers': [], + } + + if isinstance(file_bytes, bytes): + try: + content = file_bytes.decode('utf-8-sig') + except UnicodeDecodeError: + content = file_bytes.decode('latin-1') + else: + content = file_bytes + + # ── OFX / QFX ───────────────────────────────────────────────────────── + if fname_lower.endswith(('.ofx', '.qfx')) or '' in content.upper()[:1000]: + raw = _parse_ofx(content) + return { + 'rows': _enrich(raw), 'errors': [], + 'format_name': 'OFX / QFX', + 'needs_mapping': False, 'headers': [], + } + + # ── CSV ──────────────────────────────────────────────────────────────── + reader = csv.DictReader(io.StringIO(content)) + headers = [h.strip() for h in (reader.fieldnames or [])] + headers_set = {h.lower() for h in headers} + + # User-supplied mapping (from the mapping UI) + if col_map: + effective_map = col_map + format_name = 'Custom mapping' + else: + fmt = _detect_csv_format(headers_set) + if fmt: + effective_map = fmt['map'] + format_name = fmt['name'] + else: + effective_map = _build_generic_map(headers_set) + if effective_map: + format_name = 'Generic CSV' + else: + log.info('[bank_import] CSV column auto-detect failed; headers=%s', headers) + return { + 'rows': [], 'errors': [], + 'format_name': 'Unknown', + 'needs_mapping': True, 'headers': headers, + } + + raw, errors = _parse_csv_with_map(content, effective_map) + log.info('[bank_import] CSV parsed %d rows (format=%s, errors=%d)', + len(raw), format_name, len(errors)) + return { + 'rows': _enrich(raw), 'errors': errors, + 'format_name': format_name, + 'needs_mapping': False, 'headers': [], + } + + +# ── Import ───────────────────────────────────────────────────────────────────── + +def do_import(rows, default_account_id, skip_dupes=True): + """ + Persist a list of row dicts to the DB. + + Each row must have: date (ISO str), transaction_type, amount, description. + Optional: category_id, notes, source_id (bank ref for dupe detection). + + Returns (imported_count, skipped_count). + """ + from datetime import date as date_cls + from app.extensions import db + from app.models.transaction import Transaction + from app.services.account_service import calc_balance + + imported = 0 + skipped = 0 + affected = set() + + for r in rows: + txn_date = date_cls.fromisoformat(r['date']) if isinstance(r['date'], str) else r['date'] + txn_type = r['transaction_type'] + amount = float(r['amount']) + description = (r.get('description') or '').strip() or 'Imported' + category_id = r.get('category_id') or None + account_id = int(r.get('account_id') or default_account_id) + notes = (r.get('notes') or '').strip() + source_id = r.get('source_id') or None + + if skip_dupes: + if source_id: + dup = Transaction.query.filter( + Transaction.notes.like(f'%import:{source_id}%') + ).first() + else: + dup = Transaction.query.filter_by( + date=txn_date, + amount=amount, + transaction_type=txn_type, + description=description, + ).first() + if dup: + skipped += 1 + continue + + note_parts = [] + if source_id: + note_parts.append(f'import:{source_id}') + if notes: + note_parts.append(notes) + + txn = Transaction( + date=txn_date, + transaction_type=txn_type, + amount=amount, + description=description, + category_id=category_id, + account_id=account_id, + notes=' | '.join(note_parts), + ) + db.session.add(txn) + affected.add(account_id) + imported += 1 + + db.session.commit() + for aid in affected: + calc_balance(aid) + + log.info('[bank_import] do_import: imported=%d skipped=%d', imported, skipped) + return imported, skipped diff --git a/app/templates/bank_import/index.html b/app/templates/bank_import/index.html new file mode 100644 index 0000000..34bb1c3 --- /dev/null +++ b/app/templates/bank_import/index.html @@ -0,0 +1,621 @@ +{% extends 'base.html' %} +{% block title %}Import Bank Statement{% endblock %} +{% block page_title %}Import Bank Statement{% endblock %} + +{% block extra_css %} + +{% endblock %} + +{% block content %} +
+ + {# ── LEFT COLUMN: upload form + supported formats ── #} +
+
+
Upload Statement
+ + {# Drop zone #} +
+ + +
Drag & drop or click to browse
+
CSV, OFX, QFX, PDF
+
+
+ +
+ + + All transactions will be linked to this account. +
+ +
+ + +
+ + + +
+
+ + {# Supported formats #} +
+
Supported Formats
+
+ PDF + OFX / QFX + Chase + Bank of America + Citi + Capital One + Discover + USAA + Wells Fargo + Amex + Generic CSV +
+
+
+ +
PDF — Text is extracted from digital statements and parsed by the AI (Groq). + Works with any bank's PDF. Scanned/image PDFs are not supported.
+
+ OFX/QFX is the most reliable structured format — download it from your bank's website.
+ For CSV, the file must have at least a date, description, and + amount (or separate debit/credit) column. + If the format isn't recognised, you'll be asked to map columns manually. +
+
+
+ + {# ── RIGHT COLUMN: results ── #} +
+ + {# Step: column mapping #} +
+
+ + Column Mapping Required +
+

+ The CSV format wasn't recognised automatically. Please select which column contains each field. +

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+
+ +
+ + {# Step: preview #} +
+
+ + +
+ + + +
+
+ + {# Parse errors (non-fatal) #} +
+ Warnings — these rows were skipped:
+
+
+ +
+
+ + + + + + + + + + + + +
DateDescriptionTypeAmountCategory
+
+
+ +
+
+ + {# Step: done #} +
+ +
+

+
+ + View Transactions + + +
+
+ + {# Placeholder when nothing parsed yet #} +
+ +
Upload a bank statement to get started.
+
+ +
+
+{% endblock %} + +{% block extra_js %} + +{% endblock %} diff --git a/app/templates/base.html b/app/templates/base.html index 44d318d..6b59cfe 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -215,6 +215,12 @@ > Accounts + + Import Statement +
Planning