06/05 Optimize app
This commit is contained in:
@@ -572,9 +572,115 @@ def _groq_parse_statement(text):
|
||||
return raw_rows
|
||||
|
||||
|
||||
_TABLE_DATE_HDRS = {'date', 'posted', 'transaction date', 'trans date',
|
||||
'posting date', 'value date', 'effective date', 'settled'}
|
||||
_TABLE_DESC_HDRS = {'description', 'payee', 'merchant', 'memo', 'transaction',
|
||||
'details', 'name', 'narrative', 'particulars', 'reference'}
|
||||
_TABLE_DEBIT_HDRS = {'debit', 'withdrawal', 'withdrawals', 'charge', 'charges',
|
||||
'amount debited', 'payment', 'dr'}
|
||||
_TABLE_CRED_HDRS = {'credit', 'deposit', 'deposits', 'amount credited',
|
||||
'cr', 'inflow'}
|
||||
_TABLE_AMT_HDRS = {'amount', 'transaction amount', 'net amount'}
|
||||
|
||||
|
||||
def _pdfplumber_table_parse(pdf_handle):
|
||||
"""
|
||||
Try to extract transactions directly from pdfplumber table structures.
|
||||
|
||||
Iterates every page, finds tables whose headers match bank-statement
|
||||
patterns, and converts rows to raw transaction dicts.
|
||||
|
||||
Returns a (possibly empty) list of raw dicts; never raises.
|
||||
"""
|
||||
all_rows = []
|
||||
|
||||
for page in pdf_handle.pages:
|
||||
try:
|
||||
tables = page.extract_tables()
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
for table in tables:
|
||||
if not table or len(table) < 2:
|
||||
continue
|
||||
|
||||
# Normalise headers (lower-case, strip)
|
||||
raw_headers = [str(h).strip().lower() if h else '' for h in table[0]]
|
||||
|
||||
date_col = next((i for i, h in enumerate(raw_headers)
|
||||
if h in _TABLE_DATE_HDRS), None)
|
||||
desc_col = next((i for i, h in enumerate(raw_headers)
|
||||
if h in _TABLE_DESC_HDRS), None)
|
||||
amt_col = next((i for i, h in enumerate(raw_headers)
|
||||
if h in _TABLE_AMT_HDRS), None)
|
||||
debit_col = next((i for i, h in enumerate(raw_headers)
|
||||
if h in _TABLE_DEBIT_HDRS), None)
|
||||
cred_col = next((i for i, h in enumerate(raw_headers)
|
||||
if h in _TABLE_CRED_HDRS), None)
|
||||
|
||||
# Need at least date + description + one amount column
|
||||
if date_col is None or desc_col is None:
|
||||
continue
|
||||
if amt_col is None and debit_col is None and cred_col is None:
|
||||
continue
|
||||
|
||||
col_max = max(c for c in [date_col, desc_col, amt_col, debit_col, cred_col]
|
||||
if c is not None)
|
||||
|
||||
for row in table[1:]:
|
||||
if not row or len(row) <= col_max:
|
||||
continue
|
||||
|
||||
date_str = str(row[date_col]).strip() if row[date_col] else ''
|
||||
if not date_str or date_str.lower() in ('', 'none', '-', '--', 'n/a'):
|
||||
continue
|
||||
try:
|
||||
txn_date = _parse_date(date_str)
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
description = str(row[desc_col]).strip() if row[desc_col] else ''
|
||||
if not description or description.lower() in ('', 'none'):
|
||||
continue
|
||||
|
||||
if debit_col is not None or cred_col is not None:
|
||||
debit = abs(_clean_amount(row[debit_col] if debit_col is not None else ''))
|
||||
credit = abs(_clean_amount(row[cred_col] if cred_col is not None else ''))
|
||||
if debit > 0:
|
||||
amount, txn_type = debit, 'expense'
|
||||
elif credit > 0:
|
||||
amount, txn_type = credit, 'income'
|
||||
else:
|
||||
continue
|
||||
else:
|
||||
raw_amt = _clean_amount(row[amt_col] if row[amt_col] else '')
|
||||
if raw_amt == 0.0:
|
||||
continue
|
||||
txn_type = 'expense' if raw_amt < 0 else 'income'
|
||||
amount = abs(raw_amt)
|
||||
|
||||
all_rows.append({
|
||||
'date': txn_date,
|
||||
'transaction_type': txn_type,
|
||||
'amount': amount,
|
||||
'description': description,
|
||||
'notes': '',
|
||||
'source_id': None,
|
||||
})
|
||||
|
||||
return all_rows
|
||||
|
||||
|
||||
def _parse_pdf(file_bytes):
|
||||
"""
|
||||
Extract text from a digital PDF using pdfplumber, then parse with Groq.
|
||||
Extract transactions from a digital bank-statement PDF.
|
||||
|
||||
Strategy (in order):
|
||||
1. pdfplumber table extraction — fast, free, no API call needed.
|
||||
Used when structured tables with recognisable headers are found and
|
||||
yield at least 3 rows.
|
||||
2. pdfplumber text extraction → Groq LLM — handles unstructured
|
||||
or narrative-style statements.
|
||||
|
||||
Returns (raw_rows, warnings_list).
|
||||
Raises RuntimeError for unrecoverable problems (scanned PDF, bad file, etc.).
|
||||
@@ -593,10 +699,27 @@ def _parse_pdf(file_bytes):
|
||||
with pdfplumber.open(io.BytesIO(file_bytes)) as pdf:
|
||||
num_pages = len(pdf.pages)
|
||||
log.info('[bank_import] PDF has %d page(s)', num_pages)
|
||||
|
||||
# ── Strategy 1: structured table extraction ──────────────────────
|
||||
table_rows = _pdfplumber_table_parse(pdf)
|
||||
if len(table_rows) >= 3:
|
||||
log.info(
|
||||
'[bank_import] PDF table extraction: %d rows (skipping Groq)',
|
||||
len(table_rows),
|
||||
)
|
||||
return table_rows, warnings
|
||||
|
||||
log.info(
|
||||
'[bank_import] PDF table extraction yielded %d row(s) — falling back to Groq',
|
||||
len(table_rows),
|
||||
)
|
||||
|
||||
# ── Strategy 2: text extraction → Groq ──────────────────────────
|
||||
for page in pdf.pages:
|
||||
text = page.extract_text(x_tolerance=2, y_tolerance=2)
|
||||
if text:
|
||||
text_parts.append(text)
|
||||
|
||||
except Exception as exc:
|
||||
log.error('[bank_import] pdfplumber failed: %s', exc, exc_info=True)
|
||||
raise RuntimeError(
|
||||
|
||||
Reference in New Issue
Block a user