526 lines
17 KiB
Python
526 lines
17 KiB
Python
"""
|
||
routes/ai_summary.py — AI document analysis routes.
|
||
"""
|
||
|
||
import logging
|
||
import json
|
||
import io
|
||
import os
|
||
import re
|
||
import requests as http_requests
|
||
from flask import (Blueprint, render_template, request, redirect, url_for,
|
||
flash, session, jsonify)
|
||
from models import (
|
||
get_all_criteria, get_active_criteria, create_criterion, update_criterion,
|
||
delete_criterion, save_ai_analysis, get_ai_analysis_history,
|
||
get_ai_analysis_detail, delete_ai_analysis, log_action,
|
||
)
|
||
from config import get_setting
|
||
from utils.decorators import login_required, admin_required
|
||
|
||
logger = logging.getLogger("routes.ai_summary")
|
||
ai_summary_bp = Blueprint("ai_summary", __name__, url_prefix="/ai-summary")
|
||
|
||
GROQ_MODELS = [
|
||
"llama-3.3-70b-versatile",
|
||
"llama-3.1-8b-instant",
|
||
"gemma2-9b-it",
|
||
]
|
||
|
||
SUPPORTED_EXT = {".txt", ".md", ".csv", ".pdf", ".doc", ".docx", ".xlsx", ".xls"}
|
||
|
||
|
||
@ai_summary_bp.route("/")
|
||
@login_required
|
||
def ai_summary():
|
||
user = session["user"]
|
||
criteria = get_all_criteria()
|
||
history = get_ai_analysis_history(user_id=None)
|
||
groq_key = get_setting("groq.api_key", "")
|
||
groq_model = get_setting("groq.model", GROQ_MODELS[0])
|
||
return render_template("ai_summary.html",
|
||
criteria=criteria, history=history,
|
||
groq_key_set=bool(groq_key),
|
||
groq_model=groq_model,
|
||
groq_models=GROQ_MODELS,
|
||
is_admin=user["role"] == "admin")
|
||
|
||
|
||
@ai_summary_bp.route("/analyze", methods=["POST"])
|
||
@login_required
|
||
def analyze():
|
||
user = session["user"]
|
||
files = request.files.getlist("documents[]")
|
||
model = request.form.get("model", get_setting("groq.model", GROQ_MODELS[0]))
|
||
# Read API key: env var takes priority, then DB setting
|
||
api_key = (os.environ.get("GROQ_API_KEY") or "").strip() or get_setting("groq.api_key", "").strip()
|
||
|
||
if not api_key:
|
||
return jsonify({"error": "Groq API key is not configured. Set GROQ_API_KEY in .env or via Admin → Settings."}), 400
|
||
if not files or all(f.filename == "" for f in files):
|
||
return jsonify({"error": "No files uploaded."}), 400
|
||
|
||
texts = []
|
||
file_names = []
|
||
errors = []
|
||
|
||
for f in files:
|
||
if not f.filename:
|
||
continue
|
||
ext = ("." + f.filename.rsplit(".", 1)[-1].lower()) if "." in f.filename else ""
|
||
if ext not in SUPPORTED_EXT:
|
||
errors.append(f"{f.filename}: unsupported file type")
|
||
continue
|
||
file_names.append(f.filename)
|
||
try:
|
||
content = _extract_text(f, ext)
|
||
if content and content.strip():
|
||
texts.append(f"=== {f.filename} ===\n{content.strip()}")
|
||
else:
|
||
errors.append(f"{f.filename}: no text could be extracted")
|
||
except Exception as e:
|
||
logger.warning(f"Could not extract text from {f.filename}: {e}")
|
||
errors.append(f"{f.filename}: {e}")
|
||
|
||
if not texts:
|
||
detail = "; ".join(errors) if errors else "Check the file format and try again."
|
||
return jsonify({"error": f"Could not extract text from any uploaded file. {detail}"}), 400
|
||
|
||
criteria = get_active_criteria()
|
||
combined = "\n\n".join(texts)
|
||
|
||
try:
|
||
result = _call_groq(api_key, model, combined, criteria)
|
||
except Exception as e:
|
||
logger.error(f"Groq API error: {e}")
|
||
return jsonify({"error": f"AI analysis failed: {e}"}), 500
|
||
|
||
criteria_snap = json.dumps([
|
||
{"title": c["title"], "description": c["description"]} for c in criteria
|
||
])
|
||
analysis_id = save_ai_analysis(
|
||
user["id"], ", ".join(file_names), model,
|
||
result.get("verdict"), criteria_snap, result.get("summary", "")
|
||
)
|
||
log_action(user["id"], "AI_ANALYSIS", "ai_analysis_log", analysis_id,
|
||
f"AI analysis on {len(file_names)} file(s). Verdict: {result.get('verdict')}.")
|
||
|
||
return jsonify({"analysis_id": analysis_id, "model": model,
|
||
"file_count": len(file_names), "file_errors": errors, **result})
|
||
|
||
|
||
# Magic-byte signatures for each supported binary format.
|
||
# Office Open XML (docx/xlsx) and legacy OLE (doc/xls) share these headers.
|
||
_FILE_MAGIC: dict = {
|
||
".pdf": b"%PDF",
|
||
".docx": b"PK\x03\x04",
|
||
".xlsx": b"PK\x03\x04",
|
||
".doc": b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1",
|
||
".xls": b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1",
|
||
}
|
||
|
||
|
||
def _validate_magic(data: bytes, ext: str) -> None:
|
||
"""Raise ValueError if the file's actual bytes don't match its extension."""
|
||
magic = _FILE_MAGIC.get(ext)
|
||
if magic:
|
||
if data[: len(magic)] != magic:
|
||
raise ValueError(
|
||
f"File content does not match the declared {ext} format "
|
||
"(possible disguised upload)."
|
||
)
|
||
elif ext in (".txt", ".csv", ".md"):
|
||
# Text files must be decodable; binary data masquerading as text is rejected.
|
||
try:
|
||
data[:512].decode("utf-8")
|
||
except UnicodeDecodeError:
|
||
raise ValueError(
|
||
"Text file does not appear to be valid UTF-8. "
|
||
"Ensure the file is a plain text document."
|
||
)
|
||
|
||
|
||
def _extract_text(file_obj, ext: str) -> str:
|
||
"""Extract plain text from an uploaded file object."""
|
||
data = file_obj.read()
|
||
_validate_magic(data, ext)
|
||
|
||
if ext in (".txt", ".md", ".csv"):
|
||
return data.decode("utf-8", errors="replace")
|
||
|
||
if ext == ".pdf":
|
||
import pypdf
|
||
reader = pypdf.PdfReader(io.BytesIO(data))
|
||
pages = []
|
||
for page in reader.pages:
|
||
t = page.extract_text()
|
||
if t:
|
||
pages.append(t)
|
||
return "\n".join(pages)
|
||
|
||
if ext in (".docx", ".doc"):
|
||
# Use python-docx (installed as 'docx') — do NOT use docx2txt
|
||
from docx import Document
|
||
doc = Document(io.BytesIO(data))
|
||
lines = [para.text for para in doc.paragraphs if para.text.strip()]
|
||
return "\n".join(lines)
|
||
|
||
if ext in (".xlsx", ".xls"):
|
||
import openpyxl
|
||
wb = openpyxl.load_workbook(io.BytesIO(data), read_only=True, data_only=True)
|
||
lines = []
|
||
for ws in wb.worksheets:
|
||
for row in ws.iter_rows(values_only=True):
|
||
line = "\t".join(str(c) if c is not None else "" for c in row)
|
||
if line.strip():
|
||
lines.append(line)
|
||
return "\n".join(lines)
|
||
|
||
return ""
|
||
|
||
|
||
# Office address used as origin for distance/travel-time estimates.
|
||
_OFFICE_ADDRESS = "2815 Hartland Road, Falls Church, VA 22043, USA"
|
||
|
||
# Text character limit sent to the API. llama-3.3-70b supports 128K tokens (~500K chars);
|
||
# 60K chars is a safe ceiling that leaves room for the prompt and response.
|
||
_TEXT_LIMIT = 60_000
|
||
|
||
# Analyst persona injected as the system message in every API call.
|
||
_SYSTEM_PROMPT = (
|
||
"You are a senior government procurement analyst supporting a small business BD team. "
|
||
"Your role is to extract, organize, and evaluate information from solicitation documents "
|
||
"with precision and clarity. Format all output in well-structured Markdown using tables "
|
||
"and headers. Never invent or assume information not present in the source documents — "
|
||
"write \"Not specified\" for any missing field."
|
||
)
|
||
|
||
# Stage 1 — extraction prompt (always sent)
|
||
_EXTRACTION_PROMPT = """\
|
||
Analyze the {n} attached government solicitation document(s) as one combined package.
|
||
Complete every section below exactly as structured. Write **"Not specified"** for any field
|
||
not found in the documents. Do not guess or invent details.
|
||
|
||
Our office is located at: **{office}** — use this as the origin for all travel estimates.
|
||
|
||
---
|
||
|
||
## Solicitation Overview
|
||
|
||
| Field | Value |
|
||
|---|---|
|
||
| **Solicitation Number** | |
|
||
| **Solicitation Type** | (RFP / RFQ / IFB / IDIQ / BPA / etc.) |
|
||
| **Issuing Agency / Office** | |
|
||
| **Set-Aside** | (Small Business / 8(a) / SDVOSB / HUBZone / WOSB / Unrestricted / etc.) |
|
||
| **Contract Type** | (Firm-Fixed-Price / T&M / Cost-Plus / IDIQ / etc.) |
|
||
| **Contract Period** | |
|
||
| **NAICS Code** | |
|
||
| **Estimated Value** | |
|
||
|
||
---
|
||
|
||
## Scope of Work
|
||
|
||
*Describe what is required in full detail: services, deliverables, performance standards, and any technical requirements. Use bullet points.*
|
||
|
||
---
|
||
|
||
## Performance Location(s)
|
||
|
||
*List all work sites. Note whether remote or on-site work is permitted.*
|
||
|
||
---
|
||
|
||
## Key Dates & Deadlines
|
||
|
||
| Milestone | Date & Time (with timezone) |
|
||
|---|---|
|
||
| Pre-Proposal Conference / Site Visit | |
|
||
| Questions Due | |
|
||
| **Proposal Due** | |
|
||
| Award Date (if stated) | |
|
||
| Period of Performance Start | |
|
||
| Other Deadlines | |
|
||
|
||
---
|
||
|
||
## Pre-Proposal Conference / Site Visit
|
||
|
||
| | |
|
||
|---|---|
|
||
| **Date & Time** | |
|
||
| **Full Address** | |
|
||
| **Attendance** | (Mandatory / Optional / Not applicable) |
|
||
| **RSVP Required** | (Yes — deadline & method / No) |
|
||
|
||
---
|
||
|
||
## Point of Contact (POC)
|
||
|
||
| | |
|
||
|---|---|
|
||
| **Name / Title** | |
|
||
| **Phone** | |
|
||
| **Email** | |
|
||
| **Questions Submission Method** | |
|
||
|
||
---
|
||
|
||
## Proposal Requirements
|
||
|
||
*Summarize format, page limits, required sections/volumes, submission method (portal/email/mail), and number of copies. Use bullet points.*
|
||
|
||
---
|
||
|
||
## Evaluation Factors
|
||
|
||
*List evaluation criteria and their weights or order of priority as stated in the solicitation.*
|
||
|
||
---
|
||
|
||
## Travel & Logistics
|
||
|
||
| | |
|
||
|---|---|
|
||
| **Origin** | {office} |
|
||
| **Destination** | (Pre-Proposal / Primary Work Site address) |
|
||
| **Estimated Driving Distance** | |
|
||
| **Estimated Drive Time** | (normal traffic conditions) |
|
||
|
||
*(AI estimate — verify with a mapping service before scheduling travel.)*
|
||
|
||
---
|
||
|
||
## Notable Requirements & Red Flags
|
||
|
||
*List anything that affects bid/no-bid decisions: unusual insurance or bonding levels, required security clearances, certifications, teaming or subcontracting restrictions, incumbent advantage indicators, aggressive timelines, or any other risk factors.*
|
||
|
||
---
|
||
|
||
## Total Square Footage
|
||
|
||
*(If applicable to the scope of work; write "N/A" if not.)*
|
||
|
||
---
|
||
|
||
## Overall Summary
|
||
|
||
Provide a concise but thorough summary covering:
|
||
|
||
- **What is Being Procured** — Scope and key deliverables.
|
||
- **Contract Period & Estimated Value** — Duration and any stated ceiling or estimate.
|
||
- **Key Constraints & Requirements** — Timeline, location, special certifications, etc.
|
||
- **Immediate Action Items** — What the team must do and by when.
|
||
|
||
---
|
||
|
||
DOCUMENTS:
|
||
{documents}"""
|
||
|
||
# Stage 2 — criteria evaluation (sent as a separate API call when active criteria exist).
|
||
# Receives the clean Stage 1 extraction output as {summary}, not the raw documents,
|
||
# so the model can focus entirely on the evaluation without re-parsing document noise.
|
||
_CRITERIA_PROMPT = """\
|
||
Below is an extracted and summarized government solicitation opportunity.
|
||
Evaluate whether our company should pursue it based on our evaluation criteria.
|
||
|
||
---
|
||
|
||
## OPPORTUNITY SUMMARY
|
||
|
||
{summary}
|
||
|
||
---
|
||
|
||
## ALIGNMENT EVALUATION
|
||
|
||
**Our evaluation criteria:**
|
||
|
||
{criteria_list}
|
||
|
||
For **each criterion** listed above, provide:
|
||
- A status: ✅ **MEETS** / ⚠️ **PARTIALLY MEETS** / ❌ **DOES NOT MEET** / ❓ **CANNOT DETERMINE**
|
||
- A 1–2 sentence explanation citing specific details from the summary above.
|
||
|
||
---
|
||
|
||
## Overall Recommendation
|
||
|
||
State your recommendation on its own line in **exactly** this format (required for system parsing — do not alter the label):
|
||
|
||
```
|
||
RECOMMENDATION: PURSUE
|
||
```
|
||
or
|
||
```
|
||
RECOMMENDATION: PASS
|
||
```
|
||
or
|
||
```
|
||
RECOMMENDATION: UNCLEAR
|
||
```
|
||
|
||
**PURSUE** — opportunity clearly aligns with most criteria and is competitive.
|
||
**PASS** — fails one or more critical criteria or presents unacceptable risk.
|
||
**UNCLEAR** — insufficient information to make a confident determination.
|
||
|
||
---
|
||
|
||
## Executive Summary
|
||
|
||
Write 3–4 sentences in plain business language explaining your recommendation:
|
||
the strongest reasons to pursue or pass, and the single biggest risk or opportunity."""
|
||
|
||
|
||
def _call_groq_api(api_key: str, model: str, user_message: str) -> str:
|
||
"""Single Groq chat completions call. Raises on HTTP error."""
|
||
response = http_requests.post(
|
||
"https://api.groq.com/openai/v1/chat/completions",
|
||
headers={
|
||
"Authorization": f"Bearer {api_key}",
|
||
"Content-Type": "application/json",
|
||
},
|
||
json={
|
||
"model": model,
|
||
"messages": [
|
||
{"role": "system", "content": _SYSTEM_PROMPT},
|
||
{"role": "user", "content": user_message},
|
||
],
|
||
"max_tokens": 8192,
|
||
"temperature": 0.1,
|
||
},
|
||
timeout=120,
|
||
)
|
||
if not response.ok:
|
||
logger.error(f"Groq API {response.status_code}: {response.text[:300]}")
|
||
response.raise_for_status()
|
||
return response.json()["choices"][0]["message"]["content"] or ""
|
||
|
||
|
||
def _call_groq(api_key: str, model: str, text: str, criteria: list) -> dict:
|
||
"""
|
||
Two-stage Groq analysis:
|
||
Stage 1 (always): extract structured fields + overall summary from documents.
|
||
Stage 2 (optional): evaluate criteria against the Stage 1 output (not raw docs),
|
||
producing a focused RECOMMENDATION with per-criterion scoring.
|
||
"""
|
||
truncated = len(text) > _TEXT_LIMIT
|
||
doc_text = text[:_TEXT_LIMIT]
|
||
n = doc_text.count("=== ") or 1
|
||
|
||
# ── Stage 1: extraction ────────────────────────────────────────────────────
|
||
stage1_prompt = _EXTRACTION_PROMPT.format(
|
||
n=n, office=_OFFICE_ADDRESS, documents=doc_text
|
||
)
|
||
summary = _call_groq_api(api_key, model, stage1_prompt)
|
||
|
||
if not criteria:
|
||
return {"verdict": None, "summary": summary, "truncated": truncated}
|
||
|
||
# ── Stage 2: criteria evaluation (uses clean summary, not raw docs) ────────
|
||
criteria_list = "\n".join(
|
||
f"{i+1}. **{c['title']}**: {c['description']}"
|
||
for i, c in enumerate(criteria)
|
||
)
|
||
stage2_prompt = _CRITERIA_PROMPT.format(
|
||
summary=summary, criteria_list=criteria_list
|
||
)
|
||
evaluation = _call_groq_api(api_key, model, stage2_prompt)
|
||
|
||
verdict = None
|
||
match = re.search(
|
||
r"RECOMMENDATION\s*:\s*(PURSUE|PASS|UNCLEAR)",
|
||
evaluation, re.IGNORECASE,
|
||
)
|
||
if match:
|
||
verdict = match.group(1).upper()
|
||
|
||
combined = summary + "\n\n---\n\n" + evaluation
|
||
return {"verdict": verdict, "summary": combined, "truncated": truncated}
|
||
|
||
|
||
# ─── Criteria Management (admin only) ─────────────────────────────────────────
|
||
|
||
@ai_summary_bp.route("/criteria/create", methods=["POST"])
|
||
@admin_required
|
||
def create_criterion_view():
|
||
admin = session["user"]
|
||
title = request.form.get("title", "").strip()
|
||
desc = request.form.get("description", "").strip()
|
||
is_active = request.form.get("is_active", "1") == "1"
|
||
try:
|
||
sort_order = int(request.form.get("sort_order", 0) or 0)
|
||
except (ValueError, TypeError):
|
||
sort_order = 0
|
||
try:
|
||
create_criterion(admin["id"], title, desc, is_active, sort_order)
|
||
flash(f"Criterion '{title}' created.", "success")
|
||
except Exception as e:
|
||
flash(f"Error: {e}", "danger")
|
||
return redirect(url_for("ai_summary.ai_summary"))
|
||
|
||
|
||
@ai_summary_bp.route("/criteria/<int:criterion_id>/edit", methods=["POST"])
|
||
@admin_required
|
||
def edit_criterion_view(criterion_id):
|
||
admin = session["user"]
|
||
title = request.form.get("title", "").strip()
|
||
desc = request.form.get("description", "").strip()
|
||
is_active = request.form.get("is_active", "1") == "1"
|
||
try:
|
||
sort_order = int(request.form.get("sort_order", 0) or 0)
|
||
except (ValueError, TypeError):
|
||
sort_order = 0
|
||
try:
|
||
update_criterion(admin["id"], criterion_id, title, desc, is_active, sort_order)
|
||
flash(f"Criterion '{title}' updated.", "success")
|
||
except Exception as e:
|
||
flash(f"Error: {e}", "danger")
|
||
return redirect(url_for("ai_summary.ai_summary"))
|
||
|
||
|
||
@ai_summary_bp.route("/criteria/<int:criterion_id>/delete", methods=["POST"])
|
||
@admin_required
|
||
def delete_criterion_view(criterion_id):
|
||
admin = session["user"]
|
||
try:
|
||
delete_criterion(admin["id"], criterion_id)
|
||
flash("Criterion deleted.", "success")
|
||
except Exception as e:
|
||
flash(f"Error: {e}", "danger")
|
||
return redirect(url_for("ai_summary.ai_summary"))
|
||
|
||
|
||
@ai_summary_bp.route("/history/<int:analysis_id>")
|
||
@login_required
|
||
def analysis_detail(analysis_id):
|
||
record = get_ai_analysis_detail(analysis_id)
|
||
if not record:
|
||
return jsonify({"error": "Not found"}), 404
|
||
return jsonify({
|
||
"id": record["id"],
|
||
"username": record.get("username"),
|
||
"file_names": record["file_names"],
|
||
"model": record["model"],
|
||
"verdict": record["verdict"],
|
||
"analyzed_at": str(record["analyzed_at"]),
|
||
"summary_text": record["summary_text"],
|
||
"criteria_snapshot": record.get("criteria_snapshot") or "",
|
||
})
|
||
|
||
|
||
@ai_summary_bp.route("/history/<int:analysis_id>/delete", methods=["POST"])
|
||
@login_required
|
||
def delete_analysis(analysis_id):
|
||
user = session["user"]
|
||
if user["role"] != "admin":
|
||
return jsonify({"error": "Access denied"}), 403
|
||
record = get_ai_analysis_detail(analysis_id)
|
||
if not record:
|
||
return jsonify({"error": "Not found"}), 404
|
||
delete_ai_analysis(analysis_id)
|
||
log_action(user["id"], "DELETE_AI_ANALYSIS", "ai_analysis_log", analysis_id,
|
||
f"Deleted AI analysis id={analysis_id}.")
|
||
return jsonify({"ok": True})
|