Initial Codes
This commit is contained in:
@@ -0,0 +1,416 @@
|
||||
"""
|
||||
QuickBooks Online API Client
|
||||
Handles OAuth 2.0 authentication and API operations.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
from urllib.parse import urlencode
|
||||
from typing import Optional, Dict, List, Any
|
||||
from datetime import datetime, timedelta
|
||||
from dataclasses import dataclass
|
||||
import requests
|
||||
|
||||
from src.config.settings import QBOCredentials
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QBOEntity:
|
||||
"""Base class for QBO entities."""
|
||||
id: Optional[str] = None
|
||||
sync_token: Optional[str] = None
|
||||
|
||||
def to_qbo_dict(self) -> Dict[str, Any]:
|
||||
"""Convert to QBO API format."""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class QBOClient:
|
||||
"""QuickBooks Online API Client with OAuth 2.0 support."""
|
||||
|
||||
# API URLs
|
||||
SANDBOX_BASE_URL = "https://sandbox-quickbooks.api.intuit.com"
|
||||
PRODUCTION_BASE_URL = "https://quickbooks.api.intuit.com"
|
||||
|
||||
AUTHORIZATION_URL = "https://appcenter.intuit.com/connect/oauth2"
|
||||
TOKEN_URL = "https://oauth.platform.intuit.com/oauth2/v1/tokens/bearer"
|
||||
REVOKE_URL = "https://developer.api.intuit.com/v2/oauth2/tokens/revoke"
|
||||
|
||||
# Scopes
|
||||
ACCOUNTING_SCOPE = "com.intuit.quickbooks.accounting"
|
||||
|
||||
def __init__(self, credentials: QBOCredentials):
|
||||
self.credentials = credentials
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
"""Get base URL based on environment."""
|
||||
if self.credentials.environment == "production":
|
||||
return self.PRODUCTION_BASE_URL
|
||||
return self.SANDBOX_BASE_URL
|
||||
|
||||
@property
|
||||
def is_authenticated(self) -> bool:
|
||||
"""Check if we have valid tokens."""
|
||||
if not self.credentials.access_token or not self.credentials.realm_id:
|
||||
return False
|
||||
|
||||
# Check token expiry
|
||||
if self.credentials.token_expiry:
|
||||
try:
|
||||
expiry = datetime.fromisoformat(self.credentials.token_expiry)
|
||||
if datetime.now() >= expiry:
|
||||
# Try to refresh
|
||||
return self.refresh_tokens()
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
return True
|
||||
|
||||
def get_authorization_url(self, state: str = "security_token") -> str:
|
||||
"""Get OAuth authorization URL."""
|
||||
params = {
|
||||
"client_id": self.credentials.client_id,
|
||||
"response_type": "code",
|
||||
"scope": self.ACCOUNTING_SCOPE,
|
||||
"redirect_uri": self.credentials.redirect_uri,
|
||||
"state": state
|
||||
}
|
||||
return f"{self.AUTHORIZATION_URL}?{urlencode(params)}"
|
||||
|
||||
def _exchange_code_for_tokens(self, code: str) -> bool:
|
||||
"""Exchange authorization code for tokens."""
|
||||
try:
|
||||
logger.info("Exchanging authorization code for tokens")
|
||||
response = requests.post(
|
||||
self.TOKEN_URL,
|
||||
auth=(self.credentials.client_id, self.credentials.client_secret),
|
||||
data={
|
||||
"grant_type": "authorization_code",
|
||||
"code": code,
|
||||
"redirect_uri": self.credentials.redirect_uri
|
||||
},
|
||||
headers={"Accept": "application/json"}
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
token_data = response.json()
|
||||
self._update_tokens(token_data)
|
||||
logger.info("Token exchange successful")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Token exchange failed: {response.status_code} - {response.text}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Token exchange exception: {e}")
|
||||
return False
|
||||
|
||||
def refresh_tokens(self) -> bool:
|
||||
"""Refresh access token using refresh token."""
|
||||
if not self.credentials.refresh_token:
|
||||
logger.warning("No refresh token available")
|
||||
return False
|
||||
|
||||
try:
|
||||
logger.info("Refreshing access token")
|
||||
response = requests.post(
|
||||
self.TOKEN_URL,
|
||||
auth=(self.credentials.client_id, self.credentials.client_secret),
|
||||
data={
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": self.credentials.refresh_token
|
||||
},
|
||||
headers={"Accept": "application/json"}
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
token_data = response.json()
|
||||
self._update_tokens(token_data)
|
||||
logger.info("Token refresh successful")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"Token refresh failed: {response.status_code}")
|
||||
return False
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Token refresh exception: {e}")
|
||||
return False
|
||||
|
||||
def _update_tokens(self, token_data: Dict[str, Any]):
|
||||
"""Update credentials with new token data."""
|
||||
self.credentials.access_token = token_data.get("access_token")
|
||||
self.credentials.refresh_token = token_data.get("refresh_token")
|
||||
|
||||
# Calculate expiry
|
||||
expires_in = token_data.get("expires_in", 3600)
|
||||
expiry = datetime.now() + timedelta(seconds=expires_in - 300) # 5 min buffer
|
||||
self.credentials.token_expiry = expiry.isoformat()
|
||||
|
||||
def disconnect(self) -> bool:
|
||||
"""Revoke tokens and disconnect."""
|
||||
if self.credentials.access_token:
|
||||
try:
|
||||
logger.info("Revoking access token")
|
||||
requests.post(
|
||||
self.REVOKE_URL,
|
||||
auth=(self.credentials.client_id, self.credentials.client_secret),
|
||||
data={"token": self.credentials.access_token},
|
||||
headers={"Accept": "application/json"}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Token revocation failed: {e}")
|
||||
|
||||
self.credentials.access_token = None
|
||||
self.credentials.refresh_token = None
|
||||
self.credentials.realm_id = None
|
||||
self.credentials.token_expiry = None
|
||||
|
||||
return True
|
||||
|
||||
def _make_request(
|
||||
self,
|
||||
method: str,
|
||||
endpoint: str,
|
||||
data: Optional[Dict] = None,
|
||||
params: Optional[Dict] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Make authenticated API request."""
|
||||
if not self.is_authenticated:
|
||||
if not self.refresh_tokens():
|
||||
raise Exception("Not authenticated. Please connect to QuickBooks first.")
|
||||
|
||||
url = f"{self.base_url}/v3/company/{self.credentials.realm_id}/{endpoint}"
|
||||
|
||||
headers = {
|
||||
"Authorization": f"Bearer {self.credentials.access_token}",
|
||||
"Accept": "application/json",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
response = requests.request(
|
||||
method,
|
||||
url,
|
||||
headers=headers,
|
||||
json=data,
|
||||
params=params
|
||||
)
|
||||
|
||||
if response.status_code == 401:
|
||||
# Token expired, try to refresh
|
||||
if self.refresh_tokens():
|
||||
headers["Authorization"] = f"Bearer {self.credentials.access_token}"
|
||||
response = requests.request(
|
||||
method,
|
||||
url,
|
||||
headers=headers,
|
||||
json=data,
|
||||
params=params
|
||||
)
|
||||
|
||||
if response.status_code >= 400:
|
||||
error_msg = response.text
|
||||
error_code = None
|
||||
error_detail = None
|
||||
error_element = None
|
||||
|
||||
try:
|
||||
error_data = response.json()
|
||||
if "Fault" in error_data:
|
||||
fault = error_data["Fault"]
|
||||
errors = fault.get("Error", [])
|
||||
if errors:
|
||||
first_error = errors[0]
|
||||
error_code = first_error.get("code", "")
|
||||
error_msg = first_error.get("Message", error_msg)
|
||||
error_detail = first_error.get("Detail", "")
|
||||
error_element = first_error.get("element", "")
|
||||
|
||||
# Build comprehensive error message
|
||||
parts = []
|
||||
if error_code:
|
||||
parts.append(f"Code: {error_code}")
|
||||
if error_msg:
|
||||
parts.append(f"Message: {error_msg}")
|
||||
if error_detail:
|
||||
parts.append(f"Detail: {error_detail}")
|
||||
if error_element:
|
||||
parts.append(f"Element: {error_element}")
|
||||
|
||||
# Include all errors if multiple
|
||||
if len(errors) > 1:
|
||||
for i, err in enumerate(errors[1:], 2):
|
||||
parts.append(f"Error {i}: {err.get('Message', '')} - {err.get('Detail', '')}")
|
||||
|
||||
error_msg = " | ".join(parts)
|
||||
except:
|
||||
pass
|
||||
|
||||
logger.error(f"API Error ({response.status_code}): {error_msg}")
|
||||
logger.error(f"Request URL: {url}")
|
||||
if data:
|
||||
logger.error(f"Request Data: {json.dumps(data, indent=2)}")
|
||||
|
||||
raise Exception(f"QBO API Error ({response.status_code}): {error_msg}")
|
||||
|
||||
return response.json()
|
||||
|
||||
# Query operations
|
||||
def query(self, entity_type: str, where_clause: str = "", max_results: int = 1000) -> List[Dict]:
|
||||
"""Execute a query against QBO."""
|
||||
query = f"SELECT * FROM {entity_type}"
|
||||
if where_clause:
|
||||
query += f" WHERE {where_clause}"
|
||||
query += f" MAXRESULTS {max_results}"
|
||||
|
||||
result = self._make_request("GET", "query", params={"query": query})
|
||||
|
||||
query_response = result.get("QueryResponse", {})
|
||||
return query_response.get(entity_type, [])
|
||||
|
||||
def get_by_id(self, entity_type: str, entity_id: str) -> Optional[Dict]:
|
||||
"""Get entity by ID."""
|
||||
result = self._make_request("GET", f"{entity_type.lower()}/{entity_id}")
|
||||
return result.get(entity_type)
|
||||
|
||||
# Create operations
|
||||
def create(self, entity_type: str, data: Dict) -> Dict:
|
||||
"""Create a new entity."""
|
||||
logger.info(f"Creating {entity_type}")
|
||||
result = self._make_request("POST", entity_type.lower(), data=data)
|
||||
return result.get(entity_type, result)
|
||||
|
||||
def create_batch(self, operations: List[Dict]) -> Dict:
|
||||
"""Execute batch operations."""
|
||||
batch_data = {"BatchItemRequest": operations}
|
||||
result = self._make_request("POST", "batch", data=batch_data)
|
||||
return result
|
||||
|
||||
# Update operations
|
||||
def update(self, entity_type: str, data: Dict) -> Dict:
|
||||
"""Update an existing entity."""
|
||||
logger.info(f"Updating {entity_type}")
|
||||
result = self._make_request("POST", entity_type.lower(), data=data)
|
||||
return result.get(entity_type, result)
|
||||
|
||||
# Entity-specific operations
|
||||
def get_customers(self, active_only: bool = True) -> List[Dict]:
|
||||
"""Get all customers."""
|
||||
where = "Active = true" if active_only else ""
|
||||
return self.query("Customer", where)
|
||||
|
||||
def get_vendors(self, active_only: bool = True) -> List[Dict]:
|
||||
"""Get all vendors."""
|
||||
where = "Active = true" if active_only else ""
|
||||
return self.query("Vendor", where)
|
||||
|
||||
def get_accounts(self, active_only: bool = True) -> List[Dict]:
|
||||
"""Get chart of accounts."""
|
||||
where = "Active = true" if active_only else ""
|
||||
return self.query("Account", where)
|
||||
|
||||
def get_items(self, active_only: bool = True) -> List[Dict]:
|
||||
"""Get all items/products."""
|
||||
where = "Active = true" if active_only else ""
|
||||
return self.query("Item", where)
|
||||
|
||||
def get_company_info(self) -> Dict:
|
||||
"""Get company information."""
|
||||
result = self._make_request("GET", "companyinfo/" + self.credentials.realm_id)
|
||||
return result.get("CompanyInfo", {})
|
||||
|
||||
# Check-specific operations
|
||||
def create_check(self, check_data: Dict) -> Dict:
|
||||
"""Create a check/purchase."""
|
||||
return self.create("Purchase", check_data)
|
||||
|
||||
def get_checks(self, start_date: Optional[str] = None, end_date: Optional[str] = None) -> List[Dict]:
|
||||
"""Get checks (filtered purchases with PaymentType=Check)."""
|
||||
where_parts = ["PaymentType = 'Check'"]
|
||||
if start_date:
|
||||
where_parts.append(f"TxnDate >= '{start_date}'")
|
||||
if end_date:
|
||||
where_parts.append(f"TxnDate <= '{end_date}'")
|
||||
|
||||
return self.query("Purchase", " AND ".join(where_parts))
|
||||
|
||||
# Invoice operations
|
||||
def create_invoice(self, invoice_data: Dict) -> Dict:
|
||||
"""Create an invoice."""
|
||||
return self.create("Invoice", invoice_data)
|
||||
|
||||
def get_invoices(self, start_date: Optional[str] = None, end_date: Optional[str] = None) -> List[Dict]:
|
||||
"""Get invoices."""
|
||||
where_parts = []
|
||||
if start_date:
|
||||
where_parts.append(f"TxnDate >= '{start_date}'")
|
||||
if end_date:
|
||||
where_parts.append(f"TxnDate <= '{end_date}'")
|
||||
|
||||
return self.query("Invoice", " AND ".join(where_parts) if where_parts else "")
|
||||
|
||||
# Bill operations
|
||||
def create_bill(self, bill_data: Dict) -> Dict:
|
||||
"""Create a bill."""
|
||||
return self.create("Bill", bill_data)
|
||||
|
||||
def get_bills(self, start_date: Optional[str] = None, end_date: Optional[str] = None) -> List[Dict]:
|
||||
"""Get bills."""
|
||||
where_parts = []
|
||||
if start_date:
|
||||
where_parts.append(f"TxnDate >= '{start_date}'")
|
||||
if end_date:
|
||||
where_parts.append(f"TxnDate <= '{end_date}'")
|
||||
|
||||
return self.query("Bill", " AND ".join(where_parts) if where_parts else "")
|
||||
|
||||
# Customer/Vendor operations
|
||||
def create_customer(self, customer_data: Dict) -> Dict:
|
||||
"""Create a customer."""
|
||||
return self.create("Customer", customer_data)
|
||||
|
||||
def create_vendor(self, vendor_data: Dict) -> Dict:
|
||||
"""Create a vendor."""
|
||||
return self.create("Vendor", vendor_data)
|
||||
|
||||
# Account operations
|
||||
def create_account(self, account_data: Dict) -> Dict:
|
||||
"""Create an account."""
|
||||
return self.create("Account", account_data)
|
||||
|
||||
def find_entity_by_name(
|
||||
self,
|
||||
entity_type: str,
|
||||
name: str,
|
||||
name_field: str = "DisplayName"
|
||||
) -> Optional[Dict]:
|
||||
"""Find an entity by name."""
|
||||
# Escape single quotes in name
|
||||
escaped_name = name.replace("'", "\\'")
|
||||
results = self.query(entity_type, f"{name_field} = '{escaped_name}'")
|
||||
return results[0] if results else None
|
||||
|
||||
def find_or_create_reference(
|
||||
self,
|
||||
entity_type: str,
|
||||
name: str,
|
||||
create_if_missing: bool = False,
|
||||
additional_data: Optional[Dict] = None
|
||||
) -> Optional[Dict]:
|
||||
"""Find entity by name, optionally creating if not found."""
|
||||
entity = self.find_entity_by_name(entity_type, name)
|
||||
|
||||
if entity:
|
||||
return {"value": entity["Id"], "name": entity.get("DisplayName", entity.get("Name"))}
|
||||
|
||||
if create_if_missing:
|
||||
create_data = {"DisplayName": name}
|
||||
if additional_data:
|
||||
create_data.update(additional_data)
|
||||
|
||||
new_entity = self.create(entity_type, create_data)
|
||||
return {"value": new_entity["Id"], "name": name}
|
||||
|
||||
return None
|
||||
@@ -0,0 +1,351 @@
|
||||
"""
|
||||
Application settings and configuration management.
|
||||
Handles OAuth tokens, mapping templates, and user preferences.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from dataclasses import dataclass, field, asdict
|
||||
from typing import Optional, Dict, List, Any
|
||||
from datetime import datetime
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QBOCredentials:
|
||||
"""QuickBooks Online OAuth credentials."""
|
||||
client_id: str = ""
|
||||
client_secret: str = ""
|
||||
redirect_uri: str = "http://localhost:5000/oauth/callback"
|
||||
environment: str = "sandbox" # 'sandbox' or 'production'
|
||||
|
||||
# Token data (stored securely)
|
||||
access_token: Optional[str] = None
|
||||
refresh_token: Optional[str] = None
|
||||
realm_id: Optional[str] = None
|
||||
token_expiry: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class FieldMapping:
|
||||
"""Single field mapping configuration."""
|
||||
excel_column: str
|
||||
qbo_field: str
|
||||
transform: Optional[str] = None # 'date', 'currency', 'lookup', etc.
|
||||
default_value: Optional[str] = None
|
||||
required: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class MappingTemplate:
|
||||
"""Complete mapping template for a data type."""
|
||||
name: str
|
||||
data_type: str # 'check', 'invoice', 'bill', 'customer', 'vendor', 'account'
|
||||
mappings: List[FieldMapping] = field(default_factory=list)
|
||||
created_at: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
updated_at: str = field(default_factory=lambda: datetime.now().isoformat())
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportSettings:
|
||||
"""Import behavior settings."""
|
||||
skip_duplicates: bool = True
|
||||
duplicate_check_fields: List[str] = field(default_factory=lambda: ["DocNumber"])
|
||||
batch_size: int = 50
|
||||
validate_before_import: bool = True
|
||||
create_missing_references: bool = False # Auto-create missing customers/vendors
|
||||
date_format: str = "%Y-%m-%d"
|
||||
decimal_separator: str = "."
|
||||
thousand_separator: str = ","
|
||||
|
||||
|
||||
class Settings:
|
||||
"""Application settings manager."""
|
||||
|
||||
APP_NAME = "QBOExcelSyncWeb"
|
||||
|
||||
def __init__(self):
|
||||
self.config_dir = self._get_config_dir()
|
||||
self.config_file = self.config_dir / "config.json"
|
||||
self.templates_dir = self.config_dir / "templates"
|
||||
self.credentials_file = self.config_dir / ".credentials"
|
||||
|
||||
# Ensure directories exist
|
||||
self.config_dir.mkdir(parents=True, exist_ok=True)
|
||||
self.templates_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load or create default config
|
||||
self._config = self._load_config()
|
||||
|
||||
# Credentials (stored separately for security)
|
||||
self._credentials: Optional[QBOCredentials] = None
|
||||
|
||||
logger.info(f"Settings initialized. Config dir: {self.config_dir}")
|
||||
|
||||
def _get_config_dir(self) -> Path:
|
||||
"""Get platform-specific config directory."""
|
||||
# For web app, use a local directory
|
||||
base = Path(__file__).parent.parent.parent
|
||||
return base / "data"
|
||||
|
||||
def _load_config(self) -> Dict[str, Any]:
|
||||
"""Load configuration from file."""
|
||||
default_config = {
|
||||
"import_settings": asdict(ImportSettings()),
|
||||
"recent_files": [],
|
||||
"last_template": None,
|
||||
"qbo_environment": "sandbox",
|
||||
}
|
||||
|
||||
if self.config_file.exists():
|
||||
try:
|
||||
with open(self.config_file, 'r') as f:
|
||||
loaded = json.load(f)
|
||||
# Merge with defaults
|
||||
for key, value in default_config.items():
|
||||
if key not in loaded:
|
||||
loaded[key] = value
|
||||
return loaded
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
logger.warning(f"Failed to load config: {e}")
|
||||
return default_config
|
||||
|
||||
return default_config
|
||||
|
||||
def save(self):
|
||||
"""Save configuration to file."""
|
||||
try:
|
||||
with open(self.config_file, 'w') as f:
|
||||
json.dump(self._config, f, indent=2)
|
||||
logger.info("Configuration saved successfully")
|
||||
except IOError as e:
|
||||
logger.error(f"Failed to save config: {e}")
|
||||
|
||||
@property
|
||||
def import_settings(self) -> ImportSettings:
|
||||
"""Get import settings."""
|
||||
return ImportSettings(**self._config.get("import_settings", {}))
|
||||
|
||||
@import_settings.setter
|
||||
def import_settings(self, settings: ImportSettings):
|
||||
"""Set import settings."""
|
||||
self._config["import_settings"] = asdict(settings)
|
||||
self.save()
|
||||
logger.info("Import settings updated")
|
||||
|
||||
@property
|
||||
def recent_files(self) -> List[str]:
|
||||
"""Get list of recently opened files."""
|
||||
return self._config.get("recent_files", [])
|
||||
|
||||
def add_recent_file(self, filepath: str):
|
||||
"""Add file to recent files list."""
|
||||
files = self.recent_files
|
||||
if filepath in files:
|
||||
files.remove(filepath)
|
||||
files.insert(0, filepath)
|
||||
self._config["recent_files"] = files[:10] # Keep only 10 recent
|
||||
self.save()
|
||||
|
||||
@property
|
||||
def qbo_environment(self) -> str:
|
||||
"""Get QBO environment (sandbox/production)."""
|
||||
return self._config.get("qbo_environment", "sandbox")
|
||||
|
||||
@qbo_environment.setter
|
||||
def qbo_environment(self, env: str):
|
||||
"""Set QBO environment."""
|
||||
if env not in ("sandbox", "production"):
|
||||
raise ValueError("Environment must be 'sandbox' or 'production'")
|
||||
self._config["qbo_environment"] = env
|
||||
self.save()
|
||||
logger.info(f"QBO environment set to: {env}")
|
||||
|
||||
# Credential storage (file-based for web app)
|
||||
def get_credentials(self) -> QBOCredentials:
|
||||
"""Get QBO credentials from file storage."""
|
||||
if self._credentials is None:
|
||||
self._credentials = QBOCredentials()
|
||||
|
||||
try:
|
||||
if self.credentials_file.exists():
|
||||
with open(self.credentials_file, 'r') as f:
|
||||
creds_data = json.load(f)
|
||||
self._credentials = QBOCredentials(**creds_data)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load credentials: {e}")
|
||||
|
||||
return self._credentials
|
||||
|
||||
def save_credentials(self, credentials: QBOCredentials):
|
||||
"""Save QBO credentials to file storage."""
|
||||
self._credentials = credentials
|
||||
try:
|
||||
with open(self.credentials_file, 'w') as f:
|
||||
json.dump(asdict(credentials), f)
|
||||
os.chmod(self.credentials_file, 0o600) # Restrict permissions
|
||||
logger.info("Credentials saved successfully")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to save credentials: {e}")
|
||||
|
||||
def clear_credentials(self):
|
||||
"""Clear stored credentials."""
|
||||
self._credentials = None
|
||||
if self.credentials_file.exists():
|
||||
self.credentials_file.unlink()
|
||||
logger.info("Credentials cleared")
|
||||
|
||||
# Template management
|
||||
def get_templates(self, data_type: Optional[str] = None) -> List[MappingTemplate]:
|
||||
"""Get all mapping templates, optionally filtered by data type."""
|
||||
templates = []
|
||||
|
||||
for template_file in self.templates_dir.glob("*.json"):
|
||||
try:
|
||||
with open(template_file, 'r') as f:
|
||||
data = json.load(f)
|
||||
# Reconstruct FieldMapping objects
|
||||
mappings = [FieldMapping(**m) for m in data.get("mappings", [])]
|
||||
template = MappingTemplate(
|
||||
name=data["name"],
|
||||
data_type=data["data_type"],
|
||||
mappings=mappings,
|
||||
created_at=data.get("created_at", ""),
|
||||
updated_at=data.get("updated_at", "")
|
||||
)
|
||||
if data_type is None or template.data_type == data_type:
|
||||
templates.append(template)
|
||||
except (json.JSONDecodeError, IOError, KeyError) as e:
|
||||
logger.warning(f"Failed to load template {template_file}: {e}")
|
||||
continue
|
||||
|
||||
return sorted(templates, key=lambda t: t.name)
|
||||
|
||||
def save_template(self, template: MappingTemplate):
|
||||
"""Save a mapping template."""
|
||||
template.updated_at = datetime.now().isoformat()
|
||||
|
||||
# Convert to serializable format
|
||||
data = {
|
||||
"name": template.name,
|
||||
"data_type": template.data_type,
|
||||
"mappings": [asdict(m) for m in template.mappings],
|
||||
"created_at": template.created_at,
|
||||
"updated_at": template.updated_at
|
||||
}
|
||||
|
||||
# Safe filename
|
||||
safe_name = "".join(c for c in template.name if c.isalnum() or c in " -_").strip()
|
||||
filepath = self.templates_dir / f"{safe_name}.json"
|
||||
|
||||
with open(filepath, 'w') as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
logger.info(f"Template saved: {template.name}")
|
||||
|
||||
def delete_template(self, template_name: str):
|
||||
"""Delete a mapping template."""
|
||||
safe_name = "".join(c for c in template_name if c.isalnum() or c in " -_").strip()
|
||||
filepath = self.templates_dir / f"{safe_name}.json"
|
||||
|
||||
if filepath.exists():
|
||||
filepath.unlink()
|
||||
logger.info(f"Template deleted: {template_name}")
|
||||
|
||||
def get_default_templates(self) -> Dict[str, MappingTemplate]:
|
||||
"""Get built-in default templates for each data type."""
|
||||
return {
|
||||
"check": MappingTemplate(
|
||||
name="Default Check Import",
|
||||
data_type="check",
|
||||
mappings=[
|
||||
FieldMapping("Payee", "EntityRef.value", required=True),
|
||||
FieldMapping("Bank Acct", "AccountRef.value", required=True),
|
||||
FieldMapping("Date", "TxnDate", transform="date", required=True),
|
||||
FieldMapping("Amount", "Line.Amount", transform="currency", required=True),
|
||||
FieldMapping("Number", "DocNumber"),
|
||||
FieldMapping("Posting Acct", "Line.AccountBasedExpenseLineDetail.AccountRef.value"),
|
||||
FieldMapping("Period Memo", "PrivateNote"),
|
||||
FieldMapping("Period Memo", "Line.Description"),
|
||||
]
|
||||
),
|
||||
"invoice": MappingTemplate(
|
||||
name="Default Invoice Import",
|
||||
data_type="invoice",
|
||||
mappings=[
|
||||
FieldMapping("CustomerRef", "CustomerRef.value", required=True),
|
||||
FieldMapping("TxnDate", "TxnDate", transform="date", required=True),
|
||||
FieldMapping("DueDate", "DueDate", transform="date"),
|
||||
FieldMapping("DocNumber", "DocNumber"),
|
||||
FieldMapping("ItemRef", "Line.SalesItemLineDetail.ItemRef.value"),
|
||||
FieldMapping("Description", "Line.Description"),
|
||||
FieldMapping("Qty", "Line.SalesItemLineDetail.Qty", transform="number"),
|
||||
FieldMapping("UnitPrice", "Line.SalesItemLineDetail.UnitPrice", transform="currency"),
|
||||
FieldMapping("Amount", "Line.Amount", transform="currency"),
|
||||
FieldMapping("PrivateNote", "PrivateNote"),
|
||||
]
|
||||
),
|
||||
"bill": MappingTemplate(
|
||||
name="Default Bill Import",
|
||||
data_type="bill",
|
||||
mappings=[
|
||||
FieldMapping("VendorRef", "VendorRef.value", required=True),
|
||||
FieldMapping("TxnDate", "TxnDate", transform="date", required=True),
|
||||
FieldMapping("DueDate", "DueDate", transform="date"),
|
||||
FieldMapping("DocNumber", "DocNumber"),
|
||||
FieldMapping("AccountRef", "Line.AccountBasedExpenseLineDetail.AccountRef.value"),
|
||||
FieldMapping("Description", "Line.Description"),
|
||||
FieldMapping("Amount", "Line.Amount", transform="currency"),
|
||||
FieldMapping("PrivateNote", "PrivateNote"),
|
||||
]
|
||||
),
|
||||
"customer": MappingTemplate(
|
||||
name="Default Customer Import",
|
||||
data_type="customer",
|
||||
mappings=[
|
||||
FieldMapping("DisplayName", "DisplayName", required=True),
|
||||
FieldMapping("CompanyName", "CompanyName"),
|
||||
FieldMapping("GivenName", "GivenName"),
|
||||
FieldMapping("FamilyName", "FamilyName"),
|
||||
FieldMapping("Email", "PrimaryEmailAddr.Address"),
|
||||
FieldMapping("Phone", "PrimaryPhone.FreeFormNumber"),
|
||||
FieldMapping("Street", "BillAddr.Line1"),
|
||||
FieldMapping("City", "BillAddr.City"),
|
||||
FieldMapping("State", "BillAddr.CountrySubDivisionCode"),
|
||||
FieldMapping("PostalCode", "BillAddr.PostalCode"),
|
||||
FieldMapping("Country", "BillAddr.Country"),
|
||||
]
|
||||
),
|
||||
"vendor": MappingTemplate(
|
||||
name="Default Vendor Import",
|
||||
data_type="vendor",
|
||||
mappings=[
|
||||
FieldMapping("DisplayName", "DisplayName", required=True),
|
||||
FieldMapping("CompanyName", "CompanyName"),
|
||||
FieldMapping("GivenName", "GivenName"),
|
||||
FieldMapping("FamilyName", "FamilyName"),
|
||||
FieldMapping("Email", "PrimaryEmailAddr.Address"),
|
||||
FieldMapping("Phone", "PrimaryPhone.FreeFormNumber"),
|
||||
FieldMapping("Street", "BillAddr.Line1"),
|
||||
FieldMapping("City", "BillAddr.City"),
|
||||
FieldMapping("State", "BillAddr.CountrySubDivisionCode"),
|
||||
FieldMapping("PostalCode", "BillAddr.PostalCode"),
|
||||
FieldMapping("TaxIdentifier", "TaxIdentifier"),
|
||||
]
|
||||
),
|
||||
"account": MappingTemplate(
|
||||
name="Default Chart of Accounts Import",
|
||||
data_type="account",
|
||||
mappings=[
|
||||
FieldMapping("Name", "Name", required=True),
|
||||
FieldMapping("AccountType", "AccountType", required=True),
|
||||
FieldMapping("AccountSubType", "AccountSubType"),
|
||||
FieldMapping("AcctNum", "AcctNum"),
|
||||
FieldMapping("Description", "Description"),
|
||||
FieldMapping("CurrentBalance", "CurrentBalance", transform="currency"),
|
||||
]
|
||||
),
|
||||
}
|
||||
@@ -0,0 +1,680 @@
|
||||
"""
|
||||
Excel Parser and Data Processor
|
||||
Handles reading Excel files and transforming data for QBO import.
|
||||
"""
|
||||
|
||||
import re
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Any, Optional, Tuple, Callable
|
||||
from datetime import datetime, date
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
import openpyxl
|
||||
from openpyxl.utils import get_column_letter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ValidationSeverity(Enum):
|
||||
"""Validation issue severity levels."""
|
||||
ERROR = "error"
|
||||
WARNING = "warning"
|
||||
INFO = "info"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationIssue:
|
||||
"""A single validation issue."""
|
||||
row: int
|
||||
column: str
|
||||
field: str
|
||||
message: str
|
||||
severity: ValidationSeverity = ValidationSeverity.ERROR
|
||||
value: Any = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedRow:
|
||||
"""A parsed row with original and transformed data."""
|
||||
row_number: int
|
||||
original_data: Dict[str, Any]
|
||||
transformed_data: Dict[str, Any] = field(default_factory=dict)
|
||||
qbo_data: Dict[str, Any] = field(default_factory=dict)
|
||||
issues: List[ValidationIssue] = field(default_factory=list)
|
||||
is_valid: bool = True
|
||||
|
||||
@property
|
||||
def has_errors(self) -> bool:
|
||||
return any(i.severity == ValidationSeverity.ERROR for i in self.issues)
|
||||
|
||||
@property
|
||||
def has_warnings(self) -> bool:
|
||||
return any(i.severity == ValidationSeverity.WARNING for i in self.issues)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParseResult:
|
||||
"""Complete result of parsing an Excel file."""
|
||||
filepath: str
|
||||
sheet_name: str
|
||||
columns: List[str]
|
||||
rows: List[ParsedRow]
|
||||
total_rows: int
|
||||
valid_rows: int
|
||||
error_count: int
|
||||
warning_count: int
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
if self.total_rows == 0:
|
||||
return 0.0
|
||||
return (self.valid_rows / self.total_rows) * 100
|
||||
|
||||
|
||||
class DataTransformer:
|
||||
"""Transforms data between Excel and QBO formats."""
|
||||
|
||||
# Common date formats to try
|
||||
DATE_FORMATS = [
|
||||
"%Y-%m-%d",
|
||||
"%m/%d/%Y",
|
||||
"%d/%m/%Y",
|
||||
"%Y/%m/%d",
|
||||
"%m-%d-%Y",
|
||||
"%d-%m-%Y",
|
||||
"%B %d, %Y",
|
||||
"%b %d, %Y",
|
||||
"%d %B %Y",
|
||||
"%d %b %Y",
|
||||
]
|
||||
|
||||
def __init__(self, date_format: str = "%Y-%m-%d"):
|
||||
self.date_format = date_format
|
||||
|
||||
def transform_date(self, value: Any) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Transform value to QBO date format (YYYY-MM-DD).
|
||||
Returns (transformed_value, error_message)
|
||||
"""
|
||||
if value is None or value == "":
|
||||
return None, None
|
||||
|
||||
# Already a date/datetime object
|
||||
if isinstance(value, (datetime, date)):
|
||||
return value.strftime("%Y-%m-%d"), None
|
||||
|
||||
# String value - try parsing
|
||||
value_str = str(value).strip()
|
||||
|
||||
for fmt in self.DATE_FORMATS:
|
||||
try:
|
||||
parsed = datetime.strptime(value_str, fmt)
|
||||
return parsed.strftime("%Y-%m-%d"), None
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
return None, f"Unable to parse date: {value}"
|
||||
|
||||
def transform_currency(self, value: Any) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""
|
||||
Transform value to decimal currency format.
|
||||
Returns (transformed_value, error_message)
|
||||
"""
|
||||
if value is None or value == "":
|
||||
return None, None
|
||||
|
||||
# Already numeric
|
||||
if isinstance(value, (int, float, Decimal)):
|
||||
return str(round(float(value), 2)), None
|
||||
|
||||
# String - clean and parse
|
||||
value_str = str(value).strip()
|
||||
|
||||
# Remove currency symbols and formatting
|
||||
cleaned = re.sub(r'[^\d.\-,]', '', value_str)
|
||||
|
||||
# Handle European format (comma as decimal)
|
||||
if ',' in cleaned and '.' in cleaned:
|
||||
# Determine which is decimal separator based on position
|
||||
if cleaned.rfind(',') > cleaned.rfind('.'):
|
||||
# European: 1.234,56
|
||||
cleaned = cleaned.replace('.', '').replace(',', '.')
|
||||
else:
|
||||
# US: 1,234.56
|
||||
cleaned = cleaned.replace(',', '')
|
||||
elif ',' in cleaned:
|
||||
# Could be either format - assume decimal if only one comma
|
||||
if cleaned.count(',') == 1 and len(cleaned.split(',')[1]) <= 2:
|
||||
cleaned = cleaned.replace(',', '.')
|
||||
else:
|
||||
cleaned = cleaned.replace(',', '')
|
||||
|
||||
try:
|
||||
amount = Decimal(cleaned)
|
||||
return str(round(float(amount), 2)), None
|
||||
except InvalidOperation:
|
||||
return None, f"Unable to parse amount: {value}"
|
||||
|
||||
def transform_number(self, value: Any) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Transform value to number format."""
|
||||
if value is None or value == "":
|
||||
return None, None
|
||||
|
||||
if isinstance(value, (int, float)):
|
||||
return str(value), None
|
||||
|
||||
try:
|
||||
cleaned = re.sub(r'[^\d.\-]', '', str(value))
|
||||
return str(float(cleaned)), None
|
||||
except ValueError:
|
||||
return None, f"Unable to parse number: {value}"
|
||||
|
||||
def transform_text(self, value: Any) -> Tuple[Optional[str], Optional[str]]:
|
||||
"""Transform value to text format."""
|
||||
if value is None:
|
||||
return None, None
|
||||
return str(value).strip(), None
|
||||
|
||||
def transform_boolean(self, value: Any) -> Tuple[Optional[bool], Optional[str]]:
|
||||
"""Transform value to boolean."""
|
||||
if value is None or value == "":
|
||||
return None, None
|
||||
|
||||
if isinstance(value, bool):
|
||||
return value, None
|
||||
|
||||
value_str = str(value).lower().strip()
|
||||
|
||||
if value_str in ('true', 'yes', '1', 'y', 'x'):
|
||||
return True, None
|
||||
elif value_str in ('false', 'no', '0', 'n', ''):
|
||||
return False, None
|
||||
|
||||
return None, f"Unable to parse boolean: {value}"
|
||||
|
||||
def transform_value(
|
||||
self,
|
||||
value: Any,
|
||||
transform_type: Optional[str]
|
||||
) -> Tuple[Any, Optional[str]]:
|
||||
"""Apply transformation based on type."""
|
||||
if transform_type is None or transform_type == "text":
|
||||
return self.transform_text(value)
|
||||
elif transform_type == "date":
|
||||
return self.transform_date(value)
|
||||
elif transform_type == "currency":
|
||||
return self.transform_currency(value)
|
||||
elif transform_type == "number":
|
||||
return self.transform_number(value)
|
||||
elif transform_type == "boolean":
|
||||
return self.transform_boolean(value)
|
||||
else:
|
||||
return self.transform_text(value)
|
||||
|
||||
|
||||
class ExcelParser:
|
||||
"""Parses Excel files for QBO import."""
|
||||
|
||||
def __init__(self):
|
||||
self.transformer = DataTransformer()
|
||||
|
||||
def get_sheet_names(self, filepath: str) -> List[str]:
|
||||
"""Get list of sheet names in Excel file."""
|
||||
logger.info(f"Getting sheet names from: {filepath}")
|
||||
wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True)
|
||||
sheets = wb.sheetnames
|
||||
wb.close()
|
||||
return sheets
|
||||
|
||||
def get_columns(self, filepath: str, sheet_name: Optional[str] = None) -> List[str]:
|
||||
"""Get column headers from Excel file."""
|
||||
wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True)
|
||||
|
||||
if sheet_name:
|
||||
ws = wb[sheet_name]
|
||||
else:
|
||||
ws = wb.active
|
||||
|
||||
columns = []
|
||||
for cell in ws[1]:
|
||||
if cell.value:
|
||||
columns.append(str(cell.value).strip())
|
||||
else:
|
||||
# Use column letter for empty headers
|
||||
columns.append(f"Column_{get_column_letter(cell.column)}")
|
||||
|
||||
wb.close()
|
||||
return columns
|
||||
|
||||
def preview(
|
||||
self,
|
||||
filepath: str,
|
||||
sheet_name: Optional[str] = None,
|
||||
max_rows: int = 10
|
||||
) -> Tuple[List[str], List[Dict[str, Any]]]:
|
||||
"""Preview Excel data (columns and first few rows)."""
|
||||
logger.info(f"Previewing file: {filepath}, sheet: {sheet_name}")
|
||||
wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True)
|
||||
|
||||
if sheet_name:
|
||||
ws = wb[sheet_name]
|
||||
else:
|
||||
ws = wb.active
|
||||
|
||||
rows_iter = ws.iter_rows(values_only=True)
|
||||
|
||||
# Get headers
|
||||
header_row = next(rows_iter, None)
|
||||
if not header_row:
|
||||
wb.close()
|
||||
return [], []
|
||||
|
||||
columns = []
|
||||
for i, val in enumerate(header_row):
|
||||
if val:
|
||||
columns.append(str(val).strip())
|
||||
else:
|
||||
columns.append(f"Column_{get_column_letter(i + 1)}")
|
||||
|
||||
# Get data rows
|
||||
data = []
|
||||
for i, row in enumerate(rows_iter):
|
||||
if i >= max_rows:
|
||||
break
|
||||
|
||||
row_data = {}
|
||||
for j, val in enumerate(row):
|
||||
if j < len(columns):
|
||||
# Convert to JSON-serializable format
|
||||
if isinstance(val, datetime):
|
||||
row_data[columns[j]] = val.isoformat()
|
||||
elif isinstance(val, date):
|
||||
row_data[columns[j]] = val.isoformat()
|
||||
else:
|
||||
row_data[columns[j]] = val
|
||||
|
||||
# Skip completely empty rows
|
||||
if any(v is not None and v != "" for v in row_data.values()):
|
||||
data.append(row_data)
|
||||
|
||||
wb.close()
|
||||
return columns, data
|
||||
|
||||
def parse(
|
||||
self,
|
||||
filepath: str,
|
||||
mappings: List['FieldMapping'],
|
||||
sheet_name: Optional[str] = None,
|
||||
skip_empty_rows: bool = True,
|
||||
progress_callback: Optional[Callable[[int, int], None]] = None
|
||||
) -> ParseResult:
|
||||
"""
|
||||
Parse Excel file using field mappings.
|
||||
|
||||
Args:
|
||||
filepath: Path to Excel file
|
||||
mappings: List of FieldMapping objects
|
||||
sheet_name: Specific sheet to parse
|
||||
skip_empty_rows: Whether to skip empty rows
|
||||
progress_callback: Optional callback(current_row, total_rows)
|
||||
|
||||
Returns:
|
||||
ParseResult with all parsed data
|
||||
"""
|
||||
from src.config.settings import FieldMapping
|
||||
|
||||
logger.info(f"Parsing file: {filepath}, sheet: {sheet_name}")
|
||||
|
||||
wb = openpyxl.load_workbook(filepath, read_only=True, data_only=True)
|
||||
|
||||
if sheet_name:
|
||||
ws = wb[sheet_name]
|
||||
else:
|
||||
ws = wb.active
|
||||
sheet_name = ws.title
|
||||
|
||||
# Count total rows (excluding header)
|
||||
total_rows = ws.max_row - 1 if ws.max_row else 0
|
||||
|
||||
rows_iter = ws.iter_rows(values_only=True)
|
||||
|
||||
# Get headers
|
||||
header_row = next(rows_iter, None)
|
||||
if not header_row:
|
||||
wb.close()
|
||||
return ParseResult(
|
||||
filepath=filepath,
|
||||
sheet_name=sheet_name,
|
||||
columns=[],
|
||||
rows=[],
|
||||
total_rows=0,
|
||||
valid_rows=0,
|
||||
error_count=0,
|
||||
warning_count=0
|
||||
)
|
||||
|
||||
columns = []
|
||||
col_index_map = {}
|
||||
for i, val in enumerate(header_row):
|
||||
col_name = str(val).strip() if val else f"Column_{get_column_letter(i + 1)}"
|
||||
columns.append(col_name)
|
||||
col_index_map[col_name] = i
|
||||
|
||||
# Create mapping lookup
|
||||
mapping_lookup = {m.excel_column: m for m in mappings if m.excel_column}
|
||||
|
||||
# Parse rows
|
||||
parsed_rows = []
|
||||
valid_count = 0
|
||||
error_count = 0
|
||||
warning_count = 0
|
||||
|
||||
for row_num, row in enumerate(rows_iter, start=2):
|
||||
if progress_callback:
|
||||
progress_callback(row_num - 1, total_rows)
|
||||
|
||||
# Build original data dict
|
||||
original_data = {}
|
||||
for i, val in enumerate(row):
|
||||
if i < len(columns):
|
||||
original_data[columns[i]] = val
|
||||
|
||||
# Skip empty rows
|
||||
if skip_empty_rows and all(
|
||||
v is None or v == "" for v in original_data.values()
|
||||
):
|
||||
continue
|
||||
|
||||
# Create parsed row
|
||||
parsed_row = ParsedRow(
|
||||
row_number=row_num,
|
||||
original_data=original_data
|
||||
)
|
||||
|
||||
# Apply mappings and transformations
|
||||
for mapping in mappings:
|
||||
if not mapping.excel_column:
|
||||
continue
|
||||
|
||||
# Get original value
|
||||
original_value = original_data.get(mapping.excel_column)
|
||||
|
||||
# Use default if empty
|
||||
if (original_value is None or original_value == "") and mapping.default_value:
|
||||
original_value = mapping.default_value
|
||||
|
||||
# Validate required fields
|
||||
if mapping.required and (original_value is None or original_value == ""):
|
||||
parsed_row.issues.append(ValidationIssue(
|
||||
row=row_num,
|
||||
column=mapping.excel_column,
|
||||
field=mapping.qbo_field,
|
||||
message=f"Required field '{mapping.excel_column}' is empty",
|
||||
severity=ValidationSeverity.ERROR
|
||||
))
|
||||
parsed_row.is_valid = False
|
||||
continue
|
||||
|
||||
# Transform value
|
||||
transformed, error = self.transformer.transform_value(
|
||||
original_value,
|
||||
mapping.transform
|
||||
)
|
||||
|
||||
if error:
|
||||
parsed_row.issues.append(ValidationIssue(
|
||||
row=row_num,
|
||||
column=mapping.excel_column,
|
||||
field=mapping.qbo_field,
|
||||
message=error,
|
||||
severity=ValidationSeverity.ERROR,
|
||||
value=original_value
|
||||
))
|
||||
parsed_row.is_valid = False
|
||||
else:
|
||||
parsed_row.transformed_data[mapping.qbo_field] = transformed
|
||||
|
||||
# Build QBO data structure
|
||||
if parsed_row.is_valid:
|
||||
parsed_row.qbo_data = self._build_qbo_structure(
|
||||
parsed_row.transformed_data,
|
||||
mappings
|
||||
)
|
||||
|
||||
# Update counts
|
||||
if parsed_row.has_errors:
|
||||
error_count += len([i for i in parsed_row.issues if i.severity == ValidationSeverity.ERROR])
|
||||
if parsed_row.has_warnings:
|
||||
warning_count += len([i for i in parsed_row.issues if i.severity == ValidationSeverity.WARNING])
|
||||
if parsed_row.is_valid:
|
||||
valid_count += 1
|
||||
|
||||
parsed_rows.append(parsed_row)
|
||||
|
||||
wb.close()
|
||||
|
||||
logger.info(f"Parsed {len(parsed_rows)} rows, {valid_count} valid, {error_count} errors")
|
||||
|
||||
return ParseResult(
|
||||
filepath=filepath,
|
||||
sheet_name=sheet_name,
|
||||
columns=columns,
|
||||
rows=parsed_rows,
|
||||
total_rows=len(parsed_rows),
|
||||
valid_rows=valid_count,
|
||||
error_count=error_count,
|
||||
warning_count=warning_count
|
||||
)
|
||||
|
||||
def _build_qbo_structure(
|
||||
self,
|
||||
transformed_data: Dict[str, Any],
|
||||
mappings: List['FieldMapping']
|
||||
) -> Dict[str, Any]:
|
||||
"""Build nested QBO data structure from flat transformed data."""
|
||||
result = {}
|
||||
|
||||
for qbo_field, value in transformed_data.items():
|
||||
if value is None:
|
||||
continue
|
||||
|
||||
# Handle nested fields (e.g., "CustomerRef.value" or "Line.Amount")
|
||||
parts = qbo_field.split('.')
|
||||
|
||||
current = result
|
||||
for i, part in enumerate(parts[:-1]):
|
||||
if part not in current:
|
||||
current[part] = {}
|
||||
current = current[part]
|
||||
|
||||
current[parts[-1]] = value
|
||||
|
||||
return result
|
||||
|
||||
def validate_mappings(
|
||||
self,
|
||||
filepath: str,
|
||||
mappings: List['FieldMapping'],
|
||||
sheet_name: Optional[str] = None
|
||||
) -> List[ValidationIssue]:
|
||||
"""Validate that mappings match Excel columns."""
|
||||
columns = self.get_columns(filepath, sheet_name)
|
||||
issues = []
|
||||
|
||||
for mapping in mappings:
|
||||
if mapping.excel_column and mapping.excel_column not in columns:
|
||||
issues.append(ValidationIssue(
|
||||
row=0,
|
||||
column=mapping.excel_column,
|
||||
field=mapping.qbo_field,
|
||||
message=f"Column '{mapping.excel_column}' not found in Excel file",
|
||||
severity=ValidationSeverity.ERROR
|
||||
))
|
||||
|
||||
return issues
|
||||
|
||||
|
||||
class QBODataBuilder:
|
||||
"""Builds QBO-ready data structures from parsed rows."""
|
||||
|
||||
@staticmethod
|
||||
def build_check(row_data: Dict[str, Any], line_items: List[Dict] = None) -> Dict[str, Any]:
|
||||
"""Build a Check/Purchase entity."""
|
||||
check = {
|
||||
"PaymentType": "Check" # Required field - always set
|
||||
}
|
||||
|
||||
# Direct field mappings
|
||||
direct_fields = [
|
||||
"PayeeRef", "BankAccountRef", "TxnDate", "DocNumber",
|
||||
"PrivateNote", "TotalAmt", "Memo"
|
||||
]
|
||||
|
||||
for field in direct_fields:
|
||||
if field in row_data and row_data[field]:
|
||||
check[field] = row_data[field]
|
||||
|
||||
# Build line items
|
||||
if line_items:
|
||||
check["Line"] = line_items
|
||||
elif "Line" in row_data:
|
||||
# Single line from row
|
||||
line = row_data["Line"]
|
||||
if isinstance(line, dict) and ("AccountBasedExpenseLineDetail" in line or "Amount" in line):
|
||||
line_item = {
|
||||
"DetailType": "AccountBasedExpenseLineDetail",
|
||||
"Amount": line.get("Amount", 0),
|
||||
"Description": line.get("Description", "")
|
||||
}
|
||||
# Handle AccountBasedExpenseLineDetail
|
||||
if "AccountBasedExpenseLineDetail" in line:
|
||||
line_item["AccountBasedExpenseLineDetail"] = line["AccountBasedExpenseLineDetail"]
|
||||
elif "AccountRef" in line:
|
||||
line_item["AccountBasedExpenseLineDetail"] = {"AccountRef": line["AccountRef"]}
|
||||
check["Line"] = [line_item]
|
||||
|
||||
return check
|
||||
|
||||
@staticmethod
|
||||
def build_invoice(row_data: Dict[str, Any], line_items: List[Dict] = None) -> Dict[str, Any]:
|
||||
"""Build an Invoice entity."""
|
||||
invoice = {}
|
||||
|
||||
direct_fields = [
|
||||
"CustomerRef", "TxnDate", "DueDate", "DocNumber",
|
||||
"PrivateNote", "Memo", "BillEmail"
|
||||
]
|
||||
|
||||
for field in direct_fields:
|
||||
if field in row_data and row_data[field]:
|
||||
invoice[field] = row_data[field]
|
||||
|
||||
if line_items:
|
||||
invoice["Line"] = line_items
|
||||
elif "Line" in row_data:
|
||||
line = row_data["Line"]
|
||||
invoice["Line"] = [{
|
||||
"DetailType": "SalesItemLineDetail",
|
||||
"Amount": line.get("Amount", 0),
|
||||
"SalesItemLineDetail": line.get("SalesItemLineDetail", {}),
|
||||
"Description": line.get("Description", "")
|
||||
}]
|
||||
|
||||
return invoice
|
||||
|
||||
@staticmethod
|
||||
def build_bill(row_data: Dict[str, Any], line_items: List[Dict] = None) -> Dict[str, Any]:
|
||||
"""Build a Bill entity."""
|
||||
bill = {}
|
||||
|
||||
direct_fields = [
|
||||
"VendorRef", "TxnDate", "DueDate", "DocNumber",
|
||||
"PrivateNote", "Memo"
|
||||
]
|
||||
|
||||
for field in direct_fields:
|
||||
if field in row_data and row_data[field]:
|
||||
bill[field] = row_data[field]
|
||||
|
||||
if line_items:
|
||||
bill["Line"] = line_items
|
||||
elif "Line" in row_data:
|
||||
line = row_data["Line"]
|
||||
bill["Line"] = [{
|
||||
"DetailType": "AccountBasedExpenseLineDetail",
|
||||
"Amount": line.get("Amount", 0),
|
||||
"AccountBasedExpenseLineDetail": line.get("AccountBasedExpenseLineDetail", {}),
|
||||
"Description": line.get("Description", "")
|
||||
}]
|
||||
|
||||
return bill
|
||||
|
||||
@staticmethod
|
||||
def build_customer(row_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build a Customer entity."""
|
||||
customer = {}
|
||||
|
||||
direct_fields = [
|
||||
"DisplayName", "CompanyName", "GivenName", "FamilyName",
|
||||
"Title", "Suffix", "Notes"
|
||||
]
|
||||
|
||||
for field in direct_fields:
|
||||
if field in row_data and row_data[field]:
|
||||
customer[field] = row_data[field]
|
||||
|
||||
# Handle nested address
|
||||
if "BillAddr" in row_data:
|
||||
customer["BillAddr"] = row_data["BillAddr"]
|
||||
|
||||
# Handle contact info
|
||||
if "PrimaryEmailAddr" in row_data:
|
||||
customer["PrimaryEmailAddr"] = row_data["PrimaryEmailAddr"]
|
||||
|
||||
if "PrimaryPhone" in row_data:
|
||||
customer["PrimaryPhone"] = row_data["PrimaryPhone"]
|
||||
|
||||
return customer
|
||||
|
||||
@staticmethod
|
||||
def build_vendor(row_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build a Vendor entity."""
|
||||
vendor = {}
|
||||
|
||||
direct_fields = [
|
||||
"DisplayName", "CompanyName", "GivenName", "FamilyName",
|
||||
"Title", "Suffix", "Notes", "TaxIdentifier", "AcctNum"
|
||||
]
|
||||
|
||||
for field in direct_fields:
|
||||
if field in row_data and row_data[field]:
|
||||
vendor[field] = row_data[field]
|
||||
|
||||
if "BillAddr" in row_data:
|
||||
vendor["BillAddr"] = row_data["BillAddr"]
|
||||
|
||||
if "PrimaryEmailAddr" in row_data:
|
||||
vendor["PrimaryEmailAddr"] = row_data["PrimaryEmailAddr"]
|
||||
|
||||
if "PrimaryPhone" in row_data:
|
||||
vendor["PrimaryPhone"] = row_data["PrimaryPhone"]
|
||||
|
||||
return vendor
|
||||
|
||||
@staticmethod
|
||||
def build_account(row_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Build an Account entity."""
|
||||
account = {}
|
||||
|
||||
fields = [
|
||||
"Name", "AccountType", "AccountSubType", "AcctNum",
|
||||
"Description", "CurrentBalance"
|
||||
]
|
||||
|
||||
for field in fields:
|
||||
if field in row_data and row_data[field]:
|
||||
account[field] = row_data[field]
|
||||
|
||||
return account
|
||||
@@ -0,0 +1,719 @@
|
||||
"""
|
||||
Import Processor
|
||||
Handles batch import operations with validation, duplicate detection, and error handling.
|
||||
"""
|
||||
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
from typing import List, Dict, Any, Optional, Callable, Tuple
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from datetime import datetime
|
||||
|
||||
from src.core.excel_parser import ParsedRow, ParseResult, QBODataBuilder, ValidationIssue, ValidationSeverity
|
||||
from src.api.qbo_client import QBOClient
|
||||
from src.config.settings import ImportSettings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ImportStatus(Enum):
|
||||
"""Status of an individual import operation."""
|
||||
PENDING = "pending"
|
||||
PROCESSING = "processing"
|
||||
SUCCESS = "success"
|
||||
FAILED = "failed"
|
||||
SKIPPED = "skipped"
|
||||
DUPLICATE = "duplicate"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportRecord:
|
||||
"""Record of a single import attempt."""
|
||||
row_number: int
|
||||
original_data: Dict[str, Any]
|
||||
qbo_data: Dict[str, Any]
|
||||
status: ImportStatus = ImportStatus.PENDING
|
||||
qbo_id: Optional[str] = None
|
||||
error_message: Optional[str] = None
|
||||
timestamp: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ImportResult:
|
||||
"""Complete result of an import operation."""
|
||||
data_type: str
|
||||
total_records: int
|
||||
successful: int
|
||||
failed: int
|
||||
skipped: int
|
||||
duplicates: int
|
||||
records: List[ImportRecord] = field(default_factory=list)
|
||||
start_time: Optional[str] = None
|
||||
end_time: Optional[str] = None
|
||||
duration_seconds: float = 0.0
|
||||
|
||||
@property
|
||||
def success_rate(self) -> float:
|
||||
if self.total_records == 0:
|
||||
return 0.0
|
||||
return (self.successful / self.total_records) * 100
|
||||
|
||||
|
||||
class ReferenceResolver:
|
||||
"""Resolves entity references (customers, vendors, accounts) for import."""
|
||||
|
||||
def __init__(self, qbo_client: QBOClient, settings: ImportSettings):
|
||||
self.client = qbo_client
|
||||
self.settings = settings
|
||||
|
||||
# Caches
|
||||
self._customers: Optional[Dict[str, Dict]] = None
|
||||
self._vendors: Optional[Dict[str, Dict]] = None
|
||||
self._accounts: Optional[Dict[str, Dict]] = None
|
||||
self._items: Optional[Dict[str, Dict]] = None
|
||||
|
||||
def refresh_cache(self):
|
||||
"""Refresh all reference caches."""
|
||||
logger.info("Refreshing reference caches")
|
||||
self._customers = None
|
||||
self._vendors = None
|
||||
self._accounts = None
|
||||
self._items = None
|
||||
|
||||
@property
|
||||
def customers(self) -> Dict[str, Dict]:
|
||||
"""Get customers cache (name -> entity)."""
|
||||
if self._customers is None:
|
||||
self._customers = {}
|
||||
for c in self.client.get_customers():
|
||||
name = c.get("DisplayName", "").lower()
|
||||
self._customers[name] = {"value": c["Id"], "name": c.get("DisplayName")}
|
||||
return self._customers
|
||||
|
||||
@property
|
||||
def vendors(self) -> Dict[str, Dict]:
|
||||
"""Get vendors cache."""
|
||||
if self._vendors is None:
|
||||
self._vendors = {}
|
||||
for v in self.client.get_vendors():
|
||||
name = v.get("DisplayName", "").lower()
|
||||
self._vendors[name] = {"value": v["Id"], "name": v.get("DisplayName")}
|
||||
return self._vendors
|
||||
|
||||
@property
|
||||
def accounts(self) -> Dict[str, Dict]:
|
||||
"""Get accounts cache."""
|
||||
if self._accounts is None:
|
||||
self._accounts = {}
|
||||
for a in self.client.get_accounts():
|
||||
name = a.get("Name", "")
|
||||
acct_num = a.get("AcctNum", "")
|
||||
acct_id = a["Id"]
|
||||
|
||||
# Index by multiple keys for flexible matching
|
||||
self._accounts[name.lower()] = {"value": acct_id, "name": name}
|
||||
|
||||
if acct_num:
|
||||
self._accounts[acct_num.lower()] = {"value": acct_id, "name": name}
|
||||
self._accounts[str(acct_num)] = {"value": acct_id, "name": name}
|
||||
|
||||
combined_key = f"{acct_num} · {name}".lower() if acct_num else name.lower()
|
||||
self._accounts[combined_key] = {"value": acct_id, "name": name}
|
||||
|
||||
return self._accounts
|
||||
|
||||
def _extract_account_identifier(self, value: str) -> str:
|
||||
"""Extract account number or name from formatted strings."""
|
||||
if not value:
|
||||
return ""
|
||||
|
||||
value = value.strip()
|
||||
|
||||
if " · " in value:
|
||||
parts = value.split(" · ", 1)
|
||||
acct_num = parts[0].strip()
|
||||
if acct_num.isdigit():
|
||||
return acct_num
|
||||
|
||||
if " - " in value:
|
||||
parts = value.split(" - ", 1)
|
||||
acct_num = parts[0].strip()
|
||||
if acct_num.isdigit():
|
||||
return acct_num
|
||||
|
||||
return value
|
||||
|
||||
@property
|
||||
def items(self) -> Dict[str, Dict]:
|
||||
"""Get items/products cache."""
|
||||
if self._items is None:
|
||||
self._items = {}
|
||||
for i in self.client.get_items():
|
||||
name = i.get("Name", "").lower()
|
||||
self._items[name] = {"value": i["Id"], "name": i.get("Name")}
|
||||
return self._items
|
||||
|
||||
def resolve_customer(self, name: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
"""Resolve customer name to reference."""
|
||||
if not name:
|
||||
return None, "Customer name is required"
|
||||
|
||||
ref = self.customers.get(name.lower())
|
||||
if ref:
|
||||
return ref, None
|
||||
|
||||
if self.settings.create_missing_references:
|
||||
try:
|
||||
new_customer = self.client.create_customer({"DisplayName": name})
|
||||
ref = {"value": new_customer["Id"], "name": name}
|
||||
self._customers[name.lower()] = ref
|
||||
logger.info(f"Created new customer: {name}")
|
||||
return ref, None
|
||||
except Exception as e:
|
||||
return None, f"Failed to create customer: {str(e)}"
|
||||
|
||||
return None, f"Customer not found: {name}"
|
||||
|
||||
def resolve_vendor(self, name: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
"""Resolve vendor name to reference."""
|
||||
if not name:
|
||||
return None, "Vendor name is required"
|
||||
|
||||
clean_name = name.strip()
|
||||
|
||||
ref = self.vendors.get(clean_name.lower())
|
||||
if ref:
|
||||
return ref, None
|
||||
|
||||
if self.settings.create_missing_references:
|
||||
try:
|
||||
new_vendor = self.client.create_vendor({"DisplayName": clean_name})
|
||||
ref = {"value": new_vendor["Id"], "name": clean_name}
|
||||
self._vendors[clean_name.lower()] = ref
|
||||
logger.info(f"Created new vendor: {clean_name}")
|
||||
return ref, None
|
||||
except Exception as e:
|
||||
return None, f"Failed to create vendor: {str(e)}"
|
||||
|
||||
return None, f"Vendor not found: {clean_name}"
|
||||
|
||||
def resolve_account(self, name_or_num: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
"""Resolve account name or number to reference."""
|
||||
if not name_or_num:
|
||||
return None, "Account name/number is required"
|
||||
|
||||
ref = self.accounts.get(name_or_num.lower())
|
||||
if ref:
|
||||
return ref, None
|
||||
|
||||
extracted = self._extract_account_identifier(name_or_num)
|
||||
if extracted != name_or_num:
|
||||
ref = self.accounts.get(extracted.lower())
|
||||
if ref:
|
||||
return ref, None
|
||||
ref = self.accounts.get(extracted)
|
||||
if ref:
|
||||
return ref, None
|
||||
|
||||
return None, f"Account not found: {name_or_num}"
|
||||
|
||||
def resolve_item(self, name: str) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
"""Resolve item/product name to reference."""
|
||||
if not name:
|
||||
return None, "Item name is required"
|
||||
|
||||
ref = self.items.get(name.lower())
|
||||
if ref:
|
||||
return ref, None
|
||||
|
||||
return None, f"Item not found: {name}"
|
||||
|
||||
|
||||
class DuplicateChecker:
|
||||
"""Checks for duplicate records before import."""
|
||||
|
||||
def __init__(self, qbo_client: QBOClient, settings: ImportSettings):
|
||||
self.client = qbo_client
|
||||
self.settings = settings
|
||||
self._existing_doc_numbers: Dict[str, set] = {}
|
||||
|
||||
def load_existing(self, data_type: str, date_range: Tuple[str, str] = None):
|
||||
"""Load existing document numbers for duplicate checking."""
|
||||
self._existing_doc_numbers[data_type] = set()
|
||||
|
||||
try:
|
||||
if data_type == "check":
|
||||
records = self.client.get_checks(
|
||||
start_date=date_range[0] if date_range else None,
|
||||
end_date=date_range[1] if date_range else None
|
||||
)
|
||||
elif data_type == "invoice":
|
||||
records = self.client.get_invoices(
|
||||
start_date=date_range[0] if date_range else None,
|
||||
end_date=date_range[1] if date_range else None
|
||||
)
|
||||
elif data_type == "bill":
|
||||
records = self.client.get_bills(
|
||||
start_date=date_range[0] if date_range else None,
|
||||
end_date=date_range[1] if date_range else None
|
||||
)
|
||||
else:
|
||||
return
|
||||
|
||||
for record in records:
|
||||
for field in self.settings.duplicate_check_fields:
|
||||
if field in record and record[field]:
|
||||
self._existing_doc_numbers[data_type].add(str(record[field]).lower())
|
||||
|
||||
logger.info(f"Loaded {len(self._existing_doc_numbers[data_type])} existing {data_type} records for duplicate check")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load existing records for duplicate check: {e}")
|
||||
|
||||
def is_duplicate(self, data_type: str, record_data: Dict) -> bool:
|
||||
"""Check if record is a duplicate."""
|
||||
if not self.settings.skip_duplicates:
|
||||
return False
|
||||
|
||||
existing = self._existing_doc_numbers.get(data_type, set())
|
||||
|
||||
for field in self.settings.duplicate_check_fields:
|
||||
value = record_data.get(field)
|
||||
if value and str(value).lower() in existing:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def add_imported(self, data_type: str, record_data: Dict):
|
||||
"""Add newly imported record to duplicate tracker."""
|
||||
if data_type not in self._existing_doc_numbers:
|
||||
self._existing_doc_numbers[data_type] = set()
|
||||
|
||||
for field in self.settings.duplicate_check_fields:
|
||||
if field in record_data and record_data[field]:
|
||||
self._existing_doc_numbers[data_type].add(str(record_data[field]).lower())
|
||||
|
||||
|
||||
class ImportProcessor:
|
||||
"""Main import processor with validation and batch operations."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
qbo_client: QBOClient,
|
||||
settings: ImportSettings,
|
||||
progress_callback: Optional[Callable[[int, int, str], None]] = None,
|
||||
log_callback: Optional[Callable[[str, str], None]] = None
|
||||
):
|
||||
self.client = qbo_client
|
||||
self.settings = settings
|
||||
self.progress_callback = progress_callback
|
||||
self.log_callback = log_callback
|
||||
|
||||
self.resolver = ReferenceResolver(qbo_client, settings)
|
||||
self.duplicate_checker = DuplicateChecker(qbo_client, settings)
|
||||
|
||||
def _log(self, level: str, message: str):
|
||||
"""Log a message."""
|
||||
if self.log_callback:
|
||||
self.log_callback(level, message)
|
||||
|
||||
if level == "error":
|
||||
logger.error(message)
|
||||
elif level == "warning":
|
||||
logger.warning(message)
|
||||
else:
|
||||
logger.info(message)
|
||||
|
||||
def _progress(self, current: int, total: int, message: str = ""):
|
||||
"""Report progress."""
|
||||
if self.progress_callback:
|
||||
self.progress_callback(current, total, message)
|
||||
|
||||
def validate_and_resolve(
|
||||
self,
|
||||
parse_result: ParseResult,
|
||||
data_type: str
|
||||
) -> List[ImportRecord]:
|
||||
"""Validate parsed data and resolve references."""
|
||||
records = []
|
||||
|
||||
self._log("info", f"Validating {len(parse_result.rows)} records...")
|
||||
self._log("info", "Loading reference data from QuickBooks...")
|
||||
|
||||
# Refresh reference caches
|
||||
self.resolver.refresh_cache()
|
||||
|
||||
# Load existing records for duplicate check
|
||||
if self.settings.skip_duplicates:
|
||||
self._log("info", "Checking for duplicates...")
|
||||
self.duplicate_checker.load_existing(data_type)
|
||||
|
||||
for i, row in enumerate(parse_result.rows):
|
||||
self._progress(i + 1, len(parse_result.rows), f"Validating row {row.row_number}")
|
||||
|
||||
record = ImportRecord(
|
||||
row_number=row.row_number,
|
||||
original_data=row.original_data,
|
||||
qbo_data=row.qbo_data.copy()
|
||||
)
|
||||
|
||||
# Skip rows with parsing errors
|
||||
if not row.is_valid:
|
||||
record.status = ImportStatus.FAILED
|
||||
record.error_message = "; ".join(
|
||||
issue.message for issue in row.issues
|
||||
if issue.severity == ValidationSeverity.ERROR
|
||||
)
|
||||
records.append(record)
|
||||
continue
|
||||
|
||||
# Resolve references based on data type
|
||||
error = self._resolve_references(record.qbo_data, data_type)
|
||||
if error:
|
||||
record.status = ImportStatus.FAILED
|
||||
record.error_message = error
|
||||
records.append(record)
|
||||
continue
|
||||
|
||||
# Check for duplicates
|
||||
if self.duplicate_checker.is_duplicate(data_type, record.qbo_data):
|
||||
record.status = ImportStatus.DUPLICATE
|
||||
record.error_message = "Duplicate record found"
|
||||
records.append(record)
|
||||
continue
|
||||
|
||||
record.status = ImportStatus.PENDING
|
||||
records.append(record)
|
||||
|
||||
return records
|
||||
|
||||
def _resolve_references(self, qbo_data: Dict, data_type: str) -> Optional[str]:
|
||||
"""Resolve entity references in QBO data. Returns error message or None."""
|
||||
errors = []
|
||||
|
||||
if data_type == "check":
|
||||
# Resolve payee/vendor (EntityRef for Purchase API)
|
||||
entity_key = "EntityRef" if "EntityRef" in qbo_data else "PayeeRef"
|
||||
if entity_key in qbo_data:
|
||||
payee_val = qbo_data[entity_key]
|
||||
if isinstance(payee_val, dict):
|
||||
payee_name = payee_val.get("value", payee_val.get("name", ""))
|
||||
else:
|
||||
payee_name = str(payee_val)
|
||||
|
||||
if payee_name and not str(payee_name).isdigit():
|
||||
ref, err = self.resolver.resolve_vendor(payee_name)
|
||||
if err:
|
||||
errors.append(err)
|
||||
else:
|
||||
qbo_data["EntityRef"] = ref
|
||||
if entity_key == "PayeeRef":
|
||||
del qbo_data["PayeeRef"]
|
||||
elif payee_name and str(payee_name).isdigit():
|
||||
qbo_data["EntityRef"] = {"value": payee_name}
|
||||
if entity_key == "PayeeRef":
|
||||
del qbo_data["PayeeRef"]
|
||||
|
||||
# Resolve bank account
|
||||
acct_key = "AccountRef" if "AccountRef" in qbo_data else "BankAccountRef"
|
||||
if acct_key in qbo_data:
|
||||
acct_val = qbo_data[acct_key]
|
||||
if isinstance(acct_val, dict):
|
||||
acct = acct_val.get("value", acct_val.get("name", ""))
|
||||
else:
|
||||
acct = str(acct_val)
|
||||
|
||||
if acct and not str(acct).isdigit():
|
||||
ref, err = self.resolver.resolve_account(acct)
|
||||
if err:
|
||||
errors.append(err)
|
||||
else:
|
||||
qbo_data["AccountRef"] = ref
|
||||
if acct_key == "BankAccountRef":
|
||||
del qbo_data["BankAccountRef"]
|
||||
elif acct and str(acct).isdigit():
|
||||
qbo_data["AccountRef"] = {"value": acct}
|
||||
if acct_key == "BankAccountRef":
|
||||
del qbo_data["BankAccountRef"]
|
||||
|
||||
# Build Line items properly
|
||||
if "Line" in qbo_data and isinstance(qbo_data["Line"], dict):
|
||||
line = qbo_data["Line"]
|
||||
amount = line.get("Amount", 0)
|
||||
if isinstance(amount, str):
|
||||
try:
|
||||
amount = float(amount)
|
||||
except ValueError:
|
||||
amount = 0.0
|
||||
|
||||
description = line.get("Description", "")
|
||||
|
||||
account_ref = None
|
||||
if "AccountBasedExpenseLineDetail" in line:
|
||||
detail = line["AccountBasedExpenseLineDetail"]
|
||||
if "AccountRef" in detail:
|
||||
acct_val = detail["AccountRef"]
|
||||
if isinstance(acct_val, dict):
|
||||
acct = acct_val.get("value", "")
|
||||
else:
|
||||
acct = str(acct_val)
|
||||
|
||||
if acct and not str(acct).isdigit():
|
||||
ref, err = self.resolver.resolve_account(acct)
|
||||
if err:
|
||||
errors.append(err)
|
||||
else:
|
||||
account_ref = ref
|
||||
elif acct and str(acct).isdigit():
|
||||
account_ref = {"value": acct}
|
||||
|
||||
line_item = {
|
||||
"DetailType": "AccountBasedExpenseLineDetail",
|
||||
"Amount": amount,
|
||||
"Description": description,
|
||||
"AccountBasedExpenseLineDetail": {}
|
||||
}
|
||||
|
||||
if account_ref:
|
||||
line_item["AccountBasedExpenseLineDetail"]["AccountRef"] = account_ref
|
||||
|
||||
qbo_data["Line"] = [line_item]
|
||||
|
||||
elif data_type == "invoice":
|
||||
if "CustomerRef" in qbo_data:
|
||||
cust_name = qbo_data["CustomerRef"].get("value") if isinstance(qbo_data["CustomerRef"], dict) else qbo_data["CustomerRef"]
|
||||
if cust_name and not str(cust_name).isdigit():
|
||||
ref, err = self.resolver.resolve_customer(str(cust_name))
|
||||
if err:
|
||||
errors.append(err)
|
||||
else:
|
||||
qbo_data["CustomerRef"] = ref
|
||||
|
||||
if "Line" in qbo_data:
|
||||
for line in qbo_data.get("Line", []):
|
||||
detail = line.get("SalesItemLineDetail", {})
|
||||
if "ItemRef" in detail:
|
||||
item = detail["ItemRef"].get("value") if isinstance(detail["ItemRef"], dict) else detail["ItemRef"]
|
||||
if item and not str(item).isdigit():
|
||||
ref, err = self.resolver.resolve_item(str(item))
|
||||
if err:
|
||||
errors.append(err)
|
||||
else:
|
||||
detail["ItemRef"] = ref
|
||||
|
||||
elif data_type == "bill":
|
||||
if "VendorRef" in qbo_data:
|
||||
vendor_name = qbo_data["VendorRef"].get("value") if isinstance(qbo_data["VendorRef"], dict) else qbo_data["VendorRef"]
|
||||
if vendor_name and not str(vendor_name).isdigit():
|
||||
ref, err = self.resolver.resolve_vendor(str(vendor_name))
|
||||
if err:
|
||||
errors.append(err)
|
||||
else:
|
||||
qbo_data["VendorRef"] = ref
|
||||
|
||||
if "Line" in qbo_data:
|
||||
for line in qbo_data.get("Line", []):
|
||||
detail = line.get("AccountBasedExpenseLineDetail", {})
|
||||
if "AccountRef" in detail:
|
||||
acct = detail["AccountRef"].get("value") if isinstance(detail["AccountRef"], dict) else detail["AccountRef"]
|
||||
if acct and not str(acct).isdigit():
|
||||
ref, err = self.resolver.resolve_account(str(acct))
|
||||
if err:
|
||||
errors.append(err)
|
||||
else:
|
||||
detail["AccountRef"] = ref
|
||||
|
||||
return "; ".join(errors) if errors else None
|
||||
|
||||
def import_records(
|
||||
self,
|
||||
records: List[ImportRecord],
|
||||
data_type: str
|
||||
) -> ImportResult:
|
||||
"""Import validated records to QuickBooks Online."""
|
||||
start_time = datetime.now()
|
||||
|
||||
result = ImportResult(
|
||||
data_type=data_type,
|
||||
total_records=len(records),
|
||||
successful=0,
|
||||
failed=0,
|
||||
skipped=0,
|
||||
duplicates=0,
|
||||
records=records,
|
||||
start_time=start_time.isoformat()
|
||||
)
|
||||
|
||||
# Count pre-existing failures
|
||||
for record in records:
|
||||
if record.status == ImportStatus.FAILED:
|
||||
result.failed += 1
|
||||
self._log("error", f"Row {record.row_number}: Pre-validation failed - {record.error_message}")
|
||||
elif record.status == ImportStatus.DUPLICATE:
|
||||
result.duplicates += 1
|
||||
self._log("warning", f"Row {record.row_number}: Skipped (duplicate)")
|
||||
elif record.status == ImportStatus.SKIPPED:
|
||||
result.skipped += 1
|
||||
self._log("warning", f"Row {record.row_number}: Skipped - {record.error_message or 'Unknown reason'}")
|
||||
|
||||
pending_records = [r for r in records if r.status == ImportStatus.PENDING]
|
||||
|
||||
self._log("info", f"Importing {len(pending_records)} records to QuickBooks Online...")
|
||||
|
||||
# Process in batches
|
||||
batch_size = self.settings.batch_size
|
||||
for batch_start in range(0, len(pending_records), batch_size):
|
||||
batch = pending_records[batch_start:batch_start + batch_size]
|
||||
batch_num = (batch_start // batch_size) + 1
|
||||
total_batches = (len(pending_records) + batch_size - 1) // batch_size
|
||||
|
||||
self._log("info", f"Processing batch {batch_num}/{total_batches} ({len(batch)} records)")
|
||||
|
||||
for i, record in enumerate(batch):
|
||||
current = batch_start + i + 1
|
||||
self._progress(current, len(pending_records), f"Importing record {current}")
|
||||
|
||||
record.status = ImportStatus.PROCESSING
|
||||
record.timestamp = datetime.now().isoformat()
|
||||
|
||||
try:
|
||||
# Log the data being sent
|
||||
self._log("debug", f"Row {record.row_number}: Sending data - {json.dumps(record.qbo_data, default=str)[:500]}")
|
||||
|
||||
if data_type == "check":
|
||||
check_data = record.qbo_data.copy()
|
||||
check_data["PaymentType"] = "Check"
|
||||
qbo_entity = self.client.create_check(check_data)
|
||||
elif data_type == "invoice":
|
||||
qbo_entity = self.client.create_invoice(record.qbo_data)
|
||||
elif data_type == "bill":
|
||||
qbo_entity = self.client.create_bill(record.qbo_data)
|
||||
elif data_type == "customer":
|
||||
qbo_entity = self.client.create_customer(record.qbo_data)
|
||||
elif data_type == "vendor":
|
||||
qbo_entity = self.client.create_vendor(record.qbo_data)
|
||||
elif data_type == "account":
|
||||
qbo_entity = self.client.create_account(record.qbo_data)
|
||||
else:
|
||||
raise ValueError(f"Unknown data type: {data_type}")
|
||||
|
||||
record.status = ImportStatus.SUCCESS
|
||||
record.qbo_id = qbo_entity.get("Id")
|
||||
result.successful += 1
|
||||
|
||||
self.duplicate_checker.add_imported(data_type, record.qbo_data)
|
||||
|
||||
self._log("info", f"Row {record.row_number}: SUCCESS - Created {data_type} ID {record.qbo_id}")
|
||||
|
||||
except Exception as e:
|
||||
record.status = ImportStatus.FAILED
|
||||
error_str = str(e)
|
||||
record.error_message = error_str
|
||||
result.failed += 1
|
||||
|
||||
# Extract key data for error context
|
||||
key_fields = self._get_key_fields(record.qbo_data, data_type)
|
||||
|
||||
self._log("error", f"Row {record.row_number}: FAILED - {error_str}")
|
||||
self._log("error", f"Row {record.row_number}: Key fields - {key_fields}")
|
||||
|
||||
# Log original Excel data for debugging
|
||||
if record.original_data:
|
||||
orig_data_str = ", ".join(f"{k}={v}" for k, v in record.original_data.items() if v)
|
||||
self._log("error", f"Row {record.row_number}: Original data - {orig_data_str[:300]}")
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
end_time = datetime.now()
|
||||
result.end_time = end_time.isoformat()
|
||||
result.duration_seconds = (end_time - start_time).total_seconds()
|
||||
|
||||
self._log("info", "=" * 50)
|
||||
self._log("info", f"Import complete: {result.successful} successful, {result.failed} failed, "
|
||||
f"{result.duplicates} duplicates, {result.skipped} skipped")
|
||||
self._log("info", f"Duration: {result.duration_seconds:.2f} seconds")
|
||||
self._log("info", f"Success rate: {result.success_rate:.1f}%")
|
||||
|
||||
return result
|
||||
|
||||
def _get_key_fields(self, qbo_data: Dict[str, Any], data_type: str) -> str:
|
||||
"""Extract key fields from QBO data for error context."""
|
||||
key_info = []
|
||||
|
||||
if data_type == "check":
|
||||
if "EntityRef" in qbo_data:
|
||||
entity = qbo_data["EntityRef"]
|
||||
key_info.append(f"Payee: {entity.get('name') or entity.get('value', 'N/A')}")
|
||||
if "AccountRef" in qbo_data:
|
||||
acct = qbo_data["AccountRef"]
|
||||
key_info.append(f"Account: {acct.get('name') or acct.get('value', 'N/A')}")
|
||||
if "TxnDate" in qbo_data:
|
||||
key_info.append(f"Date: {qbo_data['TxnDate']}")
|
||||
if "DocNumber" in qbo_data:
|
||||
key_info.append(f"DocNumber: {qbo_data['DocNumber']}")
|
||||
if "Line" in qbo_data and qbo_data["Line"]:
|
||||
total = sum(line.get("Amount", 0) for line in qbo_data["Line"])
|
||||
key_info.append(f"Amount: ${total:,.2f}")
|
||||
|
||||
elif data_type == "invoice":
|
||||
if "CustomerRef" in qbo_data:
|
||||
cust = qbo_data["CustomerRef"]
|
||||
key_info.append(f"Customer: {cust.get('name') or cust.get('value', 'N/A')}")
|
||||
if "TxnDate" in qbo_data:
|
||||
key_info.append(f"Date: {qbo_data['TxnDate']}")
|
||||
if "DocNumber" in qbo_data:
|
||||
key_info.append(f"Invoice#: {qbo_data['DocNumber']}")
|
||||
|
||||
elif data_type == "bill":
|
||||
if "VendorRef" in qbo_data:
|
||||
vendor = qbo_data["VendorRef"]
|
||||
key_info.append(f"Vendor: {vendor.get('name') or vendor.get('value', 'N/A')}")
|
||||
if "TxnDate" in qbo_data:
|
||||
key_info.append(f"Date: {qbo_data['TxnDate']}")
|
||||
if "DocNumber" in qbo_data:
|
||||
key_info.append(f"Bill#: {qbo_data['DocNumber']}")
|
||||
|
||||
elif data_type in ("customer", "vendor"):
|
||||
if "DisplayName" in qbo_data:
|
||||
key_info.append(f"Name: {qbo_data['DisplayName']}")
|
||||
if "CompanyName" in qbo_data:
|
||||
key_info.append(f"Company: {qbo_data['CompanyName']}")
|
||||
|
||||
elif data_type == "account":
|
||||
if "Name" in qbo_data:
|
||||
key_info.append(f"Name: {qbo_data['Name']}")
|
||||
if "AccountType" in qbo_data:
|
||||
key_info.append(f"Type: {qbo_data['AccountType']}")
|
||||
|
||||
return " | ".join(key_info) if key_info else "No key fields"
|
||||
|
||||
def process(
|
||||
self,
|
||||
parse_result: ParseResult,
|
||||
data_type: str
|
||||
) -> ImportResult:
|
||||
"""Full import process: validate, resolve, and import."""
|
||||
records = self.validate_and_resolve(parse_result, data_type)
|
||||
|
||||
pending = sum(1 for r in records if r.status == ImportStatus.PENDING)
|
||||
failed = sum(1 for r in records if r.status == ImportStatus.FAILED)
|
||||
duplicates = sum(1 for r in records if r.status == ImportStatus.DUPLICATE)
|
||||
|
||||
self._log("info", f"Validation complete: {pending} ready, {failed} failed, {duplicates} duplicates")
|
||||
|
||||
if pending == 0:
|
||||
self._log("warning", "No records to import after validation")
|
||||
return ImportResult(
|
||||
data_type=data_type,
|
||||
total_records=len(records),
|
||||
successful=0,
|
||||
failed=failed,
|
||||
skipped=0,
|
||||
duplicates=duplicates,
|
||||
records=records,
|
||||
start_time=datetime.now().isoformat(),
|
||||
end_time=datetime.now().isoformat()
|
||||
)
|
||||
|
||||
return self.import_records(records, data_type)
|
||||
Reference in New Issue
Block a user