500 lines
20 KiB
Python
500 lines
20 KiB
Python
"""
|
|
config.py — Application configuration and DB connection manager.
|
|
Web version: credentials loaded from environment variables / .env file.
|
|
No OS keyring dependency — suitable for server deployment.
|
|
"""
|
|
|
|
import os
|
|
import mysql.connector
|
|
from mysql.connector import pooling
|
|
import logging
|
|
import queue
|
|
import threading
|
|
import sys
|
|
|
|
# ─── Load .env before anything reads os.environ ───────────────────────────────
|
|
# Must happen at the very top of this module — DB_CONFIG is built at import
|
|
# time, so dotenv must populate os.environ before those lines execute.
|
|
try:
|
|
from dotenv import load_dotenv
|
|
load_dotenv() # looks for .env in cwd, then parent directories
|
|
except ImportError:
|
|
pass # python-dotenv not installed — rely on real env vars
|
|
|
|
# ─── Logging Setup ────────────────────────────────────────────────────────────
|
|
_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s - %(message)s")
|
|
|
|
_stream_handler = logging.StreamHandler(stream=sys.stdout)
|
|
_stream_handler.setFormatter(_formatter)
|
|
|
|
APP_TITLE = "Website Checker"
|
|
APP_VERSION = "1.0.0"
|
|
|
|
|
|
# ─── Database Log Handler (async, queued) ─────────────────────────────────────
|
|
class _DBLogHandler(logging.Handler):
|
|
"""Async handler that writes log records to the app_log DB table."""
|
|
|
|
def __init__(self):
|
|
super().__init__()
|
|
self._queue = queue.Queue()
|
|
self._ready = False
|
|
self._thread = threading.Thread(target=self._worker, name="DBLogWorker", daemon=True)
|
|
self._thread.start()
|
|
|
|
def emit(self, record: logging.LogRecord):
|
|
if record.name in ("config", "mysql.connector"):
|
|
return
|
|
try:
|
|
self._queue.put_nowait({
|
|
"level": record.levelname,
|
|
"logger_name": record.name[:100],
|
|
"message": self.format(record),
|
|
})
|
|
except Exception:
|
|
pass
|
|
|
|
def install(self):
|
|
"""Signal worker that the DB pool is ready."""
|
|
self._ready = True
|
|
|
|
def _worker(self):
|
|
while True:
|
|
try:
|
|
record = self._queue.get(timeout=2)
|
|
except queue.Empty:
|
|
continue
|
|
if not self._ready:
|
|
self._queue.put(record)
|
|
threading.Event().wait(1)
|
|
continue
|
|
try:
|
|
conn = get_connection()
|
|
cur = conn.cursor()
|
|
cur.execute(
|
|
"INSERT INTO app_log (level, logger_name, message) VALUES (%s,%s,%s)",
|
|
(record["level"], record["logger_name"], record["message"]),
|
|
)
|
|
conn.commit()
|
|
cur.close()
|
|
conn.close()
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
db_log_handler = _DBLogHandler()
|
|
db_log_handler.setFormatter(_formatter)
|
|
|
|
logger = logging.getLogger("config")
|
|
|
|
# ─── DB Configuration (environment variables) ─────────────────────────────────
|
|
DB_CONFIG = {
|
|
"host": os.environ.get("DB_HOST", "localhost"),
|
|
"port": int(os.environ.get("DB_PORT", 3306)),
|
|
"database": os.environ.get("DB_NAME", "website_checker"),
|
|
"user": os.environ.get("DB_USER", "wc_user"),
|
|
"password": os.environ.get("DB_PASSWORD", ""),
|
|
"connection_timeout": 10,
|
|
"charset": "utf8mb4",
|
|
}
|
|
|
|
# ─── Connection Pool ──────────────────────────────────────────────────────────
|
|
_pool = None
|
|
_pool_lock = threading.Lock()
|
|
|
|
|
|
def get_connection_pool():
|
|
global _pool
|
|
if _pool is None:
|
|
with _pool_lock:
|
|
if _pool is None:
|
|
try:
|
|
_pool = pooling.MySQLConnectionPool(
|
|
pool_name="app_pool",
|
|
pool_size=10,
|
|
**DB_CONFIG,
|
|
)
|
|
logger.info("Database connection pool initialised.")
|
|
except mysql.connector.Error as e:
|
|
logger.error(f"Failed to create connection pool: {e}")
|
|
raise
|
|
return _pool
|
|
|
|
|
|
def get_connection():
|
|
"""Return a connection from the pool."""
|
|
return get_connection_pool().get_connection()
|
|
|
|
|
|
# ─── Schema DDL ───────────────────────────────────────────────────────────────
|
|
def initialize_database():
|
|
"""Create all required tables if they do not exist."""
|
|
ddl_statements = [
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS users (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
username VARCHAR(100) NOT NULL UNIQUE,
|
|
password VARCHAR(255) NOT NULL,
|
|
role ENUM('admin','user') NOT NULL DEFAULT 'user',
|
|
full_name VARCHAR(200),
|
|
email VARCHAR(255) NULL DEFAULT NULL,
|
|
is_active TINYINT(1) NOT NULL DEFAULT 1,
|
|
failed_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0,
|
|
locked_until DATETIME NULL,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS websites (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
name VARCHAR(200) NOT NULL,
|
|
url TEXT NOT NULL,
|
|
check_type ENUM('daily','weekly') NOT NULL DEFAULT 'daily',
|
|
visibility ENUM('all','assigned') NOT NULL DEFAULT 'all',
|
|
note TEXT,
|
|
is_active TINYINT(1) NOT NULL DEFAULT 1,
|
|
created_by INT,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS website_credentials (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
website_id INT NOT NULL,
|
|
label VARCHAR(100),
|
|
username VARCHAR(200),
|
|
password TEXT,
|
|
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS shift_checks (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
website_id INT NOT NULL,
|
|
user_id INT NOT NULL,
|
|
checked_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
user_note TEXT,
|
|
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS activity_log (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
user_id INT,
|
|
action VARCHAR(100) NOT NULL,
|
|
entity VARCHAR(100),
|
|
entity_id INT,
|
|
detail TEXT,
|
|
logged_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
INDEX idx_activity_log_time (logged_at),
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS shifts (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
name VARCHAR(200) NOT NULL,
|
|
days_of_week VARCHAR(7) NOT NULL DEFAULT '23456',
|
|
start_time TIME NOT NULL DEFAULT '08:00:00',
|
|
end_time TIME NOT NULL DEFAULT '17:00:00',
|
|
is_active TINYINT(1) NOT NULL DEFAULT 1,
|
|
note TEXT,
|
|
created_by INT,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS shift_users (
|
|
shift_id INT NOT NULL,
|
|
user_id INT NOT NULL,
|
|
PRIMARY KEY (shift_id, user_id),
|
|
FOREIGN KEY (shift_id) REFERENCES shifts(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS shift_websites (
|
|
shift_id INT NOT NULL,
|
|
website_id INT NOT NULL,
|
|
sort_order INT NOT NULL DEFAULT 0,
|
|
PRIMARY KEY (shift_id, website_id),
|
|
FOREIGN KEY (shift_id) REFERENCES shifts(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS login_attempts (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
username VARCHAR(100) NOT NULL,
|
|
attempted_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
ip_address VARCHAR(45),
|
|
INDEX idx_username_time (username, attempted_at)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS website_users (
|
|
website_id INT NOT NULL,
|
|
user_id INT NOT NULL,
|
|
PRIMARY KEY (website_id, user_id),
|
|
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS ai_analysis_log (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
user_id INT,
|
|
file_names TEXT NOT NULL,
|
|
model VARCHAR(100) NOT NULL,
|
|
verdict ENUM('PURSUE','PASS','UNCLEAR') NULL,
|
|
criteria_snapshot TEXT NULL,
|
|
summary_text MEDIUMTEXT NOT NULL,
|
|
analyzed_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS ai_criteria (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
title VARCHAR(200) NOT NULL,
|
|
description TEXT NOT NULL,
|
|
is_active TINYINT(1) NOT NULL DEFAULT 1,
|
|
sort_order INT NOT NULL DEFAULT 0,
|
|
created_by INT,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS app_log (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
level VARCHAR(10) NOT NULL,
|
|
logger_name VARCHAR(100) NOT NULL,
|
|
message TEXT NOT NULL,
|
|
logged_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3),
|
|
INDEX idx_app_log_level (level),
|
|
INDEX idx_app_log_time (logged_at)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS bid_tracker (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
title VARCHAR(300) NOT NULL,
|
|
url TEXT NOT NULL,
|
|
source VARCHAR(200) NULL,
|
|
solicitation_number VARCHAR(100) NULL,
|
|
status ENUM('open','monitoring','awarded','no_bid','cancelled') NOT NULL DEFAULT 'open',
|
|
due_date DATE NULL,
|
|
notes TEXT NULL,
|
|
added_by INT,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (added_by) REFERENCES users(id) ON DELETE SET NULL
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS bid_updates (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
bid_id INT NOT NULL,
|
|
user_id INT,
|
|
content TEXT NOT NULL,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (bid_id) REFERENCES bid_tracker(id) ON DELETE CASCADE,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS password_reset_tokens (
|
|
id INT AUTO_INCREMENT PRIMARY KEY,
|
|
user_id INT NOT NULL,
|
|
token VARCHAR(64) NOT NULL UNIQUE,
|
|
expires_at DATETIME NOT NULL,
|
|
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
|
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE,
|
|
INDEX idx_prt_token (token),
|
|
INDEX idx_prt_expires (expires_at)
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
"""
|
|
CREATE TABLE IF NOT EXISTS app_settings (
|
|
key_name VARCHAR(100) NOT NULL PRIMARY KEY,
|
|
value TEXT NULL,
|
|
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
|
|
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
|
|
""",
|
|
]
|
|
|
|
conn = None
|
|
try:
|
|
conn = get_connection()
|
|
cursor = conn.cursor()
|
|
for stmt in ddl_statements:
|
|
cursor.execute(stmt)
|
|
conn.commit()
|
|
logger.info("Database schema initialised successfully.")
|
|
|
|
# ── ip_address column on login_attempts (web-only addition) ───────
|
|
# The desktop does not capture IP addresses; this adds the column
|
|
# safely if deploying the web app alongside an existing desktop DB.
|
|
cursor.execute(
|
|
"""
|
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
|
WHERE TABLE_SCHEMA = DATABASE()
|
|
AND TABLE_NAME = 'login_attempts'
|
|
AND COLUMN_NAME = 'ip_address'
|
|
"""
|
|
)
|
|
(has_ip,) = cursor.fetchone()
|
|
if not has_ip:
|
|
cursor.execute(
|
|
"ALTER TABLE login_attempts "
|
|
"ADD COLUMN ip_address VARCHAR(45) NULL AFTER attempted_at"
|
|
)
|
|
conn.commit()
|
|
logger.info("Migration: added ip_address column to login_attempts table.")
|
|
|
|
# ── activity_log: rename created_at → logged_at ────────────────
|
|
# The desktop app uses logged_at; earlier web-only installs may have
|
|
# created the table with created_at. Rename it if that's the case.
|
|
cursor.execute(
|
|
"""
|
|
SELECT COUNT(*) FROM information_schema.COLUMNS
|
|
WHERE TABLE_SCHEMA = DATABASE()
|
|
AND TABLE_NAME = 'activity_log'
|
|
AND COLUMN_NAME = 'created_at'
|
|
"""
|
|
)
|
|
(has_created_at,) = cursor.fetchone()
|
|
if has_created_at:
|
|
cursor.execute(
|
|
"ALTER TABLE activity_log "
|
|
"CHANGE COLUMN created_at logged_at DATETIME DEFAULT CURRENT_TIMESTAMP"
|
|
)
|
|
conn.commit()
|
|
logger.info("Migration: renamed activity_log.created_at to logged_at.")
|
|
|
|
# ── activity_log: add index on logged_at if missing ────────────
|
|
cursor.execute(
|
|
"""
|
|
SELECT COUNT(*) FROM information_schema.STATISTICS
|
|
WHERE TABLE_SCHEMA = DATABASE()
|
|
AND TABLE_NAME = 'activity_log'
|
|
AND INDEX_NAME = 'idx_activity_log_time'
|
|
"""
|
|
)
|
|
(has_idx,) = cursor.fetchone()
|
|
if not has_idx:
|
|
cursor.execute(
|
|
"ALTER TABLE activity_log "
|
|
"ADD INDEX idx_activity_log_time (logged_at)"
|
|
)
|
|
conn.commit()
|
|
logger.info("Migration: added idx_activity_log_time index to activity_log.")
|
|
|
|
# ── Seed app_settings from environment variables (first-run bootstrap) ─
|
|
# Uses INSERT IGNORE so values already saved via the Admin UI are never
|
|
# overwritten — .env only fills in keys that are completely absent.
|
|
env_seeds = [
|
|
("groq.api_key", os.environ.get("GROQ_API_KEY", "")),
|
|
("groq.model", os.environ.get("GROQ_MODEL", "")),
|
|
("email.smtp_host", os.environ.get("SMTP_HOST", "")),
|
|
("email.smtp_port", os.environ.get("SMTP_PORT", "")),
|
|
("email.smtp_user", os.environ.get("SMTP_USER", "")),
|
|
("email.smtp_password",os.environ.get("SMTP_PASSWORD", "")),
|
|
("email.smtp_from", os.environ.get("SMTP_FROM", "")),
|
|
]
|
|
for k, v in env_seeds:
|
|
if v:
|
|
cursor.execute(
|
|
"""INSERT INTO app_settings (key_name, value) VALUES (%s, %s)
|
|
ON DUPLICATE KEY UPDATE
|
|
value = IF(value IS NULL OR value = '', VALUES(value), value)""",
|
|
(k, v),
|
|
)
|
|
conn.commit()
|
|
logger.info("app_settings seeded from environment variables (env→DB, blank-only overwrite).")
|
|
|
|
cursor.close()
|
|
except mysql.connector.Error as e:
|
|
logger.error(f"Database initialisation error: {e}")
|
|
raise
|
|
finally:
|
|
if conn:
|
|
conn.close()
|
|
|
|
|
|
# ─── app_settings helpers ─────────────────────────────────────────────────────
|
|
|
|
# Map app_settings keys → environment variable names so .env values are
|
|
# used as fallbacks when the DB row is absent or empty.
|
|
_ENV_FALLBACKS = {
|
|
"groq.api_key": "GROQ_API_KEY",
|
|
"groq.model": "GROQ_MODEL",
|
|
"email.smtp_host": "SMTP_HOST",
|
|
"email.smtp_port": "SMTP_PORT",
|
|
"email.smtp_user": "SMTP_USER",
|
|
"email.smtp_password": "SMTP_PASSWORD",
|
|
"email.smtp_from": "SMTP_FROM",
|
|
}
|
|
|
|
|
|
def get_setting(key: str, default: str = "") -> str:
|
|
"""Read a value from app_settings, falling back to env var then default."""
|
|
try:
|
|
conn = get_connection()
|
|
cur = conn.cursor()
|
|
cur.execute("SELECT value FROM app_settings WHERE key_name=%s", (key,))
|
|
row = cur.fetchone()
|
|
cur.close()
|
|
conn.close()
|
|
if row and row[0]:
|
|
return row[0]
|
|
except Exception as e:
|
|
logger.warning(f"get_setting({key!r}) failed: {e}")
|
|
# Fall back to environment variable if mapped
|
|
env_var = _ENV_FALLBACKS.get(key)
|
|
if env_var:
|
|
env_val = os.environ.get(env_var, "")
|
|
if env_val:
|
|
return env_val
|
|
return default
|
|
|
|
|
|
def set_setting(key: str, value: str) -> None:
|
|
try:
|
|
conn = get_connection()
|
|
cur = conn.cursor()
|
|
cur.execute(
|
|
"INSERT INTO app_settings (key_name, value) VALUES (%s,%s) "
|
|
"ON DUPLICATE KEY UPDATE value=%s, updated_at=CURRENT_TIMESTAMP",
|
|
(key, value, value),
|
|
)
|
|
conn.commit()
|
|
cur.close()
|
|
conn.close()
|
|
except Exception as e:
|
|
logger.error(f"set_setting({key!r}) failed: {e}")
|
|
|
|
|
|
def get_settings_dict(prefix: str) -> dict:
|
|
try:
|
|
conn = get_connection()
|
|
cur = conn.cursor()
|
|
cur.execute(
|
|
"SELECT key_name, value FROM app_settings WHERE key_name LIKE %s",
|
|
(prefix + "%",),
|
|
)
|
|
rows = cur.fetchall()
|
|
cur.close()
|
|
conn.close()
|
|
return {r[0]: (r[1] or "") for r in rows}
|
|
except Exception as e:
|
|
logger.warning(f"get_settings_dict({prefix!r}) failed: {e}")
|
|
return {} |