nngo/WebChecker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
5d6ca5a039adc0c0820e47daba33e5a90e606489
WebChecker/config.py
T

1265 lines
50 KiB
Python
Raw Blame History

"""
config.py — Application configuration and DB connection manager.
Update DB_CONFIG with your remote MySQL server credentials.
"""
import mysql.connector
from mysql.connector import pooling
import logging
# ─── Logging Setup ────────────────────────────────────────────────────────────
import sys
import queue
import threading
_formatter = logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s - %(message)s"
)
# ── Console handler (development / stdout) ────────────────────────────────────
_stream_handler = logging.StreamHandler(stream=sys.stdout)
# Force UTF-8 on Windows cp1252 consoles so Unicode chars don't raise errors.
if hasattr(_stream_handler.stream, "reconfigure"):
try:
_stream_handler.stream.reconfigure(encoding="utf-8", errors="replace")
except Exception:
pass
_stream_handler.setFormatter(_formatter)
# ── Database log handler ───────────────────────────────────────────────────────
# Writes every log record to the app_log table asynchronously via a background
# thread so DB I/O never blocks the logging call or the UI.
# The handler is installed as soon as the pool is ready (after initialize_database).
# Before the pool is ready, records are queued and flushed on first install.
class _DBLogHandler(logging.Handler):
"""
Asynchronous logging handler that inserts records into app_log.
Uses a daemon thread + Queue to keep DB writes off the main thread.
Safe to create before the DB pool exists — records are queued until
install() is called after initialize_database() succeeds.
"""
def __init__(self):
super().__init__()
self._queue = queue.Queue()
self._ready = False # True once the pool is confirmed available
self._thread = threading.Thread(
target=self._worker, name="DBLogWorker", daemon=True)
self._thread.start()
def emit(self, record: logging.LogRecord):
# Never log our own worker thread to avoid infinite recursion
if record.name in ("config", "mysql.connector"):
return
try:
self._queue.put_nowait({
"level": record.levelname,
"logger_name": record.name[:100],
"message": self.format(record),
})
except Exception:
pass
"""
config.py — Application configuration and DB connection manager.
Update DB_CONFIG with your remote MySQL server credentials.
"""
import mysql.connector
from mysql.connector import pooling
import logging
# ─── Logging Setup ────────────────────────────────────────────────────────────
import sys
import queue
import threading
_formatter = logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s - %(message)s"
)
# ── Console handler (development / stdout) ────────────────────────────────────
_stream_handler = logging.StreamHandler(stream=sys.stdout)
# Force UTF-8 on Windows cp1252 consoles so Unicode chars don't raise errors.
if hasattr(_stream_handler.stream, "reconfigure"):
try:
_stream_handler.stream.reconfigure(encoding="utf-8", errors="replace")
except Exception:
pass
_stream_handler.setFormatter(_formatter)
# ── Database log handler ───────────────────────────────────────────────────────
# Writes every log record to the app_log table asynchronously via a background
# thread so DB I/O never blocks the logging call or the UI.
# The handler is installed as soon as the pool is ready (after initialize_database).
# Before the pool is ready, records are queued and flushed on first install.
class _DBLogHandler(logging.Handler):
"""
Asynchronous logging handler that inserts records into app_log.
Uses a daemon thread + Queue to keep DB writes off the main thread.
Safe to create before the DB pool exists — records are queued until
install() is called after initialize_database() succeeds.
"""
def __init__(self):
super().__init__()
self._queue = queue.Queue()
self._ready = False # True once the pool is confirmed available
self._thread = threading.Thread(
target=self._worker, name="DBLogWorker", daemon=True)
self._thread.start()
def emit(self, record: logging.LogRecord):
# Never log our own worker thread to avoid infinite recursion
if record.name in ("config", "mysql.connector"):
return
try:
self._queue.put_nowait({
"level": record.levelname,
"logger_name": record.name[:100],
"message": self.format(record),
})
except Exception:
pass
def install(self):
"""Signal the worker that the DB pool is ready to accept writes."""
self._ready = True
def _worker(self):
"""Background thread: drain the queue into app_log."""
while True:
try:
record = self._queue.get(timeout=2)
except queue.Empty:
continue
if not self._ready:
# Put it back and wait — pool not yet initialised
self._queue.put(record)
threading.Event().wait(1)
continue
try:
conn = get_connection()
cur = conn.cursor()
cur.execute(
"INSERT INTO app_log (level, logger_name, message) "
"VALUES (%s, %s, %s)",
(record["level"], record["logger_name"], record["message"]),
)
conn.commit()
cur.close()
conn.close()
except Exception:
pass # Silently discard if DB is temporarily unavailable
db_log_handler = _DBLogHandler()
db_log_handler.setFormatter(_formatter)
logging.basicConfig(
level=logging.INFO,
handlers=[_stream_handler, db_log_handler],
)
logger = logging.getLogger("config")
# ─── DB Credential Storage (OS Keychain via keyring) ─────────────────────────
#
# DB connection details are stored in the OS native credential store:
# Windows → Windows Credential Manager
# macOS → Keychain
# Linux → Secret Service / libsecret (or plaintext fallback)
#
# This replaces config.ini entirely — no local file is needed on any machine.
# The user enters credentials once via the Settings dialog; keyring persists
# them securely and they survive reboots, upgrades, and user profile migrations.
#
# keyring is a stdlib-level cross-platform abstraction; install with:
# pip install keyring
#
import os as _os
import json as _json
APP_TITLE = "Website Checker"
APP_VERSION = "1.0.0"
_KEYRING_SVC = "WebChecker" # service name in the OS credential store
_KEYRING_KEY = "db_config" # single credential entry stores JSON blob
def _keyring_set(data: dict) -> None:
"""Persist DB config dict as a JSON blob in the OS keychain."""
import keyring
keyring.set_password(_KEYRING_SVC, _KEYRING_KEY, _json.dumps(data))
def _keyring_get() -> dict:
"""Read DB config dict from the OS keychain. Returns {} if not found."""
try:
import keyring
raw = keyring.get_password(_KEYRING_SVC, _KEYRING_KEY)
if raw:
return _json.loads(raw)
except Exception as e:
logger.warning(f"keyring read failed: {e}")
return {}
def load_config() -> dict:
"""
Load DB connection settings from the OS keychain.
Returns a dict with keys: host, port, database, user, password.
Returns empty dict if no credentials are stored yet (first run).
On first startup after migration from config.ini, automatically imports
existing credentials from config.ini into the keychain and removes them
from the file so the file is no longer required.
"""
# Try keychain first
data = _keyring_get()
if data.get("host") and data.get("database") and data.get("user"):
return data
# One-time migration: pull from config.ini if it still exists
try:
import configparser as _cp
_cfg_file = "config.ini"
if _os.path.exists(_cfg_file):
from utils.config_crypto import decrypt_value
cfg = _cp.ConfigParser()
cfg.read(_cfg_file, encoding="utf-8")
if cfg.has_section("database"):
section = cfg["database"]
migrated = {
"host": section.get("host", ""),
"port": section.getint("port", 3306),
"database": section.get("database", ""),
"user": decrypt_value(section.get("user", "")),
"password": decrypt_value(section.get("password", "")),
}
if migrated.get("host") and migrated.get("database") and migrated.get("user"):
_keyring_set(migrated)
logger.info(
"DB credentials migrated from config.ini to OS keychain. "
"config.ini [database] section is no longer needed."
)
return migrated
except Exception as e:
logger.warning(f"config.ini migration attempt failed: {e}")
return {}
def save_config(host: str, port: int, database: str, user: str, password: str):
"""Persist DB connection settings to the OS keychain."""
data = {
"host": host,
"port": port,
"database": database,
"user": user,
"password": password,
}
try:
_keyring_set(data)
logger.info(
f"DB credentials saved to OS keychain "
f"({host}:{port}/{database})."
)
except Exception as e:
logger.error(f"Failed to save credentials to OS keychain: {e}")
raise RuntimeError(
f"Could not save to the OS keychain: {e}\n\n"
"Ensure the keyring package is installed: pip install keyring"
) from e
def migrate_plaintext_config():
"""
One-time migration runner called on every startup.
- Imports config.ini [database] into the OS keychain (handled by load_config).
- Migrates config.ini [email]/[groq]/[crypto] into app_settings (DB).
Safe to call repeatedly — all operations are idempotent no-ops once done.
"""
# Trigger the config.ini → keychain migration via load_config
load_config()
def config_exists() -> bool:
"""Return True if DB credentials are stored in the OS keychain."""
data = _keyring_get()
return bool(data.get("host") and data.get("database") and data.get("user"))
def reload_db_config():
"""
Re-read credentials from the OS keychain and update DB_CONFIG in place.
Also resets the connection pool so the next get_connection() uses new creds.
"""
global _pool, DB_CONFIG
data = load_config()
DB_CONFIG.update({
"host": data.get("host", DB_CONFIG["host"]),
"port": data.get("port", DB_CONFIG["port"]),
"database": data.get("database", DB_CONFIG["database"]),
"user": data.get("user", DB_CONFIG["user"]),
"password": data.get("password", DB_CONFIG["password"]),
})
_pool = None # force pool recreation on next connection
logger.info("DB_CONFIG reloaded from OS keychain.")
# ─── Database Configuration ───────────────────────────────────────────────────
# Populated from the OS keychain at import time; falls back to placeholder
# strings so the module is importable even before first-run setup has completed.
_ini = load_config()
DB_CONFIG = {
"host": _ini.get("host", "your-mysql-host"),
"port": _ini.get("port", 3306),
"database": _ini.get("database", "website_checker"),
"user": _ini.get("user", "your-db-user"),
"password": _ini.get("password", "your-db-password"),
"connection_timeout": 10,
}
# ─── Connection Pool ──────────────────────────────────────────────────────────
_pool = None
def get_connection_pool():
global _pool
if _pool is None:
try:
_pool = pooling.MySQLConnectionPool(
pool_name="app_pool",
pool_size=5,
**DB_CONFIG
)
logger.info("Database connection pool initialised.")
except mysql.connector.Error as e:
logger.error(f"Failed to create connection pool: {e}")
raise
return _pool
def get_connection():
"""Return a connection from the pool."""
return get_connection_pool().get_connection()
def initialize_database():
"""Create all required tables if they do not exist."""
ddl_statements = [
"""
CREATE TABLE IF NOT EXISTS users (
id INT AUTO_INCREMENT PRIMARY KEY,
username VARCHAR(100) NOT NULL UNIQUE,
password VARCHAR(255) NOT NULL,
role ENUM('admin','user') NOT NULL DEFAULT 'user',
full_name VARCHAR(200),
is_active TINYINT(1) NOT NULL DEFAULT 1,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS websites (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(200) NOT NULL,
url TEXT NOT NULL,
check_type ENUM('daily','weekly') NOT NULL DEFAULT 'daily',
visibility ENUM('all','assigned') NOT NULL DEFAULT 'all',
note TEXT,
is_active TINYINT(1) NOT NULL DEFAULT 1,
created_by INT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS website_credentials (
id INT AUTO_INCREMENT PRIMARY KEY,
website_id INT NOT NULL,
label VARCHAR(100),
username VARCHAR(200),
password TEXT,
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS shift_checks (
id INT AUTO_INCREMENT PRIMARY KEY,
website_id INT NOT NULL,
user_id INT NOT NULL,
checked_at DATETIME DEFAULT CURRENT_TIMESTAMP,
user_note TEXT,
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS activity_log (
id INT AUTO_INCREMENT PRIMARY KEY,
user_id INT,
action VARCHAR(100) NOT NULL,
entity VARCHAR(100),
entity_id INT,
detail TEXT,
logged_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS shifts (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(200) NOT NULL,
days_of_week VARCHAR(7) NOT NULL DEFAULT '23456',
start_time TIME NOT NULL DEFAULT '08:00:00',
end_time TIME NOT NULL DEFAULT '17:00:00',
is_active TINYINT(1) NOT NULL DEFAULT 1,
note TEXT,
created_by INT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS shift_users (
shift_id INT NOT NULL,
user_id INT NOT NULL,
PRIMARY KEY (shift_id, user_id),
FOREIGN KEY (shift_id) REFERENCES shifts(id) ON DELETE CASCADE,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS shift_websites (
shift_id INT NOT NULL,
website_id INT NOT NULL,
sort_order INT NOT NULL DEFAULT 0,
PRIMARY KEY (shift_id, website_id),
FOREIGN KEY (shift_id) REFERENCES shifts(id) ON DELETE CASCADE,
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS login_attempts (
id INT AUTO_INCREMENT PRIMARY KEY,
username VARCHAR(100) NOT NULL,
attempted_at DATETIME DEFAULT CURRENT_TIMESTAMP,
ip_address VARCHAR(45),
INDEX idx_username_time (username, attempted_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS website_users (
website_id INT NOT NULL,
user_id INT NOT NULL,
PRIMARY KEY (website_id, user_id),
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS ai_analysis_log (
id INT AUTO_INCREMENT PRIMARY KEY,
user_id INT,
file_names TEXT NOT NULL,
model VARCHAR(100) NOT NULL,
verdict ENUM('PURSUE','PASS','UNCLEAR') NULL,
criteria_snapshot TEXT NULL,
summary_text MEDIUMTEXT NOT NULL,
analyzed_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS ai_criteria (
id INT AUTO_INCREMENT PRIMARY KEY,
title VARCHAR(200) NOT NULL,
description TEXT NOT NULL,
is_active TINYINT(1) NOT NULL DEFAULT 1,
sort_order INT NOT NULL DEFAULT 0,
created_by INT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS app_log (
id INT AUTO_INCREMENT PRIMARY KEY,
level VARCHAR(10) NOT NULL,
logger_name VARCHAR(100) NOT NULL,
message TEXT NOT NULL,
logged_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3),
INDEX idx_app_log_level (level),
INDEX idx_app_log_time (logged_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS bid_tracker (
id INT AUTO_INCREMENT PRIMARY KEY,
title VARCHAR(300) NOT NULL,
url TEXT NOT NULL,
source VARCHAR(200) NULL,
solicitation_number VARCHAR(100) NULL,
status ENUM('open','monitoring','awarded','no_bid','cancelled')
NOT NULL DEFAULT 'open',
due_date DATE NULL,
notes TEXT NULL,
added_by INT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (added_by) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS bid_updates (
id INT AUTO_INCREMENT PRIMARY KEY,
bid_id INT NOT NULL,
user_id INT,
content TEXT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (bid_id) REFERENCES bid_tracker(id) ON DELETE CASCADE,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS app_settings (
key_name VARCHAR(100) NOT NULL PRIMARY KEY,
value TEXT NULL,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
COMMENT='Key-value store for application configuration. '
'Replaces config.ini sections: email, groq, crypto.';
""",
]
conn = None
try:
conn = get_connection()
cursor = conn.cursor()
for stmt in ddl_statements:
cursor.execute(stmt)
conn.commit()
logger.info("Database schema initialised successfully.")
# ── Safe migrations for existing deployments ───────────────────────
# Add check_type column to websites if it doesn't exist yet
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'websites'
AND COLUMN_NAME = 'check_type'
"""
)
(has_col,) = cursor.fetchone()
if not has_col:
cursor.execute(
"ALTER TABLE websites ADD COLUMN check_type ENUM('daily','weekly') "
"NOT NULL DEFAULT 'daily' AFTER url"
)
conn.commit()
logger.info("Migration: added check_type column to websites table.")
# Add failed_attempts / locked_until columns to users if absent
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'users'
AND COLUMN_NAME = 'failed_attempts'
"""
)
(has_fa,) = cursor.fetchone()
if not has_fa:
cursor.execute(
"ALTER TABLE users "
"ADD COLUMN failed_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0 AFTER is_active, "
"ADD COLUMN locked_until DATETIME NULL AFTER failed_attempts"
)
conn.commit()
logger.info("Migration: added failed_attempts and locked_until columns to users table.")
# Add visibility column to websites if absent
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'websites'
AND COLUMN_NAME = 'visibility'
"""
)
(has_vis,) = cursor.fetchone()
if not has_vis:
cursor.execute(
"ALTER TABLE websites ADD COLUMN visibility "
"ENUM('all','assigned') NOT NULL DEFAULT 'all' AFTER check_type"
)
conn.commit()
logger.info("Migration: added visibility column to websites table.")
# Confirm presence of tables added post-initial-deployment
for new_table in ("ai_criteria", "ai_analysis_log", "app_log", "bid_tracker", "bid_updates"):
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = %s
""",
(new_table,)
)
(table_exists,) = cursor.fetchone()
if table_exists:
logger.info(f"Table '{new_table}' confirmed present.")
else:
logger.warning(f"Table '{new_table}' was not created -- check DDL.")
# Seed default admin if users table is empty
cursor.execute("SELECT COUNT(*) FROM users")
(count,) = cursor.fetchone()
if count == 0:
import bcrypt as _bcrypt
default_pw = _bcrypt.hashpw(b"admin123", _bcrypt.gensalt(rounds=12)).decode("utf-8")
cursor.execute(
"INSERT INTO users (username, password, role, full_name) VALUES (%s,%s,'admin','System Admin')",
("admin", default_pw)
)
conn.commit()
logger.info("Default admin account seeded (username: admin / password: admin123).")
# ── users.email column (added in this release) ─────────────────────────
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'users'
AND COLUMN_NAME = 'email'
"""
)
(has_email,) = cursor.fetchone()
if not has_email:
cursor.execute(
"ALTER TABLE users ADD COLUMN email VARCHAR(255) NULL DEFAULT NULL "
"AFTER full_name"
)
conn.commit()
logger.info("Migration: added email column to users table.")
# ── app_settings: migrate config.ini → DB on first post-upgrade start ──
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = 'app_settings'
"""
)
(has_settings,) = cursor.fetchone()
if has_settings:
logger.info("app_settings table confirmed present.")
_migrate_config_ini_to_db(cursor, conn)
cursor.close()
except mysql.connector.Error as e:
logger.error(f"Database initialisation error: {e}")
raise
finally:
if conn:
conn.close()
# ─── app_settings DB helpers ──────────────────────────────────────────────────
# These replace all config.ini reads/writes for email, groq, and crypto settings.
# The DB connection settings remain in the local .db_config file (bootstrap only).
def _migrate_config_ini_to_db(cursor, conn):
"""
One-time migration: read email, groq, and crypto sections from config.ini
(if present) and upsert them into app_settings. This runs silently on every
startup but is a no-op once the keys exist in the DB or if config.ini is absent.
"""
import configparser as _cfgp, os as _os
_cfg_file = "config.ini"
if not _os.path.exists(_cfg_file):
return
cfg = _cfgp.ConfigParser()
cfg.read(_cfg_file, encoding="utf-8")
mappings = []
# email section
if cfg.has_section("email"):
for key in ("enabled", "smtp_host", "smtp_port", "smtp_user",
"smtp_password", "security", "use_tls", "recipients",
"send_time", "last_sent_date"):
val = cfg.get("email", key, fallback=None)
if val is not None:
mappings.append((f"email.{key}", val))
# groq section
if cfg.has_section("groq"):
for key in ("api_key", "model"):
val = cfg.get("groq", key, fallback=None)
if val is not None:
mappings.append((f"groq.{key}", val))
# crypto section (salt)
if cfg.has_section("crypto"):
val = cfg.get("crypto", "salt", fallback=None)
if val is not None:
mappings.append(("crypto.salt", val))
if not mappings:
return
for k, v in mappings:
cursor.execute(
"INSERT INTO app_settings (key_name, value) VALUES (%s, %s) "
"ON DUPLICATE KEY UPDATE value=value", # don't overwrite existing
(k, v),
)
conn.commit()
logger.info(f"Migrated {len(mappings)} setting(s) from config.ini to app_settings.")
def get_setting(key: str, default: str = "") -> str:
"""Read a value from the app_settings table. Returns default if not found."""
try:
conn = get_connection()
cur = conn.cursor()
cur.execute("SELECT value FROM app_settings WHERE key_name=%s", (key,))
row = cur.fetchone()
cur.close()
conn.close()
return row[0] if row and row[0] is not None else default
except Exception as e:
logger.warning(f"get_setting({key!r}) failed: {e}")
return default
def set_setting(key: str, value: str) -> None:
"""Upsert a key-value pair into app_settings."""
try:
conn = get_connection()
cur = conn.cursor()
cur.execute(
"INSERT INTO app_settings (key_name, value) VALUES (%s, %s) "
"ON DUPLICATE KEY UPDATE value=%s, updated_at=CURRENT_TIMESTAMP",
(key, value, value),
)
conn.commit()
cur.close()
conn.close()
except Exception as e:
logger.error(f"set_setting({key!r}) failed: {e}")
def get_settings_dict(prefix: str) -> dict:
"""Return all app_settings rows whose key starts with prefix as a plain dict."""
try:
conn = get_connection()
cur = conn.cursor()
cur.execute(
"SELECT key_name, value FROM app_settings WHERE key_name LIKE %s",
(prefix + "%",),
)
rows = cur.fetchall()
cur.close()
conn.close()
return {r[0]: (r[1] or "") for r in rows}
except Exception as e:
logger.warning(f"get_settings_dict({prefix!r}) failed: {e}")
return {}
def install(self):
"""Signal the worker that the DB pool is ready to accept writes."""
self._ready = True
def _worker(self):
"""Background thread: drain the queue into app_log."""
while True:
try:
record = self._queue.get(timeout=2)
except queue.Empty:
continue
if not self._ready:
# Put it back and wait — pool not yet initialised
self._queue.put(record)
threading.Event().wait(1)
continue
try:
conn = get_connection()
cur = conn.cursor()
cur.execute(
"INSERT INTO app_log (level, logger_name, message) "
"VALUES (%s, %s, %s)",
(record["level"], record["logger_name"], record["message"]),
)
conn.commit()
cur.close()
conn.close()
except Exception:
pass # Silently discard if DB is temporarily unavailable
db_log_handler = _DBLogHandler()
db_log_handler.setFormatter(_formatter)
logging.basicConfig(
level=logging.INFO,
handlers=[_stream_handler, db_log_handler],
)
logger = logging.getLogger("config")
# ─── Config File Load / Save ──────────────────────────────────────────────────
import configparser as _cp
import os as _os
CONFIG_FILE = "config.ini"
APP_TITLE = "Website Checker"
APP_VERSION = "1.0.0"
# Sensitive fields that are DPAPI-encrypted in config.ini
_DB_SENSITIVE = {"user", "password"}
def load_config() -> dict:
"""
Load DB settings from config.ini.
Returns a dict with keys: host, port, database, user, password.
Sensitive fields (user, password) are decrypted transparently via DPAPI.
Returns empty dict if the file does not exist or is incomplete.
"""
from utils.config_crypto import decrypt_value
cfg = _cp.ConfigParser()
if not _os.path.exists(CONFIG_FILE):
return {}
cfg.read(CONFIG_FILE, encoding="utf-8")
if "database" not in cfg:
return {}
section = cfg["database"]
try:
return {
"host": section.get("host", ""),
"port": section.getint("port", 3306),
"database": section.get("database", ""),
"user": decrypt_value(section.get("user", "")),
"password": decrypt_value(section.get("password", "")),
}
except RuntimeError as exc:
logger.error(f"Failed to decrypt DB credentials: {exc}")
raise
def save_config(host: str, port: int, database: str, user: str, password: str):
"""Persist DB connection settings to config.ini (sensitive fields DPAPI-encrypted)."""
from utils.config_crypto import encrypt_value
# Preserve any existing non-database sections (email, crypto, groq)
cfg = _cp.ConfigParser()
if _os.path.exists(CONFIG_FILE):
cfg.read(CONFIG_FILE, encoding="utf-8")
cfg["database"] = {
"host": host,
"port": str(port),
"database": database,
"user": encrypt_value(user),
"password": encrypt_value(password),
}
with open(CONFIG_FILE, "w", encoding="utf-8") as fh:
cfg.write(fh)
logger.info(f"Configuration saved to {CONFIG_FILE} (credentials encrypted).")
def migrate_plaintext_config():
"""
One-time migration: if config.ini contains plain-text DB credentials
(no 'dpapi:' prefix) encrypt them in-place using Windows DPAPI.
Safe to call on every startup — is a no-op when already encrypted.
"""
from utils.config_crypto import encrypt_value, is_encrypted
if not _os.path.exists(CONFIG_FILE):
return
cfg = _cp.ConfigParser()
cfg.read(CONFIG_FILE, encoding="utf-8")
changed = False
# DB section
for key in ("user", "password"):
if cfg.has_option("database", key):
raw = cfg.get("database", key)
if raw and not is_encrypted(raw):
cfg.set("database", key, encrypt_value(raw))
changed = True
# Email section
if cfg.has_option("email", "smtp_password"):
raw = cfg.get("email", "smtp_password")
if raw and not is_encrypted(raw):
cfg.set("email", "smtp_password", encrypt_value(raw))
changed = True
# Groq section
if cfg.has_option("groq", "api_key"):
raw = cfg.get("groq", "api_key")
if raw and not is_encrypted(raw):
cfg.set("groq", "api_key", encrypt_value(raw))
changed = True
if changed:
with open(CONFIG_FILE, "w", encoding="utf-8") as fh:
cfg.write(fh)
logger.info("config.ini: plain-text credentials encrypted with Windows DPAPI.")
def config_exists() -> bool:
"""Return True if config.ini contains all required connection fields."""
ini = load_config()
return bool(ini.get("host") and ini.get("database") and ini.get("user"))
def reload_db_config():
"""
Re-read config.ini and update DB_CONFIG in place.
Also resets the connection pool so the next get_connection() uses new creds.
"""
global _pool, DB_CONFIG
ini = load_config()
DB_CONFIG.update({
"host": ini.get("host", DB_CONFIG["host"]),
"port": ini.get("port", DB_CONFIG["port"]),
"database": ini.get("database", DB_CONFIG["database"]),
"user": ini.get("user", DB_CONFIG["user"]),
"password": ini.get("password", DB_CONFIG["password"]),
})
_pool = None # force pool recreation on next connection
logger.info("DB_CONFIG reloaded from config.ini.")
# ─── Database Configuration ───────────────────────────────────────────────────
# Populated from config.ini at runtime; falls back to placeholder strings so
# the module is importable even before first-run setup has completed.
_ini = load_config()
DB_CONFIG = {
"host": _ini.get("host", "your-mysql-host"),
"port": _ini.get("port", 3306),
"database": _ini.get("database", "website_checker"),
"user": _ini.get("user", "your-db-user"),
"password": _ini.get("password", "your-db-password"),
"connection_timeout": 10,
}
# ─── Connection Pool ──────────────────────────────────────────────────────────
_pool = None
def get_connection_pool():
global _pool
if _pool is None:
try:
_pool = pooling.MySQLConnectionPool(
pool_name="app_pool",
pool_size=5,
**DB_CONFIG
)
logger.info("Database connection pool initialised.")
except mysql.connector.Error as e:
logger.error(f"Failed to create connection pool: {e}")
raise
return _pool
def get_connection():
"""Return a connection from the pool."""
return get_connection_pool().get_connection()
def initialize_database():
"""Create all required tables if they do not exist."""
ddl_statements = [
"""
CREATE TABLE IF NOT EXISTS users (
id INT AUTO_INCREMENT PRIMARY KEY,
username VARCHAR(100) NOT NULL UNIQUE,
password VARCHAR(255) NOT NULL,
role ENUM('admin','user') NOT NULL DEFAULT 'user',
full_name VARCHAR(200),
is_active TINYINT(1) NOT NULL DEFAULT 1,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS websites (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(200) NOT NULL,
url TEXT NOT NULL,
check_type ENUM('daily','weekly') NOT NULL DEFAULT 'daily',
visibility ENUM('all','assigned') NOT NULL DEFAULT 'all',
note TEXT,
is_active TINYINT(1) NOT NULL DEFAULT 1,
created_by INT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS website_credentials (
id INT AUTO_INCREMENT PRIMARY KEY,
website_id INT NOT NULL,
label VARCHAR(100),
username VARCHAR(200),
password TEXT,
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS shift_checks (
id INT AUTO_INCREMENT PRIMARY KEY,
website_id INT NOT NULL,
user_id INT NOT NULL,
checked_at DATETIME DEFAULT CURRENT_TIMESTAMP,
user_note TEXT,
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS activity_log (
id INT AUTO_INCREMENT PRIMARY KEY,
user_id INT,
action VARCHAR(100) NOT NULL,
entity VARCHAR(100),
entity_id INT,
detail TEXT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS shifts (
id INT AUTO_INCREMENT PRIMARY KEY,
name VARCHAR(200) NOT NULL,
days_of_week VARCHAR(7) NOT NULL DEFAULT '23456',
start_time TIME NOT NULL DEFAULT '08:00:00',
end_time TIME NOT NULL DEFAULT '17:00:00',
is_active TINYINT(1) NOT NULL DEFAULT 1,
note TEXT,
created_by INT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS shift_users (
shift_id INT NOT NULL,
user_id INT NOT NULL,
PRIMARY KEY (shift_id, user_id),
FOREIGN KEY (shift_id) REFERENCES shifts(id) ON DELETE CASCADE,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS shift_websites (
shift_id INT NOT NULL,
website_id INT NOT NULL,
sort_order INT NOT NULL DEFAULT 0,
PRIMARY KEY (shift_id, website_id),
FOREIGN KEY (shift_id) REFERENCES shifts(id) ON DELETE CASCADE,
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS login_attempts (
id INT AUTO_INCREMENT PRIMARY KEY,
username VARCHAR(100) NOT NULL,
attempted_at DATETIME DEFAULT CURRENT_TIMESTAMP,
ip_address VARCHAR(45),
INDEX idx_username_time (username, attempted_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS website_users (
website_id INT NOT NULL,
user_id INT NOT NULL,
PRIMARY KEY (website_id, user_id),
FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS ai_analysis_log (
id INT AUTO_INCREMENT PRIMARY KEY,
user_id INT,
file_names TEXT NOT NULL,
model VARCHAR(100) NOT NULL,
verdict ENUM('PURSUE','PASS','UNCLEAR') NULL,
criteria_snapshot TEXT NULL,
summary_text MEDIUMTEXT NOT NULL,
analyzed_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS ai_criteria (
id INT AUTO_INCREMENT PRIMARY KEY,
title VARCHAR(200) NOT NULL,
description TEXT NOT NULL,
is_active TINYINT(1) NOT NULL DEFAULT 1,
sort_order INT NOT NULL DEFAULT 0,
created_by INT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS app_log (
id INT AUTO_INCREMENT PRIMARY KEY,
level VARCHAR(10) NOT NULL,
logger_name VARCHAR(100) NOT NULL,
message TEXT NOT NULL,
logged_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3),
INDEX idx_app_log_level (level),
INDEX idx_app_log_time (logged_at)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS bid_tracker (
id INT AUTO_INCREMENT PRIMARY KEY,
title VARCHAR(300) NOT NULL,
url TEXT NOT NULL,
source VARCHAR(200) NULL,
solicitation_number VARCHAR(100) NULL,
status ENUM('open','monitoring','awarded','no_bid','cancelled')
NOT NULL DEFAULT 'open',
due_date DATE NULL,
notes TEXT NULL,
added_by INT,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
FOREIGN KEY (added_by) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
"""
CREATE TABLE IF NOT EXISTS bid_updates (
id INT AUTO_INCREMENT PRIMARY KEY,
bid_id INT NOT NULL,
user_id INT,
content TEXT NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (bid_id) REFERENCES bid_tracker(id) ON DELETE CASCADE,
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
""",
]
conn = None
try:
conn = get_connection()
cursor = conn.cursor()
for stmt in ddl_statements:
cursor.execute(stmt)
conn.commit()
logger.info("Database schema initialised successfully.")
# ── Safe migrations for existing deployments ───────────────────────
# Add check_type column to websites if it doesn't exist yet
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'websites'
AND COLUMN_NAME = 'check_type'
"""
)
(has_col,) = cursor.fetchone()
if not has_col:
cursor.execute(
"ALTER TABLE websites ADD COLUMN check_type ENUM('daily','weekly') "
"NOT NULL DEFAULT 'daily' AFTER url"
)
conn.commit()
logger.info("Migration: added check_type column to websites table.")
# Add failed_attempts / locked_until columns to users if absent
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'users'
AND COLUMN_NAME = 'failed_attempts'
"""
)
(has_fa,) = cursor.fetchone()
if not has_fa:
cursor.execute(
"ALTER TABLE users "
"ADD COLUMN failed_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0 AFTER is_active, "
"ADD COLUMN locked_until DATETIME NULL AFTER failed_attempts"
)
conn.commit()
logger.info("Migration: added failed_attempts and locked_until columns to users table.")
# Add visibility column to websites if absent
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.COLUMNS
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = 'websites'
AND COLUMN_NAME = 'visibility'
"""
)
(has_vis,) = cursor.fetchone()
if not has_vis:
cursor.execute(
"ALTER TABLE websites ADD COLUMN visibility "
"ENUM('all','assigned') NOT NULL DEFAULT 'all' AFTER check_type"
)
conn.commit()
logger.info("Migration: added visibility column to websites table.")
# Confirm presence of tables added post-initial-deployment
for new_table in ("ai_criteria", "ai_analysis_log", "app_log", "bid_tracker", "bid_updates"):
cursor.execute(
"""
SELECT COUNT(*) FROM information_schema.TABLES
WHERE TABLE_SCHEMA = DATABASE()
AND TABLE_NAME = %s
""",
(new_table,)
)
(table_exists,) = cursor.fetchone()
if table_exists:
logger.info(f"Table '{new_table}' confirmed present.")
else:
logger.warning(f"Table '{new_table}' was not created -- check DDL.")
# Seed default admin if users table is empty
cursor.execute("SELECT COUNT(*) FROM users")
(count,) = cursor.fetchone()
if count == 0:
import bcrypt as _bcrypt
default_pw = _bcrypt.hashpw(b"admin123", _bcrypt.gensalt(rounds=12)).decode("utf-8")
cursor.execute(
"INSERT INTO users (username, password, role, full_name) VALUES (%s,%s,'admin','System Admin')",
("admin", default_pw)
)
conn.commit()
logger.info("Default admin account seeded (username: admin / password: admin123).")
cursor.close()
except mysql.connector.Error as e:
logger.error(f"Database initialisation error: {e}")
raise
finally:
if conn:
conn.close()
Reference in New Issue View Git Blame Copy Permalink