diff --git a/app.py b/app.py index 1c46cdc..b95ddfa 100644 --- a/app.py +++ b/app.py @@ -68,6 +68,13 @@ class App(tk.Tk): """Initialise database schema, then proceed to login.""" try: initialize_database() + # Signal the DB log handler that the pool is ready so queued + # startup log records are flushed to app_log immediately. + try: + from config import db_log_handler + db_log_handler.install() + except Exception: + pass except Exception as e: messagebox.showerror( "Database Error", @@ -148,28 +155,37 @@ class App(tk.Tk): self._nav_buttons = {} self._active_section = None + # nav_groups is a list of (group_label, items) tuples. + # group_label=None means no section header — used for the user role. + # A separator is rendered between every group automatically. if self.current_user["role"] == "admin": - nav_items = [ - ("🏠 Dashboard", "dashboard", self._show_dashboard), - ("👤 Users", "users", self._show_users), - ("🌐 Websites", "websites", self._show_websites), - ("🗓 Shifts", "shift_mgmt", self._show_shift_mgmt), - ("📋 Activity Log", "log", self._show_log), - ("📊 My Shifts", "shifts", self._show_shifts), - ("📑 Reports", "reports", self._show_reports), - ("📧 Email Reports","email", self._show_email_settings), - ("🤖 AI Summary", "ai_summary", self._show_ai_summary), - ("📌 Bid Tracker", "bid_tracker", self._show_bid_tracker), - ("⚙ Settings", "settings", self._show_settings), + nav_groups = [ + ("MANAGEMENT", [ + ("🏠 Dashboard", "dashboard", self._show_dashboard), + ("👤 Users", "users", self._show_users), + ("🌐 Websites", "websites", self._show_websites), + ("🗓 Shifts", "shift_mgmt", self._show_shift_mgmt), + ("📑 Reports", "reports", self._show_reports), + ("📋 Activity Log", "log", self._show_log), + ("📧 Email Reports", "email", self._show_email_settings), + ("⚙ Settings", "settings", self._show_settings), + ]), + ("MY WORKSPACE", [ + ("📊 My Shifts", "shifts", self._show_shifts), + ("🤖 AI Summary", "ai_summary", self._show_ai_summary), + ("📌 Bid Tracker", "bid_tracker", self._show_bid_tracker), + ]), ] else: - nav_items = [ - ("📊 My Shifts", "shifts", self._show_shifts), - ("🤖 AI Summary", "ai_summary", self._show_ai_summary), - ("📌 Bid Tracker", "bid_tracker", self._show_bid_tracker), + nav_groups = [ + (None, [ + ("📊 My Shifts", "shifts", self._show_shifts), + ("🤖 AI Summary", "ai_summary", self._show_ai_summary), + ("📌 Bid Tracker", "bid_tracker", self._show_bid_tracker), + ]), ] - for label, key, cmd in nav_items: + def _make_nav_btn(label, key, cmd): btn = tk.Button( sidebar, text=label, @@ -182,12 +198,34 @@ class App(tk.Tk): anchor="w", font=FONT_BOLD, padx=20, - pady=12, + pady=10, cursor="hand2", ) btn.pack(fill="x") self._nav_buttons[key] = btn + for i, (group_label, items) in enumerate(nav_groups): + # Separator between groups (not before the first one) + if i > 0: + ttk.Separator(sidebar, orient="horizontal").pack( + fill="x", pady=4) + + # Section header label (skip for user role where label is None) + if group_label: + tk.Label( + sidebar, + text=group_label, + bg=COLOURS["surface"], + fg=COLOURS["text_dim"], + font=(FONT_SMALL[0], 8, "bold"), + anchor="w", + padx=20, + pady=4, + ).pack(fill="x") + + for label, key, cmd in items: + _make_nav_btn(label, key, cmd) + # Spacer + user info at bottom ttk.Separator(sidebar, orient="horizontal").pack(fill="x", side="bottom", pady=8) user_frame = tk.Frame(sidebar, bg=COLOURS["surface"], pady=10) diff --git a/config.ini b/config.ini index 22925d1..4d72d39 100644 --- a/config.ini +++ b/config.ini @@ -2,8 +2,8 @@ host = 67.217.62.199 port = 3306 database = webchecker -user = dpapi:AQAAANCMnd8BFdERjHoAwE/Cl+sBAAAAtjFleFYUHkGTELQRhDYe8AAAAAAkAAAAVwBlAGIAQwBoAGUAYwBrAGUAcgAgAGMAbwBuAGYAaQBnAAAAEGYAAAABAAAgAAAAm1sPKS5A3aczt5vkmqeO+XXJLmKwA1ojVXeBzQZ9QIwAAAAADoAAAAACAAAgAAAA6CLkACCqBm32SUXekn49rHMseysgdxMtdhywY0+zxO4QAAAA6KGNWrsRGwpImEXQZfuQikAAAABHFvkj2Z2choaPIoFwNryk31E6oZfI11uv/Ms5eIrN0uSC9VSdjYC9IU7SKCkK2z7BYQB2Uy9QoXZF6VFDXE/r -password = dpapi:AQAAANCMnd8BFdERjHoAwE/Cl+sBAAAAtjFleFYUHkGTELQRhDYe8AAAAAAkAAAAVwBlAGIAQwBoAGUAYwBrAGUAcgAgAGMAbwBuAGYAaQBnAAAAEGYAAAABAAAgAAAAO3bOCycWnlLWFscvK6zmWysxARgAyABh7eIYDzK/KN8AAAAADoAAAAACAAAgAAAAFU2hwhMfmMnWGTZL16cWfpc2rs5/Oy1lq7aEqeC5GcUQAAAAExqVIbNsn/vH99tw8NMfBUAAAAA+pf+tJVrYOMikcjrPcFLRKS+dR6+4OGBj/BiM4IgR07Zbk8GNxddhdb9Hg8/2D51oaywFNV55lDd9aJcrtnHS +user = dpapi:AQAAANCMnd8BFdERjHoAwE/Cl+sBAAAAtjFleFYUHkGTELQRhDYe8AAAAAAkAAAAVwBlAGIAQwBoAGUAYwBrAGUAcgAgAGMAbwBuAGYAaQBnAAAAEGYAAAABAAAgAAAAYidUx+/EF8dBvkm7qVrCTNghUdDMslBCZBBTjfFXnuIAAAAADoAAAAACAAAgAAAArWRcoWRv2DWdaf+Np9mP3VUerkem6fgxbPN9RP6vE+sQAAAA+IYMqcdW0pBoZ5HdhykWrEAAAADsUqFCIW6brBYISrc+XdSS0WqYBBP3jvoBXUuAtmyhI4g8j9C+R1y27Jd/VUy/5hybmayzzeAsj2a+dK+rr8yR +password = dpapi:AQAAANCMnd8BFdERjHoAwE/Cl+sBAAAAtjFleFYUHkGTELQRhDYe8AAAAAAkAAAAVwBlAGIAQwBoAGUAYwBrAGUAcgAgAGMAbwBuAGYAaQBnAAAAEGYAAAABAAAgAAAAykVXkmPXEYXlFlNlaFvKD2SdOE+KF+FwGjxyIua1vlUAAAAADoAAAAACAAAgAAAAlUEcvqr1nvxQ2liPnoA7aJc5mqZ9OpZ69l1YctCrhxcQAAAAmEg7tJjEkA+4Eg2jzvP+7EAAAADHWK0G9UJn4ZmNlsMB4WQtZduNzSeIoT/TuKB8X7hNMASwz9wXTw/KxX2iNi9C0Tg3JgqaWjv/rLJQeJF8MeTF [crypto] salt = GE453Sa0afGLfzq8EPKdXjvZyAzK499XzWxWq/ZdnFQ= diff --git a/config.py b/config.py index 8d77280..5d7a1dd 100644 --- a/config.py +++ b/config.py @@ -9,27 +9,649 @@ import logging # ─── Logging Setup ──────────────────────────────────────────────────────────── import sys +import queue +import threading -_file_handler = logging.FileHandler("app.log", encoding="utf-8") +_formatter = logging.Formatter( + "%(asctime)s [%(levelname)s] %(name)s - %(message)s" +) + +# ── Console handler (development / stdout) ──────────────────────────────────── _stream_handler = logging.StreamHandler(stream=sys.stdout) -# Force UTF-8 on the stream so Windows cp1252 consoles don't choke on -# Unicode characters (arrows, em-dashes, etc.) in log messages. +# Force UTF-8 on Windows cp1252 consoles so Unicode chars don't raise errors. if hasattr(_stream_handler.stream, "reconfigure"): try: _stream_handler.stream.reconfigure(encoding="utf-8", errors="replace") except Exception: pass +_stream_handler.setFormatter(_formatter) + +# ── Database log handler ─────────────────────────────────────────────────────── +# Writes every log record to the app_log table asynchronously via a background +# thread so DB I/O never blocks the logging call or the UI. +# The handler is installed as soon as the pool is ready (after initialize_database). +# Before the pool is ready, records are queued and flushed on first install. + +class _DBLogHandler(logging.Handler): + """ + Asynchronous logging handler that inserts records into app_log. + Uses a daemon thread + Queue to keep DB writes off the main thread. + Safe to create before the DB pool exists — records are queued until + install() is called after initialize_database() succeeds. + """ + + def __init__(self): + super().__init__() + self._queue = queue.Queue() + self._ready = False # True once the pool is confirmed available + self._thread = threading.Thread( + target=self._worker, name="DBLogWorker", daemon=True) + self._thread.start() + + def emit(self, record: logging.LogRecord): + # Never log our own worker thread to avoid infinite recursion + if record.name in ("config", "mysql.connector"): + return + try: + self._queue.put_nowait({ + "level": record.levelname, + "logger_name": record.name[:100], + "message": self.format(record), + }) + except Exception: + pass +""" +config.py — Application configuration and DB connection manager. +Update DB_CONFIG with your remote MySQL server credentials. +""" + +import mysql.connector +from mysql.connector import pooling +import logging + +# ─── Logging Setup ──────────────────────────────────────────────────────────── +import sys +import queue +import threading + _formatter = logging.Formatter( "%(asctime)s [%(levelname)s] %(name)s - %(message)s" ) -_file_handler.setFormatter(_formatter) + +# ── Console handler (development / stdout) ──────────────────────────────────── +_stream_handler = logging.StreamHandler(stream=sys.stdout) + +# Force UTF-8 on Windows cp1252 consoles so Unicode chars don't raise errors. +if hasattr(_stream_handler.stream, "reconfigure"): + try: + _stream_handler.stream.reconfigure(encoding="utf-8", errors="replace") + except Exception: + pass + _stream_handler.setFormatter(_formatter) +# ── Database log handler ─────────────────────────────────────────────────────── +# Writes every log record to the app_log table asynchronously via a background +# thread so DB I/O never blocks the logging call or the UI. +# The handler is installed as soon as the pool is ready (after initialize_database). +# Before the pool is ready, records are queued and flushed on first install. + +class _DBLogHandler(logging.Handler): + """ + Asynchronous logging handler that inserts records into app_log. + Uses a daemon thread + Queue to keep DB writes off the main thread. + Safe to create before the DB pool exists — records are queued until + install() is called after initialize_database() succeeds. + """ + + def __init__(self): + super().__init__() + self._queue = queue.Queue() + self._ready = False # True once the pool is confirmed available + self._thread = threading.Thread( + target=self._worker, name="DBLogWorker", daemon=True) + self._thread.start() + + def emit(self, record: logging.LogRecord): + # Never log our own worker thread to avoid infinite recursion + if record.name in ("config", "mysql.connector"): + return + try: + self._queue.put_nowait({ + "level": record.levelname, + "logger_name": record.name[:100], + "message": self.format(record), + }) + except Exception: + pass + + def install(self): + """Signal the worker that the DB pool is ready to accept writes.""" + self._ready = True + + def _worker(self): + """Background thread: drain the queue into app_log.""" + while True: + try: + record = self._queue.get(timeout=2) + except queue.Empty: + continue + + if not self._ready: + # Put it back and wait — pool not yet initialised + self._queue.put(record) + threading.Event().wait(1) + continue + + try: + conn = get_connection() + cur = conn.cursor() + cur.execute( + "INSERT INTO app_log (level, logger_name, message) " + "VALUES (%s, %s, %s)", + (record["level"], record["logger_name"], record["message"]), + ) + conn.commit() + cur.close() + conn.close() + except Exception: + pass # Silently discard if DB is temporarily unavailable + + +db_log_handler = _DBLogHandler() +db_log_handler.setFormatter(_formatter) + logging.basicConfig( level=logging.INFO, - handlers=[_file_handler, _stream_handler], + handlers=[_stream_handler, db_log_handler], +) +logger = logging.getLogger("config") + +# ─── Config File Load / Save ────────────────────────────────────────────────── +import configparser as _cp +import os as _os + +CONFIG_FILE = "config.ini" +APP_TITLE = "Website Checker" +APP_VERSION = "1.0.0" + + +# Sensitive fields that are DPAPI-encrypted in config.ini +_DB_SENSITIVE = {"user", "password"} + + +def load_config() -> dict: + """ + Load DB settings from config.ini. + Returns a dict with keys: host, port, database, user, password. + Sensitive fields (user, password) are decrypted transparently via DPAPI. + Returns empty dict if the file does not exist or is incomplete. + """ + from utils.config_crypto import decrypt_value + cfg = _cp.ConfigParser() + if not _os.path.exists(CONFIG_FILE): + return {} + cfg.read(CONFIG_FILE, encoding="utf-8") + if "database" not in cfg: + return {} + section = cfg["database"] + try: + return { + "host": section.get("host", ""), + "port": section.getint("port", 3306), + "database": section.get("database", ""), + "user": decrypt_value(section.get("user", "")), + "password": decrypt_value(section.get("password", "")), + } + except RuntimeError as exc: + logger.error(f"Failed to decrypt DB credentials: {exc}") + raise + + +def save_config(host: str, port: int, database: str, user: str, password: str): + """Persist DB connection settings to config.ini (sensitive fields DPAPI-encrypted).""" + from utils.config_crypto import encrypt_value + # Preserve any existing non-database sections (email, crypto, groq) + cfg = _cp.ConfigParser() + if _os.path.exists(CONFIG_FILE): + cfg.read(CONFIG_FILE, encoding="utf-8") + cfg["database"] = { + "host": host, + "port": str(port), + "database": database, + "user": encrypt_value(user), + "password": encrypt_value(password), + } + with open(CONFIG_FILE, "w", encoding="utf-8") as fh: + cfg.write(fh) + logger.info(f"Configuration saved to {CONFIG_FILE} (credentials encrypted).") + + +def migrate_plaintext_config(): + """ + One-time migration: if config.ini contains plain-text DB credentials + (no 'dpapi:' prefix) encrypt them in-place using Windows DPAPI. + Safe to call on every startup — is a no-op when already encrypted. + """ + from utils.config_crypto import encrypt_value, is_encrypted + if not _os.path.exists(CONFIG_FILE): + return + cfg = _cp.ConfigParser() + cfg.read(CONFIG_FILE, encoding="utf-8") + changed = False + + # DB section + for key in ("user", "password"): + if cfg.has_option("database", key): + raw = cfg.get("database", key) + if raw and not is_encrypted(raw): + cfg.set("database", key, encrypt_value(raw)) + changed = True + + # Email section + if cfg.has_option("email", "smtp_password"): + raw = cfg.get("email", "smtp_password") + if raw and not is_encrypted(raw): + cfg.set("email", "smtp_password", encrypt_value(raw)) + changed = True + + # Groq section + if cfg.has_option("groq", "api_key"): + raw = cfg.get("groq", "api_key") + if raw and not is_encrypted(raw): + cfg.set("groq", "api_key", encrypt_value(raw)) + changed = True + + if changed: + with open(CONFIG_FILE, "w", encoding="utf-8") as fh: + cfg.write(fh) + logger.info("config.ini: plain-text credentials encrypted with Windows DPAPI.") + + +def config_exists() -> bool: + """Return True if config.ini contains all required connection fields.""" + ini = load_config() + return bool(ini.get("host") and ini.get("database") and ini.get("user")) + + +def reload_db_config(): + """ + Re-read config.ini and update DB_CONFIG in place. + Also resets the connection pool so the next get_connection() uses new creds. + """ + global _pool, DB_CONFIG + ini = load_config() + DB_CONFIG.update({ + "host": ini.get("host", DB_CONFIG["host"]), + "port": ini.get("port", DB_CONFIG["port"]), + "database": ini.get("database", DB_CONFIG["database"]), + "user": ini.get("user", DB_CONFIG["user"]), + "password": ini.get("password", DB_CONFIG["password"]), + }) + _pool = None # force pool recreation on next connection + logger.info("DB_CONFIG reloaded from config.ini.") + + +# ─── Database Configuration ─────────────────────────────────────────────────── +# Populated from config.ini at runtime; falls back to placeholder strings so +# the module is importable even before first-run setup has completed. +_ini = load_config() +DB_CONFIG = { + "host": _ini.get("host", "your-mysql-host"), + "port": _ini.get("port", 3306), + "database": _ini.get("database", "website_checker"), + "user": _ini.get("user", "your-db-user"), + "password": _ini.get("password", "your-db-password"), + "connection_timeout": 10, +} + +# ─── Connection Pool ────────────────────────────────────────────────────────── +_pool = None + +def get_connection_pool(): + global _pool + if _pool is None: + try: + _pool = pooling.MySQLConnectionPool( + pool_name="app_pool", + pool_size=5, + **DB_CONFIG + ) + logger.info("Database connection pool initialised.") + except mysql.connector.Error as e: + logger.error(f"Failed to create connection pool: {e}") + raise + return _pool + + +def get_connection(): + """Return a connection from the pool.""" + return get_connection_pool().get_connection() + + +def initialize_database(): + """Create all required tables if they do not exist.""" + ddl_statements = [ + """ + CREATE TABLE IF NOT EXISTS users ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(100) NOT NULL UNIQUE, + password VARCHAR(255) NOT NULL, + role ENUM('admin','user') NOT NULL DEFAULT 'user', + full_name VARCHAR(200), + is_active TINYINT(1) NOT NULL DEFAULT 1, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS websites ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(200) NOT NULL, + url TEXT NOT NULL, + check_type ENUM('daily','weekly') NOT NULL DEFAULT 'daily', + visibility ENUM('all','assigned') NOT NULL DEFAULT 'all', + note TEXT, + is_active TINYINT(1) NOT NULL DEFAULT 1, + created_by INT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS website_credentials ( + id INT AUTO_INCREMENT PRIMARY KEY, + website_id INT NOT NULL, + label VARCHAR(100), + username VARCHAR(200), + password TEXT, + FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS shift_checks ( + id INT AUTO_INCREMENT PRIMARY KEY, + website_id INT NOT NULL, + user_id INT NOT NULL, + checked_at DATETIME DEFAULT CURRENT_TIMESTAMP, + user_note TEXT, + FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS activity_log ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + action VARCHAR(100) NOT NULL, + entity VARCHAR(100), + entity_id INT, + detail TEXT, + logged_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS shifts ( + id INT AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(200) NOT NULL, + days_of_week VARCHAR(7) NOT NULL DEFAULT '23456', + start_time TIME NOT NULL DEFAULT '08:00:00', + end_time TIME NOT NULL DEFAULT '17:00:00', + is_active TINYINT(1) NOT NULL DEFAULT 1, + note TEXT, + created_by INT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS shift_users ( + shift_id INT NOT NULL, + user_id INT NOT NULL, + PRIMARY KEY (shift_id, user_id), + FOREIGN KEY (shift_id) REFERENCES shifts(id) ON DELETE CASCADE, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS shift_websites ( + shift_id INT NOT NULL, + website_id INT NOT NULL, + sort_order INT NOT NULL DEFAULT 0, + PRIMARY KEY (shift_id, website_id), + FOREIGN KEY (shift_id) REFERENCES shifts(id) ON DELETE CASCADE, + FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS login_attempts ( + id INT AUTO_INCREMENT PRIMARY KEY, + username VARCHAR(100) NOT NULL, + attempted_at DATETIME DEFAULT CURRENT_TIMESTAMP, + ip_address VARCHAR(45), + INDEX idx_username_time (username, attempted_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS website_users ( + website_id INT NOT NULL, + user_id INT NOT NULL, + PRIMARY KEY (website_id, user_id), + FOREIGN KEY (website_id) REFERENCES websites(id) ON DELETE CASCADE, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS ai_analysis_log ( + id INT AUTO_INCREMENT PRIMARY KEY, + user_id INT, + file_names TEXT NOT NULL, + model VARCHAR(100) NOT NULL, + verdict ENUM('PURSUE','PASS','UNCLEAR') NULL, + criteria_snapshot TEXT NULL, + summary_text MEDIUMTEXT NOT NULL, + analyzed_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS ai_criteria ( + id INT AUTO_INCREMENT PRIMARY KEY, + title VARCHAR(200) NOT NULL, + description TEXT NOT NULL, + is_active TINYINT(1) NOT NULL DEFAULT 1, + sort_order INT NOT NULL DEFAULT 0, + created_by INT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS app_log ( + id INT AUTO_INCREMENT PRIMARY KEY, + level VARCHAR(10) NOT NULL, + logger_name VARCHAR(100) NOT NULL, + message TEXT NOT NULL, + logged_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3), + INDEX idx_app_log_level (level), + INDEX idx_app_log_time (logged_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS bid_tracker ( + id INT AUTO_INCREMENT PRIMARY KEY, + title VARCHAR(300) NOT NULL, + url TEXT NOT NULL, + source VARCHAR(200) NULL, + solicitation_number VARCHAR(100) NULL, + status ENUM('open','monitoring','awarded','no_bid','cancelled') + NOT NULL DEFAULT 'open', + due_date DATE NULL, + notes TEXT NULL, + added_by INT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + FOREIGN KEY (added_by) REFERENCES users(id) ON DELETE SET NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ + CREATE TABLE IF NOT EXISTS bid_updates ( + id INT AUTO_INCREMENT PRIMARY KEY, + bid_id INT NOT NULL, + user_id INT, + content TEXT NOT NULL, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (bid_id) REFERENCES bid_tracker(id) ON DELETE CASCADE, + FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE SET NULL + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + ] + + conn = None + try: + conn = get_connection() + cursor = conn.cursor() + for stmt in ddl_statements: + cursor.execute(stmt) + conn.commit() + logger.info("Database schema initialised successfully.") + + # ── Safe migrations for existing deployments ─────────────────────── + # Add check_type column to websites if it doesn't exist yet + cursor.execute( + """ + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'websites' + AND COLUMN_NAME = 'check_type' + """ + ) + (has_col,) = cursor.fetchone() + if not has_col: + cursor.execute( + "ALTER TABLE websites ADD COLUMN check_type ENUM('daily','weekly') " + "NOT NULL DEFAULT 'daily' AFTER url" + ) + conn.commit() + logger.info("Migration: added check_type column to websites table.") + + # Add failed_attempts / locked_until columns to users if absent + cursor.execute( + """ + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'users' + AND COLUMN_NAME = 'failed_attempts' + """ + ) + (has_fa,) = cursor.fetchone() + if not has_fa: + cursor.execute( + "ALTER TABLE users " + "ADD COLUMN failed_attempts TINYINT UNSIGNED NOT NULL DEFAULT 0 AFTER is_active, " + "ADD COLUMN locked_until DATETIME NULL AFTER failed_attempts" + ) + conn.commit() + logger.info("Migration: added failed_attempts and locked_until columns to users table.") + + # Add visibility column to websites if absent + cursor.execute( + """ + SELECT COUNT(*) FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'websites' + AND COLUMN_NAME = 'visibility' + """ + ) + (has_vis,) = cursor.fetchone() + if not has_vis: + cursor.execute( + "ALTER TABLE websites ADD COLUMN visibility " + "ENUM('all','assigned') NOT NULL DEFAULT 'all' AFTER check_type" + ) + conn.commit() + logger.info("Migration: added visibility column to websites table.") + + # Confirm presence of tables added post-initial-deployment + for new_table in ("ai_criteria", "ai_analysis_log", "app_log", "bid_tracker", "bid_updates"): + cursor.execute( + """ + SELECT COUNT(*) FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = %s + """, + (new_table,) + ) + (table_exists,) = cursor.fetchone() + if table_exists: + logger.info(f"Table '{new_table}' confirmed present.") + else: + logger.warning(f"Table '{new_table}' was not created -- check DDL.") + + # Seed default admin if users table is empty + cursor.execute("SELECT COUNT(*) FROM users") + (count,) = cursor.fetchone() + if count == 0: + import bcrypt as _bcrypt + default_pw = _bcrypt.hashpw(b"admin123", _bcrypt.gensalt(rounds=12)).decode("utf-8") + cursor.execute( + "INSERT INTO users (username, password, role, full_name) VALUES (%s,%s,'admin','System Admin')", + ("admin", default_pw) + ) + conn.commit() + logger.info("Default admin account seeded (username: admin / password: admin123).") + cursor.close() + except mysql.connector.Error as e: + logger.error(f"Database initialisation error: {e}") + raise + finally: + if conn: + conn.close() + def install(self): + """Signal the worker that the DB pool is ready to accept writes.""" + self._ready = True + + def _worker(self): + """Background thread: drain the queue into app_log.""" + while True: + try: + record = self._queue.get(timeout=2) + except queue.Empty: + continue + + if not self._ready: + # Put it back and wait — pool not yet initialised + self._queue.put(record) + threading.Event().wait(1) + continue + + try: + conn = get_connection() + cur = conn.cursor() + cur.execute( + "INSERT INTO app_log (level, logger_name, message) " + "VALUES (%s, %s, %s)", + (record["level"], record["logger_name"], record["message"]), + ) + conn.commit() + cur.close() + conn.close() + except Exception: + pass # Silently discard if DB is temporarily unavailable + + +db_log_handler = _DBLogHandler() +db_log_handler.setFormatter(_formatter) + +logging.basicConfig( + level=logging.INFO, + handlers=[_stream_handler, db_log_handler], ) logger = logging.getLogger("config") @@ -337,6 +959,17 @@ def initialize_database(): ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; """, """ + CREATE TABLE IF NOT EXISTS app_log ( + id INT AUTO_INCREMENT PRIMARY KEY, + level VARCHAR(10) NOT NULL, + logger_name VARCHAR(100) NOT NULL, + message TEXT NOT NULL, + logged_at DATETIME(3) DEFAULT CURRENT_TIMESTAMP(3), + INDEX idx_app_log_level (level), + INDEX idx_app_log_time (logged_at) + ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + """, + """ CREATE TABLE IF NOT EXISTS bid_tracker ( id INT AUTO_INCREMENT PRIMARY KEY, title VARCHAR(300) NOT NULL, @@ -432,7 +1065,7 @@ def initialize_database(): logger.info("Migration: added visibility column to websites table.") # Confirm presence of tables added post-initial-deployment - for new_table in ("ai_criteria", "ai_analysis_log", "bid_tracker", "bid_updates"): + for new_table in ("ai_criteria", "ai_analysis_log", "app_log", "bid_tracker", "bid_updates"): cursor.execute( """ SELECT COUNT(*) FROM information_schema.TABLES diff --git a/models.py b/models.py index 0552e6d..f06e69f 100644 --- a/models.py +++ b/models.py @@ -805,20 +805,47 @@ def update_check_note(user_id, website_id, user_note): conn.close() -def get_activity_log(limit=200): +def get_activity_log(limit=200, search: str = ""): + """ + Return recent activity_log entries, newest first. + The DB column is created_at; we alias it to logged_at for a consistent + key across both log tables so the view layer never needs to care. + """ conn = None try: conn = get_connection() cur = conn.cursor(dictionary=True) - cur.execute( + + where = "" + params: list = [] + if search: + where = """ + WHERE al.action LIKE %s + OR u.username LIKE %s + OR al.entity LIKE %s + OR al.detail LIKE %s """ - SELECT al.*, u.username + like = f"%{search}%" + params.extend([like, like, like, like]) + params.append(limit) + + cur.execute( + f""" + SELECT al.id, + al.user_id, + al.action, + al.entity, + al.entity_id, + al.detail, + al.logged_at, + u.username FROM activity_log al LEFT JOIN users u ON u.id = al.user_id + {where} ORDER BY al.logged_at DESC LIMIT %s """, - (limit,) + params, ) rows = cur.fetchall() cur.close() @@ -2113,3 +2140,73 @@ def delete_bid_update(user_id: int, update_id: int): finally: if conn: conn.close() + + +# ─── Application Log (DB-backed logging) ───────────────────────────────────── + +def get_app_log(limit: int = 500, level_filter: str = "", + search: str = "") -> list: + """ + Return recent application log entries from app_log. + level_filter : one of DEBUG/INFO/WARNING/ERROR/CRITICAL — empty = all + search : substring match against logger_name or message + """ + conn = None + try: + conn = get_connection() + cur = conn.cursor(dictionary=True) + + conditions = [] + params: list = [] + + if level_filter: + conditions.append("level = %s") + params.append(level_filter) + if search: + conditions.append("(logger_name LIKE %s OR message LIKE %s)") + like = f"%{search}%" + params.extend([like, like]) + + where = ("WHERE " + " AND ".join(conditions)) if conditions else "" + params.append(limit) + + cur.execute( + f""" + SELECT id, level, logger_name, message, logged_at + FROM app_log + {where} + ORDER BY logged_at DESC + LIMIT %s + """, + params, + ) + rows = cur.fetchall() + cur.close() + return rows + finally: + if conn: + conn.close() + + +def purge_app_log(older_than_days: int = 30): + """ + Delete app_log entries older than older_than_days days. + Called from the admin log view's Purge button. + """ + conn = None + try: + conn = get_connection() + cur = conn.cursor() + cur.execute( + "DELETE FROM app_log WHERE logged_at < NOW() - INTERVAL %s DAY", + (older_than_days,), + ) + deleted = cur.rowcount + conn.commit() + cur.close() + logger.info(f"App log purged: {deleted} records older than " + f"{older_than_days} days deleted.") + return deleted + finally: + if conn: + conn.close() \ No newline at end of file diff --git a/views/admin_log_view.py b/views/admin_log_view.py index 9fae7c5..f2731e2 100644 --- a/views/admin_log_view.py +++ b/views/admin_log_view.py @@ -1,10 +1,30 @@ """ -views/admin_log_view.py — Admin panel: Activity Log tab. +views/admin_log_view.py — Admin panel: Logs + +Two tabs: + Activity Log — user-action audit trail (existing, from activity_log table) + App Log — application-level logging (new, from app_log table) """ import tkinter as tk -from tkinter import ttk -from utils.ui_helpers import COLOURS, FONT_HEADING, show_error +from tkinter import ttk, messagebox +import logging + +from utils.ui_helpers import ( + COLOURS, FONT, FONT_BOLD, FONT_SMALL, FONT_HEADING, + show_error, show_info, +) + +logger = logging.getLogger("admin_log_view") + +# Colour map for log level badges +_LEVEL_COLOURS = { + "DEBUG": "text_dim", + "INFO": "text", + "WARNING": "warning", + "ERROR": "danger", + "CRITICAL": "danger", +} class AdminLogView(ttk.Frame): @@ -12,39 +32,317 @@ class AdminLogView(ttk.Frame): super().__init__(parent) self.current_user = current_user self._build_ui() - self._load() def _build_ui(self): - toolbar = ttk.Frame(self) - toolbar.pack(fill="x", pady=(0, 10)) - ttk.Label(toolbar, text="Activity Log", style="Heading.TLabel").pack(side="left") - ttk.Button(toolbar, text="↻ Refresh", style="Ghost.TButton", - command=self._load).pack(side="right") + # Page header + hdr = ttk.Frame(self) + hdr.pack(fill="x", pady=(0, 8)) + ttk.Label(hdr, text="Logs", style="Heading.TLabel").pack(side="left") + + # Notebook with two tabs + nb = ttk.Notebook(self) + nb.pack(fill="both", expand=True) + + # ── Tab 1 — Activity Log ────────────────────────────────────────────── + act_tab = ttk.Frame(nb) + nb.add(act_tab, text="📋 Activity Log") + self._build_activity_tab(act_tab) + + # ── Tab 2 — App Log ─────────────────────────────────────────────────── + app_tab = ttk.Frame(nb) + nb.add(app_tab, text="🖥 App Log") + self._build_app_log_tab(app_tab) + + nb.bind("<>", self._on_tab_changed) + self._nb = nb + + # Load the default tab + self._load_activity() + + # ── Activity Log tab ────────────────────────────────────────────────────── + + def _build_activity_tab(self, parent): + C = COLOURS + + toolbar = tk.Frame(parent, bg=C["surface"], pady=8, padx=10) + toolbar.pack(fill="x") + + ttk.Button(toolbar, text="↻ Refresh", style="Ghost.TButton", + command=self._load_activity).pack(side="right") + + # Search + tk.Label(toolbar, text="Search:", + bg=C["surface"], fg=C["text_dim"], + font=FONT_SMALL).pack(side="left", padx=(0, 4)) + self._act_search_var = tk.StringVar() + self._act_search_var.trace_add("write", lambda *_: self._load_activity()) + tk.Entry(toolbar, textvariable=self._act_search_var, + bg=C["surface2"], fg=C["text"], + insertbackground=C["text"], + relief="flat", font=FONT, width=24).pack( + side="left", ipady=4) cols = ("Time", "User", "Action", "Entity", "Entity ID", "Detail") - self.tree = ttk.Treeview(self, columns=cols, show="headings", selectmode="browse") - widths = [140, 100, 130, 100, 70, 320] + widths = [140, 100, 140, 100, 70, 320] + self._act_tree = ttk.Treeview( + parent, columns=cols, show="headings", selectmode="browse") for col, w in zip(cols, widths): - self.tree.heading(col, text=col) - self.tree.column(col, width=w, anchor="w") - self.tree.pack(fill="both", expand=True) + self._act_tree.heading(col, text=col) + self._act_tree.column(col, width=w, anchor="w") - vsb = ttk.Scrollbar(self, orient="vertical", command=self.tree.yview) - self.tree.configure(yscrollcommand=vsb.set) - vsb.place(relx=1, rely=0, relheight=1, anchor="ne") + vsb = ttk.Scrollbar(parent, orient="vertical", + command=self._act_tree.yview) + self._act_tree.configure(yscrollcommand=vsb.set) + vsb.pack(side="right", fill="y") + self._act_tree.pack(fill="both", expand=True) - def _load(self): + # Status bar + self._act_status = tk.Label( + parent, text="", bg=C["surface2"], fg=C["text_dim"], + font=FONT_SMALL, anchor="w") + self._act_status.pack(fill="x", side="bottom") + + def _load_activity(self, *_): from models import get_activity_log - self.tree.delete(*self.tree.get_children()) + search = self._act_search_var.get().strip() \ + if hasattr(self, "_act_search_var") else "" + self._act_tree.delete(*self._act_tree.get_children()) try: - for entry in get_activity_log(): - self.tree.insert("", "end", values=( - str(entry["logged_at"])[:16], - entry.get("username") or "—", - entry["action"], - entry.get("entity") or "", - entry.get("entity_id") or "", - entry.get("detail") or "", - )) + # Pass search to the DB query so filtering happens server-side. + entries = get_activity_log(limit=500, search=search) except Exception as e: show_error(f"Failed to load activity log:\n{e}") + return + for entry in entries: + self._act_tree.insert("", "end", values=( + str(entry.get("logged_at", ""))[:16], + entry.get("username") or "—", + entry.get("action") or "", + entry.get("entity") or "", + entry.get("entity_id") or "", + entry.get("detail") or "", + )) + self._act_status.config(text=f" {len(entries)} records") + + # ── App Log tab ─────────────────────────────────────────────────────────── + + def _build_app_log_tab(self, parent): + C = COLOURS + + toolbar = tk.Frame(parent, bg=C["surface"], pady=8, padx=10) + toolbar.pack(fill="x") + + ttk.Button(toolbar, text="↻ Refresh", style="Ghost.TButton", + command=self._load_app_log).pack(side="right", padx=(4, 0)) + + # Purge button + tk.Button( + toolbar, text="🗑 Purge Old Entries", + command=self._purge_app_log, + bg=C["surface2"], fg=C["danger"], + activebackground=C["danger"], activeforeground=C["white"], + relief="flat", font=FONT_SMALL, cursor="hand2", + padx=8, pady=4, + ).pack(side="right", padx=(4, 0)) + + # Level filter + tk.Label(toolbar, text="Level:", + bg=C["surface"], fg=C["text_dim"], + font=FONT_SMALL).pack(side="left", padx=(0, 4)) + self._level_var = tk.StringVar(value="All") + level_cb = ttk.Combobox( + toolbar, textvariable=self._level_var, + values=["All", "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + state="readonly", width=10, + ) + level_cb.pack(side="left") + level_cb.bind("<>", lambda _: self._load_app_log()) + + # Search + tk.Label(toolbar, text="Search:", + bg=C["surface"], fg=C["text_dim"], + font=FONT_SMALL).pack(side="left", padx=(12, 4)) + self._app_search_var = tk.StringVar() + self._app_search_var.trace_add("write", lambda *_: self._load_app_log()) + tk.Entry(toolbar, textvariable=self._app_search_var, + bg=C["surface2"], fg=C["text"], + insertbackground=C["text"], + relief="flat", font=FONT, width=24).pack( + side="left", ipady=4) + + # Treeview + cols = ("Time", "Level", "Logger", "Message") + widths = [140, 75, 130, 480] + self._app_tree = ttk.Treeview( + parent, columns=cols, show="headings", selectmode="browse") + for col, w in zip(cols, widths): + self._app_tree.heading(col, text=col) + self._app_tree.column( + col, width=w, + anchor="center" if col == "Level" else "w", + ) + + # Level colour tags + for level, colour_key in _LEVEL_COLOURS.items(): + self._app_tree.tag_configure(level, foreground=C[colour_key]) + # Bold for WARNING and above + self._app_tree.tag_configure("WARNING", foreground=C["warning"], + font=FONT_BOLD) + self._app_tree.tag_configure("ERROR", foreground=C["danger"], + font=FONT_BOLD) + self._app_tree.tag_configure("CRITICAL", foreground=C["danger"], + font=FONT_BOLD) + + vsb = ttk.Scrollbar(parent, orient="vertical", + command=self._app_tree.yview) + self._app_tree.configure(yscrollcommand=vsb.set) + vsb.pack(side="right", fill="y") + self._app_tree.pack(fill="both", expand=True) + + # Detail strip — shows full message when a row is selected + detail_frame = tk.Frame(parent, bg=C["surface2"], pady=6, padx=10) + detail_frame.pack(fill="x", side="bottom") + self._app_detail_var = tk.StringVar( + value="Select a row to see the full message.") + tk.Label( + detail_frame, textvariable=self._app_detail_var, + bg=C["surface2"], fg=C["text"], + font=FONT_SMALL, anchor="w", + justify="left", wraplength=900, + ).pack(fill="x") + self._app_tree.bind("<>", self._on_app_row_select) + + # Status bar + self._app_status = tk.Label( + parent, text="", bg=C["surface2"], fg=C["text_dim"], + font=FONT_SMALL, anchor="w") + self._app_status.pack(fill="x", side="bottom") + + # Store full messages keyed by tree iid for the detail strip + self._app_full_messages: dict[str, str] = {} + + def _load_app_log(self, *_): + from models import get_app_log + level = self._level_var.get() + level_filter = "" if level == "All" else level + search = self._app_search_var.get().strip() + + self._app_tree.delete(*self._app_tree.get_children()) + self._app_full_messages.clear() + self._app_detail_var.set("Select a row to see the full message.") + + try: + entries = get_app_log( + limit=500, + level_filter=level_filter, + search=search, + ) + except Exception as e: + show_error(f"Failed to load app log:\n{e}") + return + + for entry in entries: + lvl = entry.get("level", "INFO") + msg_full = entry.get("message", "") + # Truncate long messages in the tree; full text in detail strip + msg_short = msg_full[:120] + ("…" if len(msg_full) > 120 else "") + iid = str(entry["id"]) + self._app_tree.insert( + "", "end", iid=iid, tags=(lvl,), + values=( + str(entry.get("logged_at", ""))[:19], + lvl, + entry.get("logger_name", ""), + msg_short, + ), + ) + self._app_full_messages[iid] = msg_full + + count = len(entries) + self._app_status.config(text=f" {count} records") + + def _on_app_row_select(self, event=None): + sel = self._app_tree.selection() + if not sel: + return + iid = sel[0] + full = self._app_full_messages.get(iid, "") + self._app_detail_var.set(full) + + def _purge_app_log(self): + """Ask for confirmation, then delete log entries older than N days.""" + days_var = tk.StringVar(value="30") + + dlg = tk.Toplevel(self) + dlg.title("Purge App Log") + dlg.configure(bg=COLOURS["bg"]) + dlg.resizable(False, False) + dlg.grab_set() + + w, h = 340, 170 + x = (dlg.winfo_screenwidth() - w) // 2 + y = (dlg.winfo_screenheight() - h) // 2 + dlg.geometry(f"{w}x{h}+{x}+{y}") + + tk.Label(dlg, + text="Delete app log entries older than:", + bg=COLOURS["bg"], fg=COLOURS["text"], + font=FONT_BOLD).pack(pady=(20, 8)) + + row = tk.Frame(dlg, bg=COLOURS["bg"]) + row.pack() + ttk.Spinbox(row, from_=1, to=365, + textvariable=days_var, width=6).pack(side="left") + tk.Label(row, text=" days", + bg=COLOURS["bg"], fg=COLOURS["text"], + font=FONT).pack(side="left") + + def _do_purge(): + try: + days = int(days_var.get()) + if days < 1: + raise ValueError + except ValueError: + show_error("Please enter a valid number of days (1–365).") + return + if not messagebox.askyesno( + "Confirm Purge", + f"Delete all app log entries older than {days} days?\n" + "This cannot be undone.", + parent=dlg, + ): + return + try: + from models import purge_app_log + deleted = purge_app_log(days) + dlg.destroy() + show_info(f"{deleted} old log record(s) deleted.") + self._load_app_log() + except Exception as e: + show_error(f"Purge failed:\n{e}") + + btn_frame = tk.Frame(dlg, bg=COLOURS["bg"]) + btn_frame.pack(pady=12) + tk.Button(btn_frame, text="Purge", + command=_do_purge, + bg=COLOURS["danger"], fg=COLOURS["white"], + activebackground="#c94444", activeforeground=COLOURS["white"], + relief="flat", font=FONT_SMALL, cursor="hand2", + padx=12, pady=6).pack(side="left", padx=(0, 8)) + tk.Button(btn_frame, text="Cancel", + command=dlg.destroy, + bg=COLOURS["surface2"], fg=COLOURS["text"], + relief="flat", font=FONT_SMALL, cursor="hand2", + padx=12, pady=6).pack(side="left") + + # ── Tab switch ──────────────────────────────────────────────────────────── + + def _on_tab_changed(self, event=None): + try: + idx = self._nb.index(self._nb.select()) + if idx == 0: + self._load_activity() + else: + self._load_app_log() + except Exception: + pass