diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..8dbd11e --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,9 @@ +{ + "permissions": { + "allow": [ + "Bash(grep -vE \"^[+-]\\\\s*$\")", + "Bash(python -c \"import ast;ast.parse\\(open\\('app.py'\\).read\\(\\)\\);ast.parse\\(open\\('config.py'\\).read\\(\\)\\);print\\('AST OK'\\)\")", + "Bash(python -c ' *)" + ] + } +} diff --git a/app.py b/app.py index 5611045..7bc109c 100644 --- a/app.py +++ b/app.py @@ -314,18 +314,60 @@ def create_app() -> Flask: # ------------------------------------------------------------------ # Startup initialization (runs under gunicorn and flask run alike) # ------------------------------------------------------------------ - with app.app_context(): - try: - create_tables() - update_existing_qr_codes() - except Exception as e: - from extensions import logger_handler as _startup_lh - _startup_lh.logger.error(f"Startup initialization failed: {e}", exc_info=True) - raise + _run_startup_initialization(app) return app +def _run_startup_initialization(app) -> bool: + """Run create_tables() + update_existing_qr_codes(), tolerating a DB that + is not up yet. + + After a server reboot gunicorn and mysqld start in parallel, so the first + connection attempt can be refused. Retry for a short bounded window, then + boot the app anyway instead of re-raising: a worker that refuses to start + takes the whole site down permanently (supervisor exhausts its start + retries within seconds and gives up), while a booted worker recovers on its + own once MySQL accepts connections — pool_pre_ping discards the dead + connections. Startup work that was skipped is idempotent and runs on the + next successful restart. + """ + from extensions import logger_handler as _startup_lh + + attempts = app.config.get('DB_STARTUP_RETRY_ATTEMPTS', 5) + delay = app.config.get('DB_STARTUP_RETRY_DELAY', 3) + last_error = None + + for attempt in range(1, attempts + 1): + with app.app_context(): + try: + create_tables() + update_existing_qr_codes() + if attempt > 1: + _startup_lh.logger.info( + f"Startup initialization succeeded on attempt {attempt}/{attempts}" + ) + return True + except Exception as e: + last_error = e + try: + db.session.rollback() + except Exception: + pass + _startup_lh.logger.warning( + f"Startup initialization attempt {attempt}/{attempts} failed: {e}" + ) + if attempt < attempts: + _time.sleep(delay) + + _startup_lh.logger.error( + f"Startup initialization failed after {attempts} attempts; starting anyway " + f"so workers can serve once the database recovers: {last_error}", + exc_info=True + ) + return False + + # --------------------------------------------------------------------------- # Database initialization helpers (called at startup) # --------------------------------------------------------------------------- diff --git a/config.py b/config.py index 349f376..d25afb6 100644 --- a/config.py +++ b/config.py @@ -41,8 +41,21 @@ class Config: 'pool_timeout': int(os.environ.get('SQLALCHEMY_ENGINE_OPTIONS_POOL_TIMEOUT', '20')), 'pool_recycle': int(os.environ.get('SQLALCHEMY_ENGINE_OPTIONS_POOL_RECYCLE', '3600')), 'max_overflow': int(os.environ.get('SQLALCHEMY_ENGINE_OPTIONS_MAX_OVERFLOW', '20')), + # Validate a pooled connection before handing it out. Without this, every + # connection opened while mysqld was down/restarting stays in the pool as a + # dead socket and keeps failing requests long after MySQL has recovered. + 'pool_pre_ping': True, } + # ------------------------------------------------------------------ # + # Startup DB-connect retry (server reboot: gunicorn may start before mysqld) + # ------------------------------------------------------------------ # + # Observed worst case: an unattended-upgrades restart of mysql-server left + # mysqld down for ~15s. 8 attempts x 3s covers ~21s of downtime while + # staying inside gunicorn's 30s worker-boot timeout. + DB_STARTUP_RETRY_ATTEMPTS = int(os.environ.get('DB_STARTUP_RETRY_ATTEMPTS', '8')) + DB_STARTUP_RETRY_DELAY = int(os.environ.get('DB_STARTUP_RETRY_DELAY', '3')) + # ------------------------------------------------------------------ # # Session / cookies # ------------------------------------------------------------------ #