Jul 31 - Fix database issue which leads to the app dies

.claude/settings.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(grep -vE \"^[+-]\\\\s*$\")",
+      "Bash(python -c \"import ast;ast.parse\\(open\\('app.py'\\).read\\(\\)\\);ast.parse\\(open\\('config.py'\\).read\\(\\)\\);print\\('AST OK'\\)\")",
+      "Bash(python -c ' *)"
+    ]
+  }
+}

app.py

@@ -314,18 +314,60 @@ def create_app() -> Flask:
     # ------------------------------------------------------------------
     # Startup initialization (runs under gunicorn and flask run alike)
     # ------------------------------------------------------------------
-    with app.app_context():
-        try:
-            create_tables()
-            update_existing_qr_codes()
-        except Exception as e:
-            from extensions import logger_handler as _startup_lh
-            _startup_lh.logger.error(f"Startup initialization failed: {e}", exc_info=True)
-            raise
+    _run_startup_initialization(app)
 
     return app
 
 
+def _run_startup_initialization(app) -> bool:
+    """Run create_tables() + update_existing_qr_codes(), tolerating a DB that
+    is not up yet.
+
+    After a server reboot gunicorn and mysqld start in parallel, so the first
+    connection attempt can be refused. Retry for a short bounded window, then
+    boot the app anyway instead of re-raising: a worker that refuses to start
+    takes the whole site down permanently (supervisor exhausts its start
+    retries within seconds and gives up), while a booted worker recovers on its
+    own once MySQL accepts connections — pool_pre_ping discards the dead
+    connections. Startup work that was skipped is idempotent and runs on the
+    next successful restart.
+    """
+    from extensions import logger_handler as _startup_lh
+
+    attempts = app.config.get('DB_STARTUP_RETRY_ATTEMPTS', 5)
+    delay = app.config.get('DB_STARTUP_RETRY_DELAY', 3)
+    last_error = None
+
+    for attempt in range(1, attempts + 1):
+        with app.app_context():
+            try:
+                create_tables()
+                update_existing_qr_codes()
+                if attempt > 1:
+                    _startup_lh.logger.info(
+                        f"Startup initialization succeeded on attempt {attempt}/{attempts}"
+                    )
+                return True
+            except Exception as e:
+                last_error = e
+                try:
+                    db.session.rollback()
+                except Exception:
+                    pass
+                _startup_lh.logger.warning(
+                    f"Startup initialization attempt {attempt}/{attempts} failed: {e}"
+                )
+        if attempt < attempts:
+            _time.sleep(delay)
+
+    _startup_lh.logger.error(
+        f"Startup initialization failed after {attempts} attempts; starting anyway "
+        f"so workers can serve once the database recovers: {last_error}",
+        exc_info=True
+    )
+    return False
+
+
 # ---------------------------------------------------------------------------
 # Database initialization helpers (called at startup)
 # ---------------------------------------------------------------------------

config.py

@@ -41,8 +41,21 @@ class Config:
         'pool_timeout': int(os.environ.get('SQLALCHEMY_ENGINE_OPTIONS_POOL_TIMEOUT', '20')),
         'pool_recycle': int(os.environ.get('SQLALCHEMY_ENGINE_OPTIONS_POOL_RECYCLE', '3600')),
         'max_overflow': int(os.environ.get('SQLALCHEMY_ENGINE_OPTIONS_MAX_OVERFLOW', '20')),
+        # Validate a pooled connection before handing it out. Without this, every
+        # connection opened while mysqld was down/restarting stays in the pool as a
+        # dead socket and keeps failing requests long after MySQL has recovered.
+        'pool_pre_ping': True,
     }
 
+    # ------------------------------------------------------------------ #
+    # Startup DB-connect retry (server reboot: gunicorn may start before mysqld)
+    # ------------------------------------------------------------------ #
+    # Observed worst case: an unattended-upgrades restart of mysql-server left
+    # mysqld down for ~15s. 8 attempts x 3s covers ~21s of downtime while
+    # staying inside gunicorn's 30s worker-boot timeout.
+    DB_STARTUP_RETRY_ATTEMPTS = int(os.environ.get('DB_STARTUP_RETRY_ATTEMPTS', '8'))
+    DB_STARTUP_RETRY_DELAY    = int(os.environ.get('DB_STARTUP_RETRY_DELAY', '3'))
+
     # ------------------------------------------------------------------ #
     # Session / cookies
     # ------------------------------------------------------------------ #