Aug 26 - Add gunicorn.conf.py

gunicorn.conf.py

@@ -0,0 +1,148 @@
+"""
+gunicorn.conf.py — Gunicorn runtime configuration for PassKeeper.
+
+Loaded explicitly by the systemd unit:
+
+    ExecStart=/home/spuser/.venv/bin/gunicorn -c /home/spuser/PassKeeper/gunicorn.conf.py wsgi:app
+
+Every value can be overridden from the environment (systemd reads
+EnvironmentFile=/home/spuser/PassKeeper/.env), so tuning a production box does
+not require editing this file or the unit.
+
+── Timeout ordering (this is what causes 502s) ───────────────────────────────
+Nginx must give up BEFORE Gunicorn kills the worker:
+
+    nginx proxy_read_timeout  <  gunicorn timeout
+
+If Gunicorn kills first, the connection is severed mid-response and Nginx
+reports 502 Bad Gateway. If Nginx gives up first, the client gets a clean
+504 Gateway Timeout instead. `timeout` below is therefore set comfortably
+above the proxy_read_timeout in scripts/passkeeper-nginx.conf — raise this
+value first if you ever raise that one.
+"""
+import multiprocessing
+import os
+
+
+def _env_int(name: str, default: int) -> int:
+    """Read an int from the environment, falling back on unset/garbage values."""
+    try:
+        return int(os.environ.get(name, default))
+    except (TypeError, ValueError):
+        return default
+
+
+# ── Socket ───────────────────────────────────────────────────────────────────
+# Loopback only — Nginx is the sole public entry point and terminates TLS.
+bind = os.environ.get('GUNICORN_BIND', '127.0.0.1:5000')
+backlog = 2048
+
+# ── Worker processes ─────────────────────────────────────────────────────────
+# gthread rather than the default sync worker.
+#
+# Login is the expensive path: Argon2id at ARGON2_MEMORY_COST=65536 (64 MB) with
+# parallelism=4 blocks a sync worker completely for the duration of the hash, and
+# a sync worker serves exactly one request at a time. A handful of concurrent
+# logins was enough to saturate all workers, queue everything behind them, and
+# push requests past the worker timeout — which surfaces as 502.
+#
+# Threads let a worker keep serving while another request is blocked, and make
+# the `keepalive` setting below meaningful (sync workers ignore keep-alive
+# entirely, so Nginx's `proxy_http_version 1.1` was a no-op against them).
+worker_class = 'gthread'
+
+# Memory, not CPU, is the binding constraint: each in-flight Argon2id hash claims
+# 64 MB. Keep workers modest and add concurrency with threads instead.
+workers = _env_int('GUNICORN_WORKERS', min(4, multiprocessing.cpu_count() + 1))
+threads = _env_int('GUNICORN_THREADS', 4)
+
+# ── Timeouts ─────────────────────────────────────────────────────────────────
+# Must exceed nginx proxy_read_timeout — see the module docstring.
+# Long enough to cover the genuinely slow flows: bulk vault import/export and
+# the atomic re-encryption of every item during a password change or recovery.
+timeout = _env_int('GUNICORN_TIMEOUT', 90)
+
+# Let in-flight requests finish on reload/restart rather than cutting them off.
+graceful_timeout = _env_int('GUNICORN_GRACEFUL_TIMEOUT', 30)
+
+# Idle keep-alive window for connections from Nginx. Slightly above Nginx's
+# keepalive_timeout (15s) so Gunicorn is never the side that closes first —
+# a connection closed underneath Nginx as it reuses it is a classic 502.
+keepalive = _env_int('GUNICORN_KEEPALIVE', 20)
+
+# ── Worker recycling ─────────────────────────────────────────────────────────
+# Recycle workers periodically to bound the impact of any slow memory growth.
+# The jitter staggers restarts so workers never all recycle at once (which would
+# briefly leave nothing to serve — another source of intermittent 502s).
+max_requests = _env_int('GUNICORN_MAX_REQUESTS', 1000)
+max_requests_jitter = _env_int('GUNICORN_MAX_REQUESTS_JITTER', 100)
+
+# Heartbeat file location. The default (/tmp) is disk-backed on many systems,
+# and a slow disk makes the arbiter believe healthy workers have died and kill
+# them. /dev/shm is always tmpfs. The unit sets PrivateTmp=true, which does not
+# cover /dev/shm, so this stays writable.
+if os.path.isdir('/dev/shm'):
+    worker_tmp_dir = '/dev/shm'
+
+# ── Application loading ──────────────────────────────────────────────────────
+# preload_app MUST stay False.
+#
+# create_app() starts an APScheduler BackgroundScheduler for the hourly cleanup
+# of token_blacklist / recovery_challenges / totp_used_codes / expired shares.
+# With preload_app=True the app is built once in the arbiter and then forked —
+# and threads do not survive fork(), so the scheduler thread would exist only in
+# the arbiter, which serves no requests. The cleanup job would silently never
+# run and those tables would grow without bound.
+#
+# Note: Deloy.md's "test Gunicorn manually" step suggests --preload. That is fine
+# for a one-off smoke test, but must never reach the service definition.
+#
+# The cost of preload_app=False is one scheduler per worker, so the cleanup runs
+# `workers` times an hour instead of once. The job only DELETEs already-expired
+# rows, so it is idempotent and the redundancy is harmless.
+preload_app = False
+
+# ── Proxy ────────────────────────────────────────────────────────────────────
+# Only trust X-Forwarded-* from the local Nginx. ProxyFix(x_for=1) in the app
+# factory does the actual header parsing; this stops Gunicorn honouring
+# forwarded headers from anything else.
+forwarded_allow_ips = os.environ.get('GUNICORN_FORWARDED_ALLOW_IPS', '127.0.0.1')
+
+# ── Request limits ───────────────────────────────────────────────────────────
+# Defence in depth behind Nginx's client_max_body_size 1m.
+limit_request_line = 8190
+limit_request_fields = 100
+limit_request_field_size = 8190
+
+# ── Logging ──────────────────────────────────────────────────────────────────
+# Paths must be inside the unit's ReadWritePaths= or Gunicorn cannot start.
+accesslog = os.environ.get('GUNICORN_ACCESS_LOG', '/home/spuser/logs/access.log')
+errorlog = os.environ.get('GUNICORN_ERROR_LOG', '/home/spuser/logs/error.log')
+loglevel = os.environ.get('GUNICORN_LOG_LEVEL', 'warning')
+
+# %({X-Forwarded-For}i)s rather than %(h)s — %(h)s would log Nginx's loopback
+# address for every request. Never log Authorization headers or request bodies:
+# they carry bearer tokens and auth_hash values.
+access_log_format = (
+    '%({X-Forwarded-For}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %(D)sus'
+)
+
+proc_name = 'passkeeper'
+
+
+# ── Hooks ────────────────────────────────────────────────────────────────────
+
+def on_starting(server):
+    server.log.info(
+        '[PassKeeper] starting: %s worker(s) x %s thread(s), class=%s, timeout=%ss',
+        workers, threads, worker_class, timeout,
+    )
+
+
+def worker_abort(worker):
+    """Fires when a worker is killed for exceeding `timeout`."""
+    worker.log.error(
+        '[PassKeeper] worker %s aborted after %ss — a request exceeded the '
+        'timeout. Nginx will have reported 502 to the client.',
+        worker.pid, timeout,
+    )

scripts/passkeeper.service

@@ -9,8 +9,8 @@
 #
 # Phase 5 additions vs original:
 #   - Restart=on-failure: systemd restarts Gunicorn if the master exits non-zero
-#   - Gunicorn --timeout: workers that don't respond within 25 s are replaced
+#   - Gunicorn settings live in gunicorn.conf.py (gthread workers, timeout
-#   - Gunicorn --graceful-timeout: allows in-flight requests to finish on reload
+#     ordered above nginx proxy_read_timeout, /dev/shm heartbeat dir)
 #   - PrivateTmp, NoNewPrivileges, ProtectSystem: basic systemd sandboxing
 #   - StartLimitIntervalSec / StartLimitBurst: caps restart storm
 
@@ -29,19 +29,27 @@ Group=www-data
 WorkingDirectory=/home/spuser/PassKeeper
 EnvironmentFile=/home/spuser/PassKeeper/.env
 
+# All tuning lives in gunicorn.conf.py (worker class, counts, timeouts,
+# logging) so it is versioned with the code and documented in one place.
+# Override any of it with GUNICORN_* variables in the EnvironmentFile above
+# rather than editing this line.
+# Absolute path — do not rely on WorkingDirectory for config lookup.
 ExecStart=/home/spuser/.venv/bin/gunicorn \
-          --workers 4 \
+          -c /home/spuser/PassKeeper/gunicorn.conf.py \
-          --bind 127.0.0.1:5000 \
-          --timeout 25 \
-          --graceful-timeout 20 \
-          --keep-alive 5 \
-          --access-logfile /home/spuser/logs/access.log \
-          --error-logfile  /home/spuser/logs/error.log \
-          --log-level warning \
           wsgi:app
 
-# Reload (zero-downtime): send USR2 to Gunicorn master
+# Reload: HUP re-reads config and restarts workers in place under the
-ExecReload=/bin/kill -s USR2 $MAINPID
+# existing master.
+#
+# NOT USR2. USR2 forks a *second* master that inherits the listening socket,
+# and retiring the old one needs a follow-up WINCH + QUIT that this unit never
+# sent. The result was two masters competing for the port with systemd $MAINPID
+# tracking the stale one — a later stop/restart then signalled the wrong
+# process and the socket vanished, which Nginx reports as 502.
+#
+# HUP does not pick up changed Python source: use `systemctl restart` for code
+# deploys, reload only for config-only changes.
+ExecReload=/bin/kill -s HUP $MAINPID
 
 # NO WatchdogSec here — deliberately.
 #
@@ -52,7 +60,7 @@ ExecReload=/bin/kill -s USR2 $MAINPID
 # SIGKILLed it every ~30 s. Restart=on-failure then brought it back after RestartSec,
 # producing a repeating window of 502s from Nginx.
 #
-# Hung *workers* are already handled by Gunicorn's own --timeout above; a crashed
+# Hung *workers* are already handled by Gunicorn's own `timeout` in gunicorn.conf.py; a crashed
 # *master* is already handled by Restart=on-failure below. The watchdog added no
 # coverage, only outages.
 #