From 82dd7c5aeff62d85c8205f0d64b1bc8f5b9efa31 Mon Sep 17 00:00:00 2001 From: NguyenND Date: Wed, 26 Aug 2026 10:58:55 -0400 Subject: [PATCH] Aug 26 - Add gunicorn.conf.py --- gunicorn.conf.py | 148 +++++++++++++++++++++++++++++++++++++ scripts/passkeeper.service | 34 +++++---- 2 files changed, 169 insertions(+), 13 deletions(-) create mode 100644 gunicorn.conf.py diff --git a/gunicorn.conf.py b/gunicorn.conf.py new file mode 100644 index 0000000..2d07c7e --- /dev/null +++ b/gunicorn.conf.py @@ -0,0 +1,148 @@ +""" +gunicorn.conf.py — Gunicorn runtime configuration for PassKeeper. + +Loaded explicitly by the systemd unit: + + ExecStart=/home/spuser/.venv/bin/gunicorn -c /home/spuser/PassKeeper/gunicorn.conf.py wsgi:app + +Every value can be overridden from the environment (systemd reads +EnvironmentFile=/home/spuser/PassKeeper/.env), so tuning a production box does +not require editing this file or the unit. + +── Timeout ordering (this is what causes 502s) ─────────────────────────────── +Nginx must give up BEFORE Gunicorn kills the worker: + + nginx proxy_read_timeout < gunicorn timeout + +If Gunicorn kills first, the connection is severed mid-response and Nginx +reports 502 Bad Gateway. If Nginx gives up first, the client gets a clean +504 Gateway Timeout instead. `timeout` below is therefore set comfortably +above the proxy_read_timeout in scripts/passkeeper-nginx.conf — raise this +value first if you ever raise that one. +""" +import multiprocessing +import os + + +def _env_int(name: str, default: int) -> int: + """Read an int from the environment, falling back on unset/garbage values.""" + try: + return int(os.environ.get(name, default)) + except (TypeError, ValueError): + return default + + +# ── Socket ─────────────────────────────────────────────────────────────────── +# Loopback only — Nginx is the sole public entry point and terminates TLS. +bind = os.environ.get('GUNICORN_BIND', '127.0.0.1:5000') +backlog = 2048 + +# ── Worker processes ───────────────────────────────────────────────────────── +# gthread rather than the default sync worker. +# +# Login is the expensive path: Argon2id at ARGON2_MEMORY_COST=65536 (64 MB) with +# parallelism=4 blocks a sync worker completely for the duration of the hash, and +# a sync worker serves exactly one request at a time. A handful of concurrent +# logins was enough to saturate all workers, queue everything behind them, and +# push requests past the worker timeout — which surfaces as 502. +# +# Threads let a worker keep serving while another request is blocked, and make +# the `keepalive` setting below meaningful (sync workers ignore keep-alive +# entirely, so Nginx's `proxy_http_version 1.1` was a no-op against them). +worker_class = 'gthread' + +# Memory, not CPU, is the binding constraint: each in-flight Argon2id hash claims +# 64 MB. Keep workers modest and add concurrency with threads instead. +workers = _env_int('GUNICORN_WORKERS', min(4, multiprocessing.cpu_count() + 1)) +threads = _env_int('GUNICORN_THREADS', 4) + +# ── Timeouts ───────────────────────────────────────────────────────────────── +# Must exceed nginx proxy_read_timeout — see the module docstring. +# Long enough to cover the genuinely slow flows: bulk vault import/export and +# the atomic re-encryption of every item during a password change or recovery. +timeout = _env_int('GUNICORN_TIMEOUT', 90) + +# Let in-flight requests finish on reload/restart rather than cutting them off. +graceful_timeout = _env_int('GUNICORN_GRACEFUL_TIMEOUT', 30) + +# Idle keep-alive window for connections from Nginx. Slightly above Nginx's +# keepalive_timeout (15s) so Gunicorn is never the side that closes first — +# a connection closed underneath Nginx as it reuses it is a classic 502. +keepalive = _env_int('GUNICORN_KEEPALIVE', 20) + +# ── Worker recycling ───────────────────────────────────────────────────────── +# Recycle workers periodically to bound the impact of any slow memory growth. +# The jitter staggers restarts so workers never all recycle at once (which would +# briefly leave nothing to serve — another source of intermittent 502s). +max_requests = _env_int('GUNICORN_MAX_REQUESTS', 1000) +max_requests_jitter = _env_int('GUNICORN_MAX_REQUESTS_JITTER', 100) + +# Heartbeat file location. The default (/tmp) is disk-backed on many systems, +# and a slow disk makes the arbiter believe healthy workers have died and kill +# them. /dev/shm is always tmpfs. The unit sets PrivateTmp=true, which does not +# cover /dev/shm, so this stays writable. +if os.path.isdir('/dev/shm'): + worker_tmp_dir = '/dev/shm' + +# ── Application loading ────────────────────────────────────────────────────── +# preload_app MUST stay False. +# +# create_app() starts an APScheduler BackgroundScheduler for the hourly cleanup +# of token_blacklist / recovery_challenges / totp_used_codes / expired shares. +# With preload_app=True the app is built once in the arbiter and then forked — +# and threads do not survive fork(), so the scheduler thread would exist only in +# the arbiter, which serves no requests. The cleanup job would silently never +# run and those tables would grow without bound. +# +# Note: Deloy.md's "test Gunicorn manually" step suggests --preload. That is fine +# for a one-off smoke test, but must never reach the service definition. +# +# The cost of preload_app=False is one scheduler per worker, so the cleanup runs +# `workers` times an hour instead of once. The job only DELETEs already-expired +# rows, so it is idempotent and the redundancy is harmless. +preload_app = False + +# ── Proxy ──────────────────────────────────────────────────────────────────── +# Only trust X-Forwarded-* from the local Nginx. ProxyFix(x_for=1) in the app +# factory does the actual header parsing; this stops Gunicorn honouring +# forwarded headers from anything else. +forwarded_allow_ips = os.environ.get('GUNICORN_FORWARDED_ALLOW_IPS', '127.0.0.1') + +# ── Request limits ─────────────────────────────────────────────────────────── +# Defence in depth behind Nginx's client_max_body_size 1m. +limit_request_line = 8190 +limit_request_fields = 100 +limit_request_field_size = 8190 + +# ── Logging ────────────────────────────────────────────────────────────────── +# Paths must be inside the unit's ReadWritePaths= or Gunicorn cannot start. +accesslog = os.environ.get('GUNICORN_ACCESS_LOG', '/home/spuser/logs/access.log') +errorlog = os.environ.get('GUNICORN_ERROR_LOG', '/home/spuser/logs/error.log') +loglevel = os.environ.get('GUNICORN_LOG_LEVEL', 'warning') + +# %({X-Forwarded-For}i)s rather than %(h)s — %(h)s would log Nginx's loopback +# address for every request. Never log Authorization headers or request bodies: +# they carry bearer tokens and auth_hash values. +access_log_format = ( + '%({X-Forwarded-For}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %(D)sus' +) + +proc_name = 'passkeeper' + + +# ── Hooks ──────────────────────────────────────────────────────────────────── + +def on_starting(server): + server.log.info( + '[PassKeeper] starting: %s worker(s) x %s thread(s), class=%s, timeout=%ss', + workers, threads, worker_class, timeout, + ) + + +def worker_abort(worker): + """Fires when a worker is killed for exceeding `timeout`.""" + worker.log.error( + '[PassKeeper] worker %s aborted after %ss — a request exceeded the ' + 'timeout. Nginx will have reported 502 to the client.', + worker.pid, timeout, + ) diff --git a/scripts/passkeeper.service b/scripts/passkeeper.service index af965b4..ae7ae6d 100644 --- a/scripts/passkeeper.service +++ b/scripts/passkeeper.service @@ -9,8 +9,8 @@ # # Phase 5 additions vs original: # - Restart=on-failure: systemd restarts Gunicorn if the master exits non-zero -# - Gunicorn --timeout: workers that don't respond within 25 s are replaced -# - Gunicorn --graceful-timeout: allows in-flight requests to finish on reload +# - Gunicorn settings live in gunicorn.conf.py (gthread workers, timeout +# ordered above nginx proxy_read_timeout, /dev/shm heartbeat dir) # - PrivateTmp, NoNewPrivileges, ProtectSystem: basic systemd sandboxing # - StartLimitIntervalSec / StartLimitBurst: caps restart storm @@ -29,19 +29,27 @@ Group=www-data WorkingDirectory=/home/spuser/PassKeeper EnvironmentFile=/home/spuser/PassKeeper/.env +# All tuning lives in gunicorn.conf.py (worker class, counts, timeouts, +# logging) so it is versioned with the code and documented in one place. +# Override any of it with GUNICORN_* variables in the EnvironmentFile above +# rather than editing this line. +# Absolute path — do not rely on WorkingDirectory for config lookup. ExecStart=/home/spuser/.venv/bin/gunicorn \ - --workers 4 \ - --bind 127.0.0.1:5000 \ - --timeout 25 \ - --graceful-timeout 20 \ - --keep-alive 5 \ - --access-logfile /home/spuser/logs/access.log \ - --error-logfile /home/spuser/logs/error.log \ - --log-level warning \ + -c /home/spuser/PassKeeper/gunicorn.conf.py \ wsgi:app -# Reload (zero-downtime): send USR2 to Gunicorn master -ExecReload=/bin/kill -s USR2 $MAINPID +# Reload: HUP re-reads config and restarts workers in place under the +# existing master. +# +# NOT USR2. USR2 forks a *second* master that inherits the listening socket, +# and retiring the old one needs a follow-up WINCH + QUIT that this unit never +# sent. The result was two masters competing for the port with systemd $MAINPID +# tracking the stale one — a later stop/restart then signalled the wrong +# process and the socket vanished, which Nginx reports as 502. +# +# HUP does not pick up changed Python source: use `systemctl restart` for code +# deploys, reload only for config-only changes. +ExecReload=/bin/kill -s HUP $MAINPID # NO WatchdogSec here — deliberately. # @@ -52,7 +60,7 @@ ExecReload=/bin/kill -s USR2 $MAINPID # SIGKILLed it every ~30 s. Restart=on-failure then brought it back after RestartSec, # producing a repeating window of 502s from Nginx. # -# Hung *workers* are already handled by Gunicorn's own --timeout above; a crashed +# Hung *workers* are already handled by Gunicorn's own `timeout` in gunicorn.conf.py; a crashed # *master* is already handled by Restart=on-failure below. The watchdog added no # coverage, only outages. #