PassKeeper/gunicorn.conf.py

"""
gunicorn.conf.py — Gunicorn runtime configuration for PassKeeper.

Loaded explicitly by the systemd unit:

    ExecStart=/home/spuser/.venv/bin/gunicorn -c /home/spuser/PassKeeper/gunicorn.conf.py wsgi:app

Every value can be overridden from the environment (systemd reads
EnvironmentFile=/home/spuser/PassKeeper/.env), so tuning a production box does
not require editing this file or the unit.

── Timeout ordering (this is what causes 502s) ───────────────────────────────
Nginx must give up BEFORE Gunicorn kills the worker:

    nginx proxy_read_timeout  <  gunicorn timeout

If Gunicorn kills first, the connection is severed mid-response and Nginx
reports 502 Bad Gateway. If Nginx gives up first, the client gets a clean
504 Gateway Timeout instead. `timeout` below is therefore set comfortably
above the proxy_read_timeout in scripts/passkeeper-nginx.conf — raise this
value first if you ever raise that one.
"""
import multiprocessing
import os


def _env_int(name: str, default: int) -> int:
    """Read an int from the environment, falling back on unset/garbage values."""
    try:
        return int(os.environ.get(name, default))
    except (TypeError, ValueError):
        return default


# ── Socket ───────────────────────────────────────────────────────────────────
# Loopback only — Nginx is the sole public entry point and terminates TLS.
bind = os.environ.get('GUNICORN_BIND', '127.0.0.1:5000')
backlog = 2048

# ── Worker processes ─────────────────────────────────────────────────────────
# gthread rather than the default sync worker.
#
# Login is the expensive path: Argon2id at ARGON2_MEMORY_COST=65536 (64 MB) with
# parallelism=4 blocks a sync worker completely for the duration of the hash, and
# a sync worker serves exactly one request at a time. A handful of concurrent
# logins was enough to saturate all workers, queue everything behind them, and
# push requests past the worker timeout — which surfaces as 502.
#
# Threads let a worker keep serving while another request is blocked, and make
# the `keepalive` setting below meaningful (sync workers ignore keep-alive
# entirely, so Nginx's `proxy_http_version 1.1` was a no-op against them).
worker_class = 'gthread'

# Memory, not CPU, is the binding constraint: each in-flight Argon2id hash claims
# 64 MB. Keep workers modest and add concurrency with threads instead.
workers = _env_int('GUNICORN_WORKERS', min(4, multiprocessing.cpu_count() + 1))
threads = _env_int('GUNICORN_THREADS', 4)

# ── Timeouts ─────────────────────────────────────────────────────────────────
# Must exceed nginx proxy_read_timeout — see the module docstring.
# Long enough to cover the genuinely slow flows: bulk vault import/export and
# the atomic re-encryption of every item during a password change or recovery.
timeout = _env_int('GUNICORN_TIMEOUT', 90)

# Let in-flight requests finish on reload/restart rather than cutting them off.
graceful_timeout = _env_int('GUNICORN_GRACEFUL_TIMEOUT', 30)

# Idle keep-alive window for connections from Nginx. Slightly above Nginx's
# keepalive_timeout (15s) so Gunicorn is never the side that closes first —
# a connection closed underneath Nginx as it reuses it is a classic 502.
keepalive = _env_int('GUNICORN_KEEPALIVE', 20)

# ── Worker recycling ─────────────────────────────────────────────────────────
# Recycle workers periodically to bound the impact of any slow memory growth.
# The jitter staggers restarts so workers never all recycle at once (which would
# briefly leave nothing to serve — another source of intermittent 502s).
max_requests = _env_int('GUNICORN_MAX_REQUESTS', 1000)
max_requests_jitter = _env_int('GUNICORN_MAX_REQUESTS_JITTER', 100)

# Heartbeat file location. The default (/tmp) is disk-backed on many systems,
# and a slow disk makes the arbiter believe healthy workers have died and kill
# them. /dev/shm is always tmpfs. The unit sets PrivateTmp=true, which does not
# cover /dev/shm, so this stays writable.
if os.path.isdir('/dev/shm'):
    worker_tmp_dir = '/dev/shm'

# ── Application loading ──────────────────────────────────────────────────────
# preload_app MUST stay False.
#
# create_app() starts an APScheduler BackgroundScheduler for the hourly cleanup
# of token_blacklist / recovery_challenges / totp_used_codes / expired shares.
# With preload_app=True the app is built once in the arbiter and then forked —
# and threads do not survive fork(), so the scheduler thread would exist only in
# the arbiter, which serves no requests. The cleanup job would silently never
# run and those tables would grow without bound.
#
# Note: Deloy.md's "test Gunicorn manually" step suggests --preload. That is fine
# for a one-off smoke test, but must never reach the service definition.
#
# The cost of preload_app=False is one scheduler per worker, so the cleanup runs
# `workers` times an hour instead of once. The job only DELETEs already-expired
# rows, so it is idempotent and the redundancy is harmless.
preload_app = False

# ── Proxy ────────────────────────────────────────────────────────────────────
# Only trust X-Forwarded-* from the local Nginx. ProxyFix(x_for=1) in the app
# factory does the actual header parsing; this stops Gunicorn honouring
# forwarded headers from anything else.
forwarded_allow_ips = os.environ.get('GUNICORN_FORWARDED_ALLOW_IPS', '127.0.0.1')

# ── Request limits ───────────────────────────────────────────────────────────
# Defence in depth behind Nginx's client_max_body_size 1m.
limit_request_line = 8190
limit_request_fields = 100
limit_request_field_size = 8190

# ── Logging ──────────────────────────────────────────────────────────────────
# Paths must be inside the unit's ReadWritePaths= or Gunicorn cannot start.
accesslog = os.environ.get('GUNICORN_ACCESS_LOG', '/home/spuser/logs/access.log')
errorlog = os.environ.get('GUNICORN_ERROR_LOG', '/home/spuser/logs/error.log')
loglevel = os.environ.get('GUNICORN_LOG_LEVEL', 'warning')

# %({X-Forwarded-For}i)s rather than %(h)s — %(h)s would log Nginx's loopback
# address for every request. Never log Authorization headers or request bodies:
# they carry bearer tokens and auth_hash values.
access_log_format = (
    '%({X-Forwarded-For}i)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s" %(D)sus'
)

proc_name = 'passkeeper'


# ── Hooks ────────────────────────────────────────────────────────────────────

def on_starting(server):
    server.log.info(
        '[PassKeeper] starting: %s worker(s) x %s thread(s), class=%s, timeout=%ss',
        workers, threads, worker_class, timeout,
    )


def worker_abort(worker):
    """Fires when a worker is killed for exceeding `timeout`."""
    worker.log.error(
        '[PassKeeper] worker %s aborted after %ss — a request exceeded the '
        'timeout. Nginx will have reported 502 to the client.',
        worker.pid, timeout,
    )