nngo/JQC_multi_tenant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Aug 21 - Fernet key rotation

Browse Source
This commit is contained in:
nngo
2026-08-21 10:18:22 -04:00
parent cb7c244872
commit d04190ba09
5 changed files with 578 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
control/cli.py
+125 -1
View File
@@ -8,6 +8,8 @@ the bootstrap actions on top of an already-migrated control DB.
python -m control.cli seed
python -m control.cli create-superadmin --username admin --email you@example.com
python -m control.cli list-plans
python -m control.cli verify-keys
python -m control.cli rotate-keys --dry-run
"""
import argparse
@@ -15,7 +17,7 @@ import getpass
import sys
from control.base import control_session
from control.models import Plan, Superadmin
from control.models import Plan, Superadmin, Tenant
from control.seed import seed_plans
@@ -58,6 +60,113 @@ def _cmd_create_superadmin(args):
print(f"Superadmin '{args.username}' created.")
def _cmd_verify_keys(_args):
"""MT-23: confirm every stored credential is readable under the current key set.
Run this BEFORE retiring a key and again after rotating. It is the step
that makes rotation safe: it proves no row still depends on a key that is
about to be removed, instead of finding out at the next tenant request.
"""
from control import crypto
current = crypto.current_key_version()
print(f'Primary key fingerprint : {crypto.key_fingerprint()}')
print(f'Retired keys configured : {len(crypto.old_keys())}')
print(f'Current key version : {current}')
print('-' * 60)
ok = failed = empty = stale = 0
failures = []
with control_session() as s:
tenants = s.query(Tenant).order_by(Tenant.id).all()
for t in tenants:
if not t.db_password_enc:
empty += 1
continue
if crypto.can_decrypt(t.db_password_enc):
ok += 1
else:
failed += 1
failures.append((t.id, t.slug))
if (t.key_version or 1) < current:
stale += 1
print(f' readable : {ok}')
print(f' no credential set : {empty}')
print(f' UNREADABLE : {failed}')
print(f' awaiting rotation : {stale} (key_version < {current})')
if failures:
print('\nUnreadable rows — do NOT retire any key:')
for tid, slug in failures:
print(f' tenant {tid} ({slug})')
sys.exit(1)
if stale:
print(f'\n{stale} row(s) still on an older key. Run: '
'python -m control.cli rotate-keys')
else:
print('\nAll credentials current. Safe to drop retired keys from '
'CONTROL_FERNET_KEYS_OLD.')
def _cmd_rotate_keys(args):
"""MT-23: re-encrypt tenant credentials under the primary Fernet key.
Idempotent — rows already at the current version are skipped unless
--force is given. Each tenant is committed independently so a failure
part-way leaves a resumable state rather than an all-or-nothing rollback.
"""
from control import crypto
from cryptography.fernet import InvalidToken
current = crypto.current_key_version()
print(f'Primary key fingerprint : {crypto.key_fingerprint()}')
print(f'Target key version : {current}')
if args.dry_run:
print('MODE: dry run — nothing will be written.')
print('-' * 60)
rotated = skipped = empty = errors = 0
with control_session() as s:
tenants = s.query(Tenant).order_by(Tenant.id).all()
for t in tenants:
if (t.key_version or 1) >= current and not args.force:
skipped += 1
continue
if args.dry_run:
label = 'would rotate' if t.db_password_enc else 'would stamp (no credential)'
print(f' tenant {t.id} ({t.slug}): {label}')
rotated += 1
continue
try:
changed = t.rotate_db_password_key()
s.flush()
if changed:
rotated += 1
print(f' tenant {t.id} ({t.slug}): rotated -> v{t.key_version}')
else:
empty += 1
print(f' tenant {t.id} ({t.slug}): no credential, stamped '
f'-> v{t.key_version}')
except InvalidToken:
errors += 1
print(f' tenant {t.id} ({t.slug}): ERROR — no configured key opens '
f'this credential. Add the retiring key to '
f'CONTROL_FERNET_KEYS_OLD and re-run.')
except Exception as exc:
errors += 1
print(f' tenant {t.id} ({t.slug}): ERROR — {exc}')
print('-' * 60)
print(f'rotated={rotated} skipped={skipped} no_credential={empty} errors={errors}')
if errors:
print('\nRotation incomplete. Do NOT retire any key. Fix the errors above '
'and re-run — completed rows are skipped automatically.')
sys.exit(1)
if not args.dry_run:
print('\nNow run: python -m control.cli verify-keys')
def main(argv=None):
parser = argparse.ArgumentParser(prog='control.cli',
description='JQC control-plane CLI')
@@ -73,6 +182,21 @@ def main(argv=None):
help='Omit to be prompted securely.')
p_sa.set_defaults(func=_cmd_create_superadmin)
sub.add_parser(
'verify-keys',
help='Check every tenant credential decrypts under the current key set',
).set_defaults(func=_cmd_verify_keys)
p_rot = sub.add_parser(
'rotate-keys',
help='Re-encrypt tenant credentials under the primary Fernet key',
)
p_rot.add_argument('--dry-run', action='store_true',
help='Report what would change; write nothing.')
p_rot.add_argument('--force', action='store_true',
help='Re-encrypt even rows already at the current version.')
p_rot.set_defaults(func=_cmd_rotate_keys)
args = parser.parse_args(argv)
args.func(args)
control/crypto.py
+133 -13
View File
@@ -3,39 +3,159 @@ control/crypto.py
-----------------
Symmetric encryption for sensitive control-plane fields (tenant DB passwords).
Fernet (AES-128-CBC + HMAC) keyed by the CONTROL_FERNET_KEY env var.
Generate a key once:
Fernet (AES-128-CBC + HMAC). MT-23 adds key rotation via ``MultiFernet``.
Key configuration
-----------------
``CONTROL_FERNET_KEY``
The **primary** key. Everything is encrypted with this one.
``CONTROL_FERNET_KEYS_OLD``
Comma-separated retired keys, newest first. Decrypt-only: they open
ciphertext written before the last rotation, and are never used to encrypt.
Empty or unset in steady state.
``CONTROL_FERNET_KEY_VERSION``
Integer, default 1. Stamped onto every row as it is (re-)encrypted, so
rotation progress is a SQL query rather than a guess:
SELECT COUNT(*) FROM tenants WHERE key_version < <current>;
Generate a key:
python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
Store it in the server environment / .env (never in git). Key rotation requires
re-encrypting existing tenant rows.
Rotation procedure
------------------
1. Generate a new key.
2. Move the current ``CONTROL_FERNET_KEY`` value into ``CONTROL_FERNET_KEYS_OLD``
(prepending it if that variable already holds keys), set the new key as
``CONTROL_FERNET_KEY``, and bump ``CONTROL_FERNET_KEY_VERSION``.
3. ``python -m control.cli verify-keys`` — confirms every stored credential is
still readable under the new key set **before** anything is rewritten.
4. ``python -m control.cli rotate-keys`` — re-encrypts every row under the
primary key and stamps the new version.
5. ``python -m control.cli verify-keys`` again, then drop the retired key from
``CONTROL_FERNET_KEYS_OLD``.
Steps 3 and 5 are the point of the design: at no moment is there a window
where a key has been retired but rows still depend on it.
"""
import hashlib
import os
from cryptography.fernet import Fernet
from cryptography.fernet import Fernet, MultiFernet
_KEY_ENV = 'CONTROL_FERNET_KEY'
_OLD_KEY_ENV = 'CONTROL_FERNET_KEYS_OLD'
_VERSION_ENV = 'CONTROL_FERNET_KEY_VERSION'
def _fernet() -> Fernet:
key = os.environ.get('CONTROL_FERNET_KEY')
def _as_bytes(key):
return key.encode() if isinstance(key, str) else key
def primary_key():
"""The single key used for encryption. Raises if unset."""
key = os.environ.get(_KEY_ENV)
if not key:
raise RuntimeError(
"CONTROL_FERNET_KEY is not set. Generate one with: "
f"{_KEY_ENV} is not set. Generate one with: "
"python -c \"from cryptography.fernet import Fernet; "
"print(Fernet.generate_key().decode())\""
)
return Fernet(key.encode() if isinstance(key, str) else key)
return _as_bytes(key)
def old_keys():
"""Retired decrypt-only keys, in the order they are tried."""
raw = os.environ.get(_OLD_KEY_ENV) or ''
return [_as_bytes(k.strip()) for k in raw.split(',') if k.strip()]
def current_key_version():
"""Integer stamped onto rows encrypted under the current primary key."""
try:
return int(os.environ.get(_VERSION_ENV, 1))
except (TypeError, ValueError):
return 1
def key_fingerprint(key=None):
"""Short, non-reversible identifier for a key — safe to log.
Lets an operator confirm which key a process actually loaded without ever
putting key material in a log line or a support ticket.
"""
material = _as_bytes(key) if key is not None else primary_key()
return hashlib.sha256(material).hexdigest()[:12]
def _multi():
"""MultiFernet: primary first (it encrypts), retired keys after it.
Ordering matters — MultiFernet encrypts with the first key and tries each
in turn when decrypting.
"""
keys = [Fernet(primary_key())]
seen = {primary_key()}
for k in old_keys():
if k in seen:
continue # tolerate a stale copy of the primary in the old list
seen.add(k)
keys.append(Fernet(k))
return MultiFernet(keys)
# Kept for backward compatibility: earlier code imported _fernet() directly.
def _fernet():
return _multi()
def encrypt(plaintext):
"""Encrypt a string to a Fernet token (str). Passes None through."""
"""Encrypt a string to a Fernet token (str) under the primary key.
Passes None through.
"""
if plaintext is None:
return None
return _fernet().encrypt(plaintext.encode()).decode()
return _multi().encrypt(plaintext.encode()).decode()
def decrypt(token):
"""Decrypt a Fernet token back to the original string. Passes None through."""
"""Decrypt a Fernet token, trying the primary key then each retired key.
Passes None through. Raises ``cryptography.fernet.InvalidToken`` when no
configured key opens the token.
"""
if token is None:
return None
return _fernet().decrypt(token.encode()).decode()
return _multi().decrypt(token.encode()).decode()
def rotate(token):
"""Re-encrypt an existing token under the primary key.
``MultiFernet.rotate`` decrypts with whichever key works and re-encrypts
with the first, preserving the plaintext without it ever being handled
here. Passes None through.
"""
if token is None:
return None
return _multi().rotate(token.encode()).decode()
def can_decrypt(token):
"""True when some configured key opens this token. Never raises.
Used by ``control.cli verify-keys`` to audit every stored credential
before a key is retired.
"""
if token is None:
return True # nothing stored is not a failure
try:
decrypt(token)
return True
except Exception:
return False
control/migrations/versions/control0006_fernet_key_version.py
+51
View File
@@ -0,0 +1,51 @@
"""control0006_fernet_key_version
Adds the Fernet key-version stamp to the `tenants` table:
- key_version INT NOT NULL DEFAULT 1
MT-23: credentials are encrypted under a single control-plane Fernet key, and
before this there was no way to tell which rows had been re-encrypted after a
key change — rotation was all-or-nothing and unverifiable. Stamping the version
each time a row is written makes rotation progress a plain query:
SELECT COUNT(*) FROM tenants WHERE key_version < <current>;
Existing rows default to 1, which matches the single key in use before this
migration, so nothing needs re-encrypting to adopt it.
Guarded by an INFORMATION_SCHEMA existence check, so re-running is safe.
Revision ID: control0006_fernet_key_ver
Revises: control0005_superadmin_mfa
Create Date: 2026-08-21
"""
from alembic import op
import sqlalchemy as sa
# NOTE: revision ids must fit VARCHAR(32) — 'control0006_fernet_key_ver' is 26.
revision = 'control0006_fernet_key_ver'
down_revision = 'control0005_superadmin_mfa'
branch_labels = None
depends_on = None
def _column_exists(table, column):
result = op.get_bind().execute(sa.text(
"SELECT COUNT(*) FROM information_schema.COLUMNS "
"WHERE TABLE_SCHEMA = DATABASE() "
" AND TABLE_NAME = :tbl "
" AND COLUMN_NAME = :col"
), {'tbl': table, 'col': column})
return result.scalar() > 0
def upgrade():
if not _column_exists('tenants', 'key_version'):
op.add_column('tenants', sa.Column('key_version', sa.Integer(),
nullable=False, server_default='1'))
def downgrade():
if _column_exists('tenants', 'key_version'):
op.drop_column('tenants', 'key_version')
control/models.py
+22 -1
View File
@@ -95,6 +95,10 @@ class Tenant(ControlBase):
db_name = Column(String(64), unique=True, nullable=False)
db_user = Column(String(64), nullable=False)
db_password_enc = Column(Text, nullable=True) # Fernet token
# MT-23: which Fernet key version db_password_enc was written under, so
# rotation progress is queryable rather than assumed:
# SELECT COUNT(*) FROM tenants WHERE key_version < <current>;
key_version = Column(Integer, nullable=False, default=1, server_default='1')
alembic_head = Column(String(64), nullable=True) # last tenant-schema rev applied
created_at = Column(DateTime, nullable=False, default=now_eastern)
@@ -124,8 +128,25 @@ class Tenant(ControlBase):
# ── Encrypted credential handling ──────────────────────────────────────
def set_db_password(self, plaintext):
"""Store an encrypted tenant DB password."""
"""Store an encrypted tenant DB password, stamped with the key version."""
self.db_password_enc = crypto.encrypt(plaintext)
self.key_version = crypto.current_key_version()
def rotate_db_password_key(self):
"""MT-23: re-encrypt the stored credential under the primary key.
Returns True when the row was rewritten, False when there was nothing
stored. The plaintext is never materialized here — MultiFernet.rotate
decrypts and re-encrypts internally.
"""
if not self.db_password_enc:
# Nothing to re-encrypt, but stamp the version so the row stops
# showing up as outstanding in rotation progress queries.
self.key_version = crypto.current_key_version()
return False
self.db_password_enc = crypto.rotate(self.db_password_enc)
self.key_version = crypto.current_key_version()
return True
@property
def db_password(self):