Aug 21 - Fernet key rotation

This commit is contained in:
2026-08-21 10:18:22 -04:00
parent cb7c244872
commit d04190ba09
5 changed files with 578 additions and 15 deletions
+125 -1
View File
@@ -8,6 +8,8 @@ the bootstrap actions on top of an already-migrated control DB.
python -m control.cli seed
python -m control.cli create-superadmin --username admin --email you@example.com
python -m control.cli list-plans
python -m control.cli verify-keys
python -m control.cli rotate-keys --dry-run
"""
import argparse
@@ -15,7 +17,7 @@ import getpass
import sys
from control.base import control_session
from control.models import Plan, Superadmin
from control.models import Plan, Superadmin, Tenant
from control.seed import seed_plans
@@ -58,6 +60,113 @@ def _cmd_create_superadmin(args):
print(f"Superadmin '{args.username}' created.")
def _cmd_verify_keys(_args):
"""MT-23: confirm every stored credential is readable under the current key set.
Run this BEFORE retiring a key and again after rotating. It is the step
that makes rotation safe: it proves no row still depends on a key that is
about to be removed, instead of finding out at the next tenant request.
"""
from control import crypto
current = crypto.current_key_version()
print(f'Primary key fingerprint : {crypto.key_fingerprint()}')
print(f'Retired keys configured : {len(crypto.old_keys())}')
print(f'Current key version : {current}')
print('-' * 60)
ok = failed = empty = stale = 0
failures = []
with control_session() as s:
tenants = s.query(Tenant).order_by(Tenant.id).all()
for t in tenants:
if not t.db_password_enc:
empty += 1
continue
if crypto.can_decrypt(t.db_password_enc):
ok += 1
else:
failed += 1
failures.append((t.id, t.slug))
if (t.key_version or 1) < current:
stale += 1
print(f' readable : {ok}')
print(f' no credential set : {empty}')
print(f' UNREADABLE : {failed}')
print(f' awaiting rotation : {stale} (key_version < {current})')
if failures:
print('\nUnreadable rows — do NOT retire any key:')
for tid, slug in failures:
print(f' tenant {tid} ({slug})')
sys.exit(1)
if stale:
print(f'\n{stale} row(s) still on an older key. Run: '
'python -m control.cli rotate-keys')
else:
print('\nAll credentials current. Safe to drop retired keys from '
'CONTROL_FERNET_KEYS_OLD.')
def _cmd_rotate_keys(args):
"""MT-23: re-encrypt tenant credentials under the primary Fernet key.
Idempotent — rows already at the current version are skipped unless
--force is given. Each tenant is committed independently so a failure
part-way leaves a resumable state rather than an all-or-nothing rollback.
"""
from control import crypto
from cryptography.fernet import InvalidToken
current = crypto.current_key_version()
print(f'Primary key fingerprint : {crypto.key_fingerprint()}')
print(f'Target key version : {current}')
if args.dry_run:
print('MODE: dry run — nothing will be written.')
print('-' * 60)
rotated = skipped = empty = errors = 0
with control_session() as s:
tenants = s.query(Tenant).order_by(Tenant.id).all()
for t in tenants:
if (t.key_version or 1) >= current and not args.force:
skipped += 1
continue
if args.dry_run:
label = 'would rotate' if t.db_password_enc else 'would stamp (no credential)'
print(f' tenant {t.id} ({t.slug}): {label}')
rotated += 1
continue
try:
changed = t.rotate_db_password_key()
s.flush()
if changed:
rotated += 1
print(f' tenant {t.id} ({t.slug}): rotated -> v{t.key_version}')
else:
empty += 1
print(f' tenant {t.id} ({t.slug}): no credential, stamped '
f'-> v{t.key_version}')
except InvalidToken:
errors += 1
print(f' tenant {t.id} ({t.slug}): ERROR — no configured key opens '
f'this credential. Add the retiring key to '
f'CONTROL_FERNET_KEYS_OLD and re-run.')
except Exception as exc:
errors += 1
print(f' tenant {t.id} ({t.slug}): ERROR — {exc}')
print('-' * 60)
print(f'rotated={rotated} skipped={skipped} no_credential={empty} errors={errors}')
if errors:
print('\nRotation incomplete. Do NOT retire any key. Fix the errors above '
'and re-run — completed rows are skipped automatically.')
sys.exit(1)
if not args.dry_run:
print('\nNow run: python -m control.cli verify-keys')
def main(argv=None):
parser = argparse.ArgumentParser(prog='control.cli',
description='JQC control-plane CLI')
@@ -73,6 +182,21 @@ def main(argv=None):
help='Omit to be prompted securely.')
p_sa.set_defaults(func=_cmd_create_superadmin)
sub.add_parser(
'verify-keys',
help='Check every tenant credential decrypts under the current key set',
).set_defaults(func=_cmd_verify_keys)
p_rot = sub.add_parser(
'rotate-keys',
help='Re-encrypt tenant credentials under the primary Fernet key',
)
p_rot.add_argument('--dry-run', action='store_true',
help='Report what would change; write nothing.')
p_rot.add_argument('--force', action='store_true',
help='Re-encrypt even rows already at the current version.')
p_rot.set_defaults(func=_cmd_rotate_keys)
args = parser.parse_args(argv)
args.func(args)
+133 -13
View File
@@ -3,39 +3,159 @@ control/crypto.py
-----------------
Symmetric encryption for sensitive control-plane fields (tenant DB passwords).
Fernet (AES-128-CBC + HMAC) keyed by the CONTROL_FERNET_KEY env var.
Generate a key once:
Fernet (AES-128-CBC + HMAC). MT-23 adds key rotation via ``MultiFernet``.
Key configuration
-----------------
``CONTROL_FERNET_KEY``
The **primary** key. Everything is encrypted with this one.
``CONTROL_FERNET_KEYS_OLD``
Comma-separated retired keys, newest first. Decrypt-only: they open
ciphertext written before the last rotation, and are never used to encrypt.
Empty or unset in steady state.
``CONTROL_FERNET_KEY_VERSION``
Integer, default 1. Stamped onto every row as it is (re-)encrypted, so
rotation progress is a SQL query rather than a guess:
SELECT COUNT(*) FROM tenants WHERE key_version < <current>;
Generate a key:
python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())"
Store it in the server environment / .env (never in git). Key rotation requires
re-encrypting existing tenant rows.
Rotation procedure
------------------
1. Generate a new key.
2. Move the current ``CONTROL_FERNET_KEY`` value into ``CONTROL_FERNET_KEYS_OLD``
(prepending it if that variable already holds keys), set the new key as
``CONTROL_FERNET_KEY``, and bump ``CONTROL_FERNET_KEY_VERSION``.
3. ``python -m control.cli verify-keys`` — confirms every stored credential is
still readable under the new key set **before** anything is rewritten.
4. ``python -m control.cli rotate-keys`` — re-encrypts every row under the
primary key and stamps the new version.
5. ``python -m control.cli verify-keys`` again, then drop the retired key from
``CONTROL_FERNET_KEYS_OLD``.
Steps 3 and 5 are the point of the design: at no moment is there a window
where a key has been retired but rows still depend on it.
"""
import hashlib
import os
from cryptography.fernet import Fernet
from cryptography.fernet import Fernet, MultiFernet
_KEY_ENV = 'CONTROL_FERNET_KEY'
_OLD_KEY_ENV = 'CONTROL_FERNET_KEYS_OLD'
_VERSION_ENV = 'CONTROL_FERNET_KEY_VERSION'
def _fernet() -> Fernet:
key = os.environ.get('CONTROL_FERNET_KEY')
def _as_bytes(key):
return key.encode() if isinstance(key, str) else key
def primary_key():
"""The single key used for encryption. Raises if unset."""
key = os.environ.get(_KEY_ENV)
if not key:
raise RuntimeError(
"CONTROL_FERNET_KEY is not set. Generate one with: "
f"{_KEY_ENV} is not set. Generate one with: "
"python -c \"from cryptography.fernet import Fernet; "
"print(Fernet.generate_key().decode())\""
)
return Fernet(key.encode() if isinstance(key, str) else key)
return _as_bytes(key)
def old_keys():
"""Retired decrypt-only keys, in the order they are tried."""
raw = os.environ.get(_OLD_KEY_ENV) or ''
return [_as_bytes(k.strip()) for k in raw.split(',') if k.strip()]
def current_key_version():
"""Integer stamped onto rows encrypted under the current primary key."""
try:
return int(os.environ.get(_VERSION_ENV, 1))
except (TypeError, ValueError):
return 1
def key_fingerprint(key=None):
"""Short, non-reversible identifier for a key — safe to log.
Lets an operator confirm which key a process actually loaded without ever
putting key material in a log line or a support ticket.
"""
material = _as_bytes(key) if key is not None else primary_key()
return hashlib.sha256(material).hexdigest()[:12]
def _multi():
"""MultiFernet: primary first (it encrypts), retired keys after it.
Ordering matters — MultiFernet encrypts with the first key and tries each
in turn when decrypting.
"""
keys = [Fernet(primary_key())]
seen = {primary_key()}
for k in old_keys():
if k in seen:
continue # tolerate a stale copy of the primary in the old list
seen.add(k)
keys.append(Fernet(k))
return MultiFernet(keys)
# Kept for backward compatibility: earlier code imported _fernet() directly.
def _fernet():
return _multi()
def encrypt(plaintext):
"""Encrypt a string to a Fernet token (str). Passes None through."""
"""Encrypt a string to a Fernet token (str) under the primary key.
Passes None through.
"""
if plaintext is None:
return None
return _fernet().encrypt(plaintext.encode()).decode()
return _multi().encrypt(plaintext.encode()).decode()
def decrypt(token):
"""Decrypt a Fernet token back to the original string. Passes None through."""
"""Decrypt a Fernet token, trying the primary key then each retired key.
Passes None through. Raises ``cryptography.fernet.InvalidToken`` when no
configured key opens the token.
"""
if token is None:
return None
return _fernet().decrypt(token.encode()).decode()
return _multi().decrypt(token.encode()).decode()
def rotate(token):
"""Re-encrypt an existing token under the primary key.
``MultiFernet.rotate`` decrypts with whichever key works and re-encrypts
with the first, preserving the plaintext without it ever being handled
here. Passes None through.
"""
if token is None:
return None
return _multi().rotate(token.encode()).decode()
def can_decrypt(token):
"""True when some configured key opens this token. Never raises.
Used by ``control.cli verify-keys`` to audit every stored credential
before a key is retired.
"""
if token is None:
return True # nothing stored is not a failure
try:
decrypt(token)
return True
except Exception:
return False
@@ -0,0 +1,51 @@
"""control0006_fernet_key_version
Adds the Fernet key-version stamp to the `tenants` table:
- key_version INT NOT NULL DEFAULT 1
MT-23: credentials are encrypted under a single control-plane Fernet key, and
before this there was no way to tell which rows had been re-encrypted after a
key change — rotation was all-or-nothing and unverifiable. Stamping the version
each time a row is written makes rotation progress a plain query:
SELECT COUNT(*) FROM tenants WHERE key_version < <current>;
Existing rows default to 1, which matches the single key in use before this
migration, so nothing needs re-encrypting to adopt it.
Guarded by an INFORMATION_SCHEMA existence check, so re-running is safe.
Revision ID: control0006_fernet_key_ver
Revises: control0005_superadmin_mfa
Create Date: 2026-08-21
"""
from alembic import op
import sqlalchemy as sa
# NOTE: revision ids must fit VARCHAR(32) — 'control0006_fernet_key_ver' is 26.
revision = 'control0006_fernet_key_ver'
down_revision = 'control0005_superadmin_mfa'
branch_labels = None
depends_on = None
def _column_exists(table, column):
result = op.get_bind().execute(sa.text(
"SELECT COUNT(*) FROM information_schema.COLUMNS "
"WHERE TABLE_SCHEMA = DATABASE() "
" AND TABLE_NAME = :tbl "
" AND COLUMN_NAME = :col"
), {'tbl': table, 'col': column})
return result.scalar() > 0
def upgrade():
if not _column_exists('tenants', 'key_version'):
op.add_column('tenants', sa.Column('key_version', sa.Integer(),
nullable=False, server_default='1'))
def downgrade():
if _column_exists('tenants', 'key_version'):
op.drop_column('tenants', 'key_version')
+22 -1
View File
@@ -95,6 +95,10 @@ class Tenant(ControlBase):
db_name = Column(String(64), unique=True, nullable=False)
db_user = Column(String(64), nullable=False)
db_password_enc = Column(Text, nullable=True) # Fernet token
# MT-23: which Fernet key version db_password_enc was written under, so
# rotation progress is queryable rather than assumed:
# SELECT COUNT(*) FROM tenants WHERE key_version < <current>;
key_version = Column(Integer, nullable=False, default=1, server_default='1')
alembic_head = Column(String(64), nullable=True) # last tenant-schema rev applied
created_at = Column(DateTime, nullable=False, default=now_eastern)
@@ -124,8 +128,25 @@ class Tenant(ControlBase):
# ── Encrypted credential handling ──────────────────────────────────────
def set_db_password(self, plaintext):
"""Store an encrypted tenant DB password."""
"""Store an encrypted tenant DB password, stamped with the key version."""
self.db_password_enc = crypto.encrypt(plaintext)
self.key_version = crypto.current_key_version()
def rotate_db_password_key(self):
"""MT-23: re-encrypt the stored credential under the primary key.
Returns True when the row was rewritten, False when there was nothing
stored. The plaintext is never materialized here — MultiFernet.rotate
decrypts and re-encrypts internally.
"""
if not self.db_password_enc:
# Nothing to re-encrypt, but stamp the version so the row stops
# showing up as outstanding in rotation progress queries.
self.key_version = crypto.current_key_version()
return False
self.db_password_enc = crypto.rotate(self.db_password_enc)
self.key_version = crypto.current_key_version()
return True
@property
def db_password(self):
+247
View File
@@ -0,0 +1,247 @@
"""
tests/test_fernet_rotation.py
-----------------------------
MT-23: control-plane Fernet key rotation.
The property that matters is continuity of access: at no point during a
rotation may a stored tenant credential become unreadable. A key change that
leaves one row encrypted under a discarded key locks that tenant out of its
own database, and nothing surfaces it until the next request for that tenant.
These tests exercise the rotation the way the runbook does — old key alone,
then both keys, then new key alone — and assert the credential survives each
step. The last of those is the one that catches a premature retirement.
`control.crypto` reads its keys from the environment on every call, so the
fixtures set os.environ directly rather than patching the module.
"""
import os
import pytest
from cryptography.fernet import Fernet, InvalidToken
from control import crypto
KEY_A = Fernet.generate_key().decode() # "old" key
KEY_B = Fernet.generate_key().decode() # "new" key
KEY_C = Fernet.generate_key().decode() # a key never used to encrypt
@pytest.fixture
def clean_env():
"""Isolate the crypto env vars and restore whatever was there."""
saved = {k: os.environ.get(k) for k in (
'CONTROL_FERNET_KEY', 'CONTROL_FERNET_KEYS_OLD',
'CONTROL_FERNET_KEY_VERSION',
)}
try:
yield
finally:
for k, v in saved.items():
if v is None:
os.environ.pop(k, None)
else:
os.environ[k] = v
def _configure(primary, old=(), version=1):
os.environ['CONTROL_FERNET_KEY'] = primary
os.environ['CONTROL_FERNET_KEYS_OLD'] = ','.join(old)
os.environ['CONTROL_FERNET_KEY_VERSION'] = str(version)
# ── Baseline behaviour ───────────────────────────────────────────────────────
def test_roundtrip_with_a_single_key(clean_env):
_configure(KEY_A)
token = crypto.encrypt('s3cret-pw')
assert token != 's3cret-pw'
assert crypto.decrypt(token) == 's3cret-pw'
def test_none_passes_through(clean_env):
_configure(KEY_A)
assert crypto.encrypt(None) is None
assert crypto.decrypt(None) is None
assert crypto.rotate(None) is None
assert crypto.can_decrypt(None) is True
def test_missing_key_raises_a_clear_error(clean_env):
os.environ.pop('CONTROL_FERNET_KEY', None)
with pytest.raises(RuntimeError) as exc:
crypto.encrypt('x')
assert 'CONTROL_FERNET_KEY' in str(exc.value)
def test_unknown_key_cannot_decrypt(clean_env):
_configure(KEY_A)
token = crypto.encrypt('pw')
_configure(KEY_C) # unrelated key, no old keys
assert crypto.can_decrypt(token) is False
with pytest.raises(InvalidToken):
crypto.decrypt(token)
# ── The rotation itself ──────────────────────────────────────────────────────
def test_old_ciphertext_readable_after_key_change(clean_env):
"""Step 2 of the runbook: new primary, old key retained for decrypt."""
_configure(KEY_A, version=1)
token = crypto.encrypt('tenant-pw')
_configure(KEY_B, old=[KEY_A], version=2)
assert crypto.decrypt(token) == 'tenant-pw'
assert crypto.can_decrypt(token) is True
def test_new_writes_use_the_primary_key_only(clean_env):
"""A credential written after rotation must not depend on the old key."""
_configure(KEY_B, old=[KEY_A], version=2)
token = crypto.encrypt('fresh-pw')
# Old key alone must NOT open it — proves the primary did the encrypting.
_configure(KEY_A, version=1)
assert crypto.can_decrypt(token) is False
def test_rotate_rewrites_under_the_primary_key(clean_env):
"""Step 4: after rotate(), the old key is no longer needed."""
_configure(KEY_A, version=1)
token = crypto.encrypt('tenant-pw')
_configure(KEY_B, old=[KEY_A], version=2)
rotated = crypto.rotate(token)
assert rotated != token
assert crypto.decrypt(rotated) == 'tenant-pw'
# Step 5: retire KEY_A. The rotated token must still open.
_configure(KEY_B, version=2)
assert crypto.decrypt(rotated) == 'tenant-pw'
# And the un-rotated one must not — this is what verify-keys catches.
assert crypto.can_decrypt(token) is False
def test_rotation_preserves_plaintext_across_two_generations(clean_env):
"""A credential must survive being rotated twice, A -> B -> C."""
_configure(KEY_A, version=1)
token = crypto.encrypt('long-lived-pw')
_configure(KEY_B, old=[KEY_A], version=2)
token = crypto.rotate(token)
_configure(KEY_C, old=[KEY_B], version=3) # KEY_A already retired
token = crypto.rotate(token)
_configure(KEY_C, version=3) # all predecessors retired
assert crypto.decrypt(token) == 'long-lived-pw'
def test_stale_primary_in_old_list_is_tolerated(clean_env):
"""An operator leaving the primary in CONTROL_FERNET_KEYS_OLD is harmless."""
_configure(KEY_B, old=[KEY_B, KEY_A], version=2)
token = crypto.encrypt('pw')
assert crypto.decrypt(token) == 'pw'
def test_blank_entries_in_old_key_list_are_ignored(clean_env):
"""Trailing commas and whitespace must not break key loading."""
os.environ['CONTROL_FERNET_KEY'] = KEY_B
os.environ['CONTROL_FERNET_KEYS_OLD'] = f' {KEY_A} , , '
os.environ['CONTROL_FERNET_KEY_VERSION'] = '2'
assert crypto.old_keys() == [KEY_A.encode()]
_configure(KEY_A, version=1)
token = crypto.encrypt('pw')
os.environ['CONTROL_FERNET_KEY'] = KEY_B
os.environ['CONTROL_FERNET_KEYS_OLD'] = f' {KEY_A} , , '
assert crypto.decrypt(token) == 'pw'
# ── Version stamping ─────────────────────────────────────────────────────────
def test_key_version_defaults_to_one(clean_env):
_configure(KEY_A)
os.environ.pop('CONTROL_FERNET_KEY_VERSION', None)
assert crypto.current_key_version() == 1
def test_key_version_survives_a_garbage_value(clean_env):
_configure(KEY_A)
os.environ['CONTROL_FERNET_KEY_VERSION'] = 'not-a-number'
assert crypto.current_key_version() == 1
def test_fingerprint_identifies_a_key_without_revealing_it(clean_env):
_configure(KEY_A)
fp = crypto.key_fingerprint()
assert len(fp) == 12
assert KEY_A not in fp
assert fp == crypto.key_fingerprint(KEY_A)
assert fp != crypto.key_fingerprint(KEY_B)
# ── Model-level stamping (no DB required) ────────────────────────────────────
def test_tenant_set_password_stamps_the_current_version(clean_env):
from control.models import Tenant
_configure(KEY_A, version=1)
t = Tenant()
t.set_db_password('pw-one')
assert t.key_version == 1
assert t.db_password == 'pw-one'
_configure(KEY_B, old=[KEY_A], version=2)
t.set_db_password('pw-two')
assert t.key_version == 2
assert t.db_password == 'pw-two'
def test_tenant_rotate_updates_ciphertext_and_version(clean_env):
from control.models import Tenant
_configure(KEY_A, version=1)
t = Tenant()
t.set_db_password('rotate-me')
before = t.db_password_enc
_configure(KEY_B, old=[KEY_A], version=2)
assert t.rotate_db_password_key() is True
assert t.db_password_enc != before
assert t.key_version == 2
assert t.db_password == 'rotate-me'
# Retire the old key — the rotated row must still open.
_configure(KEY_B, version=2)
assert t.db_password == 'rotate-me'
def test_tenant_without_credential_is_stamped_not_rotated(clean_env):
from control.models import Tenant
_configure(KEY_B, old=[KEY_A], version=2)
t = Tenant()
assert t.db_password_enc is None
assert t.rotate_db_password_key() is False
# Stamped anyway, so it stops appearing as outstanding work.
assert t.key_version == 2
def test_db_uri_still_builds_after_rotation(clean_env):
"""Rotation must not disturb the URI the routing layer depends on."""
from control.models import Tenant
_configure(KEY_A, version=1)
t = Tenant(db_host='localhost', db_port=3306,
db_name='jqc_t1', db_user='jqc_t1')
t.set_db_password('p@ss word//')
uri_before = t.db_uri
_configure(KEY_B, old=[KEY_A], version=2)
t.rotate_db_password_key()
_configure(KEY_B, version=2)
assert t.db_uri == uri_before
assert 'p%40ss+word%2F%2F' in t.db_uri # still URL-encoded