diff --git a/control/cli.py b/control/cli.py index 9f6834d..2377255 100644 --- a/control/cli.py +++ b/control/cli.py @@ -8,6 +8,8 @@ the bootstrap actions on top of an already-migrated control DB. python -m control.cli seed python -m control.cli create-superadmin --username admin --email you@example.com python -m control.cli list-plans + python -m control.cli verify-keys + python -m control.cli rotate-keys --dry-run """ import argparse @@ -15,7 +17,7 @@ import getpass import sys from control.base import control_session -from control.models import Plan, Superadmin +from control.models import Plan, Superadmin, Tenant from control.seed import seed_plans @@ -58,6 +60,113 @@ def _cmd_create_superadmin(args): print(f"Superadmin '{args.username}' created.") +def _cmd_verify_keys(_args): + """MT-23: confirm every stored credential is readable under the current key set. + + Run this BEFORE retiring a key and again after rotating. It is the step + that makes rotation safe: it proves no row still depends on a key that is + about to be removed, instead of finding out at the next tenant request. + """ + from control import crypto + current = crypto.current_key_version() + print(f'Primary key fingerprint : {crypto.key_fingerprint()}') + print(f'Retired keys configured : {len(crypto.old_keys())}') + print(f'Current key version : {current}') + print('-' * 60) + + ok = failed = empty = stale = 0 + failures = [] + with control_session() as s: + tenants = s.query(Tenant).order_by(Tenant.id).all() + for t in tenants: + if not t.db_password_enc: + empty += 1 + continue + if crypto.can_decrypt(t.db_password_enc): + ok += 1 + else: + failed += 1 + failures.append((t.id, t.slug)) + if (t.key_version or 1) < current: + stale += 1 + + print(f' readable : {ok}') + print(f' no credential set : {empty}') + print(f' UNREADABLE : {failed}') + print(f' awaiting rotation : {stale} (key_version < {current})') + + if failures: + print('\nUnreadable rows — do NOT retire any key:') + for tid, slug in failures: + print(f' tenant {tid} ({slug})') + sys.exit(1) + + if stale: + print(f'\n{stale} row(s) still on an older key. Run: ' + 'python -m control.cli rotate-keys') + else: + print('\nAll credentials current. Safe to drop retired keys from ' + 'CONTROL_FERNET_KEYS_OLD.') + + +def _cmd_rotate_keys(args): + """MT-23: re-encrypt tenant credentials under the primary Fernet key. + + Idempotent — rows already at the current version are skipped unless + --force is given. Each tenant is committed independently so a failure + part-way leaves a resumable state rather than an all-or-nothing rollback. + """ + from control import crypto + from cryptography.fernet import InvalidToken + + current = crypto.current_key_version() + print(f'Primary key fingerprint : {crypto.key_fingerprint()}') + print(f'Target key version : {current}') + if args.dry_run: + print('MODE: dry run — nothing will be written.') + print('-' * 60) + + rotated = skipped = empty = errors = 0 + with control_session() as s: + tenants = s.query(Tenant).order_by(Tenant.id).all() + for t in tenants: + if (t.key_version or 1) >= current and not args.force: + skipped += 1 + continue + if args.dry_run: + label = 'would rotate' if t.db_password_enc else 'would stamp (no credential)' + print(f' tenant {t.id} ({t.slug}): {label}') + rotated += 1 + continue + try: + changed = t.rotate_db_password_key() + s.flush() + if changed: + rotated += 1 + print(f' tenant {t.id} ({t.slug}): rotated -> v{t.key_version}') + else: + empty += 1 + print(f' tenant {t.id} ({t.slug}): no credential, stamped ' + f'-> v{t.key_version}') + except InvalidToken: + errors += 1 + print(f' tenant {t.id} ({t.slug}): ERROR — no configured key opens ' + f'this credential. Add the retiring key to ' + f'CONTROL_FERNET_KEYS_OLD and re-run.') + except Exception as exc: + errors += 1 + print(f' tenant {t.id} ({t.slug}): ERROR — {exc}') + + print('-' * 60) + print(f'rotated={rotated} skipped={skipped} no_credential={empty} errors={errors}') + if errors: + print('\nRotation incomplete. Do NOT retire any key. Fix the errors above ' + 'and re-run — completed rows are skipped automatically.') + sys.exit(1) + if not args.dry_run: + print('\nNow run: python -m control.cli verify-keys') + + def main(argv=None): parser = argparse.ArgumentParser(prog='control.cli', description='JQC control-plane CLI') @@ -73,6 +182,21 @@ def main(argv=None): help='Omit to be prompted securely.') p_sa.set_defaults(func=_cmd_create_superadmin) + sub.add_parser( + 'verify-keys', + help='Check every tenant credential decrypts under the current key set', + ).set_defaults(func=_cmd_verify_keys) + + p_rot = sub.add_parser( + 'rotate-keys', + help='Re-encrypt tenant credentials under the primary Fernet key', + ) + p_rot.add_argument('--dry-run', action='store_true', + help='Report what would change; write nothing.') + p_rot.add_argument('--force', action='store_true', + help='Re-encrypt even rows already at the current version.') + p_rot.set_defaults(func=_cmd_rotate_keys) + args = parser.parse_args(argv) args.func(args) diff --git a/control/crypto.py b/control/crypto.py index a9d88b4..ea587e3 100644 --- a/control/crypto.py +++ b/control/crypto.py @@ -3,39 +3,159 @@ control/crypto.py ----------------- Symmetric encryption for sensitive control-plane fields (tenant DB passwords). -Fernet (AES-128-CBC + HMAC) keyed by the CONTROL_FERNET_KEY env var. -Generate a key once: +Fernet (AES-128-CBC + HMAC). MT-23 adds key rotation via ``MultiFernet``. + +Key configuration +----------------- +``CONTROL_FERNET_KEY`` + The **primary** key. Everything is encrypted with this one. + +``CONTROL_FERNET_KEYS_OLD`` + Comma-separated retired keys, newest first. Decrypt-only: they open + ciphertext written before the last rotation, and are never used to encrypt. + Empty or unset in steady state. + +``CONTROL_FERNET_KEY_VERSION`` + Integer, default 1. Stamped onto every row as it is (re-)encrypted, so + rotation progress is a SQL query rather than a guess: + + SELECT COUNT(*) FROM tenants WHERE key_version < ; + +Generate a key: python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" -Store it in the server environment / .env (never in git). Key rotation requires -re-encrypting existing tenant rows. +Rotation procedure +------------------ +1. Generate a new key. +2. Move the current ``CONTROL_FERNET_KEY`` value into ``CONTROL_FERNET_KEYS_OLD`` + (prepending it if that variable already holds keys), set the new key as + ``CONTROL_FERNET_KEY``, and bump ``CONTROL_FERNET_KEY_VERSION``. +3. ``python -m control.cli verify-keys`` — confirms every stored credential is + still readable under the new key set **before** anything is rewritten. +4. ``python -m control.cli rotate-keys`` — re-encrypts every row under the + primary key and stamps the new version. +5. ``python -m control.cli verify-keys`` again, then drop the retired key from + ``CONTROL_FERNET_KEYS_OLD``. + +Steps 3 and 5 are the point of the design: at no moment is there a window +where a key has been retired but rows still depend on it. """ +import hashlib import os -from cryptography.fernet import Fernet + +from cryptography.fernet import Fernet, MultiFernet + +_KEY_ENV = 'CONTROL_FERNET_KEY' +_OLD_KEY_ENV = 'CONTROL_FERNET_KEYS_OLD' +_VERSION_ENV = 'CONTROL_FERNET_KEY_VERSION' -def _fernet() -> Fernet: - key = os.environ.get('CONTROL_FERNET_KEY') +def _as_bytes(key): + return key.encode() if isinstance(key, str) else key + + +def primary_key(): + """The single key used for encryption. Raises if unset.""" + key = os.environ.get(_KEY_ENV) if not key: raise RuntimeError( - "CONTROL_FERNET_KEY is not set. Generate one with: " + f"{_KEY_ENV} is not set. Generate one with: " "python -c \"from cryptography.fernet import Fernet; " "print(Fernet.generate_key().decode())\"" ) - return Fernet(key.encode() if isinstance(key, str) else key) + return _as_bytes(key) + + +def old_keys(): + """Retired decrypt-only keys, in the order they are tried.""" + raw = os.environ.get(_OLD_KEY_ENV) or '' + return [_as_bytes(k.strip()) for k in raw.split(',') if k.strip()] + + +def current_key_version(): + """Integer stamped onto rows encrypted under the current primary key.""" + try: + return int(os.environ.get(_VERSION_ENV, 1)) + except (TypeError, ValueError): + return 1 + + +def key_fingerprint(key=None): + """Short, non-reversible identifier for a key — safe to log. + + Lets an operator confirm which key a process actually loaded without ever + putting key material in a log line or a support ticket. + """ + material = _as_bytes(key) if key is not None else primary_key() + return hashlib.sha256(material).hexdigest()[:12] + + +def _multi(): + """MultiFernet: primary first (it encrypts), retired keys after it. + + Ordering matters — MultiFernet encrypts with the first key and tries each + in turn when decrypting. + """ + keys = [Fernet(primary_key())] + seen = {primary_key()} + for k in old_keys(): + if k in seen: + continue # tolerate a stale copy of the primary in the old list + seen.add(k) + keys.append(Fernet(k)) + return MultiFernet(keys) + + +# Kept for backward compatibility: earlier code imported _fernet() directly. +def _fernet(): + return _multi() def encrypt(plaintext): - """Encrypt a string to a Fernet token (str). Passes None through.""" + """Encrypt a string to a Fernet token (str) under the primary key. + + Passes None through. + """ if plaintext is None: return None - return _fernet().encrypt(plaintext.encode()).decode() + return _multi().encrypt(plaintext.encode()).decode() def decrypt(token): - """Decrypt a Fernet token back to the original string. Passes None through.""" + """Decrypt a Fernet token, trying the primary key then each retired key. + + Passes None through. Raises ``cryptography.fernet.InvalidToken`` when no + configured key opens the token. + """ if token is None: return None - return _fernet().decrypt(token.encode()).decode() + return _multi().decrypt(token.encode()).decode() + + +def rotate(token): + """Re-encrypt an existing token under the primary key. + + ``MultiFernet.rotate`` decrypts with whichever key works and re-encrypts + with the first, preserving the plaintext without it ever being handled + here. Passes None through. + """ + if token is None: + return None + return _multi().rotate(token.encode()).decode() + + +def can_decrypt(token): + """True when some configured key opens this token. Never raises. + + Used by ``control.cli verify-keys`` to audit every stored credential + before a key is retired. + """ + if token is None: + return True # nothing stored is not a failure + try: + decrypt(token) + return True + except Exception: + return False diff --git a/control/migrations/versions/control0006_fernet_key_version.py b/control/migrations/versions/control0006_fernet_key_version.py new file mode 100644 index 0000000..21bdba5 --- /dev/null +++ b/control/migrations/versions/control0006_fernet_key_version.py @@ -0,0 +1,51 @@ +"""control0006_fernet_key_version + +Adds the Fernet key-version stamp to the `tenants` table: + - key_version INT NOT NULL DEFAULT 1 + +MT-23: credentials are encrypted under a single control-plane Fernet key, and +before this there was no way to tell which rows had been re-encrypted after a +key change — rotation was all-or-nothing and unverifiable. Stamping the version +each time a row is written makes rotation progress a plain query: + + SELECT COUNT(*) FROM tenants WHERE key_version < ; + +Existing rows default to 1, which matches the single key in use before this +migration, so nothing needs re-encrypting to adopt it. + +Guarded by an INFORMATION_SCHEMA existence check, so re-running is safe. + +Revision ID: control0006_fernet_key_ver +Revises: control0005_superadmin_mfa +Create Date: 2026-08-21 +""" + +from alembic import op +import sqlalchemy as sa + +# NOTE: revision ids must fit VARCHAR(32) — 'control0006_fernet_key_ver' is 26. +revision = 'control0006_fernet_key_ver' +down_revision = 'control0005_superadmin_mfa' +branch_labels = None +depends_on = None + + +def _column_exists(table, column): + result = op.get_bind().execute(sa.text( + "SELECT COUNT(*) FROM information_schema.COLUMNS " + "WHERE TABLE_SCHEMA = DATABASE() " + " AND TABLE_NAME = :tbl " + " AND COLUMN_NAME = :col" + ), {'tbl': table, 'col': column}) + return result.scalar() > 0 + + +def upgrade(): + if not _column_exists('tenants', 'key_version'): + op.add_column('tenants', sa.Column('key_version', sa.Integer(), + nullable=False, server_default='1')) + + +def downgrade(): + if _column_exists('tenants', 'key_version'): + op.drop_column('tenants', 'key_version') diff --git a/control/models.py b/control/models.py index 45b6fe3..aaf425e 100644 --- a/control/models.py +++ b/control/models.py @@ -95,6 +95,10 @@ class Tenant(ControlBase): db_name = Column(String(64), unique=True, nullable=False) db_user = Column(String(64), nullable=False) db_password_enc = Column(Text, nullable=True) # Fernet token + # MT-23: which Fernet key version db_password_enc was written under, so + # rotation progress is queryable rather than assumed: + # SELECT COUNT(*) FROM tenants WHERE key_version < ; + key_version = Column(Integer, nullable=False, default=1, server_default='1') alembic_head = Column(String(64), nullable=True) # last tenant-schema rev applied created_at = Column(DateTime, nullable=False, default=now_eastern) @@ -124,8 +128,25 @@ class Tenant(ControlBase): # ── Encrypted credential handling ────────────────────────────────────── def set_db_password(self, plaintext): - """Store an encrypted tenant DB password.""" + """Store an encrypted tenant DB password, stamped with the key version.""" self.db_password_enc = crypto.encrypt(plaintext) + self.key_version = crypto.current_key_version() + + def rotate_db_password_key(self): + """MT-23: re-encrypt the stored credential under the primary key. + + Returns True when the row was rewritten, False when there was nothing + stored. The plaintext is never materialized here — MultiFernet.rotate + decrypts and re-encrypts internally. + """ + if not self.db_password_enc: + # Nothing to re-encrypt, but stamp the version so the row stops + # showing up as outstanding in rotation progress queries. + self.key_version = crypto.current_key_version() + return False + self.db_password_enc = crypto.rotate(self.db_password_enc) + self.key_version = crypto.current_key_version() + return True @property def db_password(self): diff --git a/tests/test_fernet_rotation.py b/tests/test_fernet_rotation.py new file mode 100644 index 0000000..61630e6 --- /dev/null +++ b/tests/test_fernet_rotation.py @@ -0,0 +1,247 @@ +""" +tests/test_fernet_rotation.py +----------------------------- +MT-23: control-plane Fernet key rotation. + +The property that matters is continuity of access: at no point during a +rotation may a stored tenant credential become unreadable. A key change that +leaves one row encrypted under a discarded key locks that tenant out of its +own database, and nothing surfaces it until the next request for that tenant. + +These tests exercise the rotation the way the runbook does — old key alone, +then both keys, then new key alone — and assert the credential survives each +step. The last of those is the one that catches a premature retirement. + +`control.crypto` reads its keys from the environment on every call, so the +fixtures set os.environ directly rather than patching the module. +""" + +import os + +import pytest +from cryptography.fernet import Fernet, InvalidToken + +from control import crypto + + +KEY_A = Fernet.generate_key().decode() # "old" key +KEY_B = Fernet.generate_key().decode() # "new" key +KEY_C = Fernet.generate_key().decode() # a key never used to encrypt + + +@pytest.fixture +def clean_env(): + """Isolate the crypto env vars and restore whatever was there.""" + saved = {k: os.environ.get(k) for k in ( + 'CONTROL_FERNET_KEY', 'CONTROL_FERNET_KEYS_OLD', + 'CONTROL_FERNET_KEY_VERSION', + )} + try: + yield + finally: + for k, v in saved.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +def _configure(primary, old=(), version=1): + os.environ['CONTROL_FERNET_KEY'] = primary + os.environ['CONTROL_FERNET_KEYS_OLD'] = ','.join(old) + os.environ['CONTROL_FERNET_KEY_VERSION'] = str(version) + + +# ── Baseline behaviour ─────────────────────────────────────────────────────── + +def test_roundtrip_with_a_single_key(clean_env): + _configure(KEY_A) + token = crypto.encrypt('s3cret-pw') + assert token != 's3cret-pw' + assert crypto.decrypt(token) == 's3cret-pw' + + +def test_none_passes_through(clean_env): + _configure(KEY_A) + assert crypto.encrypt(None) is None + assert crypto.decrypt(None) is None + assert crypto.rotate(None) is None + assert crypto.can_decrypt(None) is True + + +def test_missing_key_raises_a_clear_error(clean_env): + os.environ.pop('CONTROL_FERNET_KEY', None) + with pytest.raises(RuntimeError) as exc: + crypto.encrypt('x') + assert 'CONTROL_FERNET_KEY' in str(exc.value) + + +def test_unknown_key_cannot_decrypt(clean_env): + _configure(KEY_A) + token = crypto.encrypt('pw') + _configure(KEY_C) # unrelated key, no old keys + assert crypto.can_decrypt(token) is False + with pytest.raises(InvalidToken): + crypto.decrypt(token) + + +# ── The rotation itself ────────────────────────────────────────────────────── + +def test_old_ciphertext_readable_after_key_change(clean_env): + """Step 2 of the runbook: new primary, old key retained for decrypt.""" + _configure(KEY_A, version=1) + token = crypto.encrypt('tenant-pw') + + _configure(KEY_B, old=[KEY_A], version=2) + assert crypto.decrypt(token) == 'tenant-pw' + assert crypto.can_decrypt(token) is True + + +def test_new_writes_use_the_primary_key_only(clean_env): + """A credential written after rotation must not depend on the old key.""" + _configure(KEY_B, old=[KEY_A], version=2) + token = crypto.encrypt('fresh-pw') + + # Old key alone must NOT open it — proves the primary did the encrypting. + _configure(KEY_A, version=1) + assert crypto.can_decrypt(token) is False + + +def test_rotate_rewrites_under_the_primary_key(clean_env): + """Step 4: after rotate(), the old key is no longer needed.""" + _configure(KEY_A, version=1) + token = crypto.encrypt('tenant-pw') + + _configure(KEY_B, old=[KEY_A], version=2) + rotated = crypto.rotate(token) + assert rotated != token + assert crypto.decrypt(rotated) == 'tenant-pw' + + # Step 5: retire KEY_A. The rotated token must still open. + _configure(KEY_B, version=2) + assert crypto.decrypt(rotated) == 'tenant-pw' + # And the un-rotated one must not — this is what verify-keys catches. + assert crypto.can_decrypt(token) is False + + +def test_rotation_preserves_plaintext_across_two_generations(clean_env): + """A credential must survive being rotated twice, A -> B -> C.""" + _configure(KEY_A, version=1) + token = crypto.encrypt('long-lived-pw') + + _configure(KEY_B, old=[KEY_A], version=2) + token = crypto.rotate(token) + + _configure(KEY_C, old=[KEY_B], version=3) # KEY_A already retired + token = crypto.rotate(token) + + _configure(KEY_C, version=3) # all predecessors retired + assert crypto.decrypt(token) == 'long-lived-pw' + + +def test_stale_primary_in_old_list_is_tolerated(clean_env): + """An operator leaving the primary in CONTROL_FERNET_KEYS_OLD is harmless.""" + _configure(KEY_B, old=[KEY_B, KEY_A], version=2) + token = crypto.encrypt('pw') + assert crypto.decrypt(token) == 'pw' + + +def test_blank_entries_in_old_key_list_are_ignored(clean_env): + """Trailing commas and whitespace must not break key loading.""" + os.environ['CONTROL_FERNET_KEY'] = KEY_B + os.environ['CONTROL_FERNET_KEYS_OLD'] = f' {KEY_A} , , ' + os.environ['CONTROL_FERNET_KEY_VERSION'] = '2' + assert crypto.old_keys() == [KEY_A.encode()] + _configure(KEY_A, version=1) + token = crypto.encrypt('pw') + os.environ['CONTROL_FERNET_KEY'] = KEY_B + os.environ['CONTROL_FERNET_KEYS_OLD'] = f' {KEY_A} , , ' + assert crypto.decrypt(token) == 'pw' + + +# ── Version stamping ───────────────────────────────────────────────────────── + +def test_key_version_defaults_to_one(clean_env): + _configure(KEY_A) + os.environ.pop('CONTROL_FERNET_KEY_VERSION', None) + assert crypto.current_key_version() == 1 + + +def test_key_version_survives_a_garbage_value(clean_env): + _configure(KEY_A) + os.environ['CONTROL_FERNET_KEY_VERSION'] = 'not-a-number' + assert crypto.current_key_version() == 1 + + +def test_fingerprint_identifies_a_key_without_revealing_it(clean_env): + _configure(KEY_A) + fp = crypto.key_fingerprint() + assert len(fp) == 12 + assert KEY_A not in fp + assert fp == crypto.key_fingerprint(KEY_A) + assert fp != crypto.key_fingerprint(KEY_B) + + +# ── Model-level stamping (no DB required) ──────────────────────────────────── + +def test_tenant_set_password_stamps_the_current_version(clean_env): + from control.models import Tenant + + _configure(KEY_A, version=1) + t = Tenant() + t.set_db_password('pw-one') + assert t.key_version == 1 + assert t.db_password == 'pw-one' + + _configure(KEY_B, old=[KEY_A], version=2) + t.set_db_password('pw-two') + assert t.key_version == 2 + assert t.db_password == 'pw-two' + + +def test_tenant_rotate_updates_ciphertext_and_version(clean_env): + from control.models import Tenant + + _configure(KEY_A, version=1) + t = Tenant() + t.set_db_password('rotate-me') + before = t.db_password_enc + + _configure(KEY_B, old=[KEY_A], version=2) + assert t.rotate_db_password_key() is True + assert t.db_password_enc != before + assert t.key_version == 2 + assert t.db_password == 'rotate-me' + + # Retire the old key — the rotated row must still open. + _configure(KEY_B, version=2) + assert t.db_password == 'rotate-me' + + +def test_tenant_without_credential_is_stamped_not_rotated(clean_env): + from control.models import Tenant + + _configure(KEY_B, old=[KEY_A], version=2) + t = Tenant() + assert t.db_password_enc is None + assert t.rotate_db_password_key() is False + # Stamped anyway, so it stops appearing as outstanding work. + assert t.key_version == 2 + + +def test_db_uri_still_builds_after_rotation(clean_env): + """Rotation must not disturb the URI the routing layer depends on.""" + from control.models import Tenant + + _configure(KEY_A, version=1) + t = Tenant(db_host='localhost', db_port=3306, + db_name='jqc_t1', db_user='jqc_t1') + t.set_db_password('p@ss word//') + uri_before = t.db_uri + + _configure(KEY_B, old=[KEY_A], version=2) + t.rotate_db_password_key() + + _configure(KEY_B, version=2) + assert t.db_uri == uri_before + assert 'p%40ss+word%2F%2F' in t.db_uri # still URL-encoded