Aug 11 - Update photo storage using r2
This commit is contained in:
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
scripts/migrate_photos_to_r2.py — Verified per-tenant photo sync to R2
|
||||
========================================================================
|
||||
|
||||
Multi-tenant port of the single-tenant script of the same name (ST §22 Phase 3).
|
||||
Copies each tenant's photo files from the shared local uploads tree to the R2
|
||||
bucket **under that tenant's object prefix**, and VERIFIES each one (MD5 + size).
|
||||
|
||||
It is:
|
||||
- COPY-ONLY : never deletes or modifies local files, never touches any DB.
|
||||
- IDEMPOTENT : re-running skips objects already present and verified.
|
||||
- RESUMABLE : safe to interrupt and re-run; picks up where it left off.
|
||||
- GATED : exits non-zero if ANY file fails verification, so you never
|
||||
flip STORAGE_BACKEND=s3 on an incomplete/corrupt copy.
|
||||
|
||||
Key mapping — the whole point of this script
|
||||
--------------------------------------------
|
||||
The DB stores an unprefixed key and always will:
|
||||
|
||||
DB value : uploads/issue_photos/ab12….jpg
|
||||
R2 object : t3/uploads/issue_photos/ab12….jpg
|
||||
|
||||
The ``t<tenant_id>/`` prefix is applied by ``S3Backend._object_key()`` at
|
||||
runtime (app/utils/storage.py). This script must therefore write to the SAME
|
||||
prefixed key, or nothing will resolve after cutover.
|
||||
|
||||
Because the local backend does NOT prefix, one shared directory holds every
|
||||
tenant's files and a file on disk carries no ownership marker. Ownership is
|
||||
resolved exactly as in scripts/audit_photos.py: by reading each tenant's own
|
||||
database and collecting the keys it references. Consequences, both intended:
|
||||
|
||||
* Files referenced by NO tenant (orphans) are NOT uploaded — no tenant's
|
||||
prefix could legitimately claim them, and nothing reads them.
|
||||
* A key referenced by two tenants is copied under BOTH prefixes. Run
|
||||
audit_photos.py first; it flags that case.
|
||||
|
||||
R2 credentials come from the environment / .env: R2_ENDPOINT_URL,
|
||||
R2_ACCESS_KEY_ID, R2_SECRET_ACCESS_KEY, R2_BUCKET. Control-plane access needs
|
||||
CONTROL_DATABASE_URL and CONTROL_FERNET_KEY. Run this while the app is still
|
||||
serving on STORAGE_BACKEND=local — it only writes to R2.
|
||||
|
||||
Why put_object (not upload_file): a single-part PUT makes the R2 ETag equal the
|
||||
object's MD5, so verification is a direct hash comparison. Photos are <= 50 MB
|
||||
(MAX_CONTENT_LENGTH), well within a single PUT.
|
||||
|
||||
Usage
|
||||
-----
|
||||
cd /home/jqc/lt_janitorial_quality_control
|
||||
source venv/bin/activate
|
||||
pip install boto3 # if not already
|
||||
|
||||
python scripts/migrate_photos_to_r2.py --dry-run # plan only
|
||||
python scripts/migrate_photos_to_r2.py # all tenants
|
||||
python scripts/migrate_photos_to_r2.py --tenant lts # one tenant
|
||||
python scripts/migrate_photos_to_r2.py # again at cutover (delta)
|
||||
python scripts/migrate_photos_to_r2.py --force # re-upload everything
|
||||
python scripts/migrate_photos_to_r2.py --log /home/jqc/r2_sync.json
|
||||
|
||||
Exit code: 0 only when every referenced file that exists on disk is verified in
|
||||
R2 under its tenant prefix, with zero mismatches and zero errors. Non-zero
|
||||
otherwise. References missing on disk are reported and do NOT fail the gate —
|
||||
they were already broken before this move (audit_photos.py is the baseline).
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import hashlib
|
||||
import argparse
|
||||
from datetime import datetime
|
||||
|
||||
_REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
|
||||
sys.path.insert(0, _REPO_ROOT)
|
||||
|
||||
from dotenv import load_dotenv # noqa: E402
|
||||
load_dotenv(os.path.join(_REPO_ROOT, '.env'))
|
||||
|
||||
# Reuse the audit script's reference collection so the two can never diverge.
|
||||
from scripts.audit_photos import ( # noqa: E402
|
||||
AUDITABLE_STATUSES,
|
||||
STATIC_FOLDER,
|
||||
UPLOADS_ROOT,
|
||||
collect_referenced,
|
||||
select_tenants,
|
||||
)
|
||||
|
||||
_CONTENT_TYPES = {
|
||||
'jpg': 'image/jpeg', 'jpeg': 'image/jpeg',
|
||||
'png': 'image/png', 'gif': 'image/gif',
|
||||
}
|
||||
_CHUNK = 1024 * 1024 # 1 MB
|
||||
|
||||
|
||||
def _md5_and_size(path):
|
||||
h = hashlib.md5()
|
||||
size = 0
|
||||
with open(path, 'rb') as fh:
|
||||
while True:
|
||||
chunk = fh.read(_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
h.update(chunk)
|
||||
size += len(chunk)
|
||||
return h.hexdigest(), size
|
||||
|
||||
|
||||
def _content_type(key):
|
||||
ext = key.rsplit('.', 1)[-1].lower() if '.' in key else ''
|
||||
return _CONTENT_TYPES.get(ext, 'application/octet-stream')
|
||||
|
||||
|
||||
def _remote_state(client, bucket, key):
|
||||
"""Return (exists, etag_no_quotes, size) for a key in the bucket."""
|
||||
try:
|
||||
resp = client.head_object(Bucket=bucket, Key=key)
|
||||
return True, resp['ETag'].strip('"'), resp['ContentLength']
|
||||
except Exception:
|
||||
return False, None, None
|
||||
|
||||
|
||||
def human(n):
|
||||
n = float(n)
|
||||
for unit in ('B', 'KB', 'MB', 'GB', 'TB'):
|
||||
if abs(n) < 1024.0:
|
||||
return f'{n:.1f} {unit}'
|
||||
n /= 1024.0
|
||||
return f'{n:.1f} PB'
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Verified per-tenant photo sync to R2 (prefix t<tenant_id>/).')
|
||||
parser.add_argument('--tenant', default='all',
|
||||
help="Tenant id or slug, or 'all' (default).")
|
||||
parser.add_argument('--dry-run', action='store_true',
|
||||
help='Report what would happen; upload nothing.')
|
||||
parser.add_argument('--force', action='store_true',
|
||||
help='Re-upload even if the object already verifies.')
|
||||
parser.add_argument('--log', default=None,
|
||||
help='Path to write the JSON result log (default: /tmp/...).')
|
||||
args = parser.parse_args()
|
||||
|
||||
missing_cfg = [k for k in ('R2_ENDPOINT_URL', 'R2_ACCESS_KEY_ID',
|
||||
'R2_SECRET_ACCESS_KEY', 'R2_BUCKET')
|
||||
if not os.environ.get(k)]
|
||||
if missing_cfg:
|
||||
print(f'ERROR: missing R2 config: {", ".join(missing_cfg)}')
|
||||
print('Set them in .env before running the sync.')
|
||||
sys.exit(2)
|
||||
|
||||
try:
|
||||
import boto3
|
||||
from botocore.config import Config as BotoConfig
|
||||
except ImportError:
|
||||
print('ERROR: boto3 not installed. Run: pip install boto3')
|
||||
sys.exit(2)
|
||||
|
||||
bucket = os.environ['R2_BUCKET']
|
||||
client = boto3.client(
|
||||
's3',
|
||||
endpoint_url = os.environ['R2_ENDPOINT_URL'],
|
||||
aws_access_key_id = os.environ['R2_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key = os.environ['R2_SECRET_ACCESS_KEY'],
|
||||
region_name = 'auto',
|
||||
config = BotoConfig(signature_version='s3v4'),
|
||||
)
|
||||
|
||||
if not os.path.isdir(UPLOADS_ROOT):
|
||||
print(f'ERROR: uploads folder not found: {UPLOADS_ROOT}')
|
||||
sys.exit(2)
|
||||
|
||||
tenants = select_tenants(args.tenant)
|
||||
if not tenants:
|
||||
print(f'No tenants matched --tenant {args.tenant!r} '
|
||||
f'(statuses: {", ".join(AUDITABLE_STATUSES)}).')
|
||||
sys.exit(2)
|
||||
|
||||
totals = {'referenced': 0, 'missing_on_disk': 0, 'skipped_verified': 0,
|
||||
'uploaded': 0, 'verify_failed': 0, 'errors': 0, 'bytes_uploaded': 0}
|
||||
failures = []
|
||||
per_tenant = []
|
||||
|
||||
print('=' * 72)
|
||||
print(f' R2 PHOTO SYNC — MULTI-TENANT{" (DRY RUN)" if args.dry_run else ""}')
|
||||
print(f' bucket : {bucket}')
|
||||
print(f' source : {UPLOADS_ROOT}')
|
||||
print(f' tenants : {len(tenants)}')
|
||||
print('=' * 72)
|
||||
|
||||
for tid, slug, db_uri in tenants:
|
||||
prefix = f't{tid}/'
|
||||
counts = {'referenced': 0, 'missing_on_disk': 0, 'skipped_verified': 0,
|
||||
'uploaded': 0, 'verify_failed': 0, 'errors': 0, 'bytes_uploaded': 0}
|
||||
|
||||
print(f'-- tenant {tid} ({slug}) prefix={prefix}')
|
||||
|
||||
try:
|
||||
referenced, _by_source = collect_referenced(db_uri)
|
||||
except Exception as exc:
|
||||
print(f' ERROR reading tenant DB: {exc}')
|
||||
totals['errors'] += 1
|
||||
failures.append({'tenant_id': tid, 'slug': slug,
|
||||
'stage': 'tenant_db', 'error': str(exc)})
|
||||
per_tenant.append({'tenant_id': tid, 'slug': slug, 'error': str(exc)})
|
||||
continue
|
||||
|
||||
for key in sorted(referenced.keys()):
|
||||
counts['referenced'] += 1
|
||||
abs_path = os.path.normpath(os.path.join(STATIC_FOLDER, key))
|
||||
|
||||
if not os.path.isfile(abs_path):
|
||||
# Pre-existing broken reference — surfaced by audit_photos.py.
|
||||
counts['missing_on_disk'] += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
local_md5, local_size = _md5_and_size(abs_path)
|
||||
except OSError as e:
|
||||
counts['errors'] += 1
|
||||
failures.append({'tenant_id': tid, 'key': key,
|
||||
'stage': 'read', 'error': str(e)})
|
||||
print(f' ERROR read {key}: {e}')
|
||||
continue
|
||||
|
||||
object_key = f'{prefix}{key}'
|
||||
|
||||
# Resumable skip: already present and matching?
|
||||
if not args.force:
|
||||
exists, r_etag, r_size = _remote_state(client, bucket, object_key)
|
||||
if exists and r_etag == local_md5 and r_size == local_size:
|
||||
counts['skipped_verified'] += 1
|
||||
continue
|
||||
|
||||
if args.dry_run:
|
||||
counts['uploaded'] += 1 # would upload
|
||||
print(f' WOULD PUT {object_key} ({human(local_size)})')
|
||||
continue
|
||||
|
||||
# Upload (single-part PUT so ETag == MD5)
|
||||
try:
|
||||
with open(abs_path, 'rb') as body:
|
||||
client.put_object(
|
||||
Bucket=bucket, Key=object_key, Body=body,
|
||||
ContentType=_content_type(key),
|
||||
)
|
||||
except Exception as e:
|
||||
counts['errors'] += 1
|
||||
failures.append({'tenant_id': tid, 'key': object_key,
|
||||
'stage': 'upload', 'error': str(e)})
|
||||
print(f' ERROR upload {object_key}: {e}')
|
||||
continue
|
||||
|
||||
# Verify: re-HEAD and compare ETag(MD5) + size
|
||||
exists, r_etag, r_size = _remote_state(client, bucket, object_key)
|
||||
if exists and r_etag == local_md5 and r_size == local_size:
|
||||
counts['uploaded'] += 1
|
||||
counts['bytes_uploaded'] += local_size
|
||||
else:
|
||||
counts['verify_failed'] += 1
|
||||
failures.append({
|
||||
'tenant_id': tid, 'key': object_key, 'stage': 'verify',
|
||||
'local_md5': local_md5, 'local_size': local_size,
|
||||
'remote_etag': r_etag, 'remote_size': r_size,
|
||||
})
|
||||
print(f' VERIFY FAIL {object_key} local_md5={local_md5} '
|
||||
f'remote_etag={r_etag} local_size={local_size} remote_size={r_size}')
|
||||
|
||||
for k in totals:
|
||||
totals[k] += counts[k]
|
||||
per_tenant.append({'tenant_id': tid, 'slug': slug,
|
||||
'prefix': prefix, 'counts': counts})
|
||||
|
||||
verified_here = counts['skipped_verified'] + (0 if args.dry_run else counts['uploaded'])
|
||||
print(f' referenced={counts["referenced"]} '
|
||||
f'missing_on_disk={counts["missing_on_disk"]} '
|
||||
f'verified={verified_here} '
|
||||
f'uploaded={counts["uploaded"]} ({human(counts["bytes_uploaded"])}) '
|
||||
f'verify_failed={counts["verify_failed"]} errors={counts["errors"]}')
|
||||
|
||||
# ── summary ──
|
||||
print('-' * 72)
|
||||
if args.dry_run:
|
||||
action_line = f' would upload : {totals["uploaded"]}'
|
||||
else:
|
||||
action_line = (f' uploaded + verified : {totals["uploaded"]}'
|
||||
f' ({human(totals["bytes_uploaded"])})')
|
||||
print(f' referenced keys (all tenants) : {totals["referenced"]}')
|
||||
print(f' referenced but missing on disk: {totals["missing_on_disk"]} '
|
||||
f'(pre-existing broken refs)')
|
||||
print(f' already verified (skipped) : {totals["skipped_verified"]}')
|
||||
print(action_line)
|
||||
print(f' verify failures : {totals["verify_failed"]}')
|
||||
print(f' errors : {totals["errors"]}')
|
||||
print('=' * 72)
|
||||
|
||||
verified_total = totals['skipped_verified'] + (0 if args.dry_run else totals['uploaded'])
|
||||
syncable = totals['referenced'] - totals['missing_on_disk']
|
||||
safe = (totals['verify_failed'] == 0 and totals['errors'] == 0)
|
||||
|
||||
if args.dry_run:
|
||||
print(' DRY RUN — nothing uploaded. Re-run without --dry-run to sync.')
|
||||
elif safe:
|
||||
print(f' GATE: {verified_total}/{syncable} referenced files verified in R2 under '
|
||||
f'their tenant prefix, 0 mismatches, 0 errors.')
|
||||
print(' ✅ SAFE to flip STORAGE_BACKEND=s3. Local files are untouched.')
|
||||
else:
|
||||
print(f' GATE: {totals["verify_failed"]} verify failure(s), {totals["errors"]} error(s).')
|
||||
print(' ❌ NOT safe to cut over. Investigate failures (see log), then re-run.')
|
||||
|
||||
log_path = args.log or os.path.join(
|
||||
'/tmp', f'jqc_r2_sync_{datetime.utcnow().strftime("%Y%m%d_%H%M%S")}.json')
|
||||
try:
|
||||
with open(log_path, 'w') as fh:
|
||||
json.dump({
|
||||
'generated_at': datetime.utcnow().isoformat() + 'Z',
|
||||
'bucket': bucket, 'dry_run': args.dry_run,
|
||||
'tenant_selector': args.tenant,
|
||||
'totals': totals, 'tenants': per_tenant, 'failures': failures,
|
||||
}, fh, indent=2)
|
||||
print(f'\n Log written to: {log_path}')
|
||||
except OSError as e:
|
||||
print(f'\n Could not write log to {log_path}: {e}')
|
||||
|
||||
sys.exit(0 if safe else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user