Jul 14 - Preparation for using CDN
This commit is contained in:
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
scripts/audit_photos.py — Pre-migration photo audit (READ-ONLY)
|
||||
=================================================================
|
||||
|
||||
Phase 0 of the R2 object-storage migration. This script writes NOTHING to the
|
||||
database and NOTHING to the uploads tree. It only reads, and emits a report.
|
||||
|
||||
What it does
|
||||
------------
|
||||
1. Collects every photo path REFERENCED by the database, across all 5 sources:
|
||||
- Issue.photo_path (string)
|
||||
- Issue.mobile_photo_paths[] (JSON list — iPad evidence)
|
||||
- Issue.result_photos[] (JSON list — web resolution photos)
|
||||
- Inspection.photo_path (string)
|
||||
- Inspection.form_data{} (any 'uploads/...' string, incl. nested — image fields)
|
||||
(Signatures are stored inline as 'data:' base64, NOT files — correctly ignored.)
|
||||
2. Checks each referenced path against the file actually on disk.
|
||||
3. Walks the uploads tree and finds files NOT referenced by any record (orphans).
|
||||
4. Prints a summary and writes a detailed JSON report you can keep as the baseline.
|
||||
|
||||
The number that matters: **distinct referenced photos that resolve on disk**.
|
||||
That count must be identical after the migration. Any "missing on disk" here is a
|
||||
PRE-EXISTING broken reference — surfaced now so it can never be blamed on the move.
|
||||
|
||||
Usage
|
||||
-----
|
||||
cd /home/jqc/janitorial_qc
|
||||
source venv/bin/activate # whatever your venv is
|
||||
python scripts/audit_photos.py # report -> /tmp
|
||||
python scripts/audit_photos.py --report /home/jqc/photo_audit.json
|
||||
python scripts/audit_photos.py --list-orphans # also print orphan paths
|
||||
|
||||
Exit code is always 0 (audit is informational). Nothing is modified.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
# Make the app package importable when run from the repo root.
|
||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
|
||||
|
||||
from app import create_app, db # noqa: E402
|
||||
from app.models.issue import Issue # noqa: E402
|
||||
from app.models.inspection import Inspection # noqa: E402
|
||||
|
||||
|
||||
# ── Path helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
def normalize(path):
|
||||
"""
|
||||
Return a canonical 'uploads/...' key, or None if this value is not a stored
|
||||
photo file (base64 signatures, unsynced iPad sentinels, remote URLs, blanks).
|
||||
"""
|
||||
if not isinstance(path, str):
|
||||
return None
|
||||
p = path.strip()
|
||||
if not p:
|
||||
return None
|
||||
# Not on-disk files:
|
||||
if p.startswith('data:') or p.startswith('local://'):
|
||||
return None
|
||||
if p.startswith('http://') or p.startswith('https://'):
|
||||
return None
|
||||
# Tolerate a few stored variants and reduce to 'uploads/...'
|
||||
for prefix in ('/static/', 'static/', '/'):
|
||||
if p.startswith(prefix):
|
||||
p = p[len(prefix):]
|
||||
if not p.startswith('uploads/'):
|
||||
return None
|
||||
return p
|
||||
|
||||
|
||||
def walk_strings(value):
|
||||
"""Yield every string found in an arbitrarily nested JSON value (dict/list/str)."""
|
||||
if isinstance(value, str):
|
||||
yield value
|
||||
elif isinstance(value, dict):
|
||||
for v in value.values():
|
||||
yield from walk_strings(v)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for v in value:
|
||||
yield from walk_strings(v)
|
||||
|
||||
|
||||
# ── Reference collection ──────────────────────────────────────────────────────
|
||||
|
||||
def collect_referenced():
|
||||
"""
|
||||
Return (referenced, by_source):
|
||||
referenced : { normalized_path -> [ "issue:123:photo_path", ... ] }
|
||||
by_source : { source_name -> count_of_refs }
|
||||
"""
|
||||
referenced = defaultdict(list)
|
||||
by_source = defaultdict(int)
|
||||
|
||||
def add(path, ref_label, source_name):
|
||||
norm = normalize(path)
|
||||
if norm is None:
|
||||
return
|
||||
referenced[norm].append(ref_label)
|
||||
by_source[source_name] += 1
|
||||
|
||||
# --- Issues ---
|
||||
for iss in Issue.query.yield_per(500):
|
||||
add(iss.photo_path, f'issue:{iss.id}:photo_path', 'issue.photo_path')
|
||||
for pth in (iss.mobile_photo_paths or []):
|
||||
add(pth, f'issue:{iss.id}:mobile_photo_paths', 'issue.mobile_photo_paths')
|
||||
for pth in (iss.result_photos or []):
|
||||
add(pth, f'issue:{iss.id}:result_photos', 'issue.result_photos')
|
||||
|
||||
# --- Inspections ---
|
||||
for insp in Inspection.query.yield_per(500):
|
||||
add(insp.photo_path, f'inspection:{insp.id}:photo_path', 'inspection.photo_path')
|
||||
fd = insp.form_data or {}
|
||||
for s in walk_strings(fd):
|
||||
# Any 'uploads/...' string inside form_data is an uploaded image value.
|
||||
add(s, f'inspection:{insp.id}:form_data', 'inspection.form_data(image)')
|
||||
|
||||
return referenced, by_source
|
||||
|
||||
|
||||
# ── Disk scan ─────────────────────────────────────────────────────────────────
|
||||
|
||||
def scan_disk(uploads_root):
|
||||
"""Return { relative_uploads_key -> size_bytes } for every file under uploads/."""
|
||||
on_disk = {}
|
||||
if not os.path.isdir(uploads_root):
|
||||
return on_disk
|
||||
base = os.path.dirname(uploads_root) # .../static (so keys start with 'uploads/')
|
||||
for dirpath, _dirs, files in os.walk(uploads_root):
|
||||
for name in files:
|
||||
full = os.path.join(dirpath, name)
|
||||
rel = os.path.relpath(full, base).replace(os.sep, '/')
|
||||
try:
|
||||
on_disk[rel] = os.path.getsize(full)
|
||||
except OSError:
|
||||
on_disk[rel] = -1
|
||||
return on_disk
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def human(nbytes):
|
||||
n = float(nbytes)
|
||||
for unit in ('B', 'KB', 'MB', 'GB', 'TB'):
|
||||
if abs(n) < 1024.0:
|
||||
return f'{n:.1f} {unit}'
|
||||
n /= 1024.0
|
||||
return f'{n:.1f} PB'
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Read-only photo audit (R2 migration Phase 0).')
|
||||
parser.add_argument('--report', default=None,
|
||||
help='Path to write the JSON report (default: /tmp/jqc_photo_audit_<ts>.json)')
|
||||
parser.add_argument('--list-orphans', action='store_true',
|
||||
help='Also print orphan file paths to stdout')
|
||||
args = parser.parse_args()
|
||||
|
||||
app = create_app(os.getenv('FLASK_ENV') or 'production')
|
||||
with app.app_context():
|
||||
static_folder = os.path.join(app.root_path, 'static')
|
||||
uploads_root = os.path.join(static_folder, 'uploads')
|
||||
|
||||
referenced, by_source = collect_referenced()
|
||||
on_disk = scan_disk(uploads_root)
|
||||
on_disk_keys = set(on_disk.keys())
|
||||
|
||||
referenced_keys = set(referenced.keys())
|
||||
present = referenced_keys & on_disk_keys
|
||||
missing = referenced_keys - on_disk_keys # broken references (pre-existing)
|
||||
orphans = on_disk_keys - referenced_keys # on disk, unreferenced
|
||||
|
||||
present_bytes = sum(on_disk[k] for k in present if on_disk[k] > 0)
|
||||
orphan_bytes = sum(on_disk[k] for k in orphans if on_disk[k] > 0)
|
||||
total_bytes = sum(v for v in on_disk.values() if v > 0)
|
||||
|
||||
# Per-subfolder disk breakdown
|
||||
by_folder = defaultdict(lambda: [0, 0]) # folder -> [count, bytes]
|
||||
for k, sz in on_disk.items():
|
||||
parts = k.split('/')
|
||||
folder = parts[1] if len(parts) > 2 else '(root)'
|
||||
by_folder[folder][0] += 1
|
||||
by_folder[folder][1] += sz if sz > 0 else 0
|
||||
|
||||
report = {
|
||||
'generated_at': datetime.utcnow().isoformat() + 'Z',
|
||||
'static_folder': static_folder,
|
||||
'uploads_root': uploads_root,
|
||||
'referenced': {
|
||||
'distinct_paths': len(referenced_keys),
|
||||
'by_source': dict(by_source),
|
||||
'present_on_disk': len(present),
|
||||
'missing_on_disk': len(missing),
|
||||
},
|
||||
'disk': {
|
||||
'total_files': len(on_disk),
|
||||
'total_bytes': total_bytes,
|
||||
'by_folder': {k: {'files': v[0], 'bytes': v[1]} for k, v in by_folder.items()},
|
||||
},
|
||||
'orphans': {
|
||||
'count': len(orphans),
|
||||
'bytes': orphan_bytes,
|
||||
},
|
||||
# Full detail for follow-up / verification after migration:
|
||||
'missing_on_disk': sorted(
|
||||
[{'path': k, 'referenced_by': sorted(set(referenced[k]))} for k in missing],
|
||||
key=lambda d: d['path']
|
||||
),
|
||||
'orphan_files': sorted(
|
||||
[{'path': k, 'bytes': on_disk[k]} for k in orphans],
|
||||
key=lambda d: d['path']
|
||||
),
|
||||
}
|
||||
|
||||
# ── stdout summary ──
|
||||
print('=' * 68)
|
||||
print(' JQC PHOTO AUDIT (read-only — nothing was modified)')
|
||||
print('=' * 68)
|
||||
print(f' uploads root : {uploads_root}')
|
||||
print(f' generated : {report["generated_at"]}')
|
||||
print('-' * 68)
|
||||
print(' REFERENCED BY DATABASE')
|
||||
for src, cnt in sorted(by_source.items()):
|
||||
print(f' {src:<34} {cnt:>8} refs')
|
||||
print(f' {"distinct photo files referenced":<34} {len(referenced_keys):>8}')
|
||||
print('-' * 68)
|
||||
print(' ON DISK')
|
||||
for folder, (cnt, b) in sorted(by_folder.items()):
|
||||
print(f' {folder:<24} {cnt:>8} files {human(b):>12}')
|
||||
print(f' {"TOTAL":<24} {len(on_disk):>8} files {human(total_bytes):>12}')
|
||||
print('-' * 68)
|
||||
print(' RECONCILIATION')
|
||||
print(f' referenced AND present on disk : {len(present):>8} <-- must be unchanged after migration')
|
||||
print(f' referenced but MISSING on disk : {len(missing):>8} <-- pre-existing broken refs (investigate)')
|
||||
print(f' on disk but UNREFERENCED : {len(orphans):>8} ({human(orphan_bytes)}, migrated anyway)')
|
||||
print('=' * 68)
|
||||
|
||||
if missing:
|
||||
print(f'\n WARNING: {len(missing)} referenced photo(s) are already missing on disk.')
|
||||
print(' These are NOT caused by migration. First few:')
|
||||
for m in report['missing_on_disk'][:10]:
|
||||
print(f' - {m["path"]} (by {", ".join(m["referenced_by"][:3])})')
|
||||
if len(missing) > 10:
|
||||
print(f' ... and {len(missing) - 10} more (see JSON report)')
|
||||
|
||||
if args.list_orphans and orphans:
|
||||
print(f'\n Orphan files ({len(orphans)}):')
|
||||
for o in report['orphan_files']:
|
||||
print(f' - {o["path"]} ({human(o["bytes"])})')
|
||||
|
||||
# ── write JSON report ──
|
||||
report_path = args.report or os.path.join(
|
||||
'/tmp', f'jqc_photo_audit_{datetime.utcnow().strftime("%Y%m%d_%H%M%S")}.json'
|
||||
)
|
||||
try:
|
||||
with open(report_path, 'w') as fh:
|
||||
json.dump(report, fh, indent=2)
|
||||
print(f'\n Full JSON report written to: {report_path}')
|
||||
except OSError as e:
|
||||
print(f'\n Could not write report to {report_path}: {e}')
|
||||
|
||||
print('\n BASELINE TO REMEMBER: '
|
||||
f'{len(present)} photos must still resolve after cutover.\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user