LT_Janitorial_Quality_Control/scripts/audit_photos.py

#!/usr/bin/env python3
"""
scripts/audit_photos.py  —  Pre-migration photo audit (READ-ONLY)
=================================================================

Phase 0 of the R2 object-storage migration. This script writes NOTHING to the
database and NOTHING to the uploads tree. It only reads, and emits a report.

What it does
------------
1. Collects every photo path REFERENCED by the database, across all 5 sources:
     - Issue.photo_path            (string)
     - Issue.mobile_photo_paths[]  (JSON list — iPad evidence)
     - Issue.result_photos[]       (JSON list — web resolution photos)
     - Inspection.photo_path       (string)
     - Inspection.form_data{}      (any 'uploads/...' string, incl. nested — image fields)
   (Signatures are stored inline as 'data:' base64, NOT files — correctly ignored.)
2. Checks each referenced path against the file actually on disk.
3. Walks the uploads tree and finds files NOT referenced by any record (orphans).
4. Prints a summary and writes a detailed JSON report you can keep as the baseline.

The number that matters: **distinct referenced photos that resolve on disk**.
That count must be identical after the migration. Any "missing on disk" here is a
PRE-EXISTING broken reference — surfaced now so it can never be blamed on the move.

Usage
-----
    cd /home/jqc/janitorial_qc
    source venv/bin/activate            # whatever your venv is
    python scripts/audit_photos.py                       # report -> /tmp
    python scripts/audit_photos.py --report /home/jqc/photo_audit.json
    python scripts/audit_photos.py --list-orphans        # also print orphan paths

Exit code is always 0 (audit is informational). Nothing is modified.
"""

import os
import sys
import json
import argparse
from collections import defaultdict
from datetime import datetime

# Make the app package importable when run from the repo root.
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))

from app import create_app, db                       # noqa: E402
from app.models.issue import Issue                   # noqa: E402
from app.models.inspection import Inspection         # noqa: E402


# ── Path helpers ──────────────────────────────────────────────────────────────

def normalize(path):
    """
    Return a canonical 'uploads/...' key, or None if this value is not a stored
    photo file (base64 signatures, unsynced iPad sentinels, remote URLs, blanks).
    """
    if not isinstance(path, str):
        return None
    p = path.strip()
    if not p:
        return None
    # Not on-disk files:
    if p.startswith('data:') or p.startswith('local://'):
        return None
    if p.startswith('http://') or p.startswith('https://'):
        return None
    # Tolerate a few stored variants and reduce to 'uploads/...'
    for prefix in ('/static/', 'static/', '/'):
        if p.startswith(prefix):
            p = p[len(prefix):]
    if not p.startswith('uploads/'):
        return None
    return p


def walk_strings(value):
    """Yield every string found in an arbitrarily nested JSON value (dict/list/str)."""
    if isinstance(value, str):
        yield value
    elif isinstance(value, dict):
        for v in value.values():
            yield from walk_strings(v)
    elif isinstance(value, (list, tuple)):
        for v in value:
            yield from walk_strings(v)


# ── Reference collection ──────────────────────────────────────────────────────

def collect_referenced():
    """
    Return (referenced, by_source):
      referenced : { normalized_path -> [ "issue:123:photo_path", ... ] }
      by_source  : { source_name -> count_of_refs }
    """
    referenced = defaultdict(list)
    by_source  = defaultdict(int)

    def add(path, ref_label, source_name):
        norm = normalize(path)
        if norm is None:
            return
        referenced[norm].append(ref_label)
        by_source[source_name] += 1

    # --- Issues ---
    for iss in Issue.query.yield_per(500):
        add(iss.photo_path, f'issue:{iss.id}:photo_path', 'issue.photo_path')
        for pth in (iss.mobile_photo_paths or []):
            add(pth, f'issue:{iss.id}:mobile_photo_paths', 'issue.mobile_photo_paths')
        for pth in (iss.result_photos or []):
            add(pth, f'issue:{iss.id}:result_photos', 'issue.result_photos')

    # --- Inspections ---
    for insp in Inspection.query.yield_per(500):
        add(insp.photo_path, f'inspection:{insp.id}:photo_path', 'inspection.photo_path')
        fd = insp.form_data or {}
        for s in walk_strings(fd):
            # Any 'uploads/...' string inside form_data is an uploaded image value.
            add(s, f'inspection:{insp.id}:form_data', 'inspection.form_data(image)')

    return referenced, by_source


# ── Disk scan ─────────────────────────────────────────────────────────────────

def scan_disk(uploads_root):
    """Return { relative_uploads_key -> size_bytes } for every file under uploads/."""
    on_disk = {}
    if not os.path.isdir(uploads_root):
        return on_disk
    base = os.path.dirname(uploads_root)  # .../static  (so keys start with 'uploads/')
    for dirpath, _dirs, files in os.walk(uploads_root):
        for name in files:
            full = os.path.join(dirpath, name)
            rel  = os.path.relpath(full, base).replace(os.sep, '/')
            try:
                on_disk[rel] = os.path.getsize(full)
            except OSError:
                on_disk[rel] = -1
    return on_disk


# ── Main ──────────────────────────────────────────────────────────────────────

def human(nbytes):
    n = float(nbytes)
    for unit in ('B', 'KB', 'MB', 'GB', 'TB'):
        if abs(n) < 1024.0:
            return f'{n:.1f} {unit}'
        n /= 1024.0
    return f'{n:.1f} PB'


def main():
    parser = argparse.ArgumentParser(description='Read-only photo audit (R2 migration Phase 0).')
    parser.add_argument('--report', default=None,
                        help='Path to write the JSON report (default: /tmp/jqc_photo_audit_<ts>.json)')
    parser.add_argument('--list-orphans', action='store_true',
                        help='Also print orphan file paths to stdout')
    args = parser.parse_args()

    app = create_app(os.getenv('FLASK_ENV') or 'production')
    with app.app_context():
        static_folder = os.path.join(app.root_path, 'static')
        uploads_root  = os.path.join(static_folder, 'uploads')

        referenced, by_source = collect_referenced()
        on_disk = scan_disk(uploads_root)
        on_disk_keys = set(on_disk.keys())

        referenced_keys = set(referenced.keys())
        present = referenced_keys & on_disk_keys
        missing = referenced_keys - on_disk_keys            # broken references (pre-existing)
        orphans = on_disk_keys - referenced_keys            # on disk, unreferenced

        present_bytes = sum(on_disk[k] for k in present if on_disk[k] > 0)
        orphan_bytes  = sum(on_disk[k] for k in orphans if on_disk[k] > 0)
        total_bytes   = sum(v for v in on_disk.values() if v > 0)

        # Per-subfolder disk breakdown
        by_folder = defaultdict(lambda: [0, 0])  # folder -> [count, bytes]
        for k, sz in on_disk.items():
            parts = k.split('/')
            folder = parts[1] if len(parts) > 2 else '(root)'
            by_folder[folder][0] += 1
            by_folder[folder][1] += sz if sz > 0 else 0

        report = {
            'generated_at':   datetime.utcnow().isoformat() + 'Z',
            'static_folder':  static_folder,
            'uploads_root':   uploads_root,
            'referenced': {
                'distinct_paths': len(referenced_keys),
                'by_source':      dict(by_source),
                'present_on_disk': len(present),
                'missing_on_disk': len(missing),
            },
            'disk': {
                'total_files': len(on_disk),
                'total_bytes': total_bytes,
                'by_folder':   {k: {'files': v[0], 'bytes': v[1]} for k, v in by_folder.items()},
            },
            'orphans': {
                'count': len(orphans),
                'bytes': orphan_bytes,
            },
            # Full detail for follow-up / verification after migration:
            'missing_on_disk': sorted(
                [{'path': k, 'referenced_by': sorted(set(referenced[k]))} for k in missing],
                key=lambda d: d['path']
            ),
            'orphan_files': sorted(
                [{'path': k, 'bytes': on_disk[k]} for k in orphans],
                key=lambda d: d['path']
            ),
        }

        # ── stdout summary ──
        print('=' * 68)
        print(' JQC PHOTO AUDIT  (read-only — nothing was modified)')
        print('=' * 68)
        print(f' uploads root : {uploads_root}')
        print(f' generated    : {report["generated_at"]}')
        print('-' * 68)
        print(' REFERENCED BY DATABASE')
        for src, cnt in sorted(by_source.items()):
            print(f'   {src:<34} {cnt:>8} refs')
        print(f'   {"distinct photo files referenced":<34} {len(referenced_keys):>8}')
        print('-' * 68)
        print(' ON DISK')
        for folder, (cnt, b) in sorted(by_folder.items()):
            print(f'   {folder:<24} {cnt:>8} files   {human(b):>12}')
        print(f'   {"TOTAL":<24} {len(on_disk):>8} files   {human(total_bytes):>12}')
        print('-' * 68)
        print(' RECONCILIATION')
        print(f'   referenced AND present on disk : {len(present):>8}   <-- must be unchanged after migration')
        print(f'   referenced but MISSING on disk : {len(missing):>8}   <-- pre-existing broken refs (investigate)')
        print(f'   on disk but UNREFERENCED       : {len(orphans):>8}   ({human(orphan_bytes)}, migrated anyway)')
        print('=' * 68)

        if missing:
            print(f'\n WARNING: {len(missing)} referenced photo(s) are already missing on disk.')
            print(' These are NOT caused by migration. First few:')
            for m in report['missing_on_disk'][:10]:
                print(f'   - {m["path"]}  (by {", ".join(m["referenced_by"][:3])})')
            if len(missing) > 10:
                print(f'   ... and {len(missing) - 10} more (see JSON report)')

        if args.list_orphans and orphans:
            print(f'\n Orphan files ({len(orphans)}):')
            for o in report['orphan_files']:
                print(f'   - {o["path"]}  ({human(o["bytes"])})')

        # ── write JSON report ──
        report_path = args.report or os.path.join(
            '/tmp', f'jqc_photo_audit_{datetime.utcnow().strftime("%Y%m%d_%H%M%S")}.json'
        )
        try:
            with open(report_path, 'w') as fh:
                json.dump(report, fh, indent=2)
            print(f'\n Full JSON report written to: {report_path}')
        except OSError as e:
            print(f'\n Could not write report to {report_path}: {e}')

        print('\n BASELINE TO REMEMBER: '
              f'{len(present)} photos must still resolve after cutover.\n')


if __name__ == '__main__':
    main()