Jul 16 - Fill the gaps between Single-tenant mode and Multi-tenant mode - MT2
This commit is contained in:
@@ -0,0 +1,371 @@
|
||||
"""
|
||||
app/utils/storage.py
|
||||
---------------------
|
||||
Storage abstraction for uploaded photo files (R2 object-storage migration).
|
||||
|
||||
Ported from the single-tenant tree (§22 Phase 1/2) with one multi-tenant
|
||||
addition: per-tenant object-key prefixing. See "Tenant isolation" below.
|
||||
|
||||
One interface, backend selected by config ``STORAGE_BACKEND``:
|
||||
- ``local`` (default): files under ``app/static/uploads``, served via
|
||||
``url_for('static', ...)``. **Byte-for-byte identical** to the behavior
|
||||
before this abstraction existed — MT-2 is a no-op.
|
||||
- ``s3``: Cloudflare R2 / any S3-compatible store. Private bucket; browser/API
|
||||
URLs are short-lived presigned GETs. Requires ``boto3`` and the ``R2_*``
|
||||
config keys. boto3 is imported lazily, so a ``local`` deploy needs neither.
|
||||
Inert until MT-8 flips STORAGE_BACKEND per tenant.
|
||||
|
||||
The stored **key** is always the relative path ``uploads/<subfolder>/<file>`` —
|
||||
the exact string persisted in the DB (`Issue.photo_path`, `result_photos[]`,
|
||||
`mobile_photo_paths[]`, `Inspection.form_data` image values,
|
||||
`InspectionResult.photo_path`). It never changes between backends or between
|
||||
tenants, so there is **no schema migration** and no data rewrite.
|
||||
|
||||
Tenant isolation
|
||||
----------------
|
||||
Every tenant's DB holds keys in the same ``uploads/...`` namespace, so on a
|
||||
shared object store those keys would collide across tenants. The prefix
|
||||
``t<tenant_id>/`` is therefore applied **inside the S3 backend, on the wire**
|
||||
(``_object_key``) — never in the DB, never in a template, never in an API
|
||||
payload. Callers stay tenant-agnostic; the DB stays portable.
|
||||
|
||||
DB value: uploads/issue_photos/ab12….jpg
|
||||
R2 object: t3/uploads/issue_photos/ab12….jpg
|
||||
|
||||
The **local** backend deliberately does NOT prefix: its layout is the existing
|
||||
on-disk tree, and prefixing would relocate every existing file (that is not a
|
||||
no-op, and MT-2 must be one). Local mode therefore keeps today's shared
|
||||
``app/static/uploads`` directory across tenants — an isolation weakness that
|
||||
predates this module and is retired when a tenant moves to ``s3`` in MT-8.
|
||||
|
||||
Public module-level API (delegates to the active backend):
|
||||
|
||||
save(file_obj, subfolder) -> key # write an uploaded file, return its key
|
||||
media_url(key) -> str # browser/API URL for a key ('' if falsy)
|
||||
read(key) -> bytes # raw bytes (e.g. PDF export image embed)
|
||||
exists(key) -> bool
|
||||
delete(key) -> None # best-effort file removal
|
||||
abs_local_path(key) -> str # physical path under static/ (local backend)
|
||||
materialize_to_dir(keys) -> (base_dir, cleanup)
|
||||
|
||||
NOTE: extension / magic-byte validation stays in the CALLER (e.g. `_save_photo`,
|
||||
`api/photos.upload_photo`) — this module only moves bytes and builds URLs.
|
||||
"""
|
||||
|
||||
import os
|
||||
import uuid
|
||||
import logging
|
||||
|
||||
from flask import current_app, g, url_for
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _ext_of(filename):
|
||||
"""Lower-case extension of a filename, defaulting to 'jpg' when absent."""
|
||||
name = filename or ''
|
||||
return name.rsplit('.', 1)[-1].lower() if '.' in name else 'jpg'
|
||||
|
||||
|
||||
def tenant_key_prefix():
|
||||
"""Object-key prefix isolating one tenant's media from another's.
|
||||
|
||||
Returns ``'t<id>/'`` when multi-tenancy is enabled and a tenant is bound to
|
||||
the current context, else ``''``.
|
||||
|
||||
The empty case covers three legitimate situations:
|
||||
- MULTI_TENANT_ENABLED is false (plain single-tenant deploy),
|
||||
- a cross-tenant cron/CLI context with no tenant bound,
|
||||
- request contexts exempt from the tenant middleware.
|
||||
|
||||
Callers that must not silently write to an unprefixed key guard on this
|
||||
themselves — see ``S3Backend.save``.
|
||||
"""
|
||||
if not current_app.config.get('MULTI_TENANT_ENABLED'):
|
||||
return ''
|
||||
tenant = getattr(g, 'tenant', None) if g else None
|
||||
return f't{tenant.id}/' if tenant is not None else ''
|
||||
|
||||
|
||||
# ── Local filesystem backend (current behavior) ───────────────────────────────
|
||||
|
||||
class LocalBackend:
|
||||
"""Files under ``app/static/uploads``, served by Flask/Nginx static route.
|
||||
|
||||
No tenant prefixing — see module docstring ("Tenant isolation").
|
||||
"""
|
||||
|
||||
name = 'local'
|
||||
|
||||
def _uploads_root(self):
|
||||
# config UPLOAD_FOLDER == <root>/app/static/uploads
|
||||
return current_app.config['UPLOAD_FOLDER']
|
||||
|
||||
def _static_folder(self):
|
||||
# <root>/app/static — key 'uploads/...' resolves under here
|
||||
return os.path.join(current_app.root_path, 'static')
|
||||
|
||||
def save(self, file_obj, subfolder):
|
||||
"""Write ``file_obj`` under ``subfolder``; return the ``uploads/...`` key.
|
||||
|
||||
Replicates the previous inline logic in `_save_photo` / `upload_photo`
|
||||
exactly: random uuid filename, `os.makedirs(exist_ok=True)`, same key.
|
||||
"""
|
||||
ext = _ext_of(file_obj.filename)
|
||||
filename = f'{uuid.uuid4().hex}.{ext}'
|
||||
dest_dir = os.path.join(self._uploads_root(), subfolder)
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
file_obj.save(os.path.join(dest_dir, filename))
|
||||
return f'uploads/{subfolder}/{filename}'
|
||||
|
||||
def abs_local_path(self, key):
|
||||
return os.path.normpath(os.path.join(self._static_folder(), key))
|
||||
|
||||
def media_url(self, key, external=False):
|
||||
if not key:
|
||||
return ''
|
||||
# external=True yields an absolute URL (scheme+host) for API responses
|
||||
# consumed off-origin (the iPad); templates call with external=False.
|
||||
return url_for('static', filename=key, _external=external)
|
||||
|
||||
def read(self, key):
|
||||
with open(self.abs_local_path(key), 'rb') as fh:
|
||||
return fh.read()
|
||||
|
||||
def exists(self, key):
|
||||
return bool(key) and os.path.isfile(self.abs_local_path(key))
|
||||
|
||||
def delete(self, key):
|
||||
if not key:
|
||||
return
|
||||
try:
|
||||
path = self.abs_local_path(key)
|
||||
if os.path.isfile(path):
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def materialize_to_dir(self, keys):
|
||||
# Files already live under the static folder — no copy needed.
|
||||
return self._static_folder(), (lambda: None)
|
||||
|
||||
|
||||
# ── Cloudflare R2 / S3-compatible backend ─────────────────────────────────────
|
||||
|
||||
_CONTENT_TYPES = {
|
||||
'jpg': 'image/jpeg', 'jpeg': 'image/jpeg',
|
||||
'png': 'image/png', 'gif': 'image/gif',
|
||||
}
|
||||
|
||||
|
||||
class S3Backend:
|
||||
"""
|
||||
Cloudflare R2 (or any S3-compatible store). Objects are private; browser/API
|
||||
URLs are short-lived presigned GETs. The object key == tenant prefix + the
|
||||
DB path string (see module docstring).
|
||||
|
||||
boto3 is imported lazily so a 'local' deployment doesn't require it installed.
|
||||
"""
|
||||
|
||||
name = 's3'
|
||||
|
||||
def __init__(self):
|
||||
import boto3 # lazy — only when s3 is active
|
||||
from botocore.config import Config as _BotoConfig
|
||||
|
||||
cfg = current_app.config
|
||||
missing = [k for k in ('R2_ENDPOINT_URL', 'R2_ACCESS_KEY_ID',
|
||||
'R2_SECRET_ACCESS_KEY', 'R2_BUCKET') if not cfg.get(k)]
|
||||
if missing:
|
||||
raise RuntimeError(f'STORAGE_BACKEND=s3 but missing config: {", ".join(missing)}')
|
||||
|
||||
self.bucket = cfg['R2_BUCKET']
|
||||
self.ttl = int(cfg.get('R2_PRESIGN_TTL', 86400))
|
||||
self.fallback = bool(cfg.get('R2_MEDIA_FALLBACK', False))
|
||||
self._client = boto3.client(
|
||||
's3',
|
||||
endpoint_url = cfg['R2_ENDPOINT_URL'],
|
||||
aws_access_key_id = cfg['R2_ACCESS_KEY_ID'],
|
||||
aws_secret_access_key = cfg['R2_SECRET_ACCESS_KEY'],
|
||||
region_name = 'auto',
|
||||
config = _BotoConfig(signature_version='s3v4'),
|
||||
)
|
||||
# A LocalBackend for transition fallback (read/media_url when object absent).
|
||||
self._local = LocalBackend()
|
||||
|
||||
# ── Key mapping ──────────────────────────────────────────────────────────
|
||||
|
||||
def _object_key(self, key):
|
||||
"""DB key -> on-the-wire object key (tenant-prefixed)."""
|
||||
return f'{tenant_key_prefix()}{key}'
|
||||
|
||||
def _read_candidates(self, key):
|
||||
"""Object keys to try when reading, most-specific first.
|
||||
|
||||
The unprefixed key is retained as a fallback so objects written before a
|
||||
tenant prefix existed (or by a no-tenant context) stay readable.
|
||||
"""
|
||||
prefixed = self._object_key(key)
|
||||
return [prefixed] if prefixed == key else [prefixed, key]
|
||||
|
||||
def _content_type(self, key):
|
||||
return _CONTENT_TYPES.get(_ext_of(key), 'application/octet-stream')
|
||||
|
||||
# ── Interface ────────────────────────────────────────────────────────────
|
||||
|
||||
def save(self, file_obj, subfolder):
|
||||
ext = _ext_of(file_obj.filename)
|
||||
key = f'uploads/{subfolder}/{uuid.uuid4().hex}.{ext}'
|
||||
|
||||
# Fail loudly rather than write a key that could collide with another
|
||||
# tenant's. A missing tenant here means the caller reached an upload
|
||||
# path outside tenant resolution — a bug, not a fallback case.
|
||||
if current_app.config.get('MULTI_TENANT_ENABLED') and not tenant_key_prefix():
|
||||
raise RuntimeError(
|
||||
'storage.save() called with STORAGE_BACKEND=s3 and no tenant '
|
||||
'bound — refusing to write an unprefixed object key.'
|
||||
)
|
||||
|
||||
file_obj.stream.seek(0)
|
||||
body = file_obj.stream.read()
|
||||
self._client.put_object(
|
||||
Bucket=self.bucket, Key=self._object_key(key), Body=body,
|
||||
ContentType=_CONTENT_TYPES.get(ext, 'application/octet-stream'),
|
||||
)
|
||||
# The DB always stores the unprefixed key.
|
||||
return key
|
||||
|
||||
def abs_local_path(self, key):
|
||||
# No local path for an S3 object; expose the local mirror path (may or may
|
||||
# not exist) so transition code / callers can still reference it.
|
||||
return self._local.abs_local_path(key)
|
||||
|
||||
def media_url(self, key, external=False):
|
||||
if not key:
|
||||
return ''
|
||||
obj = self._object_key(key)
|
||||
if self.fallback and not self._head(obj):
|
||||
# Object not at the tenant-prefixed key — try the legacy unprefixed
|
||||
# key, then the local copy, during transition.
|
||||
if obj != key and self._head(key):
|
||||
obj = key
|
||||
elif self._local.exists(key):
|
||||
return self._local.media_url(key, external=external)
|
||||
# Presigned R2 URLs are always absolute; the external flag is moot.
|
||||
return self._client.generate_presigned_url(
|
||||
'get_object',
|
||||
Params={'Bucket': self.bucket, 'Key': obj},
|
||||
ExpiresIn=self.ttl,
|
||||
)
|
||||
|
||||
def read(self, key):
|
||||
for candidate in self._read_candidates(key):
|
||||
try:
|
||||
resp = self._client.get_object(Bucket=self.bucket, Key=candidate)
|
||||
return resp['Body'].read()
|
||||
except Exception:
|
||||
continue
|
||||
# Transition safety: fall back to a local copy if present.
|
||||
if self._local.exists(key):
|
||||
return self._local.read(key)
|
||||
raise FileNotFoundError(f'storage: no object for key {key!r}')
|
||||
|
||||
def _head(self, key):
|
||||
try:
|
||||
self._client.head_object(Bucket=self.bucket, Key=key)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def exists(self, key):
|
||||
if not key:
|
||||
return False
|
||||
return any(self._head(k) for k in self._read_candidates(key))
|
||||
|
||||
def delete(self, key):
|
||||
if not key:
|
||||
return
|
||||
for candidate in self._read_candidates(key):
|
||||
try:
|
||||
self._client.delete_object(Bucket=self.bucket, Key=candidate)
|
||||
except Exception as e:
|
||||
logger.warning('S3 delete failed for %s: %s', candidate, e)
|
||||
|
||||
def materialize_to_dir(self, keys):
|
||||
"""Download the given keys to a temp dir mirroring the key layout.
|
||||
|
||||
Returns (base_dir, cleanup); each key resolves at base_dir/key so tools
|
||||
that need real file paths (ReportLab/PIL) work unchanged. The temp tree
|
||||
uses the **DB** key layout (unprefixed) because that is what callers
|
||||
join against. Missing or unreadable keys are skipped — callers already
|
||||
guard with os.path.exists.
|
||||
"""
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
base = tempfile.mkdtemp(prefix='jqc_media_')
|
||||
for key in {k for k in keys if k}:
|
||||
try:
|
||||
data = self.read(key)
|
||||
except Exception:
|
||||
continue # skip missing; caller guards existence
|
||||
dest = os.path.join(base, key)
|
||||
os.makedirs(os.path.dirname(dest), exist_ok=True)
|
||||
with open(dest, 'wb') as fh:
|
||||
fh.write(data)
|
||||
|
||||
def _cleanup():
|
||||
shutil.rmtree(base, ignore_errors=True)
|
||||
|
||||
return base, _cleanup
|
||||
|
||||
|
||||
_BACKENDS = {
|
||||
'local': LocalBackend,
|
||||
's3': S3Backend,
|
||||
}
|
||||
|
||||
|
||||
def get_backend():
|
||||
"""Return the active storage backend (one cached instance per app)."""
|
||||
name = (current_app.config.get('STORAGE_BACKEND') or 'local').lower()
|
||||
cls = _BACKENDS.get(name)
|
||||
if cls is None:
|
||||
logger.warning('Unknown STORAGE_BACKEND=%r — falling back to local', name)
|
||||
cls = LocalBackend
|
||||
cache_key = f'_storage_backend_{cls.name}'
|
||||
inst = current_app.extensions.get(cache_key)
|
||||
if inst is None:
|
||||
inst = cls()
|
||||
current_app.extensions[cache_key] = inst
|
||||
return inst
|
||||
|
||||
|
||||
# ── Module-level convenience delegates ────────────────────────────────────────
|
||||
|
||||
def save(file_obj, subfolder):
|
||||
return get_backend().save(file_obj, subfolder)
|
||||
|
||||
|
||||
def media_url(key, external=False):
|
||||
return get_backend().media_url(key, external=external)
|
||||
|
||||
|
||||
def read(key):
|
||||
return get_backend().read(key)
|
||||
|
||||
|
||||
def exists(key):
|
||||
return get_backend().exists(key)
|
||||
|
||||
|
||||
def delete(key):
|
||||
return get_backend().delete(key)
|
||||
|
||||
|
||||
def abs_local_path(key):
|
||||
return get_backend().abs_local_path(key)
|
||||
|
||||
|
||||
def materialize_to_dir(keys):
|
||||
return get_backend().materialize_to_dir(keys)
|
||||
Reference in New Issue
Block a user