183 lines
7.1 KiB
Python
183 lines
7.1 KiB
Python
"""Private S3 access shared by attachment routes and maintenance jobs."""
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import unicodedata
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from tempfile import SpooledTemporaryFile
|
|
from uuid import UUID, uuid4
|
|
|
|
from fastapi import HTTPException, Request
|
|
from minio import Minio
|
|
from minio.error import S3Error
|
|
from opentelemetry import trace
|
|
from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
|
|
from PIL import Image, UnidentifiedImageError
|
|
from urllib3 import PoolManager, Retry, Timeout
|
|
|
|
from .pipeline_jobs import queue_attachment, renew_upload, reserve_upload
|
|
from .store import connection, store, timestamp
|
|
|
|
|
|
BUCKET = "messenger-attachments"
|
|
MAX_BYTES = 10 * 1024 * 1024
|
|
MAX_PIXELS = 40_000_000
|
|
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
|
logger = logging.getLogger(__name__)
|
|
tracer = trace.get_tracer(__name__)
|
|
trace_context = TraceContextTextMapPropagator()
|
|
|
|
|
|
def secret(name: str) -> str:
|
|
return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip()
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def storage() -> Minio:
|
|
return Minio(
|
|
os.environ.get("S3_ENDPOINT", "s3:9000"),
|
|
access_key=secret("S3_ACCESS_KEY_ID"),
|
|
secret_key=secret("S3_SECRET_ACCESS_KEY"),
|
|
secure=False,
|
|
region="us-east-1",
|
|
http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)),
|
|
)
|
|
|
|
|
|
def storage_ready() -> None:
|
|
try:
|
|
storage().stat_object(BUCKET, "health/ready-probe")
|
|
except S3Error as exc:
|
|
if exc.code != "NoSuchKey":
|
|
raise HTTPException(503) from None
|
|
except Exception:
|
|
raise HTTPException(503) from None
|
|
|
|
|
|
def attachment_view(row: dict) -> dict:
|
|
return {
|
|
"id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]),
|
|
"filename": row["filename"], "content_type": row["content_type"],
|
|
"size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"],
|
|
"variants": [], "error_code": row["error_code"],
|
|
"created_at": timestamp(row["created_at"]), "updated_at": timestamp(row["updated_at"]),
|
|
}
|
|
|
|
|
|
def validate_filename(filename: str) -> None:
|
|
if not 1 <= len(filename) <= 255 or filename in {".", ".."}:
|
|
raise HTTPException(400)
|
|
if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename):
|
|
raise HTTPException(400)
|
|
|
|
|
|
def validate_image(stream, declared_type: str) -> str:
|
|
try:
|
|
with Image.open(stream) as image:
|
|
actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format)
|
|
if actual_type is None or actual_type != declared_type:
|
|
raise HTTPException(415)
|
|
if image.width * image.height > MAX_PIXELS:
|
|
raise HTTPException(413)
|
|
image.verify()
|
|
stream.seek(0)
|
|
with Image.open(stream) as image:
|
|
image.load()
|
|
return actual_type
|
|
except Image.DecompressionBombError:
|
|
raise HTTPException(413) from None
|
|
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
|
|
raise HTTPException(415) from None
|
|
|
|
|
|
def verify_staging(client: Minio, key: str, size: int, sha256: str, content_type: str) -> None:
|
|
"""Confirm complete durable bytes; an S3 ETag is not a SHA-256 checksum."""
|
|
info = client.stat_object(BUCKET, key)
|
|
if info.size != size or info.content_type.split(";", 1)[0] != content_type:
|
|
raise ValueError("staging metadata mismatch")
|
|
obj = client.get_object(BUCKET, key)
|
|
try:
|
|
digest = hashlib.sha256()
|
|
received = 0
|
|
for chunk in obj.stream(64 * 1024):
|
|
received += len(chunk)
|
|
if received > size:
|
|
raise ValueError("staging size mismatch")
|
|
digest.update(chunk)
|
|
if received != size or digest.hexdigest() != sha256:
|
|
raise ValueError("staging checksum mismatch")
|
|
finally:
|
|
obj.close()
|
|
obj.release_conn()
|
|
|
|
|
|
def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: str, size: int, sha256: str, stream) -> dict:
|
|
attachment_id = uuid4()
|
|
staging_key = f"staging/{attachment_id}"
|
|
object_key = f"originals/{attachment_id}"
|
|
with connection() as conn:
|
|
reserve_upload(conn, attachment_id, staging_key)
|
|
|
|
try:
|
|
stream.seek(0)
|
|
client = storage()
|
|
with tracer.start_as_current_span("s3.stage_upload", record_exception=False, set_status_on_exception=False):
|
|
client.put_object(BUCKET, staging_key, stream, size, content_type=content_type)
|
|
with connection() as conn:
|
|
if not renew_upload(conn, attachment_id):
|
|
raise HTTPException(503)
|
|
with tracer.start_as_current_span("s3.verify_staging", record_exception=False, set_status_on_exception=False):
|
|
verify_staging(client, staging_key, size, sha256, content_type)
|
|
carrier = {}
|
|
trace_context.inject(carrier)
|
|
with connection() as conn:
|
|
with tracer.start_as_current_span("db.queue_attachment", record_exception=False,
|
|
set_status_on_exception=False):
|
|
row, _job_id = queue_attachment(
|
|
conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id),
|
|
filename=filename, staging_key=staging_key, object_key=object_key,
|
|
content_type=content_type, size_bytes=size, sha256=sha256,
|
|
traceparent=carrier.get("traceparent"), tracestate=carrier.get("tracestate"),
|
|
)
|
|
except HTTPException:
|
|
raise
|
|
except Exception as exc:
|
|
# A lost COMMIT acknowledgement may leave a valid queued job. Keep the
|
|
# staging object; cleanup must check PostgreSQL before deleting it.
|
|
logger.warning("Attachment handoff failed for %s (%s)", attachment_id, type(exc).__name__)
|
|
raise HTTPException(503) from None
|
|
return attachment_view(row)
|
|
|
|
|
|
async def upload(request: Request, chat_id: str, actor_id: str, filename: str) -> dict:
|
|
await asyncio.to_thread(store.chat, actor_id, chat_id)
|
|
validate_filename(filename)
|
|
declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
|
if declared_type not in {"image/png", "image/jpeg"}:
|
|
raise HTTPException(415)
|
|
length = request.headers.get("content-length")
|
|
if length is not None:
|
|
try:
|
|
if int(length) > MAX_BYTES:
|
|
raise HTTPException(413)
|
|
except ValueError:
|
|
raise HTTPException(400) from None
|
|
|
|
with SpooledTemporaryFile(max_size=1024 * 1024) as stream:
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
async for chunk in request.stream():
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise HTTPException(413)
|
|
stream.write(chunk)
|
|
digest.update(chunk)
|
|
stream.seek(0)
|
|
validate_image(stream, declared_type)
|
|
return await asyncio.to_thread(
|
|
persist_attachment, chat_id, actor_id, filename, declared_type, size, digest.hexdigest(), stream
|
|
)
|