"""Private S3 access shared by attachment routes and maintenance jobs.""" import asyncio import hashlib import logging import os import unicodedata from functools import lru_cache from pathlib import Path from tempfile import SpooledTemporaryFile from uuid import UUID, uuid4 from fastapi import HTTPException, Request from minio import Minio from minio.error import S3Error from opentelemetry import trace from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator from PIL import Image, UnidentifiedImageError from urllib3 import PoolManager, Retry, Timeout from .pipeline_jobs import queue_attachment, renew_upload, reserve_upload from .store import connection, store, timestamp BUCKET = "messenger-attachments" MAX_BYTES = 10 * 1024 * 1024 MAX_PIXELS = 40_000_000 Image.MAX_IMAGE_PIXELS = MAX_PIXELS logger = logging.getLogger(__name__) tracer = trace.get_tracer(__name__) trace_context = TraceContextTextMapPropagator() def secret(name: str) -> str: return Path(os.environ[f"{name}_FILE"]).read_text(encoding="utf-8").strip() @lru_cache(maxsize=1) def storage() -> Minio: return Minio( os.environ.get("S3_ENDPOINT", "s3:9000"), access_key=secret("S3_ACCESS_KEY_ID"), secret_key=secret("S3_SECRET_ACCESS_KEY"), secure=False, region="us-east-1", http_client=PoolManager(timeout=Timeout(connect=2, read=10), retries=Retry(total=0)), ) def storage_ready() -> None: try: storage().stat_object(BUCKET, "health/ready-probe") except S3Error as exc: if exc.code != "NoSuchKey": raise HTTPException(503) from None except Exception: raise HTTPException(503) from None def attachment_view(row: dict) -> dict: return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"], "content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], "variants": [], "error_code": row["error_code"], "created_at": timestamp(row["created_at"]), "updated_at": timestamp(row["updated_at"]), } def validate_filename(filename: str) -> None: if not 1 <= len(filename) <= 255 or filename in {".", ".."}: raise HTTPException(400) if any(char in "/\\" or unicodedata.category(char) == "Cc" for char in filename): raise HTTPException(400) def validate_image(stream, declared_type: str) -> str: try: with Image.open(stream) as image: actual_type = {"PNG": "image/png", "JPEG": "image/jpeg"}.get(image.format) if actual_type is None or actual_type != declared_type: raise HTTPException(415) if image.width * image.height > MAX_PIXELS: raise HTTPException(413) image.verify() stream.seek(0) with Image.open(stream) as image: image.load() return actual_type except Image.DecompressionBombError: raise HTTPException(413) from None except (UnidentifiedImageError, OSError, ValueError, SyntaxError): raise HTTPException(415) from None def verify_staging(client: Minio, key: str, size: int, sha256: str, content_type: str) -> None: """Confirm complete durable bytes; an S3 ETag is not a SHA-256 checksum.""" info = client.stat_object(BUCKET, key) if info.size != size or info.content_type.split(";", 1)[0] != content_type: raise ValueError("staging metadata mismatch") obj = client.get_object(BUCKET, key) try: digest = hashlib.sha256() received = 0 for chunk in obj.stream(64 * 1024): received += len(chunk) if received > size: raise ValueError("staging size mismatch") digest.update(chunk) if received != size or digest.hexdigest() != sha256: raise ValueError("staging checksum mismatch") finally: obj.close() obj.release_conn() def persist_attachment(chat_id: str, actor_id: str, filename: str, content_type: str, size: int, sha256: str, stream) -> dict: attachment_id = uuid4() staging_key = f"staging/{attachment_id}" object_key = f"originals/{attachment_id}" with connection() as conn: reserve_upload(conn, attachment_id, staging_key) try: stream.seek(0) client = storage() with tracer.start_as_current_span("s3.stage_upload", record_exception=False, set_status_on_exception=False): client.put_object(BUCKET, staging_key, stream, size, content_type=content_type) with connection() as conn: if not renew_upload(conn, attachment_id): raise HTTPException(503) with tracer.start_as_current_span("s3.verify_staging", record_exception=False, set_status_on_exception=False): verify_staging(client, staging_key, size, sha256, content_type) carrier = {} trace_context.inject(carrier) with connection() as conn: with tracer.start_as_current_span("db.queue_attachment", record_exception=False, set_status_on_exception=False): row, _job_id = queue_attachment( conn, attachment_id=attachment_id, chat_id=UUID(chat_id), owner_id=UUID(actor_id), filename=filename, staging_key=staging_key, object_key=object_key, content_type=content_type, size_bytes=size, sha256=sha256, traceparent=carrier.get("traceparent"), tracestate=carrier.get("tracestate"), ) except HTTPException: raise except Exception as exc: # A lost COMMIT acknowledgement may leave a valid queued job. Keep the # staging object; cleanup must check PostgreSQL before deleting it. logger.warning("Attachment handoff failed for %s (%s)", attachment_id, type(exc).__name__) raise HTTPException(503) from None return attachment_view(row) async def upload(request: Request, chat_id: str, actor_id: str, filename: str) -> dict: await asyncio.to_thread(store.chat, actor_id, chat_id) validate_filename(filename) declared_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() if declared_type not in {"image/png", "image/jpeg"}: raise HTTPException(415) length = request.headers.get("content-length") if length is not None: try: if int(length) > MAX_BYTES: raise HTTPException(413) except ValueError: raise HTTPException(400) from None with SpooledTemporaryFile(max_size=1024 * 1024) as stream: digest = hashlib.sha256() size = 0 async for chunk in request.stream(): size += len(chunk) if size > MAX_BYTES: raise HTTPException(413) stream.write(chunk) digest.update(chunk) stream.seek(0) validate_image(stream, declared_type) return await asyncio.to_thread( persist_attachment, chat_id, actor_id, filename, declared_type, size, digest.hexdigest(), stream )