"""Lab5 attachments: synchronous durable admission, asynchronous image pipeline. 202 means verified bytes were staged to private S3 and metadata + a job + an outbox event were committed in ONE PostgreSQL transaction. No broker availability is required for admission; the dispatcher replays unsent intents. """ import hashlib import os import tempfile import threading import uuid from contextlib import closing import boto3 from boto3.s3.transfer import TransferConfig from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError from fastapi import APIRouter, Query, Request from fastapi.responses import StreamingResponse from PIL import Image, ImageFile, UnidentifiedImageError from starlette.concurrency import run_in_threadpool from .auth import current_user from .core import APIError, invalid, iso, parse_uuid from .storage import db from . import telemetry as ot router = APIRouter() BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") MAX_BYTES = 10 * 1024 * 1024 MAX_PIXELS = 40_000_000 SPOOL_MEMORY_BYTES = 1024 * 1024 S3_BUFFER_BYTES = 256 * 1024 DECODE_LIMIT = threading.BoundedSemaphore(1) Image.MAX_IMAGE_PIXELS = MAX_PIXELS ImageFile.LOAD_TRUNCATED_IMAGES = False def s3_client(): return ot.S3Tracing(boto3.client( "s3", endpoint_url=os.environ["S3_ENDPOINT"], aws_access_key_id=os.environ["S3_ACCESS_KEY"], aws_secret_access_key=os.environ["S3_SECRET_KEY"], region_name="us-east-1", use_ssl=False, config=Config(s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=10, retries={"max_attempts": 1}), )) def s3_error(exc): return APIError(503, "unavailable", "Object storage unavailable") def format_attachment(row, variants=()): return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"], "content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": row["status"], "variants": [{k: item[k] for k in ("name", "content_type", "size_bytes", "width", "height", "sha256")} for item in variants] if row["status"] == "ready" else [], "error_code": row["error_code"] if row["status"] == "failed" else None, "created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]), } def require_member(conn, chat_id, user_id): if not conn.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id)).fetchone(): raise APIError(404, "not_found", "Object not found") def attachment_row(conn, attachment_id, user_id): row = conn.execute(""" SELECT a.* FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=%s AND cm.user_id=%s """, (attachment_id, user_id)).fetchone() if not row: raise APIError(404, "not_found", "Object not found") return row def validate_filename(filename: str): if not (1 <= len(filename) <= 255) or any(c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename): raise invalid("Invalid filename") def validate_image(fileobj, mime: str): """Validate MIME, magic, dimensions and complete decode; worker does the resize.""" try: fileobj.seek(0) with Image.open(fileobj) as im: if im.format not in ("PNG", "JPEG"): raise APIError(415, "unsupported_media_type", "Unsupported image encoding") actual = "image/png" if im.format == "PNG" else "image/jpeg" if actual != mime: raise APIError(415, "unsupported_media_type", "Content-Type does not match image") if im.width * im.height > MAX_PIXELS: raise APIError(413, "payload_too_large", "Image dimensions too large") im.verify() fileobj.seek(0) with DECODE_LIMIT: with Image.open(fileobj) as im: im.load() except APIError: raise except (Image.DecompressionBombError, Image.DecompressionBombWarning): raise APIError(413, "payload_too_large", "Image dimensions too large") from None except (UnidentifiedImageError, OSError, ValueError, SyntaxError): raise APIError(415, "unsupported_media_type", "Invalid image") from None finally: fileobj.seek(0) def _admit_upload(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest): from .jobs import job_payload stage_key = f"staging/{attachment_id}/original" key = f"attachments/{attachment_id}/original" backup = f"recovery/{attachment_id}/original" job_id = str(uuid.uuid4()) outbox_id = str(uuid.uuid4()) client = s3_client() try: spool.seek(0) client.upload_fileobj(spool, BUCKET, stage_key, ExtraArgs={"ContentType": mime}, Config=TransferConfig(use_threads=False, multipart_threshold=16*1024*1024, io_chunksize=S3_BUFFER_BYTES)) except (BotoCoreError, ClientError, EndpointConnectionError, OSError) as exc: raise s3_error(exc) from exc # Atomic DB metadata + intent. If DB commit is ambiguous, do not delete the # staged object: an accepted task may already exist; operator cleanup handles it. with db() as conn: with conn.transaction(): require_member(conn, chat_id, user_id) # Small educational admission bound; pre-existing queued tasks remain safe. pending = conn.execute("SELECT count(*) AS c FROM processing_jobs WHERE status IN ('queued','processing')").fetchone()["c"] if pending >= int(os.getenv("JOB_BACKLOG_LIMIT", "1000")): raise APIError(429, "rate_limited", "Processing queue is full") row = conn.execute(""" INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename, content_type,size_bytes,sha256,status,staging_key,pipeline_version) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,1) RETURNING * """, (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest, stage_key)).fetchone() conn.execute(""" INSERT INTO processing_jobs(id,attachment_id,pipeline_version,status,attempt) VALUES (%s,%s,1,'queued',1) """, (job_id, attachment_id)) payload = job_payload(job_id, attachment_id, 1, 1) conn.execute(""" INSERT INTO job_outbox(id,job_id,payload) VALUES (%s,%s,%s::jsonb) """, (outbox_id, job_id, __import__('json').dumps(payload))) ot.event("upload.accepted", job_id=job_id) return format_attachment(row) # snapshot = queued, even if worker is very fast @router.post("/api/v1/chats/{chat_id}/attachments", status_code=202) async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)): from .api import actor, require_chat who = actor(req)["id"] chat_id = parse_uuid(chat_id) validate_filename(filename) mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower() if mime not in ("image/png", "image/jpeg"): raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported") with db() as conn: require_chat(conn, chat_id, who) announced = req.headers.get("content-length") if announced: try: if int(announced) > MAX_BYTES: raise APIError(413, "payload_too_large", "Image too large") if int(announced) < 0: raise invalid("Invalid Content-Length") except ValueError: raise invalid("Invalid Content-Length") from None size = 0 digest = hashlib.sha256() with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool: async for chunk in req.stream(): size += len(chunk) if size > MAX_BYTES: raise APIError(413, "payload_too_large", "Image too large") digest.update(chunk) spool.write(chunk) if not size: raise APIError(415, "unsupported_media_type", "Empty image") await run_in_threadpool(validate_image, spool, mime) return await run_in_threadpool(_admit_upload, spool, attachment_id=str(uuid.uuid4()), chat_id=chat_id, user_id=who, filename=filename, mime=mime, size=size, digest=digest.hexdigest()) @router.get("/api/v1/attachments/{attachment_id}") def get_attachment(req: Request, attachment_id: str): who = getattr(req.state, "actor", None) or current_user(req) with db() as conn: row = attachment_row(conn, parse_uuid(attachment_id), who["id"]) variants = conn.execute(""" SELECT name,content_type,size_bytes,width,height,sha256 FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s ORDER BY CASE name WHEN 'thumbnail' THEN 0 WHEN 'crop' THEN 1 ELSE 2 END """, (row["id"], row["pipeline_version"])).fetchall() if row["status"] == "ready" else [] return format_attachment(row, variants) @router.get("/api/v1/attachments/{attachment_id}/content") def download_attachment(req: Request, attachment_id: str, variant: str = "original"): who = getattr(req.state, "actor", None) or current_user(req) with db() as conn: row = attachment_row(conn, parse_uuid(attachment_id), who["id"]) if row["status"] != "ready": raise APIError(409, "conflict", "Attachment is not ready") if variant == "original": key, mime, size = row["object_key"], row["content_type"], row["size_bytes"] else: v = conn.execute(""" SELECT * FROM attachment_variants WHERE attachment_id=%s AND pipeline_version=%s AND name=%s """, (row["id"], row["pipeline_version"], variant)).fetchone() if not v: raise APIError(404, "not_found", "Object not found") key, mime, size = v["object_key"], v["content_type"], v["size_bytes"] client = s3_client() try: response = client.get_object(Bucket=BUCKET, Key=key) except ClientError as exc: if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"): raise APIError(503, "unavailable", "Stored object missing") from exc raise s3_error(exc) from exc except BotoCoreError as exc: raise s3_error(exc) from exc def iterator(): with closing(response["Body"]) as body: while chunk := body.read(S3_BUFFER_BYTES): yield chunk return StreamingResponse(iterator(), media_type=mime, headers={"Content-Length": str(size)})