"""Lab4 synchronous attachment API: streamed ingress, S3 storage and per-chat ACL. The spool is temporary request workspace, not an authoritative file store. No original survives a completed request on either API replica's local disk. """ import hashlib import io import os import tempfile import threading import uuid from contextlib import closing import boto3 from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError, EndpointConnectionError from fastapi import APIRouter, Query, Request from fastapi.responses import StreamingResponse from PIL import Image, ImageFile, UnidentifiedImageError from starlette.concurrency import run_in_threadpool from .auth import current_user from .core import APIError, invalid, iso, parse_uuid from .storage import db router = APIRouter() BUCKET = os.getenv("S3_BUCKET", "messenger-attachments") MAX_BYTES = 10 * 1024 * 1024 MAX_PIXELS = 40_000_000 SPOOL_MEMORY_BYTES = 1024 * 1024 S3_BUFFER_BYTES = 256 * 1024 DECODE_LIMIT = threading.BoundedSemaphore(1) # bound expensive Pillow decoding per replica Image.MAX_IMAGE_PIXELS = MAX_PIXELS ImageFile.LOAD_TRUNCATED_IMAGES = False def s3_client(): return boto3.client( "s3", endpoint_url=os.environ["S3_ENDPOINT"], aws_access_key_id=os.environ["S3_ACCESS_KEY"], aws_secret_access_key=os.environ["S3_SECRET_KEY"], region_name="us-east-1", use_ssl=False, config=Config(s3={"addressing_style": "path"}, connect_timeout=3, read_timeout=10, retries={"max_attempts": 1}), ) def s3_error(exc): return APIError(503, "unavailable", "Object storage unavailable") def format_attachment(row): return { "id": str(row["id"]), "chat_id": str(row["chat_id"]), "owner_id": str(row["owner_id"]), "filename": row["filename"], "content_type": row["content_type"], "size_bytes": row["size_bytes"], "sha256": row["sha256"], "status": "ready", "variants": [], "error_code": None, "created_at": iso(row["created_at"]), "updated_at": iso(row["updated_at"]), } def require_member(conn, chat_id, user_id): row = conn.execute( "SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user_id) ).fetchone() if not row: raise APIError(404, "not_found", "Object not found") def attachment_row(conn, attachment_id, user_id): row = conn.execute(""" SELECT a.* FROM attachments a JOIN chat_members cm ON cm.chat_id=a.chat_id WHERE a.id=%s AND cm.user_id=%s """, (attachment_id, user_id)).fetchone() if not row: raise APIError(404, "not_found", "Object not found") return row def validate_filename(filename: str): if not (1 <= len(filename) <= 255) or any( c in "/\\" or ord(c) < 32 or ord(c) == 127 for c in filename ): raise invalid("Invalid filename") def validate_image(fileobj, mime: str): """Check full decode, dimensions, magic and MIME before writing to S3.""" try: fileobj.seek(0) with Image.open(fileobj) as im: fmt = im.format if fmt not in ("PNG", "JPEG"): raise APIError(415, "unsupported_media_type", "Unsupported image encoding") actual_mime = "image/png" if fmt == "PNG" else "image/jpeg" if actual_mime != mime: raise APIError(415, "unsupported_media_type", "Content-Type does not match image") if im.width * im.height > MAX_PIXELS: raise APIError(413, "payload_too_large", "Image dimensions too large") im.verify() fileobj.seek(0) with DECODE_LIMIT: with Image.open(fileobj) as im: im.load() # reject corrupt/truncated pixel data, not just headers except APIError: raise except Image.DecompressionBombError: raise APIError(413, "payload_too_large", "Image dimensions too large") from None except (UnidentifiedImageError, OSError, ValueError, SyntaxError): raise APIError(415, "unsupported_media_type", "Invalid image") from None except Image.DecompressionBombWarning: raise APIError(413, "payload_too_large", "Image dimensions too large") from None finally: fileobj.seek(0) def _upload_checked(spool, *, attachment_id, chat_id, user_id, filename, mime, size, digest): """Blocking S3 and DB steps run outside the event loop.""" key = f"attachments/{attachment_id}/original" backup = f"recovery/{attachment_id}/original" client = s3_client() wrote_original = False wrote_backup = False committed = False try: # A recovery copy enables non-destructive repair if the primary key is lost. spool.seek(0) client.upload_fileobj(spool, BUCKET, key, ExtraArgs={"ContentType": mime}, Config=__import__('boto3.s3.transfer', fromlist=['TransferConfig']).TransferConfig( use_threads=False, multipart_threshold=16*1024*1024, io_chunksize=S3_BUFFER_BYTES)) wrote_original = True client.copy_object(Bucket=BUCKET, Key=backup, CopySource={"Bucket": BUCKET, "Key": key}, ContentType=mime, MetadataDirective="REPLACE") wrote_backup = True with db() as conn: with conn.transaction(): require_member(conn, chat_id, user_id) row = conn.execute(""" INSERT INTO attachments(id,chat_id,owner_id,object_key,backup_key,filename, content_type,size_bytes,sha256) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s) RETURNING * """, (attachment_id, chat_id, user_id, key, backup, filename, mime, size, digest)).fetchone() committed = True return format_attachment(row) except (BotoCoreError, ClientError, EndpointConnectionError) as exc: raise s3_error(exc) from exc finally: # Never delete after an ambiguous COMMIT unless a fresh read proves # metadata is absent. A lost COMMIT acknowledgement can still mean # PostgreSQL committed successfully. Reconciler handles uncertainty. if not committed: confirmed_absent = False try: with db() as conn: confirmed_absent = conn.execute( "SELECT 1 FROM attachments WHERE id=%s", (attachment_id,) ).fetchone() is None except Exception: pass if confirmed_absent: for written, object_key in ((wrote_backup, backup), (wrote_original, key)): if written: try: client.delete_object(Bucket=BUCKET, Key=object_key) except (BotoCoreError, ClientError, EndpointConnectionError): pass @router.post("/api/v1/chats/{chat_id}/attachments", status_code=201) async def upload_attachment(req: Request, chat_id: str, filename: str = Query(...)): from .api import actor, require_chat who = actor(req)["id"] chat_id = parse_uuid(chat_id) validate_filename(filename) mime = req.headers.get("content-type", "").split(";", 1)[0].strip().lower() if mime not in ("image/png", "image/jpeg"): raise APIError(415, "unsupported_media_type", "Only PNG and JPEG are supported") with db() as conn: require_chat(conn, chat_id, who) announced_size = req.headers.get("content-length") if announced_size: try: if int(announced_size) > MAX_BYTES: raise APIError(413, "payload_too_large", "Image too large") except ValueError: raise invalid("Invalid Content-Length") from None size = 0 digest = hashlib.sha256() with tempfile.SpooledTemporaryFile(max_size=SPOOL_MEMORY_BYTES, mode="w+b") as spool: async for chunk in req.stream(): size += len(chunk) if size > MAX_BYTES: raise APIError(413, "payload_too_large", "Image too large") digest.update(chunk) spool.write(chunk) if size == 0: raise APIError(415, "unsupported_media_type", "Empty image") await run_in_threadpool(validate_image, spool, mime) return await run_in_threadpool(_upload_checked, spool, attachment_id=str(uuid.uuid4()), chat_id=chat_id, user_id=who, filename=filename, mime=mime, size=size, digest=digest.hexdigest()) @router.get("/api/v1/attachments/{attachment_id}") def get_attachment(req: Request, attachment_id: str): who = getattr(req.state, "actor", None) or current_user(req) with db() as conn: return format_attachment(attachment_row(conn, parse_uuid(attachment_id), who["id"])) @router.get("/api/v1/attachments/{attachment_id}/content") def download_attachment(req: Request, attachment_id: str, variant: str = "original"): who = getattr(req.state, "actor", None) or current_user(req) with db() as conn: row = attachment_row(conn, parse_uuid(attachment_id), who["id"]) if variant != "original": raise APIError(404, "not_found", "Object not found") client = s3_client() try: response = client.get_object(Bucket=BUCKET, Key=row["object_key"]) except ClientError as exc: if exc.response.get("Error", {}).get("Code") in ("NoSuchKey", "404", "NotFound"): raise APIError(503, "unavailable", "Stored object missing") from exc raise s3_error(exc) from exc except BotoCoreError as exc: raise s3_error(exc) from exc def iterator(): with closing(response["Body"]) as body: while True: chunk = body.read(S3_BUFFER_BYTES) if not chunk: break yield chunk return StreamingResponse(iterator(), media_type=row["content_type"], headers={"Content-Length": str(row["size_bytes"])})