"""Bounded synchronous uploads into private S3, with durable PostgreSQL intents.""" import asyncio import hashlib import os from pathlib import Path import tempfile import threading import time import unicodedata import uuid import warnings import boto3 from botocore.config import Config from botocore.exceptions import BotoCoreError, ClientError from fastapi import Request from fastapi.responses import StreamingResponse from PIL import Image, UnidentifiedImageError from psycopg.rows import dict_row from starlette.concurrency import run_in_threadpool from starlette.requests import ClientDisconnect MAX_BYTES = 10485760 MAX_PIXELS = 40000000 CHUNK = 65536 # Header parsing checks our exact cap before decoding. Pillow's own bomb guard is # retained as another bound, but its warning threshold is not the contract cap. Image.MAX_IMAGE_PIXELS = MAX_PIXELS COLUMNS = 'id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,error_code,created_at,updated_at' def canonical_id(value, error): try: return str(uuid.UUID(value)) except (ValueError, TypeError, AttributeError) as exc: raise error(400, 'invalid_request', 'invalid id') from exc def object_lock(conn, key, *, wait=True): function = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock' return conn.execute(f'SELECT {function}(hashtextextended(%s,44))', (key,)).fetchone()[0] class ObjectStore: def __init__(self, store, error): self.store, self.error = store, error self.bucket = os.getenv('S3_BUCKET', 'messenger-attachments') self.client = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'), region_name=os.getenv('S3_REGION', 'us-east-1'), aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'unset'), aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'unset'), config=Config(signature_version='s3v4', connect_timeout=1, read_timeout=3, retries={'total_max_attempts': 1, 'mode': 'standard'}, max_pool_connections=8, s3={'addressing_style': 'path'})) self.decode_lock = threading.BoundedSemaphore(1) # At most two payload spools per process; queued requests receive ASGI # backpressure instead of being read into an unbounded request.body(). self.upload_slots = asyncio.Semaphore(2) def close(self): self.client.close() def ready(self): try: self.client.head_bucket(Bucket=self.bucket) except (BotoCoreError, ClientError) as exc: raise self.error(503, 'unavailable', 'object storage unavailable') from exc def authorize_chat(self, request, chat_id, authenticate): actor = authenticate(request) chat_id = canonical_id(chat_id, self.error) with self.store.connect() as conn: if not self.store.fetch_chat_membership(chat_id, actor['user_id'], conn): raise self.error(404, 'not_found', 'chat not found') return actor, chat_id def validate_filename(self, name): if not name or len(name) > 255 or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in name): raise self.error(400, 'invalid_request', 'invalid filename') def inspect_image(self, stream, claimed_type): try: with self.decode_lock, warnings.catch_warnings(): warnings.simplefilter('error', Image.DecompressionBombWarning) stream.seek(0) with Image.open(stream, formats=['PNG', 'JPEG']) as image: actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format) if image.width * image.height > MAX_PIXELS: raise self.error(413, 'payload_too_large', 'too many image pixels') if actual != claimed_type: raise self.error(415, 'unsupported_media_type', 'content type mismatch') image.verify() stream.seek(0) with Image.open(stream, formats=['PNG', 'JPEG']) as image: image.load() # JPEG.verify alone does not ensure a full decode. stream.seek(0) except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc: raise self.error(413, 'payload_too_large', 'too many image pixels') from exc except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc: raise self.error(415, 'unsupported_media_type', 'invalid PNG/JPEG') from exc def fetch(self, aid, conn): with conn.cursor(row_factory=dict_row) as cur: cur.execute('SELECT '+COLUMNS+' FROM attachments WHERE id=%s', (aid,)) return cur.fetchone() def public(self, row): return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']), 'filename': row['filename'], 'content_type': row['content_type'], 'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': 'ready', 'variants': [], 'error_code': None, 'created_at': self.store.isoformat_utc(row['created_at']), 'updated_at': self.store.isoformat_utc(row['updated_at'])} def authorized(self, request, aid, authenticate): actor = authenticate(request) aid = canonical_id(aid, self.error) with self.store.connect() as conn: row = self.fetch(aid, conn) if row is None or not self.store.fetch_chat_membership(str(row['chat_id']), actor['user_id'], conn): raise self.error(404, 'not_found', 'attachment not found') if row['state'] != 'ready': raise self.error(409, 'conflict', 'attachment unavailable') return row def persist(self, stream, actor_id, chat_id, filename, content_type, size, checksum, *, after_put=None): aid = str(uuid.uuid4()) key = 'originals/' + aid now = self.store.utc_now() # Intent is durable BEFORE any S3 write. A crash leaves a known pending # row; periodic reconciliation cleans it only after a safe grace period. with self.store.connect() as conn: conn.execute('INSERT INTO attachments('+COLUMNS+') VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,NULL,%s,%s)', (aid, chat_id, actor_id, key, filename, content_type, size, checksum, 'pending', now, now)) with self.store.connect() as conn: object_lock(conn, key) try: stream.seek(0) self.client.put_object(Bucket=self.bucket, Key=key, Body=stream, ContentLength=size, ContentType=content_type, Metadata={'sha256': checksum}) # Optional in-process hook for isolated CLI fault injection only; # HTTP clients cannot select failpoints. if after_put: after_put(aid, key) conn.execute("UPDATE attachments SET state='ready',error_code=NULL,updated_at=NOW() WHERE id=%s", (aid,)) row = self.fetch(aid, conn) except (BotoCoreError, ClientError) as exc: raise self.error(503, 'unavailable', 'object storage unavailable') from exc return self.public(row) def download(self, row): try: response = self.client.get_object(Bucket=self.bucket, Key=row['object_key']) except (BotoCoreError, ClientError) as exc: raise self.error(503, 'unavailable', 'object storage unavailable') from exc body = response['Body'] if response['ContentLength'] != row['size_bytes']: body.close() raise self.error(503, 'unavailable', 'object unavailable') def chunks(): try: while True: part = body.read(CHUNK) if not part: break yield part finally: body.close() return StreamingResponse(chunks(), media_type=row['content_type'], headers={ 'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff', 'Content-Disposition': 'attachment'}) def register_routes(app, objects, authenticate): @app.post('/api/v1/chats/{chat_id}/attachments', status_code=201) async def upload(request: Request, chat_id: str): actor, chat_id = await run_in_threadpool(objects.authorize_chat, request, chat_id, authenticate) filename = request.query_params.get('filename', '') objects.validate_filename(filename) claimed = request.headers.get('content-type', '').split(';',1)[0].strip().lower() if claimed not in ('image/png', 'image/jpeg'): raise objects.error(415, 'unsupported_media_type', 'PNG/JPEG required') length = request.headers.get('content-length') if length: try: expected = int(length) except ValueError as exc: raise objects.error(400, 'invalid_request', 'invalid content length') from exc if expected > MAX_BYTES: raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB') if expected < 1: raise objects.error(415, 'unsupported_media_type', 'empty image') try: await asyncio.wait_for(objects.upload_slots.acquire(), timeout=30) except asyncio.TimeoutError as exc: raise objects.error(503, 'unavailable', 'upload capacity exceeded') from exc try: # Temporary payloads are never persistent API storage. Every path, # including disconnect/cancellation/error, closes and unlinks the spool. with tempfile.SpooledTemporaryFile(max_size=262144, mode='w+b') as stream: checksum, size = hashlib.sha256(), 0 try: iterator = request.stream().__aiter__() async with asyncio.timeout(30): while True: try: chunk = await asyncio.wait_for(iterator.__anext__(), timeout=5) except StopAsyncIteration: break size += len(chunk) if size > MAX_BYTES: raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB') checksum.update(chunk) await run_in_threadpool(stream.write, chunk) except (asyncio.TimeoutError, ClientDisconnect) as exc: raise objects.error(400, 'invalid_request', 'upload interrupted or timed out') from exc if size < 1: raise objects.error(415, 'unsupported_media_type', 'empty image') await run_in_threadpool(objects.inspect_image, stream, claimed) return await run_in_threadpool(objects.persist, stream, actor['user_id'], chat_id, filename, claimed, size, checksum.hexdigest()) finally: objects.upload_slots.release() @app.get('/api/v1/attachments/{aid}') def metadata(request: Request, aid: str): return objects.public(objects.authorized(request, aid, authenticate)) @app.get('/api/v1/attachments/{aid}/content') def content(request: Request, aid: str): row = objects.authorized(request, aid, authenticate) if request.query_params.get('variant', 'original') != 'original': raise objects.error(404, 'not_found', 'variant not found') return objects.download(row)