"""Private S3 originals and durable upload intents.""" import asyncio from contextlib import asynccontextmanager import hashlib import os from pathlib import Path import tempfile import warnings from uuid import uuid4, UUID import boto3 from botocore.config import Config from PIL import Image, UnidentifiedImageError from starlette.responses import StreamingResponse from .errors import APIError, invalid from .store import public MAX_BYTES = 10 * 1024 * 1024 MAX_PIXELS = 40_000_000 Image.MAX_IMAGE_PIXELS = MAX_PIXELS BUCKET = 'attachments' def secret(name): return Path('/run/secrets/' + name).read_text().strip() def client(): return boto3.client('s3', endpoint_url='http://minio:9000', region_name='us-east-1', aws_access_key_id='messenger-api', aws_secret_access_key=secret('s3_secret_key'), config=Config(connect_timeout=2, read_timeout=3, retries={'max_attempts': 0}, s3={'addressing_style': 'path'}, max_pool_connections=4)) def validate_image(file, content_type): try: with warnings.catch_warnings(): warnings.simplefilter('error', Image.DecompressionBombWarning) file.seek(0) with Image.open(file) as image: if image.width * image.height > MAX_PIXELS: raise APIError(413, 'payload_too_large', 'Too many pixels') if image.format not in ('PNG', 'JPEG') or Image.MIME[image.format] != content_type: raise APIError(415, 'unsupported_media_type', 'Image MIME does not match') image.verify() file.seek(0) with Image.open(file) as image: image.load() except (Image.DecompressionBombError, Image.DecompressionBombWarning): raise APIError(413, 'payload_too_large', 'Too many pixels') from None except (UnidentifiedImageError, OSError, SyntaxError, ValueError): raise APIError(415, 'unsupported_media_type', 'Invalid image') from None finally: file.seek(0) def metadata(row): row = dict(row) row['owner_id'] = row['uploader_id'] row['updated_at'] = row['created_at'] return public({k: row[k] for k in ('id','chat_id','owner_id','filename','content_type', 'size_bytes','sha256','status','created_at','updated_at')}) | {'variants': [], 'error_code': None} async def blocking(function, *args, **kwargs): # Cancellation must not close the spool or release the SQL lock while a thread uses it. task = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs)) try: return await asyncio.shield(task) except asyncio.CancelledError: await task raise class Attachments: def __init__(self, store): self.store = store self.s3 = client() # One decode per replica bounds decoded RGBA memory to about 160 MB. self.slots = asyncio.Semaphore(1) @asynccontextmanager async def slot(self): try: await asyncio.wait_for(self.slots.acquire(), timeout=10) except TimeoutError: raise APIError(503, 'unavailable', 'Upload capacity is busy') from None try: yield finally: self.slots.release() async def get(self, actor, attachment_id): async with self.store.pool.connection() as conn: cur = await conn.execute('''SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s''', (attachment_id, UUID(actor))) row = await cur.fetchone() if row is None: raise APIError(404, 'not_found', 'Attachment not found') if row['status'] != 'ready': raise APIError(409, 'conflict', 'Attachment is not ready') return row async def upload(self, actor, chat_id, request): await self.store.get_chat(actor, chat_id) name = request.query_params.get('filename', '') if not 1 <= len(name) <= 255 or any(c in '/\\' or ord(c) < 32 or ord(c) == 127 for c in name): raise invalid('Invalid filename') mime = request.headers.get('content-type', '').strip().lower() if mime not in ('image/png','image/jpeg'): raise APIError(415, 'unsupported_media_type', 'Use PNG or JPEG') length = request.headers.get('content-length') if length is not None: if not length.isascii() or not length.isdecimal(): raise invalid('Invalid Content-Length') if int(length) > MAX_BYTES: raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB') async with self.slot(): # Temporary spool is only staging, never the source of confirmed originals. with tempfile.TemporaryFile() as file: size = 0 digest = hashlib.sha256() async with asyncio.timeout(20): async for chunk in request.stream(): size += len(chunk) if size > MAX_BYTES: raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB') file.write(chunk) digest.update(chunk) await blocking(validate_image, file, mime) attachment_id = uuid4() key = 'original/' + attachment_id.hex # Commit the intent BEFORE PUT: even a killed process leaves a tracked key. async with self.store.pool.connection() as conn: await conn.execute('''INSERT INTO attachments (id,chat_id,uploader_id,object_key,filename,content_type,size_bytes,sha256,status) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')''', (attachment_id,chat_id,UUID(actor),key,name,mime,size,digest.hexdigest())) # Row lock also excludes the cleaner, including slow or cancelled PUTs. async with self.store.pool.connection() as conn: await conn.execute('SELECT id FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,)) await blocking(self.s3.put_object, Bucket=BUCKET, Key=key, Body=file, ContentLength=size, ContentType=mime) cur = await conn.execute("UPDATE attachments SET status='ready' WHERE id=%s RETURNING *", (attachment_id,)) result = metadata(await cur.fetchone()) return result async def download(self, row): obj = await asyncio.to_thread(self.s3.get_object, Bucket=BUCKET, Key=row['object_key']) stream = obj['Body'] async def chunks(): try: while chunk := await asyncio.to_thread(stream.read, 64 * 1024): yield chunk finally: stream.close() return StreamingResponse(chunks(), media_type=row['content_type'], headers={ 'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff'}) async def cleanup(self): removed = 0 async with self.store.pool.connection() as conn: cur = await conn.execute("""SELECT id,object_key FROM attachments WHERE status='pending' AND created_at < clock_timestamp()-interval '1 hour' ORDER BY created_at LIMIT 100 FOR UPDATE SKIP LOCKED""") for row in await cur.fetchall(): await asyncio.to_thread(self.s3.delete_object, Bucket=BUCKET, Key=row['object_key']) await conn.execute("DELETE FROM attachments WHERE id=%s AND status='pending'", (row['id'],)) removed += 1 return removed async def cleaner(): from .config import Settings from .store import Store store = Store(Settings.load()) await store.open() attachments = Attachments(store) try: while True: try: print({'removed_pending': await attachments.cleanup()}, flush=True) except Exception: print('Cleanup deferred; storage unavailable', flush=True) await asyncio.sleep(60) finally: await store.close() if __name__ == '__main__': asyncio.run(cleaner())