Files
mtusi2026lab4/app/attachments.py
T

187 lines
8.1 KiB
Python

"""Private S3 originals and durable upload intents."""
import asyncio
from contextlib import asynccontextmanager
import hashlib
import os
from pathlib import Path
import tempfile
import warnings
from uuid import uuid4, UUID
import boto3
from botocore.config import Config
from PIL import Image, UnidentifiedImageError
from starlette.responses import StreamingResponse
from .errors import APIError, invalid
from .store import public
MAX_BYTES = 10 * 1024 * 1024
MAX_PIXELS = 40_000_000
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
BUCKET = 'attachments'
def secret(name):
return Path('/run/secrets/' + name).read_text().strip()
def client():
return boto3.client('s3', endpoint_url='http://minio:9000', region_name='us-east-1',
aws_access_key_id='messenger-api', aws_secret_access_key=secret('s3_secret_key'),
config=Config(connect_timeout=2, read_timeout=3, retries={'max_attempts': 0},
s3={'addressing_style': 'path'}, max_pool_connections=4))
def validate_image(file, content_type):
try:
with warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
file.seek(0)
with Image.open(file) as image:
if image.width * image.height > MAX_PIXELS:
raise APIError(413, 'payload_too_large', 'Too many pixels')
if image.format not in ('PNG', 'JPEG') or Image.MIME[image.format] != content_type:
raise APIError(415, 'unsupported_media_type', 'Image MIME does not match')
image.verify()
file.seek(0)
with Image.open(file) as image:
image.load()
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
raise APIError(413, 'payload_too_large', 'Too many pixels') from None
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
raise APIError(415, 'unsupported_media_type', 'Invalid image') from None
finally:
file.seek(0)
def metadata(row):
row = dict(row)
row['owner_id'] = row['uploader_id']
row['updated_at'] = row['created_at']
return public({k: row[k] for k in ('id','chat_id','owner_id','filename','content_type',
'size_bytes','sha256','status','created_at','updated_at')}) | {'variants': [], 'error_code': None}
async def blocking(function, *args, **kwargs):
# Cancellation must not close the spool or release the SQL lock while a thread uses it.
task = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs))
try:
return await asyncio.shield(task)
except asyncio.CancelledError:
await task
raise
class Attachments:
def __init__(self, store):
self.store = store
self.s3 = client()
# One decode per replica bounds decoded RGBA memory to about 160 MB.
self.slots = asyncio.Semaphore(1)
@asynccontextmanager
async def slot(self):
try:
await asyncio.wait_for(self.slots.acquire(), timeout=10)
except TimeoutError:
raise APIError(503, 'unavailable', 'Upload capacity is busy') from None
try:
yield
finally:
self.slots.release()
async def get(self, actor, attachment_id):
async with self.store.pool.connection() as conn:
cur = await conn.execute('''SELECT a.* FROM attachments a JOIN chat_members m
ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s''',
(attachment_id, UUID(actor)))
row = await cur.fetchone()
if row is None:
raise APIError(404, 'not_found', 'Attachment not found')
if row['status'] != 'ready':
raise APIError(409, 'conflict', 'Attachment is not ready')
return row
async def upload(self, actor, chat_id, request):
await self.store.get_chat(actor, chat_id)
name = request.query_params.get('filename', '')
if not 1 <= len(name) <= 255 or any(c in '/\\' or ord(c) < 32 or ord(c) == 127 for c in name):
raise invalid('Invalid filename')
mime = request.headers.get('content-type', '').strip().lower()
if mime not in ('image/png','image/jpeg'):
raise APIError(415, 'unsupported_media_type', 'Use PNG or JPEG')
length = request.headers.get('content-length')
if length is not None:
if not length.isascii() or not length.isdecimal():
raise invalid('Invalid Content-Length')
if int(length) > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB')
async with self.slot():
# Temporary spool is only staging, never the source of confirmed originals.
with tempfile.TemporaryFile() as file:
size = 0
digest = hashlib.sha256()
async with asyncio.timeout(20):
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB')
file.write(chunk)
digest.update(chunk)
await blocking(validate_image, file, mime)
attachment_id = uuid4()
key = 'original/' + attachment_id.hex
# Commit the intent BEFORE PUT: even a killed process leaves a tracked key.
async with self.store.pool.connection() as conn:
await conn.execute('''INSERT INTO attachments
(id,chat_id,uploader_id,object_key,filename,content_type,size_bytes,sha256,status)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')''',
(attachment_id,chat_id,UUID(actor),key,name,mime,size,digest.hexdigest()))
# Row lock also excludes the cleaner, including slow or cancelled PUTs.
async with self.store.pool.connection() as conn:
await conn.execute('SELECT id FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,))
await blocking(self.s3.put_object, Bucket=BUCKET, Key=key,
Body=file, ContentLength=size, ContentType=mime)
cur = await conn.execute("UPDATE attachments SET status='ready' WHERE id=%s RETURNING *",
(attachment_id,))
result = metadata(await cur.fetchone())
return result
async def download(self, row):
obj = await asyncio.to_thread(self.s3.get_object, Bucket=BUCKET, Key=row['object_key'])
stream = obj['Body']
async def chunks():
try:
while chunk := await asyncio.to_thread(stream.read, 64 * 1024):
yield chunk
finally:
stream.close()
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff'})
async def cleanup(self):
removed = 0
async with self.store.pool.connection() as conn:
cur = await conn.execute("""SELECT id,object_key FROM attachments WHERE status='pending'
AND created_at < clock_timestamp()-interval '1 hour' ORDER BY created_at
LIMIT 100 FOR UPDATE SKIP LOCKED""")
for row in await cur.fetchall():
await asyncio.to_thread(self.s3.delete_object, Bucket=BUCKET, Key=row['object_key'])
await conn.execute("DELETE FROM attachments WHERE id=%s AND status='pending'", (row['id'],))
removed += 1
return removed
async def cleaner():
from .config import Settings
from .store import Store
store = Store(Settings.load())
await store.open()
attachments = Attachments(store)
try:
while True:
try:
print({'removed_pending': await attachments.cleanup()}, flush=True)
except Exception:
print('Cleanup deferred; storage unavailable', flush=True)
await asyncio.sleep(60)
finally:
await store.close()
if __name__ == '__main__':
asyncio.run(cleaner())