Files
mtusi2026lab5/attachments.py
T

241 lines
12 KiB
Python

"""Bounded uploads into durable S3 staging and transactional job/outbox."""
import asyncio
import hashlib
import os
from pathlib import Path
import tempfile
import threading
import time
import unicodedata
import uuid
import warnings
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from fastapi import Request
from fastapi.responses import StreamingResponse
from PIL import Image, UnidentifiedImageError
from psycopg.rows import dict_row
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
MAX_BYTES = 10485760
MAX_PIXELS = 40000000
CHUNK = 65536
# Header parsing checks our exact cap before decoding. Pillow's own bomb guard is
# retained as another bound, but its warning threshold is not the contract cap.
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
COLUMNS = 'id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,error_code,created_at,updated_at,pipeline_version,staging_key,variants'
def canonical_id(value, error):
try:
return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError) as exc:
raise error(400, 'invalid_request', 'invalid id') from exc
def object_lock(conn, key, *, wait=True):
function = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
return conn.execute(f'SELECT {function}(hashtextextended(%s,44))', (key,)).fetchone()[0]
class ObjectStore:
def __init__(self, store, error):
self.store, self.error = store, error
self.bucket = os.getenv('S3_BUCKET', 'messenger-attachments')
self.client = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'unset'),
aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'unset'),
config=Config(signature_version='s3v4', connect_timeout=1, read_timeout=3,
retries={'total_max_attempts': 1, 'mode': 'standard'},
max_pool_connections=8, s3={'addressing_style': 'path'}))
self.decode_lock = threading.BoundedSemaphore(1)
# At most two payload spools per process; queued requests receive ASGI
# backpressure instead of being read into an unbounded request.body().
self.upload_slots = asyncio.Semaphore(2)
def close(self):
self.client.close()
def ready(self):
try:
self.client.head_bucket(Bucket=self.bucket)
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
def authorize_chat(self, request, chat_id, authenticate):
actor = authenticate(request)
chat_id = canonical_id(chat_id, self.error)
with self.store.connect() as conn:
if not self.store.fetch_chat_membership(chat_id, actor['user_id'], conn):
raise self.error(404, 'not_found', 'chat not found')
return actor, chat_id
def validate_filename(self, name):
if not name or len(name) > 255 or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in name):
raise self.error(400, 'invalid_request', 'invalid filename')
def inspect_image(self, stream, claimed_type):
try:
with self.decode_lock, warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
stream.seek(0)
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
if image.width * image.height > MAX_PIXELS:
raise self.error(413, 'payload_too_large', 'too many image pixels')
if actual != claimed_type:
raise self.error(415, 'unsupported_media_type', 'content type mismatch')
image.verify()
stream.seek(0)
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
image.load() # JPEG.verify alone does not ensure a full decode.
stream.seek(0)
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
raise self.error(413, 'payload_too_large', 'too many image pixels') from exc
except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc:
raise self.error(415, 'unsupported_media_type', 'invalid PNG/JPEG') from exc
def fetch(self, aid, conn):
with conn.cursor(row_factory=dict_row) as cur:
cur.execute('SELECT '+COLUMNS+' FROM attachments WHERE id=%s', (aid,))
return cur.fetchone()
def public(self, row):
return {'id': str(row['id']), 'chat_id': str(row['chat_id']),
'owner_id': str(row['owner_id']), 'filename': row['filename'],
'content_type': row['content_type'], 'size_bytes': row['size_bytes'],
'sha256': row['sha256'], 'status': row['state'], 'variants': [{k:v for k,v in x.items() if k != 'object_key'} for x in row['variants']], 'error_code': row['error_code'],
'created_at': self.store.isoformat_utc(row['created_at']),
'updated_at': self.store.isoformat_utc(row['updated_at'])}
def authorized(self, request, aid, authenticate):
actor = authenticate(request)
aid = canonical_id(aid, self.error)
with self.store.connect() as conn:
row = self.fetch(aid, conn)
if row is None or not self.store.fetch_chat_membership(str(row['chat_id']), actor['user_id'], conn):
raise self.error(404, 'not_found', 'attachment not found')
return row
def persist(self, stream, actor_id, chat_id, filename, content_type, size, checksum):
from pipeline import create_job
aid = str(uuid.uuid4())
key = 'staging/' + aid
now = self.store.utc_now()
# The capacity lock serializes admission across every API replica.
# S3 is written before COMMIT: an ambiguous commit may leave an orphan,
# but a returned 202 always has both staging bytes and a durable job.
with self.store.connect() as conn:
conn.execute('SELECT pg_advisory_xact_lock(202605)')
count = conn.execute("SELECT count(*) FROM jobs WHERE state IN ('queued','processing')").fetchone()[0]
if count >= int(os.getenv('MAX_PENDING_JOBS', '100')):
raise self.error(429, 'rate_limited', 'processing capacity exceeded')
try:
stream.seek(0)
self.client.put_object(Bucket=self.bucket, Key=key, Body=stream,
ContentLength=size, ContentType=content_type, Metadata={'sha256': checksum})
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
conn.execute("""INSERT INTO attachments(id,chat_id,owner_id,object_key,filename,
content_type,size_bytes,sha256,state,created_at,updated_at,staging_key)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'queued',%s,%s,%s)""",
(aid,chat_id,actor_id,'results/'+aid+'/v1/original',filename,content_type,size,checksum,now,now,key))
create_job(conn, aid, 1)
row = self.fetch(aid, conn)
return self.public(row)
def download(self, row):
try:
response = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
body = response['Body']
if response['ContentLength'] != row['size_bytes']:
body.close()
raise self.error(503, 'unavailable', 'object unavailable')
def chunks():
try:
while True:
part = body.read(CHUNK)
if not part:
break
yield part
finally:
body.close()
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff',
'Content-Disposition': 'attachment'})
def register_routes(app, objects, authenticate):
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202)
async def upload(request: Request, chat_id: str):
actor, chat_id = await run_in_threadpool(objects.authorize_chat, request, chat_id, authenticate)
filename = request.query_params.get('filename', '')
objects.validate_filename(filename)
claimed = request.headers.get('content-type', '').split(';',1)[0].strip().lower()
if claimed not in ('image/png', 'image/jpeg'):
raise objects.error(415, 'unsupported_media_type', 'PNG/JPEG required')
length = request.headers.get('content-length')
if length:
try:
expected = int(length)
except ValueError as exc:
raise objects.error(400, 'invalid_request', 'invalid content length') from exc
if expected > MAX_BYTES:
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
if expected < 1:
raise objects.error(415, 'unsupported_media_type', 'empty image')
try:
await asyncio.wait_for(objects.upload_slots.acquire(), timeout=30)
except asyncio.TimeoutError as exc:
raise objects.error(503, 'unavailable', 'upload capacity exceeded') from exc
try:
# Temporary payloads are never persistent API storage. Every path,
# including disconnect/cancellation/error, closes and unlinks the spool.
with tempfile.SpooledTemporaryFile(max_size=262144, mode='w+b') as stream:
checksum, size = hashlib.sha256(), 0
try:
iterator = request.stream().__aiter__()
async with asyncio.timeout(30):
while True:
try:
chunk = await asyncio.wait_for(iterator.__anext__(), timeout=5)
except StopAsyncIteration:
break
size += len(chunk)
if size > MAX_BYTES:
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
checksum.update(chunk)
await run_in_threadpool(stream.write, chunk)
except (asyncio.TimeoutError, ClientDisconnect) as exc:
raise objects.error(400, 'invalid_request', 'upload interrupted or timed out') from exc
if size < 1:
raise objects.error(415, 'unsupported_media_type', 'empty image')
await run_in_threadpool(objects.inspect_image, stream, claimed)
return await run_in_threadpool(objects.persist, stream, actor['user_id'], chat_id,
filename, claimed, size, checksum.hexdigest())
finally:
objects.upload_slots.release()
@app.get('/api/v1/attachments/{aid}')
def metadata(request: Request, aid: str):
return objects.public(objects.authorized(request, aid, authenticate))
@app.get('/api/v1/attachments/{aid}/content')
def content(request: Request, aid: str):
row = objects.authorized(request, aid, authenticate)
if row['state'] != 'ready':
raise objects.error(409, 'conflict', 'attachment unavailable')
variant = request.query_params.get('variant', 'original')
if variant != 'original':
selected = next((v for v in row['variants'] if v['name'] == variant), None)
if selected is None:
raise objects.error(404, 'not_found', 'variant not found')
row = {**row, **selected}
return objects.download(row)