forked from mtusi2026/mtusi2026lab4
235 lines
11 KiB
Python
235 lines
11 KiB
Python
"""Bounded synchronous uploads into private S3, with durable PostgreSQL intents."""
|
|
import asyncio
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import unicodedata
|
|
import uuid
|
|
import warnings
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
from fastapi import Request
|
|
from fastapi.responses import StreamingResponse
|
|
from PIL import Image, UnidentifiedImageError
|
|
from psycopg.rows import dict_row
|
|
from starlette.concurrency import run_in_threadpool
|
|
from starlette.requests import ClientDisconnect
|
|
|
|
MAX_BYTES = 10485760
|
|
MAX_PIXELS = 40000000
|
|
CHUNK = 65536
|
|
# Header parsing checks our exact cap before decoding. Pillow's own bomb guard is
|
|
# retained as another bound, but its warning threshold is not the contract cap.
|
|
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
|
COLUMNS = 'id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,error_code,created_at,updated_at'
|
|
|
|
|
|
def canonical_id(value, error):
|
|
try:
|
|
return str(uuid.UUID(value))
|
|
except (ValueError, TypeError, AttributeError) as exc:
|
|
raise error(400, 'invalid_request', 'invalid id') from exc
|
|
|
|
|
|
def object_lock(conn, key, *, wait=True):
|
|
function = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
|
|
return conn.execute(f'SELECT {function}(hashtextextended(%s,44))', (key,)).fetchone()[0]
|
|
|
|
|
|
class ObjectStore:
|
|
def __init__(self, store, error):
|
|
self.store, self.error = store, error
|
|
self.bucket = os.getenv('S3_BUCKET', 'messenger-attachments')
|
|
self.client = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
|
|
region_name=os.getenv('S3_REGION', 'us-east-1'),
|
|
aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'unset'),
|
|
aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'unset'),
|
|
config=Config(signature_version='s3v4', connect_timeout=1, read_timeout=3,
|
|
retries={'total_max_attempts': 1, 'mode': 'standard'},
|
|
max_pool_connections=8, s3={'addressing_style': 'path'}))
|
|
self.decode_lock = threading.BoundedSemaphore(1)
|
|
# At most two payload spools per process; queued requests receive ASGI
|
|
# backpressure instead of being read into an unbounded request.body().
|
|
self.upload_slots = asyncio.Semaphore(2)
|
|
|
|
def close(self):
|
|
self.client.close()
|
|
|
|
def ready(self):
|
|
try:
|
|
self.client.head_bucket(Bucket=self.bucket)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
|
|
|
|
def authorize_chat(self, request, chat_id, authenticate):
|
|
actor = authenticate(request)
|
|
chat_id = canonical_id(chat_id, self.error)
|
|
with self.store.connect() as conn:
|
|
if not self.store.fetch_chat_membership(chat_id, actor['user_id'], conn):
|
|
raise self.error(404, 'not_found', 'chat not found')
|
|
return actor, chat_id
|
|
|
|
def validate_filename(self, name):
|
|
if not name or len(name) > 255 or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in name):
|
|
raise self.error(400, 'invalid_request', 'invalid filename')
|
|
|
|
def inspect_image(self, stream, claimed_type):
|
|
try:
|
|
with self.decode_lock, warnings.catch_warnings():
|
|
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
|
stream.seek(0)
|
|
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
|
|
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
|
|
if image.width * image.height > MAX_PIXELS:
|
|
raise self.error(413, 'payload_too_large', 'too many image pixels')
|
|
if actual != claimed_type:
|
|
raise self.error(415, 'unsupported_media_type', 'content type mismatch')
|
|
image.verify()
|
|
stream.seek(0)
|
|
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
|
|
image.load() # JPEG.verify alone does not ensure a full decode.
|
|
stream.seek(0)
|
|
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
|
|
raise self.error(413, 'payload_too_large', 'too many image pixels') from exc
|
|
except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc:
|
|
raise self.error(415, 'unsupported_media_type', 'invalid PNG/JPEG') from exc
|
|
|
|
def fetch(self, aid, conn):
|
|
with conn.cursor(row_factory=dict_row) as cur:
|
|
cur.execute('SELECT '+COLUMNS+' FROM attachments WHERE id=%s', (aid,))
|
|
return cur.fetchone()
|
|
|
|
def public(self, row):
|
|
return {'id': str(row['id']), 'chat_id': str(row['chat_id']),
|
|
'owner_id': str(row['owner_id']), 'filename': row['filename'],
|
|
'content_type': row['content_type'], 'size_bytes': row['size_bytes'],
|
|
'sha256': row['sha256'], 'status': 'ready', 'variants': [], 'error_code': None,
|
|
'created_at': self.store.isoformat_utc(row['created_at']),
|
|
'updated_at': self.store.isoformat_utc(row['updated_at'])}
|
|
|
|
def authorized(self, request, aid, authenticate):
|
|
actor = authenticate(request)
|
|
aid = canonical_id(aid, self.error)
|
|
with self.store.connect() as conn:
|
|
row = self.fetch(aid, conn)
|
|
if row is None or not self.store.fetch_chat_membership(str(row['chat_id']), actor['user_id'], conn):
|
|
raise self.error(404, 'not_found', 'attachment not found')
|
|
if row['state'] != 'ready':
|
|
raise self.error(409, 'conflict', 'attachment unavailable')
|
|
return row
|
|
|
|
def persist(self, stream, actor_id, chat_id, filename, content_type, size, checksum, *, after_put=None):
|
|
aid = str(uuid.uuid4())
|
|
key = 'originals/' + aid
|
|
now = self.store.utc_now()
|
|
# Intent is durable BEFORE any S3 write. A crash leaves a known pending
|
|
# row; periodic reconciliation cleans it only after a safe grace period.
|
|
with self.store.connect() as conn:
|
|
conn.execute('INSERT INTO attachments('+COLUMNS+') VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,NULL,%s,%s)',
|
|
(aid, chat_id, actor_id, key, filename, content_type, size, checksum, 'pending', now, now))
|
|
with self.store.connect() as conn:
|
|
object_lock(conn, key)
|
|
try:
|
|
stream.seek(0)
|
|
self.client.put_object(Bucket=self.bucket, Key=key, Body=stream,
|
|
ContentLength=size, ContentType=content_type, Metadata={'sha256': checksum})
|
|
# Optional in-process hook for isolated CLI fault injection only;
|
|
# HTTP clients cannot select failpoints.
|
|
if after_put:
|
|
after_put(aid, key)
|
|
conn.execute("UPDATE attachments SET state='ready',error_code=NULL,updated_at=NOW() WHERE id=%s", (aid,))
|
|
row = self.fetch(aid, conn)
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
|
|
return self.public(row)
|
|
|
|
def download(self, row):
|
|
try:
|
|
response = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
|
|
except (BotoCoreError, ClientError) as exc:
|
|
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
|
|
body = response['Body']
|
|
if response['ContentLength'] != row['size_bytes']:
|
|
body.close()
|
|
raise self.error(503, 'unavailable', 'object unavailable')
|
|
def chunks():
|
|
try:
|
|
while True:
|
|
part = body.read(CHUNK)
|
|
if not part:
|
|
break
|
|
yield part
|
|
finally:
|
|
body.close()
|
|
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
|
|
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff',
|
|
'Content-Disposition': 'attachment'})
|
|
|
|
|
|
def register_routes(app, objects, authenticate):
|
|
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
|
|
async def upload(request: Request, chat_id: str):
|
|
actor, chat_id = await run_in_threadpool(objects.authorize_chat, request, chat_id, authenticate)
|
|
filename = request.query_params.get('filename', '')
|
|
objects.validate_filename(filename)
|
|
claimed = request.headers.get('content-type', '').split(';',1)[0].strip().lower()
|
|
if claimed not in ('image/png', 'image/jpeg'):
|
|
raise objects.error(415, 'unsupported_media_type', 'PNG/JPEG required')
|
|
length = request.headers.get('content-length')
|
|
if length:
|
|
try:
|
|
expected = int(length)
|
|
except ValueError as exc:
|
|
raise objects.error(400, 'invalid_request', 'invalid content length') from exc
|
|
if expected > MAX_BYTES:
|
|
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
|
|
if expected < 1:
|
|
raise objects.error(415, 'unsupported_media_type', 'empty image')
|
|
try:
|
|
await asyncio.wait_for(objects.upload_slots.acquire(), timeout=30)
|
|
except asyncio.TimeoutError as exc:
|
|
raise objects.error(503, 'unavailable', 'upload capacity exceeded') from exc
|
|
try:
|
|
# Temporary payloads are never persistent API storage. Every path,
|
|
# including disconnect/cancellation/error, closes and unlinks the spool.
|
|
with tempfile.SpooledTemporaryFile(max_size=262144, mode='w+b') as stream:
|
|
checksum, size = hashlib.sha256(), 0
|
|
try:
|
|
iterator = request.stream().__aiter__()
|
|
async with asyncio.timeout(30):
|
|
while True:
|
|
try:
|
|
chunk = await asyncio.wait_for(iterator.__anext__(), timeout=5)
|
|
except StopAsyncIteration:
|
|
break
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
|
|
checksum.update(chunk)
|
|
await run_in_threadpool(stream.write, chunk)
|
|
except (asyncio.TimeoutError, ClientDisconnect) as exc:
|
|
raise objects.error(400, 'invalid_request', 'upload interrupted or timed out') from exc
|
|
if size < 1:
|
|
raise objects.error(415, 'unsupported_media_type', 'empty image')
|
|
await run_in_threadpool(objects.inspect_image, stream, claimed)
|
|
return await run_in_threadpool(objects.persist, stream, actor['user_id'], chat_id,
|
|
filename, claimed, size, checksum.hexdigest())
|
|
finally:
|
|
objects.upload_slots.release()
|
|
|
|
@app.get('/api/v1/attachments/{aid}')
|
|
def metadata(request: Request, aid: str):
|
|
return objects.public(objects.authorized(request, aid, authenticate))
|
|
|
|
@app.get('/api/v1/attachments/{aid}/content')
|
|
def content(request: Request, aid: str):
|
|
row = objects.authorized(request, aid, authenticate)
|
|
if request.query_params.get('variant', 'original') != 'original':
|
|
raise objects.error(404, 'not_found', 'variant not found')
|
|
return objects.download(row)
|