Files

235 lines
11 KiB
Python

"""Bounded synchronous uploads into private S3, with durable PostgreSQL intents."""
import asyncio
import hashlib
import os
from pathlib import Path
import tempfile
import threading
import time
import unicodedata
import uuid
import warnings
import boto3
from botocore.config import Config
from botocore.exceptions import BotoCoreError, ClientError
from fastapi import Request
from fastapi.responses import StreamingResponse
from PIL import Image, UnidentifiedImageError
from psycopg.rows import dict_row
from starlette.concurrency import run_in_threadpool
from starlette.requests import ClientDisconnect
MAX_BYTES = 10485760
MAX_PIXELS = 40000000
CHUNK = 65536
# Header parsing checks our exact cap before decoding. Pillow's own bomb guard is
# retained as another bound, but its warning threshold is not the contract cap.
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
COLUMNS = 'id,chat_id,owner_id,object_key,filename,content_type,size_bytes,sha256,state,error_code,created_at,updated_at'
def canonical_id(value, error):
try:
return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError) as exc:
raise error(400, 'invalid_request', 'invalid id') from exc
def object_lock(conn, key, *, wait=True):
function = 'pg_advisory_xact_lock' if wait else 'pg_try_advisory_xact_lock'
return conn.execute(f'SELECT {function}(hashtextextended(%s,44))', (key,)).fetchone()[0]
class ObjectStore:
def __init__(self, store, error):
self.store, self.error = store, error
self.bucket = os.getenv('S3_BUCKET', 'messenger-attachments')
self.client = boto3.client('s3', endpoint_url=os.getenv('S3_ENDPOINT', 'http://s3:9000'),
region_name=os.getenv('S3_REGION', 'us-east-1'),
aws_access_key_id=os.getenv('S3_ACCESS_KEY_ID', 'unset'),
aws_secret_access_key=os.getenv('S3_SECRET_ACCESS_KEY', 'unset'),
config=Config(signature_version='s3v4', connect_timeout=1, read_timeout=3,
retries={'total_max_attempts': 1, 'mode': 'standard'},
max_pool_connections=8, s3={'addressing_style': 'path'}))
self.decode_lock = threading.BoundedSemaphore(1)
# At most two payload spools per process; queued requests receive ASGI
# backpressure instead of being read into an unbounded request.body().
self.upload_slots = asyncio.Semaphore(2)
def close(self):
self.client.close()
def ready(self):
try:
self.client.head_bucket(Bucket=self.bucket)
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
def authorize_chat(self, request, chat_id, authenticate):
actor = authenticate(request)
chat_id = canonical_id(chat_id, self.error)
with self.store.connect() as conn:
if not self.store.fetch_chat_membership(chat_id, actor['user_id'], conn):
raise self.error(404, 'not_found', 'chat not found')
return actor, chat_id
def validate_filename(self, name):
if not name or len(name) > 255 or any(c in '/\\' or unicodedata.category(c).startswith('C') for c in name):
raise self.error(400, 'invalid_request', 'invalid filename')
def inspect_image(self, stream, claimed_type):
try:
with self.decode_lock, warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
stream.seek(0)
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
actual = {'PNG': 'image/png', 'JPEG': 'image/jpeg'}.get(image.format)
if image.width * image.height > MAX_PIXELS:
raise self.error(413, 'payload_too_large', 'too many image pixels')
if actual != claimed_type:
raise self.error(415, 'unsupported_media_type', 'content type mismatch')
image.verify()
stream.seek(0)
with Image.open(stream, formats=['PNG', 'JPEG']) as image:
image.load() # JPEG.verify alone does not ensure a full decode.
stream.seek(0)
except (Image.DecompressionBombError, Image.DecompressionBombWarning) as exc:
raise self.error(413, 'payload_too_large', 'too many image pixels') from exc
except (UnidentifiedImageError, OSError, SyntaxError, ValueError) as exc:
raise self.error(415, 'unsupported_media_type', 'invalid PNG/JPEG') from exc
def fetch(self, aid, conn):
with conn.cursor(row_factory=dict_row) as cur:
cur.execute('SELECT '+COLUMNS+' FROM attachments WHERE id=%s', (aid,))
return cur.fetchone()
def public(self, row):
return {'id': str(row['id']), 'chat_id': str(row['chat_id']),
'owner_id': str(row['owner_id']), 'filename': row['filename'],
'content_type': row['content_type'], 'size_bytes': row['size_bytes'],
'sha256': row['sha256'], 'status': 'ready', 'variants': [], 'error_code': None,
'created_at': self.store.isoformat_utc(row['created_at']),
'updated_at': self.store.isoformat_utc(row['updated_at'])}
def authorized(self, request, aid, authenticate):
actor = authenticate(request)
aid = canonical_id(aid, self.error)
with self.store.connect() as conn:
row = self.fetch(aid, conn)
if row is None or not self.store.fetch_chat_membership(str(row['chat_id']), actor['user_id'], conn):
raise self.error(404, 'not_found', 'attachment not found')
if row['state'] != 'ready':
raise self.error(409, 'conflict', 'attachment unavailable')
return row
def persist(self, stream, actor_id, chat_id, filename, content_type, size, checksum, *, after_put=None):
aid = str(uuid.uuid4())
key = 'originals/' + aid
now = self.store.utc_now()
# Intent is durable BEFORE any S3 write. A crash leaves a known pending
# row; periodic reconciliation cleans it only after a safe grace period.
with self.store.connect() as conn:
conn.execute('INSERT INTO attachments('+COLUMNS+') VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,NULL,%s,%s)',
(aid, chat_id, actor_id, key, filename, content_type, size, checksum, 'pending', now, now))
with self.store.connect() as conn:
object_lock(conn, key)
try:
stream.seek(0)
self.client.put_object(Bucket=self.bucket, Key=key, Body=stream,
ContentLength=size, ContentType=content_type, Metadata={'sha256': checksum})
# Optional in-process hook for isolated CLI fault injection only;
# HTTP clients cannot select failpoints.
if after_put:
after_put(aid, key)
conn.execute("UPDATE attachments SET state='ready',error_code=NULL,updated_at=NOW() WHERE id=%s", (aid,))
row = self.fetch(aid, conn)
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
return self.public(row)
def download(self, row):
try:
response = self.client.get_object(Bucket=self.bucket, Key=row['object_key'])
except (BotoCoreError, ClientError) as exc:
raise self.error(503, 'unavailable', 'object storage unavailable') from exc
body = response['Body']
if response['ContentLength'] != row['size_bytes']:
body.close()
raise self.error(503, 'unavailable', 'object unavailable')
def chunks():
try:
while True:
part = body.read(CHUNK)
if not part:
break
yield part
finally:
body.close()
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff',
'Content-Disposition': 'attachment'})
def register_routes(app, objects, authenticate):
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
async def upload(request: Request, chat_id: str):
actor, chat_id = await run_in_threadpool(objects.authorize_chat, request, chat_id, authenticate)
filename = request.query_params.get('filename', '')
objects.validate_filename(filename)
claimed = request.headers.get('content-type', '').split(';',1)[0].strip().lower()
if claimed not in ('image/png', 'image/jpeg'):
raise objects.error(415, 'unsupported_media_type', 'PNG/JPEG required')
length = request.headers.get('content-length')
if length:
try:
expected = int(length)
except ValueError as exc:
raise objects.error(400, 'invalid_request', 'invalid content length') from exc
if expected > MAX_BYTES:
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
if expected < 1:
raise objects.error(415, 'unsupported_media_type', 'empty image')
try:
await asyncio.wait_for(objects.upload_slots.acquire(), timeout=30)
except asyncio.TimeoutError as exc:
raise objects.error(503, 'unavailable', 'upload capacity exceeded') from exc
try:
# Temporary payloads are never persistent API storage. Every path,
# including disconnect/cancellation/error, closes and unlinks the spool.
with tempfile.SpooledTemporaryFile(max_size=262144, mode='w+b') as stream:
checksum, size = hashlib.sha256(), 0
try:
iterator = request.stream().__aiter__()
async with asyncio.timeout(30):
while True:
try:
chunk = await asyncio.wait_for(iterator.__anext__(), timeout=5)
except StopAsyncIteration:
break
size += len(chunk)
if size > MAX_BYTES:
raise objects.error(413, 'payload_too_large', 'upload exceeds 10 MiB')
checksum.update(chunk)
await run_in_threadpool(stream.write, chunk)
except (asyncio.TimeoutError, ClientDisconnect) as exc:
raise objects.error(400, 'invalid_request', 'upload interrupted or timed out') from exc
if size < 1:
raise objects.error(415, 'unsupported_media_type', 'empty image')
await run_in_threadpool(objects.inspect_image, stream, claimed)
return await run_in_threadpool(objects.persist, stream, actor['user_id'], chat_id,
filename, claimed, size, checksum.hexdigest())
finally:
objects.upload_slots.release()
@app.get('/api/v1/attachments/{aid}')
def metadata(request: Request, aid: str):
return objects.public(objects.authorized(request, aid, authenticate))
@app.get('/api/v1/attachments/{aid}/content')
def content(request: Request, aid: str):
row = objects.authorized(request, aid, authenticate)
if request.query_params.get('variant', 'original') != 'original':
raise objects.error(404, 'not_found', 'variant not found')
return objects.download(row)