187 lines
8.1 KiB
Python
187 lines
8.1 KiB
Python
"""Private S3 originals and durable upload intents."""
|
|
import asyncio
|
|
from contextlib import asynccontextmanager
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
import tempfile
|
|
import warnings
|
|
from uuid import uuid4, UUID
|
|
|
|
import boto3
|
|
from botocore.config import Config
|
|
from PIL import Image, UnidentifiedImageError
|
|
from starlette.responses import StreamingResponse
|
|
|
|
from .errors import APIError, invalid
|
|
from .store import public
|
|
|
|
MAX_BYTES = 10 * 1024 * 1024
|
|
MAX_PIXELS = 40_000_000
|
|
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
|
|
BUCKET = 'attachments'
|
|
|
|
def secret(name):
|
|
return Path('/run/secrets/' + name).read_text().strip()
|
|
|
|
def client():
|
|
return boto3.client('s3', endpoint_url='http://minio:9000', region_name='us-east-1',
|
|
aws_access_key_id='messenger-api', aws_secret_access_key=secret('s3_secret_key'),
|
|
config=Config(connect_timeout=2, read_timeout=3, retries={'max_attempts': 0},
|
|
s3={'addressing_style': 'path'}, max_pool_connections=4))
|
|
|
|
def validate_image(file, content_type):
|
|
try:
|
|
with warnings.catch_warnings():
|
|
warnings.simplefilter('error', Image.DecompressionBombWarning)
|
|
file.seek(0)
|
|
with Image.open(file) as image:
|
|
if image.width * image.height > MAX_PIXELS:
|
|
raise APIError(413, 'payload_too_large', 'Too many pixels')
|
|
if image.format not in ('PNG', 'JPEG') or Image.MIME[image.format] != content_type:
|
|
raise APIError(415, 'unsupported_media_type', 'Image MIME does not match')
|
|
image.verify()
|
|
file.seek(0)
|
|
with Image.open(file) as image:
|
|
image.load()
|
|
except (Image.DecompressionBombError, Image.DecompressionBombWarning):
|
|
raise APIError(413, 'payload_too_large', 'Too many pixels') from None
|
|
except (UnidentifiedImageError, OSError, SyntaxError, ValueError):
|
|
raise APIError(415, 'unsupported_media_type', 'Invalid image') from None
|
|
finally:
|
|
file.seek(0)
|
|
|
|
def metadata(row):
|
|
row = dict(row)
|
|
row['owner_id'] = row['uploader_id']
|
|
row['updated_at'] = row['created_at']
|
|
return public({k: row[k] for k in ('id','chat_id','owner_id','filename','content_type',
|
|
'size_bytes','sha256','status','created_at','updated_at')}) | {'variants': [], 'error_code': None}
|
|
|
|
async def blocking(function, *args, **kwargs):
|
|
# Cancellation must not close the spool or release the SQL lock while a thread uses it.
|
|
task = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs))
|
|
try:
|
|
return await asyncio.shield(task)
|
|
except asyncio.CancelledError:
|
|
await task
|
|
raise
|
|
|
|
class Attachments:
|
|
def __init__(self, store):
|
|
self.store = store
|
|
self.s3 = client()
|
|
# One decode per replica bounds decoded RGBA memory to about 160 MB.
|
|
self.slots = asyncio.Semaphore(1)
|
|
|
|
@asynccontextmanager
|
|
async def slot(self):
|
|
try:
|
|
await asyncio.wait_for(self.slots.acquire(), timeout=10)
|
|
except TimeoutError:
|
|
raise APIError(503, 'unavailable', 'Upload capacity is busy') from None
|
|
try:
|
|
yield
|
|
finally:
|
|
self.slots.release()
|
|
|
|
async def get(self, actor, attachment_id):
|
|
async with self.store.pool.connection() as conn:
|
|
cur = await conn.execute('''SELECT a.* FROM attachments a JOIN chat_members m
|
|
ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s''',
|
|
(attachment_id, UUID(actor)))
|
|
row = await cur.fetchone()
|
|
if row is None:
|
|
raise APIError(404, 'not_found', 'Attachment not found')
|
|
if row['status'] != 'ready':
|
|
raise APIError(409, 'conflict', 'Attachment is not ready')
|
|
return row
|
|
|
|
async def upload(self, actor, chat_id, request):
|
|
await self.store.get_chat(actor, chat_id)
|
|
name = request.query_params.get('filename', '')
|
|
if not 1 <= len(name) <= 255 or any(c in '/\\' or ord(c) < 32 or ord(c) == 127 for c in name):
|
|
raise invalid('Invalid filename')
|
|
mime = request.headers.get('content-type', '').strip().lower()
|
|
if mime not in ('image/png','image/jpeg'):
|
|
raise APIError(415, 'unsupported_media_type', 'Use PNG or JPEG')
|
|
length = request.headers.get('content-length')
|
|
if length is not None:
|
|
if not length.isascii() or not length.isdecimal():
|
|
raise invalid('Invalid Content-Length')
|
|
if int(length) > MAX_BYTES:
|
|
raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB')
|
|
async with self.slot():
|
|
# Temporary spool is only staging, never the source of confirmed originals.
|
|
with tempfile.TemporaryFile() as file:
|
|
size = 0
|
|
digest = hashlib.sha256()
|
|
async with asyncio.timeout(20):
|
|
async for chunk in request.stream():
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise APIError(413, 'payload_too_large', 'File exceeds 10 MiB')
|
|
file.write(chunk)
|
|
digest.update(chunk)
|
|
await blocking(validate_image, file, mime)
|
|
attachment_id = uuid4()
|
|
key = 'original/' + attachment_id.hex
|
|
# Commit the intent BEFORE PUT: even a killed process leaves a tracked key.
|
|
async with self.store.pool.connection() as conn:
|
|
await conn.execute('''INSERT INTO attachments
|
|
(id,chat_id,uploader_id,object_key,filename,content_type,size_bytes,sha256,status)
|
|
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,'pending')''',
|
|
(attachment_id,chat_id,UUID(actor),key,name,mime,size,digest.hexdigest()))
|
|
# Row lock also excludes the cleaner, including slow or cancelled PUTs.
|
|
async with self.store.pool.connection() as conn:
|
|
await conn.execute('SELECT id FROM attachments WHERE id=%s FOR UPDATE', (attachment_id,))
|
|
await blocking(self.s3.put_object, Bucket=BUCKET, Key=key,
|
|
Body=file, ContentLength=size, ContentType=mime)
|
|
cur = await conn.execute("UPDATE attachments SET status='ready' WHERE id=%s RETURNING *",
|
|
(attachment_id,))
|
|
result = metadata(await cur.fetchone())
|
|
return result
|
|
|
|
async def download(self, row):
|
|
obj = await asyncio.to_thread(self.s3.get_object, Bucket=BUCKET, Key=row['object_key'])
|
|
stream = obj['Body']
|
|
async def chunks():
|
|
try:
|
|
while chunk := await asyncio.to_thread(stream.read, 64 * 1024):
|
|
yield chunk
|
|
finally:
|
|
stream.close()
|
|
return StreamingResponse(chunks(), media_type=row['content_type'], headers={
|
|
'Content-Length': str(row['size_bytes']), 'X-Content-Type-Options': 'nosniff'})
|
|
|
|
async def cleanup(self):
|
|
removed = 0
|
|
async with self.store.pool.connection() as conn:
|
|
cur = await conn.execute("""SELECT id,object_key FROM attachments WHERE status='pending'
|
|
AND created_at < clock_timestamp()-interval '1 hour' ORDER BY created_at
|
|
LIMIT 100 FOR UPDATE SKIP LOCKED""")
|
|
for row in await cur.fetchall():
|
|
await asyncio.to_thread(self.s3.delete_object, Bucket=BUCKET, Key=row['object_key'])
|
|
await conn.execute("DELETE FROM attachments WHERE id=%s AND status='pending'", (row['id'],))
|
|
removed += 1
|
|
return removed
|
|
|
|
async def cleaner():
|
|
from .config import Settings
|
|
from .store import Store
|
|
store = Store(Settings.load())
|
|
await store.open()
|
|
attachments = Attachments(store)
|
|
try:
|
|
while True:
|
|
try:
|
|
print({'removed_pending': await attachments.cleanup()}, flush=True)
|
|
except Exception:
|
|
print('Cleanup deferred; storage unavailable', flush=True)
|
|
await asyncio.sleep(60)
|
|
finally:
|
|
await store.close()
|
|
|
|
if __name__ == '__main__':
|
|
asyncio.run(cleaner())
|