Files
mtusi2026lab4/src/app.py
T

509 lines
22 KiB
Python

"""Synchronous durable messenger with bounded binary ingress and shared S3 originals."""
import asyncio
import base64
import datetime as dt
import hashlib
import hmac
import json
import logging
import os
import re
import secrets
import tempfile
import unicodedata
import uuid
import warnings
from contextlib import asynccontextmanager
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError
from botocore.exceptions import BotoCoreError, ClientError
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response, StreamingResponse
from PIL import Image, UnidentifiedImageError
import psycopg
from starlette.concurrency import run_in_threadpool
from starlette.exceptions import HTTPException
from starlette.requests import ClientDisconnect
from .storage import BUCKET, PREFIX, STORAGE_LOCK, db, s3, setting
MAX_BYTES = min(int(os.getenv('MAX_UPLOAD_BYTES', '10485760')), 10485760)
MAX_PIXELS = min(int(os.getenv('MAX_IMAGE_PIXELS', '40000000')), 40000000)
Image.MAX_IMAGE_PIXELS = MAX_PIXELS
INSTANCE = os.getenv('INSTANCE_ID', uuid.uuid4().hex)
PH = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
DUMMY_HASH = PH.hash(secrets.token_hex(32))
CODES = {400:'invalid_request',401:'unauthorized',404:'not_found',409:'conflict',
413:'payload_too_large',415:'unsupported_media_type',429:'rate_limited',
503:'unavailable',500:'internal_error'}
MESSAGES = {400:'Invalid request',401:'Authentication required',404:'Resource not found',
409:'Conflicting request',413:'Upload limit exceeded',415:'Invalid image or media type',
429:'Upload capacity exhausted',503:'Storage temporarily unavailable',500:'Internal error'}
class APIError(Exception):
def __init__(self, status):
self.status = status
def fail(status):
raise APIError(status)
def stamp(value):
return value.astimezone(dt.timezone.utc).isoformat().replace('+00:00', 'Z')
def ident(value):
if not isinstance(value, str):
fail(400)
try:
return str(uuid.UUID(value))
except ValueError:
fail(400)
def string(value, lo, hi, nonblank=True):
if not isinstance(value, str) or not lo <= len(value) <= hi or (nonblank and not value.strip()):
fail(400)
if any(0xD800 <= ord(c) <= 0xDFFF or c == '\x00' for c in value):
fail(400)
return value
def ids(value, maximum, minimum=0):
if not isinstance(value, list) or not minimum <= len(value) <= maximum:
fail(400)
result = [ident(x) for x in value]
if len(set(result)) != len(result):
fail(400)
return result
def strict_json(data, required, optional=()):
try:
def pairs(items):
result = {}
for k, v in items:
if k in result:
raise ValueError('Duplicate key')
result[k] = v
return result
value = json.loads(data, object_pairs_hook=pairs,
parse_constant=lambda x: (_ for _ in ()).throw(ValueError()))
except (ValueError, UnicodeError, RecursionError):
fail(400)
if not isinstance(value, dict) or not set(required) <= value.keys() or value.keys() - set(required) - set(optional):
fail(400)
return value
def user(row):
return {k: str(row[k]) for k in ('id','username','display_name')} | {'created_at':stamp(row['created_at'])}
def chat(conn, row):
members = conn.execute('SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()
return {'id':str(row['id']), 'title':row['title'], 'created_at':stamp(row['created_at']),
'member_ids':[str(x['user_id']) for x in members]}
def message(conn, row):
attached = conn.execute('SELECT attachment_id FROM message_attachments WHERE message_id=%s ORDER BY position', (row['id'],)).fetchall()
return {'id':str(row['id']), 'chat_id':str(row['chat_id']), 'sender_id':str(row['sender_id']),
'text':row['text'], 'created_at':stamp(row['created_at']),
'attachment_ids':[str(x['attachment_id']) for x in attached]}
def attachment(row):
return {k:str(row[k]) for k in ('id','chat_id','owner_id','filename','content_type','sha256')} | {
'size_bytes':row['size_bytes'], 'status':'ready','variants':[], 'error_code':None,
'created_at':stamp(row['created_at']), 'updated_at':stamp(row['updated_at'])}
def authenticate(conn, authorization):
if not authorization.startswith('Bearer ') or not authorization[7:]:
fail(401)
token_hash = hashlib.sha256(authorization[7:].encode()).hexdigest()
row = conn.execute('SELECT u.* FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()', (token_hash,)).fetchone()
if not row:
fail(401)
return row, token_hash
def access(conn, uid, cid):
row = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s', (cid,uid)).fetchone()
if not row:
fail(404)
return row
def authorize_upload(authorization, cid):
with db() as conn:
actor, _ = authenticate(conn, authorization)
cid = ident(cid)
access(conn, actor['id'], cid)
return str(actor['id']), cid
def query(request, allowed):
if set(request.query_params) - set(allowed) or any(len(request.query_params.getlist(k)) != 1 for k in request.query_params):
fail(400)
return dict(request.query_params)
def page(conn, kind, resource, actor, params):
raw = params.get('limit','50')
if len(raw)>3 or not re.fullmatch(r'[0-9]+', raw) or not 1 <= int(raw) <= 100:
fail(400)
limit = int(raw)
boundary = dt.datetime.min.replace(tzinfo=dt.timezone.utc)
bid = '00000000-0000-0000-0000-000000000000'
scope = kind + ':' + resource
if 'cursor' in params:
if len(params['cursor'])>2048:
fail(400)
try:
encoded, signature = params['cursor'].split('.')
expected = hmac.new(setting('CURSOR_SECRET').encode(), encoded.encode(), hashlib.sha256).hexdigest()
if not hmac.compare_digest(signature, expected):
fail(400)
payload = json.loads(base64.urlsafe_b64decode(encoded + '=' * (-len(encoded) % 4)))
if payload['scope'] != scope:
fail(400)
boundary = dt.datetime.fromisoformat(payload['time'])
bid = ident(payload['id'])
except (ValueError, KeyError, TypeError):
fail(400)
if kind == 'chats':
rows = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s AND (c.created_at,c.id)>(%s,%s::uuid) ORDER BY c.created_at,c.id LIMIT %s', (actor,boundary,bid,limit+1)).fetchall()
items = [chat(conn,x) for x in rows[:limit]]
else:
rows = conn.execute('SELECT * FROM messages WHERE chat_id=%s AND (created_at,id)>(%s,%s::uuid) ORDER BY created_at,id LIMIT %s', (resource,boundary,bid,limit+1)).fetchall()
items = [message(conn,x) for x in rows[:limit]]
cursor = None
if len(rows) > limit:
last = rows[limit-1]
encoded = base64.urlsafe_b64encode(json.dumps({'scope':scope,'time':stamp(last['created_at']),'id':str(last['id'])}, sort_keys=True).encode()).decode().rstrip('=')
cursor = encoded + '.' + hmac.new(setting('CURSOR_SECRET').encode(), encoded.encode(), hashlib.sha256).hexdigest()
return {'items':items, 'next_cursor':cursor}
@asynccontextmanager
async def lifespan(app):
app.state.upload_slots = asyncio.Semaphore(4)
app.state.json_slots = asyncio.Semaphore(16)
yield
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
def error_response(request, status):
headers = {}
if status == 401:
headers['WWW-Authenticate'] = ('Basic realm="messenger", charset="UTF-8"' if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"')
return JSONResponse({'error':{'code':CODES[status], 'message':MESSAGES[status],
'request_id':request.state.request_id}}, status, headers=headers)
@app.middleware('http')
async def errors_and_identity(request, call_next):
request.state.request_id = uuid.uuid4().hex
try:
result = await call_next(request)
except APIError as exc:
result = error_response(request, exc.status)
except (psycopg.Error, BotoCoreError, ClientError) as exc:
code = exc.response.get('Error',{}).get('Code','') if isinstance(exc,ClientError) else getattr(exc,'sqlstate','')
logging.warning('Storage failure: %s code=%s',type(exc).__name__,code)
result = error_response(request, 503)
except Exception:
result = error_response(request, 500)
result.headers['X-Request-Id'] = request.state.request_id
result.headers['X-Instance-Id'] = INSTANCE
return result
@app.exception_handler(APIError)
async def api_error(request, exc):
return error_response(request, exc.status)
@app.exception_handler(HTTPException)
async def http_error(request, exc):
return error_response(request, 404 if exc.status_code == 404 else 400)
@app.get('/health/live')
def live():
return {'status':'ok'}
@app.get('/internal-test/memory')
def test_memory():
if os.getenv('ENABLE_TEST_METRICS') != '1':
fail(404)
with open('/proc/self/status') as status:
rss = next(int(line.split()[1])*1024 for line in status if line.startswith('VmRSS:'))
usage = os.times()
with open('/sys/fs/cgroup/memory.current') as current:
container_memory = int(current.read())
with open('/sys/fs/cgroup/memory.peak') as peak:
container_peak = int(peak.read())
return {'rss_bytes':rss, 'cpu_seconds':usage.user+usage.system,
'container_memory_bytes':container_memory, 'container_peak_bytes':container_peak}
@app.get('/health/ready')
def ready():
with db() as conn:
conn.execute('SELECT 1 FROM attachments LIMIT 1')
s3().head_bucket(Bucket=BUCKET)
return {'status':'ok'}
def validate_image(file, declared):
try:
with warnings.catch_warnings():
warnings.simplefilter('error', Image.DecompressionBombWarning)
file.seek(0)
with Image.open(file) as im:
if im.format not in ('PNG','JPEG') or Image.MIME[im.format] != declared:
fail(415)
if im.width * im.height > MAX_PIXELS:
fail(413)
if getattr(im,'n_frames',1) != 1:
fail(415)
im.verify()
file.seek(0)
with Image.open(file) as im:
im.load() # JPEG verify() alone does not detect truncation.
except (Image.DecompressionBombWarning, Image.DecompressionBombError):
fail(413)
except (UnidentifiedImageError, OSError, ValueError, SyntaxError):
fail(415)
file.seek(0)
def persist_upload(file, authorization, cid, filename, declared, size, sha):
validate_image(file, declared)
client = s3()
aid = str(uuid.uuid4())
key = PREFIX + aid
# Shared transaction lock covers PUT -> COMMIT; reconciler takes the exclusive lock.
# If COMMIT outcome is ambiguous, never delete here: the reconciler checks the committed DB.
with db() as conn:
actor, _ = authenticate(conn, authorization)
access(conn, actor['id'], cid)
conn.execute('SELECT pg_advisory_xact_lock_shared(%s)', (STORAGE_LOCK,))
md5 = base64.b64encode(hashlib.file_digest(file, 'md5').digest()).decode()
file.seek(0)
client.put_object(Bucket=BUCKET, Key=key, Body=file, ContentLength=size,
ContentType=declared, Metadata={'sha256':sha},
ContentMD5=md5)
# Test-only fault injection is disabled in the regular Compose configuration.
if os.getenv('FAULT_AFTER_PUT') == '1':
fail(503)
row = conn.execute('INSERT INTO attachments(id,chat_id,owner_id,filename,object_key,content_type,size_bytes,sha256) VALUES(%s,%s,%s,%s,%s,%s,%s,%s) RETURNING *',
(aid,cid,actor['id'],filename,key,declared,size,sha)).fetchone()
result = attachment(row)
return result
@app.post('/api/v1/chats/{cid}/attachments')
async def upload(request: Request, cid: str):
authorization = request.headers.get('authorization','')
_, cid = await run_in_threadpool(authorize_upload, authorization, cid)
params = query(request, ('filename',))
filename = string(params.get('filename'),1,255)
if any(c in '/\\' or unicodedata.category(c).startswith('C') for c in filename):
fail(400)
declared = request.headers.get('content-type','').lower()
if declared not in ('image/png','image/jpeg'):
fail(415)
length = request.headers.get('content-length')
if length is not None:
if len(length)>10 or not re.fullmatch('[0-9]+', length):
fail(400)
if int(length) > MAX_BYTES:
fail(413)
try:
await asyncio.wait_for(request.app.state.upload_slots.acquire(), timeout=1)
except TimeoutError:
fail(429)
try:
with tempfile.TemporaryFile() as file: # ephemeral bounded staging, never required after 201
digest = hashlib.sha256()
size = 0
async def receive():
nonlocal size
async for chunk in request.stream():
size += len(chunk)
if size > MAX_BYTES:
fail(413)
digest.update(chunk)
file.write(chunk)
try:
await asyncio.wait_for(receive(), timeout=30)
except (TimeoutError, ClientDisconnect):
fail(400)
if not size or (length is not None and size != int(length)):
fail(415 if not size else 400)
file.seek(0)
# file_digest leaves position at EOF; compute MD5 without moving the PUT body.
result = await run_in_threadpool(persist_upload, file, authorization, cid, filename, declared, size, digest.hexdigest())
return JSONResponse(result,201)
finally:
request.app.state.upload_slots.release()
def dispatch(method, path, authorization, data, params, headers):
with db() as conn:
if path == '/api/v1/users' and method == 'POST':
value = strict_json(data, ('username','display_name','password'))
name = string(value['username'],3,32)
if not re.fullmatch('[a-z0-9_]{3,32}',name):
fail(400)
display = string(value['display_name'],1,100)
password = string(value['password'],12,128,False)
try:
row = conn.execute('INSERT INTO users(id,username,display_name,password_hash) VALUES(%s,%s,%s,%s) RETURNING *',
(str(uuid.uuid4()),name,display,PH.hash(password))).fetchone()
except psycopg.errors.UniqueViolation:
fail(409)
return 201, user(row)
if path == '/api/v1/auth/sessions' and method == 'POST':
if data or not authorization.startswith('Basic '):
fail(401)
try:
name,password = base64.b64decode(authorization[6:],validate=True).decode('utf-8').split(':',1)
except (ValueError, UnicodeError):
fail(401)
row = conn.execute('SELECT * FROM users WHERE username=%s',(name,)).fetchone()
try:
PH.verify(row['password_hash'] if row else DUMMY_HASH, password)
except VerificationError:
fail(401)
if not row:
fail(401)
token = secrets.token_urlsafe(32)
expires = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=int(os.getenv('SESSION_TTL_SECONDS','3600')))
conn.execute('INSERT INTO sessions VALUES(%s,%s,%s)',(hashlib.sha256(token.encode()).hexdigest(),row['id'],expires))
return 201, {'token':token,'token_type':'Bearer','expires_at':stamp(expires),'user':user(row)}
actor, token_hash = authenticate(conn, authorization)
uid = actor['id']
if path == '/api/v1/users/me' and method == 'GET':
return 200, user(actor)
if path == '/api/v1/auth/sessions/current' and method == 'DELETE':
conn.execute('DELETE FROM sessions WHERE token_hash=%s',(token_hash,))
return 204, None
if path == '/api/v1/chats':
if method == 'POST':
value = strict_json(data,('title','member_ids'))
title = string(value['title'],1,100)
members = ids(value['member_ids'],100,1)
if str(uid) not in members or conn.execute('SELECT count(*) AS n FROM users WHERE id=ANY(%s::uuid[])',(members,)).fetchone()['n'] != len(members):
fail(400)
row = conn.execute('INSERT INTO chats(id,title) VALUES(%s,%s) RETURNING *',(str(uuid.uuid4()),title)).fetchone()
for member in members:
conn.execute('INSERT INTO members VALUES(%s,%s)',(row['id'],member))
return 201, chat(conn,row)
if method == 'GET':
return 200, page(conn,'chats',str(uid),uid,params)
match = re.fullmatch(r'/api/v1/chats/([^/]+)(/messages)?', path)
if match:
cid = ident(match[1])
current_chat = access(conn,uid,cid)
if not match[2] and method == 'GET':
return 200, chat(conn,current_chat)
if match[2] and method == 'GET':
return 200, page(conn,'messages',cid,uid,params)
if match[2] and method == 'POST':
value = strict_json(data,('text',),('attachment_ids',))
text = string(value['text'],0,4000,False)
attachments = ids(value.get('attachment_ids',[]),10)
if not text.strip() and not attachments:
fail(400)
key = headers.get('idempotency-key','')
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}',key):
fail(400)
canonical = json.dumps({'text':text,'attachment_ids':attachments},sort_keys=True,ensure_ascii=False)
body_hash = hashlib.sha256(canonical.encode()).hexdigest()
# One transaction lock per user/chat/key, shared across API replicas.
lock = int.from_bytes(hashlib.sha256((str(uid)+cid+key).encode()).digest()[:8],'big',signed=True)
conn.execute('SELECT pg_advisory_xact_lock(%s)',(lock,))
for aid in attachments:
a = conn.execute('SELECT * FROM attachments WHERE id=%s AND chat_id=%s',(aid,cid)).fetchone()
if not a:
fail(404)
if a['availability'] != 'present':
fail(503)
old = conn.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s AND expires_at>clock_timestamp()', (uid,cid,key)).fetchone()
if old:
if old['body_hash'] != body_hash:
fail(409)
return 201, message(conn,conn.execute('SELECT * FROM messages WHERE id=%s',(old['message_id'],)).fetchone())
conn.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s',(uid,cid,key))
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES(%s,%s,%s,%s) RETURNING *',(str(uuid.uuid4()),cid,uid,text)).fetchone()
for pos,aid in enumerate(attachments):
conn.execute('INSERT INTO message_attachments VALUES(%s,%s,%s)',(row['id'],aid,pos))
conn.execute("INSERT INTO idempotency VALUES(%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')",(uid,cid,key,body_hash,row['id']))
return 201, message(conn,row)
match = re.fullmatch(r'/api/v1/attachments/([^/]+)(/content)?',path)
if match and method == 'GET':
aid = ident(match[1])
row = conn.execute('SELECT a.* FROM attachments a JOIN members m ON m.chat_id=a.chat_id WHERE a.id=%s AND m.user_id=%s',(aid,uid)).fetchone()
if not row:
fail(404)
if not match[2]:
return 200, attachment(row)
if params.get('variant','original') != 'original':
fail(404)
if row['availability'] != 'present':
fail(503)
content = s3().get_object(Bucket=BUCKET,Key=row['object_key'])
body = content['Body']
def stream():
try:
while chunk := body.read(65536):
yield chunk
finally:
body.close()
return 200, StreamingResponse(stream(),media_type=row['content_type'],
headers={'Content-Length':str(row['size_bytes']), 'X-Content-Type-Options':'nosniff'})
fail(404)
@app.api_route('/api/v1/{path:path}', methods=['GET','POST','DELETE','PUT','PATCH'])
async def api(request: Request, path: str):
if not (path == 'users' and request.method == 'POST') and not (path == 'auth/sessions' and request.method == 'POST'):
def check_auth():
with db() as conn:
authenticate(conn,request.headers.get('authorization',''))
await run_in_threadpool(check_auth)
params = query(request, ('limit','cursor') if request.method == 'GET' and (path == 'chats' or path.endswith('/messages')) else ('variant',) if path.endswith('/content') else ())
try:
await asyncio.wait_for(request.app.state.json_slots.acquire(),timeout=1)
except TimeoutError:
fail(429)
try:
data = bytearray()
async def receive():
async for chunk in request.stream():
data.extend(chunk)
if len(data) > 65536:
fail(400)
try:
await asyncio.wait_for(receive(),timeout=10)
except (TimeoutError, ClientDisconnect):
fail(400)
# Authentication for protected routes is performed before parsing JSON in dispatch.
status, result = await run_in_threadpool(dispatch, request.method, '/api/v1/'+path,
request.headers.get('authorization',''),bytes(data),params,dict(request.headers))
if isinstance(result, Response):
return result
return Response(status_code=204) if status == 204 else JSONResponse(result,status)
finally:
request.app.state.json_slots.release()