forked from mtusi2026/mtusi2026lab7
388 lines
19 KiB
Python
388 lines
19 KiB
Python
import asyncio
|
|
import base64
|
|
import binascii
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
import re
|
|
import secrets
|
|
import time
|
|
from threading import BoundedSemaphore
|
|
import uuid
|
|
from argon2 import PasswordHasher
|
|
from argon2.exceptions import VerifyMismatchError
|
|
from contextlib import asynccontextmanager
|
|
from datetime import datetime, timedelta, timezone
|
|
from urllib.parse import parse_qs
|
|
import psycopg
|
|
from psycopg.types.json import Jsonb
|
|
from fastapi import FastAPI, Request
|
|
from fastapi.responses import JSONResponse, Response
|
|
from starlette.concurrency import run_in_threadpool
|
|
from anyio import to_thread
|
|
from opentelemetry import propagate, trace
|
|
from opentelemetry.trace import SpanKind, Status, StatusCode
|
|
from app.common import (db, get, put, now, config, s3, bucket, sha,
|
|
inspect_image, InvalidImage, ImageTooLarge)
|
|
from app.telemetry import setup, INSTANCE
|
|
|
|
tracer, meter, logger, providers = setup('messenger-api')
|
|
requests_count = meter.create_counter('http_requests', unit='{request}')
|
|
http_duration = meter.create_histogram('http_duration', unit='s')
|
|
|
|
|
|
class APIError(Exception):
|
|
def __init__(self, status, code, retry_after=None):
|
|
self.status, self.code = status, code
|
|
self.retry_after = retry_after
|
|
|
|
|
|
def require(condition, status=400, code='invalid_request'):
|
|
if not condition:
|
|
raise APIError(status, code)
|
|
|
|
|
|
def uid(value):
|
|
try:
|
|
return str(uuid.UUID(value))
|
|
except (ValueError, TypeError, AttributeError):
|
|
raise APIError(400, 'invalid_request') from None
|
|
|
|
|
|
def text(value, maximum, minimum=1):
|
|
require(isinstance(value, str) and minimum <= len(value) <= maximum)
|
|
require(bool(value.strip()) or minimum == 0)
|
|
return value
|
|
|
|
|
|
def fields(body, required, optional=()):
|
|
require(isinstance(body, dict) and set(required) <= body.keys() and not body.keys()-set(required)-set(optional))
|
|
|
|
|
|
def ids(value, maximum, minimum=0):
|
|
require(isinstance(value, list) and minimum <= len(value) <= maximum)
|
|
result = [uid(x) for x in value]
|
|
require(len(result) == len(set(result)))
|
|
return result
|
|
|
|
|
|
def public_user(conn, user_id):
|
|
return get(conn, 'user', user_id)
|
|
|
|
|
|
def authenticate(conn, headers):
|
|
auth = headers.get('authorization', '')
|
|
require(auth.startswith('Bearer ') and bool(auth[7:]), 401, 'unauthorized')
|
|
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>now()', (sha(auth[7:].encode()),)).fetchone()
|
|
require(row is not None, 401, 'unauthorized')
|
|
return row['user_id']
|
|
|
|
|
|
def chat_access(conn, chat_id, actor):
|
|
chat = get(conn, 'chat', uid(chat_id))
|
|
require(chat is not None and actor in chat['member_ids'], 404, 'not_found')
|
|
return chat
|
|
|
|
|
|
def attachment_access(conn, attachment_id, actor):
|
|
att = get(conn, 'attachment', uid(attachment_id))
|
|
require(att is not None, 404, 'not_found')
|
|
chat_access(conn, att['chat_id'], actor)
|
|
return att
|
|
|
|
|
|
def page(conn, kind, scope, query, actor):
|
|
raw_limit = query.get('limit', ['50'])
|
|
require(len(raw_limit) == 1 and re.fullmatch(r'[0-9]+', raw_limit[0]) is not None)
|
|
limit = int(raw_limit[0])
|
|
require(1 <= limit <= 100)
|
|
resource = scope if kind == 'message' else 'chats:'+actor
|
|
boundary = None
|
|
if 'cursor' in query:
|
|
try:
|
|
require(len(query['cursor']) == 1)
|
|
cursor = json.loads(base64.urlsafe_b64decode(query['cursor'][0].encode()+b'==='))
|
|
require(isinstance(cursor, list) and len(cursor) == 3 and cursor[0] == resource)
|
|
uid(cursor[2])
|
|
require(isinstance(cursor[1], str) and cursor[1].endswith('Z'))
|
|
datetime.fromisoformat(cursor[1].replace('Z', '+00:00'))
|
|
boundary = cursor[1:]
|
|
except (ValueError, TypeError, binascii.Error, UnicodeError):
|
|
raise APIError(400, 'invalid_request') from None
|
|
if kind == 'message':
|
|
condition, params = "data->>'chat_id'=%s", [scope]
|
|
else:
|
|
condition, params = "data->'member_ids' ? %s", [actor]
|
|
if boundary:
|
|
condition += " AND (data->>'created_at',id)>(%s,%s)"
|
|
params.extend(boundary)
|
|
rows = conn.execute("SELECT data FROM objects WHERE kind=%s AND "+condition+" ORDER BY data->>'created_at',id LIMIT %s",
|
|
[kind]+params+[limit+1]).fetchall()
|
|
items = [r['data'] for r in rows[:limit]]
|
|
next_cursor = None
|
|
if len(rows)>limit:
|
|
next_cursor = base64.urlsafe_b64encode(json.dumps([resource, items[-1]['created_at'], items[-1]['id']]).encode()).decode()
|
|
return {'items': items, 'next_cursor': next_cursor}
|
|
|
|
|
|
def dispatch(method, path, params, query, headers, raw):
|
|
if path == '/health/live':
|
|
return 200, {'status': 'ok'}
|
|
if path == '/health/ready':
|
|
with db() as conn:
|
|
conn.execute('SELECT 1')
|
|
s3().head_bucket(Bucket=bucket())
|
|
# Durable DB outbox accepts uploads while RabbitMQ/worker are unavailable.
|
|
return 200, {'status': 'ok'}
|
|
if path == '/api/v1/auth/sessions' and method == 'POST':
|
|
# The LB overwrites this header, and no other TLS client can reach the API.
|
|
key = headers.get('x-lab-client-ip', 'unknown')
|
|
with db() as conn:
|
|
attempts = conn.execute(
|
|
"""INSERT INTO login_limits(client, started, attempts) VALUES (%s, now(), 1)
|
|
ON CONFLICT(client) DO UPDATE SET
|
|
attempts = CASE WHEN login_limits.started <= now() - interval '1 minute'
|
|
THEN 1 ELSE least(login_limits.attempts + 1, 31) END,
|
|
started = CASE WHEN login_limits.started <= now() - interval '1 minute'
|
|
THEN now() ELSE login_limits.started END
|
|
RETURNING attempts, greatest(1, ceil(extract(epoch FROM
|
|
started + interval '1 minute' - now())))::int AS retry_after""",
|
|
(key,)).fetchone()
|
|
if attempts['attempts'] > 30:
|
|
raise APIError(429, 'rate_limited', attempts['retry_after'])
|
|
with db() as conn:
|
|
if path == '/api/v1/auth/sessions' and method == 'POST':
|
|
require(not raw, 400)
|
|
try:
|
|
auth = headers.get('authorization', '')
|
|
require(auth.startswith('Basic '), 401, 'unauthorized')
|
|
username, password = base64.b64decode(auth[6:], validate=True).decode('utf-8').split(':', 1)
|
|
except (ValueError, UnicodeError, binascii.Error):
|
|
raise APIError(401, 'unauthorized') from None
|
|
row = conn.execute('SELECT * FROM usernames WHERE username=%s', (username,)).fetchone()
|
|
encoded = row['password_hash'] if row else DUMMY_HASH
|
|
valid = False
|
|
if 12 <= len(password) <= 128:
|
|
if encoded.startswith('$argon2id$'):
|
|
try:
|
|
valid = HASHER.verify(encoded, password)
|
|
except VerifyMismatchError:
|
|
pass
|
|
else:
|
|
# Lab6 accounts remain usable after an explicit data migration.
|
|
salt, expected = encoded.split(':')
|
|
computed = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=16384, r=8, p=1).hex()
|
|
valid = hmac.compare_digest(computed, expected)
|
|
if valid:
|
|
conn.execute('UPDATE usernames SET password_hash=%s WHERE username=%s AND password_hash=%s',
|
|
(HASHER.hash(password), username, encoded))
|
|
require(row is not None and valid, 401, 'unauthorized')
|
|
token = secrets.token_urlsafe(32)
|
|
expires = datetime.now(timezone.utc)+timedelta(seconds=int(config('SESSION_TTL_SECONDS', '3600')))
|
|
conn.execute('INSERT INTO sessions VALUES(%s,%s,%s)', (sha(token.encode()), row['user_id'], expires))
|
|
return 201, {'token': token, 'token_type': 'Bearer', 'expires_at': expires.isoformat().replace('+00:00','Z'),
|
|
'user': public_user(conn, row['user_id'])}
|
|
# Authentication precedes validation of resource identifiers / body.
|
|
actor = None if path == '/api/v1/users' and method == 'POST' else authenticate(conn, headers)
|
|
body = None
|
|
if method == 'POST' and not path.endswith('/attachments'):
|
|
try:
|
|
require(headers.get('content-type', '').split(';')[0] == 'application/json')
|
|
body = json.loads(raw)
|
|
except (ValueError, UnicodeError):
|
|
raise APIError(400, 'invalid_request') from None
|
|
if path == '/api/v1/users' and method == 'POST':
|
|
fields(body, ('username','display_name','password'))
|
|
username = text(body['username'], 32, 3)
|
|
require(re.fullmatch('[a-z0-9_]{3,32}', username) is not None)
|
|
display = text(body['display_name'], 100)
|
|
password = text(body['password'], 128, 12)
|
|
hashed = HASHER.hash(password)
|
|
user = {'id': str(uuid.uuid4()), 'username': username, 'display_name': display, 'created_at': now()}
|
|
conn.execute('INSERT INTO usernames VALUES(%s,%s,%s)', (username, user['id'], hashed))
|
|
put(conn, 'user', user)
|
|
return 201, user
|
|
if path == '/api/v1/users/me':
|
|
return 200, public_user(conn, actor)
|
|
if path == '/api/v1/auth/sessions/current':
|
|
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (sha(headers['authorization'][7:].encode()),))
|
|
return 204, None
|
|
if path == '/api/v1/chats':
|
|
if method == 'GET':
|
|
return 200, page(conn, 'chat', None, query, actor)
|
|
fields(body, ('title','member_ids'))
|
|
members = ids(body['member_ids'], 100, 1)
|
|
require(actor in members and all(public_user(conn, x) for x in members))
|
|
chat = {'id': str(uuid.uuid4()), 'title': text(body['title'], 100), 'member_ids': members, 'created_at': now()}
|
|
put(conn, 'chat', chat)
|
|
return 201, chat
|
|
if 'chat_id' in params:
|
|
chat = chat_access(conn, params['chat_id'], actor)
|
|
if path == '/api/v1/chats/{chat_id}':
|
|
return 200, chat
|
|
if path.endswith('/messages'):
|
|
if method == 'GET':
|
|
return 200, page(conn, 'message', chat['id'], query, actor)
|
|
fields(body, ('text',), ('attachment_ids',))
|
|
message_text = text(body['text'], 4000, 0)
|
|
attachments = ids(body.get('attachment_ids', []), 10)
|
|
require(message_text.strip() or attachments)
|
|
key = headers.get('idempotency-key', '')
|
|
require(re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key) is not None)
|
|
payload = {'text': message_text, 'attachment_ids': attachments}
|
|
scope = actor+':POST:'+chat['id']+':'+key
|
|
# Cross-replica serialization. Hash collision merely serializes unrelated keys.
|
|
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))', (scope,))
|
|
old = conn.execute("SELECT * FROM idempotency WHERE scope=%s AND created_at>now()-interval '24 hours'", (scope,)).fetchone()
|
|
if old:
|
|
require(old['payload'] == payload, 409, 'conflict')
|
|
return 201, old['response']
|
|
for aid in attachments:
|
|
att = attachment_access(conn, aid, actor)
|
|
require(att['chat_id'] == chat['id'], 404, 'not_found')
|
|
require(att['status'] != 'failed', 409, 'conflict')
|
|
message = {'id': str(uuid.uuid4()), 'chat_id': chat['id'], 'sender_id': actor,
|
|
**payload, 'created_at': now()}
|
|
put(conn, 'message', message)
|
|
conn.execute('DELETE FROM idempotency WHERE scope=%s', (scope,))
|
|
conn.execute('INSERT INTO idempotency(scope,payload,response) VALUES(%s,%s,%s)', (scope, Jsonb(payload), Jsonb(message)))
|
|
return 201, message
|
|
if path.endswith('/attachments'):
|
|
filename = query.get('filename', [''])[0]
|
|
text(filename, 255)
|
|
require(not any(x in filename for x in ('/', '\\')) and all(ord(x)>=32 and ord(x)!=127 for x in filename))
|
|
content_type = headers.get('content-type', '').split(';')[0]
|
|
require(content_type in ('image/png','image/jpeg'), 415, 'unsupported_media_type')
|
|
require(bool(raw), 415, 'unsupported_media_type')
|
|
with IMAGE_SLOTS:
|
|
inspect_image(raw, content_type)
|
|
aid, stamp = str(uuid.uuid4()), now()
|
|
att = {'id': aid, 'chat_id': chat['id'], 'owner_id': actor, 'filename': filename,
|
|
'content_type': content_type, 'size_bytes': len(raw), 'sha256': sha(raw),
|
|
'status': 'queued', 'variants': [], 'error_code': None, 'created_at': stamp, 'updated_at': stamp}
|
|
with tracer.start_as_current_span('s3.stage', record_exception=False, set_status_on_exception=False):
|
|
s3().put_object(Bucket=bucket(), Key='staging/'+aid, Body=raw, ContentType=content_type)
|
|
put(conn, 'attachment', att)
|
|
job = {'schema_version': 1, 'job_id': str(uuid.uuid4()), 'type': 'attachment.process.v1',
|
|
'attachment_id': aid, 'pipeline_version': 1, 'attempt': 1, 'created_at': stamp}
|
|
propagate.inject(job)
|
|
conn.execute('INSERT INTO outbox(id,payload) VALUES(%s,%s)', (job['job_id'], Jsonb(job)))
|
|
return 202, att
|
|
if 'attachment_id' in params:
|
|
att = attachment_access(conn, params['attachment_id'], actor)
|
|
if not path.endswith('/content'):
|
|
return 200, att
|
|
require(att['status'] == 'ready', 409, 'conflict')
|
|
variant = query.get('variant', ['original'])[0]
|
|
meta = att if variant == 'original' else next((v for v in att['variants'] if v['name']==variant), None)
|
|
require(meta is not None, 404, 'not_found')
|
|
with tracer.start_as_current_span('s3.download', record_exception=False, set_status_on_exception=False):
|
|
data = s3().get_object(Bucket=bucket(), Key='final/'+att['id']+'/'+variant)['Body'].read()
|
|
return 200, (data, meta['content_type'])
|
|
raise APIError(404, 'not_found')
|
|
|
|
|
|
UPLOAD_SLOTS = asyncio.Semaphore(4)
|
|
IMAGE_SLOTS = BoundedSemaphore(1)
|
|
HASHER = PasswordHasher(memory_cost=19456, time_cost=2, parallelism=1)
|
|
DUMMY_HASH = HASHER.hash('nonexistent-password')
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app):
|
|
to_thread.current_default_thread_limiter().total_tokens = 8
|
|
yield
|
|
for provider in providers:
|
|
provider.shutdown()
|
|
|
|
|
|
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
|
|
|
|
|
async def endpoint(request: Request):
|
|
if request.scope['route'].path.endswith('/attachments'):
|
|
async with UPLOAD_SLOTS:
|
|
return await dispatch_request(request, int(config('MAX_UPLOAD_BYTES', '10485760')))
|
|
return await dispatch_request(request, 65536)
|
|
|
|
|
|
async def dispatch_request(request, limit):
|
|
length = request.headers.get('content-length')
|
|
if length and length.isdecimal() and int(length) > limit:
|
|
raise APIError(413, 'payload_too_large')
|
|
raw = bytearray()
|
|
async for chunk in request.stream():
|
|
if len(raw) + len(chunk) > limit:
|
|
raise APIError(413, 'payload_too_large')
|
|
raw.extend(chunk)
|
|
code, result = await run_in_threadpool(dispatch, request.method, request.scope['route'].path,
|
|
request.path_params, parse_qs(request.url.query, keep_blank_values=True), request.headers, bytes(raw))
|
|
if code == 204:
|
|
return Response(status_code=204)
|
|
if isinstance(result, tuple):
|
|
return Response(result[0], media_type=result[1], status_code=code)
|
|
return JSONResponse(result, status_code=code)
|
|
|
|
|
|
for path, methods in [('/health/live',['GET']),('/health/ready',['GET']),('/api/v1/users',['POST']),
|
|
('/api/v1/users/me',['GET']),('/api/v1/auth/sessions',['POST']),('/api/v1/auth/sessions/current',['DELETE']),
|
|
('/api/v1/chats',['GET','POST']),('/api/v1/chats/{chat_id}',['GET']),
|
|
('/api/v1/chats/{chat_id}/messages',['GET','POST']),('/api/v1/chats/{chat_id}/attachments',['POST']),
|
|
('/api/v1/attachments/{attachment_id}',['GET']),('/api/v1/attachments/{attachment_id}/content',['GET'])]:
|
|
app.add_api_route(path, endpoint, methods=methods)
|
|
|
|
|
|
def error_response(request, status, code):
|
|
response = JSONResponse({'error': {'code': code, 'message': code.replace('_',' '),
|
|
'request_id': request.state.request_id}}, status_code=status)
|
|
if status == 401:
|
|
scheme = 'Basic' if request.url.path == '/api/v1/auth/sessions' else 'Bearer'
|
|
response.headers['WWW-Authenticate'] = scheme+' realm="messenger"'+(', charset="UTF-8"' if scheme=='Basic' else '')
|
|
return response
|
|
|
|
|
|
@app.exception_handler(APIError)
|
|
async def api_error(request, exc):
|
|
response = error_response(request, exc.status, exc.code)
|
|
if exc.retry_after is not None:
|
|
response.headers['Retry-After'] = str(exc.retry_after)
|
|
return response
|
|
|
|
|
|
@app.middleware('http')
|
|
async def observe(request, call_next):
|
|
request.state.request_id = uuid.uuid4().hex
|
|
started = time.monotonic()
|
|
with tracer.start_as_current_span('HTTP', context=propagate.extract(dict(request.headers)), kind=SpanKind.SERVER,
|
|
record_exception=False, set_status_on_exception=False) as span:
|
|
try:
|
|
response = await call_next(request)
|
|
except psycopg.errors.UniqueViolation:
|
|
response = error_response(request, 409, 'conflict')
|
|
except ImageTooLarge:
|
|
response = error_response(request, 413, 'payload_too_large')
|
|
except InvalidImage:
|
|
response = error_response(request, 415, 'unsupported_media_type')
|
|
except Exception as exc:
|
|
from botocore.exceptions import BotoCoreError, ClientError
|
|
unavailable = isinstance(exc, (psycopg.Error, BotoCoreError, ClientError))
|
|
response = error_response(request, 503 if unavailable else 500, 'unavailable' if unavailable else 'internal_error')
|
|
route = getattr(request.scope.get('route'), 'path', 'unmatched')
|
|
# Method is allowlisted too, to avoid attacker-controlled metric cardinality.
|
|
method = request.method if request.method in ('GET','POST','DELETE','HEAD','OPTIONS','PUT','PATCH') else 'OTHER'
|
|
labels = {'method': method, 'route': route, 'status_class': str(response.status_code//100)+'xx'}
|
|
elapsed = time.monotonic()-started
|
|
requests_count.add(1, labels)
|
|
http_duration.record(elapsed, labels)
|
|
span.update_name(method+' '+route)
|
|
span.set_attributes({**labels, 'http.response.status_code': response.status_code, 'request_id': request.state.request_id})
|
|
if response.status_code>=500:
|
|
span.set_status(Status(StatusCode.ERROR))
|
|
logger.log(40 if response.status_code>=500 else 20, 'http.request', extra={
|
|
'event': 'http.request', 'request_id': request.state.request_id, **labels,
|
|
'status_code': response.status_code, 'duration_seconds': elapsed})
|
|
response.headers['X-Request-Id'] = request.state.request_id
|
|
response.headers['X-Instance-Id'] = INSTANCE
|
|
response.headers['X-Trace-Id'] = format(span.get_span_context().trace_id, '032x')
|
|
return response
|