import asyncio import base64 import binascii import hashlib import hmac import json import re import secrets import time from threading import BoundedSemaphore import uuid from argon2 import PasswordHasher from argon2.exceptions import VerifyMismatchError from contextlib import asynccontextmanager from datetime import datetime, timedelta, timezone from urllib.parse import parse_qs import psycopg from psycopg.types.json import Jsonb from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, Response from starlette.concurrency import run_in_threadpool from anyio import to_thread from opentelemetry import propagate, trace from opentelemetry.trace import SpanKind, Status, StatusCode from app.common import (db, get, put, now, config, s3, bucket, sha, inspect_image, InvalidImage, ImageTooLarge) from app.telemetry import setup, INSTANCE tracer, meter, logger, providers = setup('messenger-api') requests_count = meter.create_counter('http_requests', unit='{request}') http_duration = meter.create_histogram('http_duration', unit='s') class APIError(Exception): def __init__(self, status, code, retry_after=None): self.status, self.code = status, code self.retry_after = retry_after def require(condition, status=400, code='invalid_request'): if not condition: raise APIError(status, code) def uid(value): try: return str(uuid.UUID(value)) except (ValueError, TypeError, AttributeError): raise APIError(400, 'invalid_request') from None def text(value, maximum, minimum=1): require(isinstance(value, str) and minimum <= len(value) <= maximum) require(bool(value.strip()) or minimum == 0) return value def fields(body, required, optional=()): require(isinstance(body, dict) and set(required) <= body.keys() and not body.keys()-set(required)-set(optional)) def ids(value, maximum, minimum=0): require(isinstance(value, list) and minimum <= len(value) <= maximum) result = [uid(x) for x in value] require(len(result) == len(set(result))) return result def public_user(conn, user_id): return get(conn, 'user', user_id) def authenticate(conn, headers): auth = headers.get('authorization', '') require(auth.startswith('Bearer ') and bool(auth[7:]), 401, 'unauthorized') row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>now()', (sha(auth[7:].encode()),)).fetchone() require(row is not None, 401, 'unauthorized') return row['user_id'] def chat_access(conn, chat_id, actor): chat = get(conn, 'chat', uid(chat_id)) require(chat is not None and actor in chat['member_ids'], 404, 'not_found') return chat def attachment_access(conn, attachment_id, actor): att = get(conn, 'attachment', uid(attachment_id)) require(att is not None, 404, 'not_found') chat_access(conn, att['chat_id'], actor) return att def page(conn, kind, scope, query, actor): raw_limit = query.get('limit', ['50']) require(len(raw_limit) == 1 and re.fullmatch(r'[0-9]+', raw_limit[0]) is not None) limit = int(raw_limit[0]) require(1 <= limit <= 100) resource = scope if kind == 'message' else 'chats:'+actor boundary = None if 'cursor' in query: try: require(len(query['cursor']) == 1) cursor = json.loads(base64.urlsafe_b64decode(query['cursor'][0].encode()+b'===')) require(isinstance(cursor, list) and len(cursor) == 3 and cursor[0] == resource) uid(cursor[2]) require(isinstance(cursor[1], str) and cursor[1].endswith('Z')) datetime.fromisoformat(cursor[1].replace('Z', '+00:00')) boundary = cursor[1:] except (ValueError, TypeError, binascii.Error, UnicodeError): raise APIError(400, 'invalid_request') from None if kind == 'message': condition, params = "data->>'chat_id'=%s", [scope] else: condition, params = "data->'member_ids' ? %s", [actor] if boundary: condition += " AND (data->>'created_at',id)>(%s,%s)" params.extend(boundary) rows = conn.execute("SELECT data FROM objects WHERE kind=%s AND "+condition+" ORDER BY data->>'created_at',id LIMIT %s", [kind]+params+[limit+1]).fetchall() items = [r['data'] for r in rows[:limit]] next_cursor = None if len(rows)>limit: next_cursor = base64.urlsafe_b64encode(json.dumps([resource, items[-1]['created_at'], items[-1]['id']]).encode()).decode() return {'items': items, 'next_cursor': next_cursor} def dispatch(method, path, params, query, headers, raw): if path == '/health/live': return 200, {'status': 'ok'} if path == '/health/ready': with db() as conn: conn.execute('SELECT 1') s3().head_bucket(Bucket=bucket()) # Durable DB outbox accepts uploads while RabbitMQ/worker are unavailable. return 200, {'status': 'ok'} if path == '/api/v1/auth/sessions' and method == 'POST': # The LB overwrites this header, and no other TLS client can reach the API. key = headers.get('x-lab-client-ip', 'unknown') with db() as conn: attempts = conn.execute( """INSERT INTO login_limits(client, started, attempts) VALUES (%s, now(), 1) ON CONFLICT(client) DO UPDATE SET attempts = CASE WHEN login_limits.started <= now() - interval '1 minute' THEN 1 ELSE least(login_limits.attempts + 1, 31) END, started = CASE WHEN login_limits.started <= now() - interval '1 minute' THEN now() ELSE login_limits.started END RETURNING attempts, greatest(1, ceil(extract(epoch FROM started + interval '1 minute' - now())))::int AS retry_after""", (key,)).fetchone() if attempts['attempts'] > 30: raise APIError(429, 'rate_limited', attempts['retry_after']) with db() as conn: if path == '/api/v1/auth/sessions' and method == 'POST': require(not raw, 400) try: auth = headers.get('authorization', '') require(auth.startswith('Basic '), 401, 'unauthorized') username, password = base64.b64decode(auth[6:], validate=True).decode('utf-8').split(':', 1) except (ValueError, UnicodeError, binascii.Error): raise APIError(401, 'unauthorized') from None row = conn.execute('SELECT * FROM usernames WHERE username=%s', (username,)).fetchone() encoded = row['password_hash'] if row else DUMMY_HASH valid = False if 12 <= len(password) <= 128: if encoded.startswith('$argon2id$'): try: valid = HASHER.verify(encoded, password) except VerifyMismatchError: pass else: # Lab6 accounts remain usable after an explicit data migration. salt, expected = encoded.split(':') computed = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=16384, r=8, p=1).hex() valid = hmac.compare_digest(computed, expected) if valid: conn.execute('UPDATE usernames SET password_hash=%s WHERE username=%s AND password_hash=%s', (HASHER.hash(password), username, encoded)) require(row is not None and valid, 401, 'unauthorized') token = secrets.token_urlsafe(32) expires = datetime.now(timezone.utc)+timedelta(seconds=int(config('SESSION_TTL_SECONDS', '3600'))) conn.execute('INSERT INTO sessions VALUES(%s,%s,%s)', (sha(token.encode()), row['user_id'], expires)) return 201, {'token': token, 'token_type': 'Bearer', 'expires_at': expires.isoformat().replace('+00:00','Z'), 'user': public_user(conn, row['user_id'])} # Authentication precedes validation of resource identifiers / body. actor = None if path == '/api/v1/users' and method == 'POST' else authenticate(conn, headers) body = None if method == 'POST' and not path.endswith('/attachments'): try: require(headers.get('content-type', '').split(';')[0] == 'application/json') body = json.loads(raw) except (ValueError, UnicodeError): raise APIError(400, 'invalid_request') from None if path == '/api/v1/users' and method == 'POST': fields(body, ('username','display_name','password')) username = text(body['username'], 32, 3) require(re.fullmatch('[a-z0-9_]{3,32}', username) is not None) display = text(body['display_name'], 100) password = text(body['password'], 128, 12) hashed = HASHER.hash(password) user = {'id': str(uuid.uuid4()), 'username': username, 'display_name': display, 'created_at': now()} conn.execute('INSERT INTO usernames VALUES(%s,%s,%s)', (username, user['id'], hashed)) put(conn, 'user', user) return 201, user if path == '/api/v1/users/me': return 200, public_user(conn, actor) if path == '/api/v1/auth/sessions/current': conn.execute('DELETE FROM sessions WHERE token_hash=%s', (sha(headers['authorization'][7:].encode()),)) return 204, None if path == '/api/v1/chats': if method == 'GET': return 200, page(conn, 'chat', None, query, actor) fields(body, ('title','member_ids')) members = ids(body['member_ids'], 100, 1) require(actor in members and all(public_user(conn, x) for x in members)) chat = {'id': str(uuid.uuid4()), 'title': text(body['title'], 100), 'member_ids': members, 'created_at': now()} put(conn, 'chat', chat) return 201, chat if 'chat_id' in params: chat = chat_access(conn, params['chat_id'], actor) if path == '/api/v1/chats/{chat_id}': return 200, chat if path.endswith('/messages'): if method == 'GET': return 200, page(conn, 'message', chat['id'], query, actor) fields(body, ('text',), ('attachment_ids',)) message_text = text(body['text'], 4000, 0) attachments = ids(body.get('attachment_ids', []), 10) require(message_text.strip() or attachments) key = headers.get('idempotency-key', '') require(re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key) is not None) payload = {'text': message_text, 'attachment_ids': attachments} scope = actor+':POST:'+chat['id']+':'+key # Cross-replica serialization. Hash collision merely serializes unrelated keys. conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))', (scope,)) old = conn.execute("SELECT * FROM idempotency WHERE scope=%s AND created_at>now()-interval '24 hours'", (scope,)).fetchone() if old: require(old['payload'] == payload, 409, 'conflict') return 201, old['response'] for aid in attachments: att = attachment_access(conn, aid, actor) require(att['chat_id'] == chat['id'], 404, 'not_found') require(att['status'] != 'failed', 409, 'conflict') message = {'id': str(uuid.uuid4()), 'chat_id': chat['id'], 'sender_id': actor, **payload, 'created_at': now()} put(conn, 'message', message) conn.execute('DELETE FROM idempotency WHERE scope=%s', (scope,)) conn.execute('INSERT INTO idempotency(scope,payload,response) VALUES(%s,%s,%s)', (scope, Jsonb(payload), Jsonb(message))) return 201, message if path.endswith('/attachments'): filename = query.get('filename', [''])[0] text(filename, 255) require(not any(x in filename for x in ('/', '\\')) and all(ord(x)>=32 and ord(x)!=127 for x in filename)) content_type = headers.get('content-type', '').split(';')[0] require(content_type in ('image/png','image/jpeg'), 415, 'unsupported_media_type') require(bool(raw), 415, 'unsupported_media_type') with IMAGE_SLOTS: inspect_image(raw, content_type) aid, stamp = str(uuid.uuid4()), now() att = {'id': aid, 'chat_id': chat['id'], 'owner_id': actor, 'filename': filename, 'content_type': content_type, 'size_bytes': len(raw), 'sha256': sha(raw), 'status': 'queued', 'variants': [], 'error_code': None, 'created_at': stamp, 'updated_at': stamp} with tracer.start_as_current_span('s3.stage', record_exception=False, set_status_on_exception=False): s3().put_object(Bucket=bucket(), Key='staging/'+aid, Body=raw, ContentType=content_type) put(conn, 'attachment', att) job = {'schema_version': 1, 'job_id': str(uuid.uuid4()), 'type': 'attachment.process.v1', 'attachment_id': aid, 'pipeline_version': 1, 'attempt': 1, 'created_at': stamp} propagate.inject(job) conn.execute('INSERT INTO outbox(id,payload) VALUES(%s,%s)', (job['job_id'], Jsonb(job))) return 202, att if 'attachment_id' in params: att = attachment_access(conn, params['attachment_id'], actor) if not path.endswith('/content'): return 200, att require(att['status'] == 'ready', 409, 'conflict') variant = query.get('variant', ['original'])[0] meta = att if variant == 'original' else next((v for v in att['variants'] if v['name']==variant), None) require(meta is not None, 404, 'not_found') with tracer.start_as_current_span('s3.download', record_exception=False, set_status_on_exception=False): data = s3().get_object(Bucket=bucket(), Key='final/'+att['id']+'/'+variant)['Body'].read() return 200, (data, meta['content_type']) raise APIError(404, 'not_found') UPLOAD_SLOTS = asyncio.Semaphore(4) IMAGE_SLOTS = BoundedSemaphore(1) HASHER = PasswordHasher(memory_cost=19456, time_cost=2, parallelism=1) DUMMY_HASH = HASHER.hash('nonexistent-password') @asynccontextmanager async def lifespan(app): to_thread.current_default_thread_limiter().total_tokens = 8 yield for provider in providers: provider.shutdown() app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) async def endpoint(request: Request): if request.scope['route'].path.endswith('/attachments'): async with UPLOAD_SLOTS: return await dispatch_request(request, int(config('MAX_UPLOAD_BYTES', '10485760'))) return await dispatch_request(request, 65536) async def dispatch_request(request, limit): length = request.headers.get('content-length') if length and length.isdecimal() and int(length) > limit: raise APIError(413, 'payload_too_large') raw = bytearray() async for chunk in request.stream(): if len(raw) + len(chunk) > limit: raise APIError(413, 'payload_too_large') raw.extend(chunk) code, result = await run_in_threadpool(dispatch, request.method, request.scope['route'].path, request.path_params, parse_qs(request.url.query, keep_blank_values=True), request.headers, bytes(raw)) if code == 204: return Response(status_code=204) if isinstance(result, tuple): return Response(result[0], media_type=result[1], status_code=code) return JSONResponse(result, status_code=code) for path, methods in [('/health/live',['GET']),('/health/ready',['GET']),('/api/v1/users',['POST']), ('/api/v1/users/me',['GET']),('/api/v1/auth/sessions',['POST']),('/api/v1/auth/sessions/current',['DELETE']), ('/api/v1/chats',['GET','POST']),('/api/v1/chats/{chat_id}',['GET']), ('/api/v1/chats/{chat_id}/messages',['GET','POST']),('/api/v1/chats/{chat_id}/attachments',['POST']), ('/api/v1/attachments/{attachment_id}',['GET']),('/api/v1/attachments/{attachment_id}/content',['GET'])]: app.add_api_route(path, endpoint, methods=methods) def error_response(request, status, code): response = JSONResponse({'error': {'code': code, 'message': code.replace('_',' '), 'request_id': request.state.request_id}}, status_code=status) if status == 401: scheme = 'Basic' if request.url.path == '/api/v1/auth/sessions' else 'Bearer' response.headers['WWW-Authenticate'] = scheme+' realm="messenger"'+(', charset="UTF-8"' if scheme=='Basic' else '') return response @app.exception_handler(APIError) async def api_error(request, exc): response = error_response(request, exc.status, exc.code) if exc.retry_after is not None: response.headers['Retry-After'] = str(exc.retry_after) return response @app.middleware('http') async def observe(request, call_next): request.state.request_id = uuid.uuid4().hex started = time.monotonic() with tracer.start_as_current_span('HTTP', context=propagate.extract(dict(request.headers)), kind=SpanKind.SERVER, record_exception=False, set_status_on_exception=False) as span: try: response = await call_next(request) except psycopg.errors.UniqueViolation: response = error_response(request, 409, 'conflict') except ImageTooLarge: response = error_response(request, 413, 'payload_too_large') except InvalidImage: response = error_response(request, 415, 'unsupported_media_type') except Exception as exc: from botocore.exceptions import BotoCoreError, ClientError unavailable = isinstance(exc, (psycopg.Error, BotoCoreError, ClientError)) response = error_response(request, 503 if unavailable else 500, 'unavailable' if unavailable else 'internal_error') route = getattr(request.scope.get('route'), 'path', 'unmatched') # Method is allowlisted too, to avoid attacker-controlled metric cardinality. method = request.method if request.method in ('GET','POST','DELETE','HEAD','OPTIONS','PUT','PATCH') else 'OTHER' labels = {'method': method, 'route': route, 'status_class': str(response.status_code//100)+'xx'} elapsed = time.monotonic()-started requests_count.add(1, labels) http_duration.record(elapsed, labels) span.update_name(method+' '+route) span.set_attributes({**labels, 'http.response.status_code': response.status_code, 'request_id': request.state.request_id}) if response.status_code>=500: span.set_status(Status(StatusCode.ERROR)) logger.log(40 if response.status_code>=500 else 20, 'http.request', extra={ 'event': 'http.request', 'request_id': request.state.request_id, **labels, 'status_code': response.status_code, 'duration_seconds': elapsed}) response.headers['X-Request-Id'] = request.state.request_id response.headers['X-Instance-Id'] = INSTANCE response.headers['X-Trace-Id'] = format(span.get_span_context().trace_id, '032x') return response