import base64 import binascii import hashlib import hmac import json import re import secrets import time import uuid from contextlib import asynccontextmanager from datetime import datetime, timedelta, timezone from urllib.parse import parse_qs import psycopg from psycopg.types.json import Jsonb from fastapi import FastAPI, Request from fastapi.responses import JSONResponse, Response from starlette.concurrency import run_in_threadpool from opentelemetry import propagate, trace from opentelemetry.trace import SpanKind, Status, StatusCode from app.common import (db, migrate, get, put, now, config, s3, bucket, sha, inspect_image, InvalidImage, ImageTooLarge) from app.telemetry import setup, INSTANCE tracer, meter, logger, providers = setup('messenger-api') requests_count = meter.create_counter('http_requests', unit='{request}') http_duration = meter.create_histogram('http_duration', unit='s') class APIError(Exception): def __init__(self, status, code): self.status, self.code = status, code def require(condition, status=400, code='invalid_request'): if not condition: raise APIError(status, code) def uid(value): try: return str(uuid.UUID(value)) except (ValueError, TypeError, AttributeError): raise APIError(400, 'invalid_request') from None def text(value, maximum, minimum=1): require(isinstance(value, str) and minimum <= len(value) <= maximum) require(bool(value.strip()) or minimum == 0) return value def fields(body, required, optional=()): require(isinstance(body, dict) and set(required) <= body.keys() and not body.keys()-set(required)-set(optional)) def ids(value, maximum, minimum=0): require(isinstance(value, list) and minimum <= len(value) <= maximum) result = [uid(x) for x in value] require(len(result) == len(set(result))) return result def public_user(conn, user_id): return get(conn, 'user', user_id) def authenticate(conn, headers): auth = headers.get('authorization', '') require(auth.startswith('Bearer ') and bool(auth[7:]), 401, 'unauthorized') row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>now()', (sha(auth[7:].encode()),)).fetchone() require(row is not None, 401, 'unauthorized') return row['user_id'] def chat_access(conn, chat_id, actor): chat = get(conn, 'chat', uid(chat_id)) require(chat is not None and actor in chat['member_ids'], 404, 'not_found') return chat def attachment_access(conn, attachment_id, actor): att = get(conn, 'attachment', uid(attachment_id)) require(att is not None, 404, 'not_found') chat_access(conn, att['chat_id'], actor) return att def page(conn, kind, scope, query, actor): raw_limit = query.get('limit', ['50']) require(len(raw_limit) == 1 and re.fullmatch(r'[0-9]+', raw_limit[0]) is not None) limit = int(raw_limit[0]) require(1 <= limit <= 100) resource = scope if kind == 'message' else 'chats:'+actor boundary = None if 'cursor' in query: try: require(len(query['cursor']) == 1) cursor = json.loads(base64.urlsafe_b64decode(query['cursor'][0].encode()+b'===')) require(isinstance(cursor, list) and len(cursor) == 3 and cursor[0] == resource) uid(cursor[2]) require(isinstance(cursor[1], str) and cursor[1].endswith('Z')) datetime.fromisoformat(cursor[1].replace('Z', '+00:00')) boundary = cursor[1:] except (ValueError, TypeError, binascii.Error, UnicodeError): raise APIError(400, 'invalid_request') from None if kind == 'message': condition, params = "data->>'chat_id'=%s", [scope] else: condition, params = "data->'member_ids' ? %s", [actor] if boundary: condition += " AND (data->>'created_at',id)>(%s,%s)" params.extend(boundary) rows = conn.execute("SELECT data FROM objects WHERE kind=%s AND "+condition+" ORDER BY data->>'created_at',id LIMIT %s", [kind]+params+[limit+1]).fetchall() items = [r['data'] for r in rows[:limit]] next_cursor = None if len(rows)>limit: next_cursor = base64.urlsafe_b64encode(json.dumps([resource, items[-1]['created_at'], items[-1]['id']]).encode()).decode() return {'items': items, 'next_cursor': next_cursor} def dispatch(method, path, params, query, headers, raw): if path == '/health/live': return 200, {'status': 'ok'} if path == '/health/ready': with db() as conn: conn.execute('SELECT 1') s3().head_bucket(Bucket=bucket()) # Durable DB outbox accepts uploads while RabbitMQ/worker are unavailable. return 200, {'status': 'ok'} with db() as conn: if path == '/api/v1/auth/sessions' and method == 'POST': require(not raw, 400) try: auth = headers.get('authorization', '') require(auth.startswith('Basic '), 401, 'unauthorized') username, password = base64.b64decode(auth[6:], validate=True).decode('utf-8').split(':', 1) except (ValueError, UnicodeError, binascii.Error): raise APIError(401, 'unauthorized') from None row = conn.execute('SELECT * FROM usernames WHERE username=%s', (username,)).fetchone() encoded = row['password_hash'] if row else DUMMY_HASH salt, expected = encoded.split(':') valid = False if 12 <= len(password) <= 128: computed = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=16384, r=8, p=1).hex() valid = hmac.compare_digest(computed, expected) require(row is not None and valid, 401, 'unauthorized') token = secrets.token_urlsafe(32) expires = datetime.now(timezone.utc)+timedelta(seconds=int(config('SESSION_TTL_SECONDS', '3600'))) conn.execute('INSERT INTO sessions VALUES(%s,%s,%s)', (sha(token.encode()), row['user_id'], expires)) return 201, {'token': token, 'token_type': 'Bearer', 'expires_at': expires.isoformat().replace('+00:00','Z'), 'user': public_user(conn, row['user_id'])} # Authentication precedes validation of resource identifiers / body. actor = None if path == '/api/v1/users' and method == 'POST' else authenticate(conn, headers) body = None if method == 'POST' and not path.endswith('/attachments'): try: require(headers.get('content-type', '').split(';')[0] == 'application/json') body = json.loads(raw) except (ValueError, UnicodeError): raise APIError(400, 'invalid_request') from None if path == '/api/v1/users' and method == 'POST': fields(body, ('username','display_name','password')) username = text(body['username'], 32, 3) require(re.fullmatch('[a-z0-9_]{3,32}', username) is not None) display = text(body['display_name'], 100) password = text(body['password'], 128, 12) salt = secrets.token_bytes(16) hashed = salt.hex()+':'+hashlib.scrypt(password.encode(), salt=salt, n=16384, r=8, p=1).hex() user = {'id': str(uuid.uuid4()), 'username': username, 'display_name': display, 'created_at': now()} conn.execute('INSERT INTO usernames VALUES(%s,%s,%s)', (username, user['id'], hashed)) put(conn, 'user', user) return 201, user if path == '/api/v1/users/me': return 200, public_user(conn, actor) if path == '/api/v1/auth/sessions/current': conn.execute('DELETE FROM sessions WHERE token_hash=%s', (sha(headers['authorization'][7:].encode()),)) return 204, None if path == '/api/v1/chats': if method == 'GET': return 200, page(conn, 'chat', None, query, actor) fields(body, ('title','member_ids')) members = ids(body['member_ids'], 100, 1) require(actor in members and all(public_user(conn, x) for x in members)) chat = {'id': str(uuid.uuid4()), 'title': text(body['title'], 100), 'member_ids': members, 'created_at': now()} put(conn, 'chat', chat) return 201, chat if 'chat_id' in params: chat = chat_access(conn, params['chat_id'], actor) if path == '/api/v1/chats/{chat_id}': return 200, chat if path.endswith('/messages'): if method == 'GET': return 200, page(conn, 'message', chat['id'], query, actor) fields(body, ('text',), ('attachment_ids',)) message_text = text(body['text'], 4000, 0) attachments = ids(body.get('attachment_ids', []), 10) require(message_text.strip() or attachments) key = headers.get('idempotency-key', '') require(re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key) is not None) payload = {'text': message_text, 'attachment_ids': attachments} scope = actor+':POST:'+chat['id']+':'+key # Cross-replica serialization. Hash collision merely serializes unrelated keys. conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))', (scope,)) old = conn.execute("SELECT * FROM idempotency WHERE scope=%s AND created_at>now()-interval '24 hours'", (scope,)).fetchone() if old: require(old['payload'] == payload, 409, 'conflict') return 201, old['response'] for aid in attachments: att = attachment_access(conn, aid, actor) require(att['chat_id'] == chat['id'], 404, 'not_found') require(att['status'] != 'failed', 409, 'conflict') message = {'id': str(uuid.uuid4()), 'chat_id': chat['id'], 'sender_id': actor, **payload, 'created_at': now()} put(conn, 'message', message) conn.execute('DELETE FROM idempotency WHERE scope=%s', (scope,)) conn.execute('INSERT INTO idempotency(scope,payload,response) VALUES(%s,%s,%s)', (scope, Jsonb(payload), Jsonb(message))) return 201, message if path.endswith('/attachments'): filename = query.get('filename', [''])[0] text(filename, 255) require(not any(x in filename for x in ('/', '\\')) and all(ord(x)>=32 and ord(x)!=127 for x in filename)) content_type = headers.get('content-type', '').split(';')[0] require(content_type in ('image/png','image/jpeg'), 415, 'unsupported_media_type') require(bool(raw), 415, 'unsupported_media_type') inspect_image(raw, content_type) aid, stamp = str(uuid.uuid4()), now() att = {'id': aid, 'chat_id': chat['id'], 'owner_id': actor, 'filename': filename, 'content_type': content_type, 'size_bytes': len(raw), 'sha256': sha(raw), 'status': 'queued', 'variants': [], 'error_code': None, 'created_at': stamp, 'updated_at': stamp} with tracer.start_as_current_span('s3.stage', record_exception=False, set_status_on_exception=False): s3().put_object(Bucket=bucket(), Key='staging/'+aid, Body=raw, ContentType=content_type) put(conn, 'attachment', att) job = {'schema_version': 1, 'job_id': str(uuid.uuid4()), 'type': 'attachment.process.v1', 'attachment_id': aid, 'pipeline_version': 1, 'attempt': 1, 'created_at': stamp} propagate.inject(job) conn.execute('INSERT INTO outbox(id,payload) VALUES(%s,%s)', (job['job_id'], Jsonb(job))) return 202, att if 'attachment_id' in params: att = attachment_access(conn, params['attachment_id'], actor) if not path.endswith('/content'): return 200, att require(att['status'] == 'ready', 409, 'conflict') variant = query.get('variant', ['original'])[0] meta = att if variant == 'original' else next((v for v in att['variants'] if v['name']==variant), None) require(meta is not None, 404, 'not_found') with tracer.start_as_current_span('s3.download', record_exception=False, set_status_on_exception=False): data = s3().get_object(Bucket=bucket(), Key='final/'+att['id']+'/'+variant)['Body'].read() return 200, (data, meta['content_type']) raise APIError(404, 'not_found') DUMMY_HASH = '00'*16+':'+hashlib.scrypt(b'nonexistent-password', salt=bytes(16), n=16384, r=8, p=1).hex() @asynccontextmanager async def lifespan(app): await run_in_threadpool(migrate) yield for provider in providers: provider.shutdown() app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) async def endpoint(request: Request): raw = bytearray() async for chunk in request.stream(): raw.extend(chunk) if len(raw)>int(config('MAX_UPLOAD_BYTES', '10485760')): raise APIError(413, 'payload_too_large') code, result = await run_in_threadpool(dispatch, request.method, request.scope['route'].path, request.path_params, parse_qs(request.url.query, keep_blank_values=True), request.headers, bytes(raw)) if code == 204: return Response(status_code=204) if isinstance(result, tuple): return Response(result[0], media_type=result[1], status_code=code) return JSONResponse(result, status_code=code) for path, methods in [('/health/live',['GET']),('/health/ready',['GET']),('/api/v1/users',['POST']), ('/api/v1/users/me',['GET']),('/api/v1/auth/sessions',['POST']),('/api/v1/auth/sessions/current',['DELETE']), ('/api/v1/chats',['GET','POST']),('/api/v1/chats/{chat_id}',['GET']), ('/api/v1/chats/{chat_id}/messages',['GET','POST']),('/api/v1/chats/{chat_id}/attachments',['POST']), ('/api/v1/attachments/{attachment_id}',['GET']),('/api/v1/attachments/{attachment_id}/content',['GET'])]: app.add_api_route(path, endpoint, methods=methods) def error_response(request, status, code): response = JSONResponse({'error': {'code': code, 'message': code.replace('_',' '), 'request_id': request.state.request_id}}, status_code=status) if status == 401: scheme = 'Basic' if request.url.path == '/api/v1/auth/sessions' else 'Bearer' response.headers['WWW-Authenticate'] = scheme+' realm="messenger"'+(', charset="UTF-8"' if scheme=='Basic' else '') return response @app.exception_handler(APIError) async def api_error(request, exc): return error_response(request, exc.status, exc.code) @app.middleware('http') async def observe(request, call_next): request.state.request_id = uuid.uuid4().hex started = time.monotonic() with tracer.start_as_current_span('HTTP', context=propagate.extract(dict(request.headers)), kind=SpanKind.SERVER, record_exception=False, set_status_on_exception=False) as span: try: response = await call_next(request) except psycopg.errors.UniqueViolation: response = error_response(request, 409, 'conflict') except ImageTooLarge: response = error_response(request, 413, 'payload_too_large') except InvalidImage: response = error_response(request, 415, 'unsupported_media_type') except Exception as exc: from botocore.exceptions import BotoCoreError, ClientError unavailable = isinstance(exc, (psycopg.Error, BotoCoreError, ClientError)) response = error_response(request, 503 if unavailable else 500, 'unavailable' if unavailable else 'internal_error') route = getattr(request.scope.get('route'), 'path', 'unmatched') # Method is allowlisted too, to avoid attacker-controlled metric cardinality. method = request.method if request.method in ('GET','POST','DELETE','HEAD','OPTIONS','PUT','PATCH') else 'OTHER' labels = {'method': method, 'route': route, 'status_class': str(response.status_code//100)+'xx'} elapsed = time.monotonic()-started requests_count.add(1, labels) http_duration.record(elapsed, labels) span.update_name(method+' '+route) span.set_attributes({**labels, 'http.response.status_code': response.status_code, 'request_id': request.state.request_id}) if response.status_code>=500: span.set_status(Status(StatusCode.ERROR)) logger.log(40 if response.status_code>=500 else 20, 'http.request', extra={ 'event': 'http.request', 'request_id': request.state.request_id, **labels, 'status_code': response.status_code, 'duration_seconds': elapsed}) response.headers['X-Request-Id'] = request.state.request_id response.headers['X-Instance-Id'] = INSTANCE response.headers['X-Trace-Id'] = format(span.get_span_context().trace_id, '032x') return response