Files
mtusi2026lab7/app/api.py
T
2026-11-20 20:37:41 +03:00

388 lines
19 KiB
Python

import asyncio
import base64
import binascii
import hashlib
import hmac
import json
import re
import secrets
import time
from threading import BoundedSemaphore
import uuid
from argon2 import PasswordHasher
from argon2.exceptions import VerifyMismatchError
from contextlib import asynccontextmanager
from datetime import datetime, timedelta, timezone
from urllib.parse import parse_qs
import psycopg
from psycopg.types.json import Jsonb
from fastapi import FastAPI, Request
from fastapi.responses import JSONResponse, Response
from starlette.concurrency import run_in_threadpool
from anyio import to_thread
from opentelemetry import propagate, trace
from opentelemetry.trace import SpanKind, Status, StatusCode
from app.common import (db, get, put, now, config, s3, bucket, sha,
inspect_image, InvalidImage, ImageTooLarge)
from app.telemetry import setup, INSTANCE
tracer, meter, logger, providers = setup('messenger-api')
requests_count = meter.create_counter('http_requests', unit='{request}')
http_duration = meter.create_histogram('http_duration', unit='s')
class APIError(Exception):
def __init__(self, status, code, retry_after=None):
self.status, self.code = status, code
self.retry_after = retry_after
def require(condition, status=400, code='invalid_request'):
if not condition:
raise APIError(status, code)
def uid(value):
try:
return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError):
raise APIError(400, 'invalid_request') from None
def text(value, maximum, minimum=1):
require(isinstance(value, str) and minimum <= len(value) <= maximum)
require(bool(value.strip()) or minimum == 0)
return value
def fields(body, required, optional=()):
require(isinstance(body, dict) and set(required) <= body.keys() and not body.keys()-set(required)-set(optional))
def ids(value, maximum, minimum=0):
require(isinstance(value, list) and minimum <= len(value) <= maximum)
result = [uid(x) for x in value]
require(len(result) == len(set(result)))
return result
def public_user(conn, user_id):
return get(conn, 'user', user_id)
def authenticate(conn, headers):
auth = headers.get('authorization', '')
require(auth.startswith('Bearer ') and bool(auth[7:]), 401, 'unauthorized')
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>now()', (sha(auth[7:].encode()),)).fetchone()
require(row is not None, 401, 'unauthorized')
return row['user_id']
def chat_access(conn, chat_id, actor):
chat = get(conn, 'chat', uid(chat_id))
require(chat is not None and actor in chat['member_ids'], 404, 'not_found')
return chat
def attachment_access(conn, attachment_id, actor):
att = get(conn, 'attachment', uid(attachment_id))
require(att is not None, 404, 'not_found')
chat_access(conn, att['chat_id'], actor)
return att
def page(conn, kind, scope, query, actor):
raw_limit = query.get('limit', ['50'])
require(len(raw_limit) == 1 and re.fullmatch(r'[0-9]+', raw_limit[0]) is not None)
limit = int(raw_limit[0])
require(1 <= limit <= 100)
resource = scope if kind == 'message' else 'chats:'+actor
boundary = None
if 'cursor' in query:
try:
require(len(query['cursor']) == 1)
cursor = json.loads(base64.urlsafe_b64decode(query['cursor'][0].encode()+b'==='))
require(isinstance(cursor, list) and len(cursor) == 3 and cursor[0] == resource)
uid(cursor[2])
require(isinstance(cursor[1], str) and cursor[1].endswith('Z'))
datetime.fromisoformat(cursor[1].replace('Z', '+00:00'))
boundary = cursor[1:]
except (ValueError, TypeError, binascii.Error, UnicodeError):
raise APIError(400, 'invalid_request') from None
if kind == 'message':
condition, params = "data->>'chat_id'=%s", [scope]
else:
condition, params = "data->'member_ids' ? %s", [actor]
if boundary:
condition += " AND (data->>'created_at',id)>(%s,%s)"
params.extend(boundary)
rows = conn.execute("SELECT data FROM objects WHERE kind=%s AND "+condition+" ORDER BY data->>'created_at',id LIMIT %s",
[kind]+params+[limit+1]).fetchall()
items = [r['data'] for r in rows[:limit]]
next_cursor = None
if len(rows)>limit:
next_cursor = base64.urlsafe_b64encode(json.dumps([resource, items[-1]['created_at'], items[-1]['id']]).encode()).decode()
return {'items': items, 'next_cursor': next_cursor}
def dispatch(method, path, params, query, headers, raw):
if path == '/health/live':
return 200, {'status': 'ok'}
if path == '/health/ready':
with db() as conn:
conn.execute('SELECT 1')
s3().head_bucket(Bucket=bucket())
# Durable DB outbox accepts uploads while RabbitMQ/worker are unavailable.
return 200, {'status': 'ok'}
if path == '/api/v1/auth/sessions' and method == 'POST':
# The LB overwrites this header, and no other TLS client can reach the API.
key = headers.get('x-lab-client-ip', 'unknown')
with db() as conn:
attempts = conn.execute(
"""INSERT INTO login_limits(client, started, attempts) VALUES (%s, now(), 1)
ON CONFLICT(client) DO UPDATE SET
attempts = CASE WHEN login_limits.started <= now() - interval '1 minute'
THEN 1 ELSE least(login_limits.attempts + 1, 31) END,
started = CASE WHEN login_limits.started <= now() - interval '1 minute'
THEN now() ELSE login_limits.started END
RETURNING attempts, greatest(1, ceil(extract(epoch FROM
started + interval '1 minute' - now())))::int AS retry_after""",
(key,)).fetchone()
if attempts['attempts'] > 30:
raise APIError(429, 'rate_limited', attempts['retry_after'])
with db() as conn:
if path == '/api/v1/auth/sessions' and method == 'POST':
require(not raw, 400)
try:
auth = headers.get('authorization', '')
require(auth.startswith('Basic '), 401, 'unauthorized')
username, password = base64.b64decode(auth[6:], validate=True).decode('utf-8').split(':', 1)
except (ValueError, UnicodeError, binascii.Error):
raise APIError(401, 'unauthorized') from None
row = conn.execute('SELECT * FROM usernames WHERE username=%s', (username,)).fetchone()
encoded = row['password_hash'] if row else DUMMY_HASH
valid = False
if 12 <= len(password) <= 128:
if encoded.startswith('$argon2id$'):
try:
valid = HASHER.verify(encoded, password)
except VerifyMismatchError:
pass
else:
# Lab6 accounts remain usable after an explicit data migration.
salt, expected = encoded.split(':')
computed = hashlib.scrypt(password.encode(), salt=bytes.fromhex(salt), n=16384, r=8, p=1).hex()
valid = hmac.compare_digest(computed, expected)
if valid:
conn.execute('UPDATE usernames SET password_hash=%s WHERE username=%s AND password_hash=%s',
(HASHER.hash(password), username, encoded))
require(row is not None and valid, 401, 'unauthorized')
token = secrets.token_urlsafe(32)
expires = datetime.now(timezone.utc)+timedelta(seconds=int(config('SESSION_TTL_SECONDS', '3600')))
conn.execute('INSERT INTO sessions VALUES(%s,%s,%s)', (sha(token.encode()), row['user_id'], expires))
return 201, {'token': token, 'token_type': 'Bearer', 'expires_at': expires.isoformat().replace('+00:00','Z'),
'user': public_user(conn, row['user_id'])}
# Authentication precedes validation of resource identifiers / body.
actor = None if path == '/api/v1/users' and method == 'POST' else authenticate(conn, headers)
body = None
if method == 'POST' and not path.endswith('/attachments'):
try:
require(headers.get('content-type', '').split(';')[0] == 'application/json')
body = json.loads(raw)
except (ValueError, UnicodeError):
raise APIError(400, 'invalid_request') from None
if path == '/api/v1/users' and method == 'POST':
fields(body, ('username','display_name','password'))
username = text(body['username'], 32, 3)
require(re.fullmatch('[a-z0-9_]{3,32}', username) is not None)
display = text(body['display_name'], 100)
password = text(body['password'], 128, 12)
hashed = HASHER.hash(password)
user = {'id': str(uuid.uuid4()), 'username': username, 'display_name': display, 'created_at': now()}
conn.execute('INSERT INTO usernames VALUES(%s,%s,%s)', (username, user['id'], hashed))
put(conn, 'user', user)
return 201, user
if path == '/api/v1/users/me':
return 200, public_user(conn, actor)
if path == '/api/v1/auth/sessions/current':
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (sha(headers['authorization'][7:].encode()),))
return 204, None
if path == '/api/v1/chats':
if method == 'GET':
return 200, page(conn, 'chat', None, query, actor)
fields(body, ('title','member_ids'))
members = ids(body['member_ids'], 100, 1)
require(actor in members and all(public_user(conn, x) for x in members))
chat = {'id': str(uuid.uuid4()), 'title': text(body['title'], 100), 'member_ids': members, 'created_at': now()}
put(conn, 'chat', chat)
return 201, chat
if 'chat_id' in params:
chat = chat_access(conn, params['chat_id'], actor)
if path == '/api/v1/chats/{chat_id}':
return 200, chat
if path.endswith('/messages'):
if method == 'GET':
return 200, page(conn, 'message', chat['id'], query, actor)
fields(body, ('text',), ('attachment_ids',))
message_text = text(body['text'], 4000, 0)
attachments = ids(body.get('attachment_ids', []), 10)
require(message_text.strip() or attachments)
key = headers.get('idempotency-key', '')
require(re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key) is not None)
payload = {'text': message_text, 'attachment_ids': attachments}
scope = actor+':POST:'+chat['id']+':'+key
# Cross-replica serialization. Hash collision merely serializes unrelated keys.
conn.execute('SELECT pg_advisory_xact_lock(hashtextextended(%s,0))', (scope,))
old = conn.execute("SELECT * FROM idempotency WHERE scope=%s AND created_at>now()-interval '24 hours'", (scope,)).fetchone()
if old:
require(old['payload'] == payload, 409, 'conflict')
return 201, old['response']
for aid in attachments:
att = attachment_access(conn, aid, actor)
require(att['chat_id'] == chat['id'], 404, 'not_found')
require(att['status'] != 'failed', 409, 'conflict')
message = {'id': str(uuid.uuid4()), 'chat_id': chat['id'], 'sender_id': actor,
**payload, 'created_at': now()}
put(conn, 'message', message)
conn.execute('DELETE FROM idempotency WHERE scope=%s', (scope,))
conn.execute('INSERT INTO idempotency(scope,payload,response) VALUES(%s,%s,%s)', (scope, Jsonb(payload), Jsonb(message)))
return 201, message
if path.endswith('/attachments'):
filename = query.get('filename', [''])[0]
text(filename, 255)
require(not any(x in filename for x in ('/', '\\')) and all(ord(x)>=32 and ord(x)!=127 for x in filename))
content_type = headers.get('content-type', '').split(';')[0]
require(content_type in ('image/png','image/jpeg'), 415, 'unsupported_media_type')
require(bool(raw), 415, 'unsupported_media_type')
with IMAGE_SLOTS:
inspect_image(raw, content_type)
aid, stamp = str(uuid.uuid4()), now()
att = {'id': aid, 'chat_id': chat['id'], 'owner_id': actor, 'filename': filename,
'content_type': content_type, 'size_bytes': len(raw), 'sha256': sha(raw),
'status': 'queued', 'variants': [], 'error_code': None, 'created_at': stamp, 'updated_at': stamp}
with tracer.start_as_current_span('s3.stage', record_exception=False, set_status_on_exception=False):
s3().put_object(Bucket=bucket(), Key='staging/'+aid, Body=raw, ContentType=content_type)
put(conn, 'attachment', att)
job = {'schema_version': 1, 'job_id': str(uuid.uuid4()), 'type': 'attachment.process.v1',
'attachment_id': aid, 'pipeline_version': 1, 'attempt': 1, 'created_at': stamp}
propagate.inject(job)
conn.execute('INSERT INTO outbox(id,payload) VALUES(%s,%s)', (job['job_id'], Jsonb(job)))
return 202, att
if 'attachment_id' in params:
att = attachment_access(conn, params['attachment_id'], actor)
if not path.endswith('/content'):
return 200, att
require(att['status'] == 'ready', 409, 'conflict')
variant = query.get('variant', ['original'])[0]
meta = att if variant == 'original' else next((v for v in att['variants'] if v['name']==variant), None)
require(meta is not None, 404, 'not_found')
with tracer.start_as_current_span('s3.download', record_exception=False, set_status_on_exception=False):
data = s3().get_object(Bucket=bucket(), Key='final/'+att['id']+'/'+variant)['Body'].read()
return 200, (data, meta['content_type'])
raise APIError(404, 'not_found')
UPLOAD_SLOTS = asyncio.Semaphore(4)
IMAGE_SLOTS = BoundedSemaphore(1)
HASHER = PasswordHasher(memory_cost=19456, time_cost=2, parallelism=1)
DUMMY_HASH = HASHER.hash('nonexistent-password')
@asynccontextmanager
async def lifespan(app):
to_thread.current_default_thread_limiter().total_tokens = 8
yield
for provider in providers:
provider.shutdown()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
async def endpoint(request: Request):
if request.scope['route'].path.endswith('/attachments'):
async with UPLOAD_SLOTS:
return await dispatch_request(request, int(config('MAX_UPLOAD_BYTES', '10485760')))
return await dispatch_request(request, 65536)
async def dispatch_request(request, limit):
length = request.headers.get('content-length')
if length and length.isdecimal() and int(length) > limit:
raise APIError(413, 'payload_too_large')
raw = bytearray()
async for chunk in request.stream():
if len(raw) + len(chunk) > limit:
raise APIError(413, 'payload_too_large')
raw.extend(chunk)
code, result = await run_in_threadpool(dispatch, request.method, request.scope['route'].path,
request.path_params, parse_qs(request.url.query, keep_blank_values=True), request.headers, bytes(raw))
if code == 204:
return Response(status_code=204)
if isinstance(result, tuple):
return Response(result[0], media_type=result[1], status_code=code)
return JSONResponse(result, status_code=code)
for path, methods in [('/health/live',['GET']),('/health/ready',['GET']),('/api/v1/users',['POST']),
('/api/v1/users/me',['GET']),('/api/v1/auth/sessions',['POST']),('/api/v1/auth/sessions/current',['DELETE']),
('/api/v1/chats',['GET','POST']),('/api/v1/chats/{chat_id}',['GET']),
('/api/v1/chats/{chat_id}/messages',['GET','POST']),('/api/v1/chats/{chat_id}/attachments',['POST']),
('/api/v1/attachments/{attachment_id}',['GET']),('/api/v1/attachments/{attachment_id}/content',['GET'])]:
app.add_api_route(path, endpoint, methods=methods)
def error_response(request, status, code):
response = JSONResponse({'error': {'code': code, 'message': code.replace('_',' '),
'request_id': request.state.request_id}}, status_code=status)
if status == 401:
scheme = 'Basic' if request.url.path == '/api/v1/auth/sessions' else 'Bearer'
response.headers['WWW-Authenticate'] = scheme+' realm="messenger"'+(', charset="UTF-8"' if scheme=='Basic' else '')
return response
@app.exception_handler(APIError)
async def api_error(request, exc):
response = error_response(request, exc.status, exc.code)
if exc.retry_after is not None:
response.headers['Retry-After'] = str(exc.retry_after)
return response
@app.middleware('http')
async def observe(request, call_next):
request.state.request_id = uuid.uuid4().hex
started = time.monotonic()
with tracer.start_as_current_span('HTTP', context=propagate.extract(dict(request.headers)), kind=SpanKind.SERVER,
record_exception=False, set_status_on_exception=False) as span:
try:
response = await call_next(request)
except psycopg.errors.UniqueViolation:
response = error_response(request, 409, 'conflict')
except ImageTooLarge:
response = error_response(request, 413, 'payload_too_large')
except InvalidImage:
response = error_response(request, 415, 'unsupported_media_type')
except Exception as exc:
from botocore.exceptions import BotoCoreError, ClientError
unavailable = isinstance(exc, (psycopg.Error, BotoCoreError, ClientError))
response = error_response(request, 503 if unavailable else 500, 'unavailable' if unavailable else 'internal_error')
route = getattr(request.scope.get('route'), 'path', 'unmatched')
# Method is allowlisted too, to avoid attacker-controlled metric cardinality.
method = request.method if request.method in ('GET','POST','DELETE','HEAD','OPTIONS','PUT','PATCH') else 'OTHER'
labels = {'method': method, 'route': route, 'status_class': str(response.status_code//100)+'xx'}
elapsed = time.monotonic()-started
requests_count.add(1, labels)
http_duration.record(elapsed, labels)
span.update_name(method+' '+route)
span.set_attributes({**labels, 'http.response.status_code': response.status_code, 'request_id': request.state.request_id})
if response.status_code>=500:
span.set_status(Status(StatusCode.ERROR))
logger.log(40 if response.status_code>=500 else 20, 'http.request', extra={
'event': 'http.request', 'request_id': request.state.request_id, **labels,
'status_code': response.status_code, 'duration_seconds': elapsed})
response.headers['X-Request-Id'] = request.state.request_id
response.headers['X-Instance-Id'] = INSTANCE
response.headers['X-Trace-Id'] = format(span.get_span_context().trace_id, '032x')
return response