Files
mtusi2026lab4/app/main.py
T
RainakandClaude Opus 5.5 2dca5c114b ЛР4 БВТ2304 Запорожец на 3
Вложения: приватный S3-bucket (RustFS 1.0.4) + метаданные в PostgreSQL, upload PNG/JPEG
с лимитами до декодирования, авторизованный download, attachment_ids в сообщениях,
идемпотентный bootstrap bucket, failover/persistence/S3-down тесты и REPORT.md.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-12 18:49:12 +03:00

538 lines
26 KiB
Python

"""HTTP API мессенджера lab4 по contracts/openapi.json (stateless-реплика за балансировщиком).
lab4: вложения. Метаданные — PostgreSQL, байты — приватный S3-bucket (MinIO). Локальных файлов у API нет.
"""
import asyncio
import base64
import binascii
import hashlib
import json
import logging
import re
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from urllib.parse import quote
from uuid import UUID
import asyncpg
from fastapi import FastAPI, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response
from starlette.exceptions import HTTPException as StarletteHTTPException
from . import config, images, lifecycle, validation as v
from .db import DB_UNAVAILABLE, Database
from .errors import (ApiError, error_response, invalid, not_found, request_id_var,
unauthorized_basic, unauthorized_bearer, unavailable)
from .security import PasswordService, new_token
from .sessions import SessionStore
from .storage import Storage
log = logging.getLogger('messenger')
settings = config.load()
db = Database(settings)
sessions = SessionStore(settings, db)
passwords = PasswordService(settings.hash_workers)
storage = Storage(settings)
image_checker = images.ImageChecker(settings.max_image_pixels, settings.image_workers)
async def purge_loop() -> None:
while True:
await asyncio.sleep(3600)
try:
removed = await sessions.purge_expired()
keys = await purge_idempotency_keys()
log.info('purged %d expired sessions, %d idempotency keys', removed, keys)
except Exception: # очистка — фоновая гигиена, её сбой не должен ронять процесс
log.warning('purge failed', exc_info=True)
async def purge_idempotency_keys() -> int:
# Удаление по одному ключу индекса created_at; параллельный запуск на нескольких репликах безопасен.
async with db.connection() as conn:
status = await conn.execute(
'DELETE FROM idempotency_keys WHERE created_at < clock_timestamp() - make_interval(secs => $1)',
settings.idempotency_ttl)
return int(status.split()[-1])
@asynccontextmanager
async def lifespan(_app: FastAPI):
await db.start()
task = asyncio.create_task(purge_loop())
yield
task.cancel()
await sessions.cache.close()
await db.close()
passwords.close()
app = FastAPI(lifespan=lifespan, openapi_url=None, docs_url=None, redoc_url=None)
# ---------- request id, ошибки ----------
REQUEST_ID_RE = re.compile(r'[A-Za-z0-9._:-]{1,128}')
@app.middleware('http')
async def request_id_middleware(request: Request, call_next):
incoming = request.headers.get('x-request-id', '')
rid = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
request_id_var.set(rid)
try:
response = await call_next(request)
except Exception:
log.exception('unhandled error, request_id=%s', rid)
response = error_response(500, 'internal_error', 'Внутренняя ошибка сервера')
response.headers['X-Request-Id'] = rid
response.headers['X-Instance-Id'] = lifecycle.INSTANCE_ID
if lifecycle.draining:
# Во время drain закрываем keep-alive: следующий запрос LB откроет соединение уже к живой реплике.
response.headers['Connection'] = 'close'
return response
@app.exception_handler(ApiError)
async def api_error_handler(_request: Request, exc: ApiError):
return error_response(exc.status, exc.code, exc.message, exc.headers)
@app.exception_handler(StarletteHTTPException)
async def http_error_handler(_request: Request, exc: StarletteHTTPException):
if exc.status_code == 404:
return error_response(404, 'not_found', 'Объект не найден')
if exc.status_code == 405:
return error_response(405, 'invalid_request', 'Метод не поддерживается', dict(exc.headers or {}))
return error_response(exc.status_code, 'invalid_request', 'Некорректный запрос')
@app.exception_handler(RequestValidationError)
async def validation_error_handler(_request: Request, _exc: RequestValidationError):
return error_response(400, 'invalid_request', 'Некорректный запрос')
async def db_unavailable_handler(_request: Request, _exc: Exception):
# Страховка: сбой связи с БД вне db.connection() тоже даёт ограниченный 503, а не 500.
return error_response(503, 'unavailable', 'Сервис временно недоступен')
for _exc_type in DB_UNAVAILABLE:
app.add_exception_handler(_exc_type, db_unavailable_handler)
# ---------- представление ----------
def ts(value: datetime) -> str:
# RFC 3339 UTC с микросекундами: точность совпадает с timestamptz и с ключом пагинации.
return value.astimezone(timezone.utc).strftime('%Y-%m-%dT%H:%M:%S.%fZ')
def user_json(row) -> dict:
return {'id': str(row['id']), 'username': row['username'],
'display_name': row['display_name'], 'created_at': ts(row['created_at'])}
def chat_json(row) -> dict:
return {'id': str(row['id']), 'title': row['title'],
'member_ids': [str(m) for m in row['member_ids']], 'created_at': ts(row['created_at'])}
def message_json(row) -> dict:
return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'sender_id': str(row['sender_id']),
'text': row['text'], 'created_at': ts(row['created_at']),
'attachment_ids': [str(a) for a in row['attachment_ids']]}
def attachment_json(row) -> dict:
# object_key и внутренние адреса S3 в ответ не попадают.
return {'id': str(row['id']), 'chat_id': str(row['chat_id']), 'owner_id': str(row['owner_id']),
'filename': row['filename'], 'content_type': row['content_type'],
'size_bytes': row['size_bytes'], 'sha256': row['sha256'], 'status': row['status'],
'variants': [], 'error_code': None,
'created_at': ts(row['created_at']), 'updated_at': ts(row['updated_at'])}
USER_COLUMNS = 'id, username, display_name, created_at'
CHAT_COLUMNS = ('c.id, c.title, c.created_at, '
'ARRAY(SELECT cm.user_id FROM chat_members cm WHERE cm.chat_id = c.id ORDER BY cm.user_id) AS member_ids')
MESSAGE_ATTACHMENTS = ('ARRAY(SELECT ma.attachment_id FROM message_attachments ma '
'WHERE ma.message_id = {m}.id ORDER BY ma.position) AS attachment_ids')
MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, created_at, ' + MESSAGE_ATTACHMENTS.format(m='messages')
ATTACHMENT_COLUMNS = ('a.id, a.chat_id, a.owner_id, a.filename, a.content_type, a.size_bytes, a.sha256, '
'a.status, a.object_key, a.created_at, a.updated_at')
# ---------- аутентификация ----------
TOKEN_RE = re.compile(r'[A-Za-z0-9_-]{32,128}')
def bearer_token(request: Request) -> str:
"""Только Authorization: Bearer. X-User-Id и Basic здесь игнорируются/отклоняются."""
scheme, _, token = request.headers.get('authorization', '').partition(' ')
token = token.strip()
if scheme.lower() != 'bearer' or not TOKEN_RE.fullmatch(token):
raise unauthorized_bearer()
return token
async def current_user(request: Request) -> UUID:
session = await sessions.resolve(bearer_token(request))
if session is None:
raise unauthorized_bearer()
return session.user_id
def basic_credentials(request: Request) -> tuple[str, str]:
scheme, _, encoded = request.headers.get('authorization', '').partition(' ')
if scheme.lower() != 'basic':
raise unauthorized_basic()
try:
decoded = base64.b64decode(encoded.strip(), validate=True)
username, sep, password = decoded.partition(b':') # двоеточие допустимо внутри пароля
if not sep:
raise ValueError
return username.decode('ascii'), password.decode('utf-8')
except (ValueError, binascii.Error, UnicodeDecodeError):
raise unauthorized_basic() from None
# ---------- health ----------
@app.get('/health/live')
async def live():
return {'status': 'ok'}
@app.get('/health/ready')
async def ready():
# Критична только PostgreSQL (источник истины). Valkey — кэш: без него API работает медленнее, но корректно.
# Во время drain (получен SIGTERM) реплика честно сообщает LB, что новых запросов ей давать не надо.
if lifecycle.draining or not await db.ping():
raise unavailable()
return {'status': 'ok'}
# ---------- пользователи и сессии ----------
@app.post('/api/v1/users', status_code=201)
async def create_user(request: Request):
data = await v.read_json(request)
v.fields(data, {'username', 'display_name', 'password'})
username = v.username(data)
display_name = v.text_field(data, 'display_name', 1, 100)
password = v.text_field(data, 'password', 12, 128)
password_hash = await passwords.hash(password)
try:
async with db.connection() as conn:
row = await conn.fetchrow(
f'INSERT INTO users (username, display_name, password_hash) VALUES ($1, $2, $3) RETURNING {USER_COLUMNS}',
username, display_name, password_hash)
except asyncpg.UniqueViolationError:
# Гонка двух одинаковых регистраций решается UNIQUE-индексом: COMMIT пройдёт только у одной.
raise ApiError(409, 'conflict', 'username уже занят') from None
return JSONResponse(user_json(row), status_code=201)
@app.post('/api/v1/auth/sessions', status_code=201)
async def create_session(request: Request):
if await request.body():
raise invalid('Тело запроса не допускается; используйте Authorization: Basic')
username, password = basic_credentials(request)
row = None
if v.USERNAME_RE.fullmatch(username) and 12 <= len(password) <= 128:
async with db.connection() as conn:
row = await conn.fetchrow(
f'SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = $1', username)
# verify выполняется всегда (и для несуществующего пользователя) — одинаковое время ответа.
if not await passwords.verify(row['password_hash'] if row else None, password):
raise unauthorized_basic()
token = new_token()
session = await sessions.create(row['id'], token)
return JSONResponse({'token': token, 'token_type': 'Bearer', 'expires_at': ts(session.expires_at),
'user': user_json(row)}, status_code=201, headers={'Cache-Control': 'no-store'})
@app.delete('/api/v1/auth/sessions/current', status_code=204)
async def delete_session(request: Request):
token = bearer_token(request)
if not await sessions.revoke(token):
raise unauthorized_bearer()
return Response(status_code=204)
@app.get('/api/v1/users/me')
async def me(request: Request):
user_id = await current_user(request)
async with db.connection() as conn:
row = await conn.fetchrow(f'SELECT {USER_COLUMNS} FROM users WHERE id = $1', user_id)
if row is None:
raise not_found()
return user_json(row)
# ---------- чаты ----------
@app.post('/api/v1/chats', status_code=201)
async def create_chat(request: Request):
user_id = await current_user(request)
data = await v.read_json(request)
v.fields(data, {'title', 'member_ids'})
title = v.text_field(data, 'title', 1, 100)
raw_members = data['member_ids']
if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100:
raise invalid('member_ids должен быть массивом из 1..100 UUID')
members = [v.uuid_value(m, 'member_id') for m in raw_members]
if len(set(members)) != len(members):
raise invalid('member_ids содержит дубликаты')
if user_id not in members:
raise invalid('Создатель должен входить в member_ids')
async with db.connection() as conn:
# Одна транзакция: чат и все участники появляются вместе или не появляются вовсе.
async with conn.transaction():
chat = await conn.fetchrow(
'INSERT INTO chats (title, created_by) VALUES ($1, $2) RETURNING id, created_at', title, user_id)
added = await conn.fetch(
'INSERT INTO chat_members (chat_id, user_id) '
'SELECT $1, u.id FROM users u WHERE u.id = ANY($2::uuid[]) RETURNING user_id',
chat['id'], members)
if len(added) != len(members):
# Исключение внутри conn.transaction() -> ROLLBACK, строка chats тоже откатывается.
raise invalid('member_ids содержит неизвестных пользователей')
return JSONResponse({'id': str(chat['id']), 'title': title,
'member_ids': sorted(str(m) for m in members),
'created_at': ts(chat['created_at'])}, status_code=201)
@app.get('/api/v1/chats')
async def list_chats(request: Request):
user_id = await current_user(request)
limit = v.limit_param(request)
after = v.decode_cursor(request, 'chats')
query = (f'SELECT {CHAT_COLUMNS} FROM chats c '
'JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $1 ')
params = [user_id, limit + 1]
if after:
query += 'WHERE (c.created_at, c.id) > ($3, $4) '
params += after
query += 'ORDER BY c.created_at, c.id LIMIT $2'
async with db.connection() as conn:
rows = await conn.fetch(query, *params)
return page(rows, limit, 'chats', chat_json)
def page(rows, limit: int, scope: str, render) -> dict:
# Читаем limit+1 строку: если лишняя есть — следующая страница существует.
items = rows[:limit]
cursor = (v.encode_cursor(scope, items[-1]['created_at'], items[-1]['id'])
if len(rows) > limit else None)
return {'items': [render(r) for r in items], 'next_cursor': cursor}
async def chat_for_member(conn, chat_id: UUID, user_id: UUID):
return await conn.fetchrow(
f'SELECT {CHAT_COLUMNS} FROM chats c '
'WHERE c.id = $1 AND EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = c.id AND m.user_id = $2)',
chat_id, user_id)
@app.get('/api/v1/chats/{chat_id}')
async def get_chat(chat_id: str, request: Request):
user_id = await current_user(request)
cid = v.uuid_value(chat_id, 'chat_id')
async with db.connection() as conn:
row = await chat_for_member(conn, cid, user_id)
if row is None:
raise not_found()
return chat_json(row)
# ---------- сообщения ----------
IDEMPOTENCY_KEY_RE = re.compile(r'[A-Za-z0-9._:-]{1,128}')
def idempotency_key(request: Request) -> str:
key = request.headers.get('idempotency-key')
if key is None or not IDEMPOTENCY_KEY_RE.fullmatch(key):
raise invalid('Требуется заголовок Idempotency-Key: [A-Za-z0-9._:-]{1,128}')
return key
def request_fingerprint(text: str, attachments: list[UUID]) -> bytes:
# Хешируем разобранное тело, а не байты: порядок ключей и пробелы JSON не делают запрос "другим",
# отсутствующий attachment_ids равен []. Порядок вложений значим — он сохраняется в сообщении.
canonical = json.dumps({'text': text, 'attachment_ids': [str(a) for a in attachments]},
ensure_ascii=False, sort_keys=True, separators=(',', ':'))
return hashlib.sha256(canonical.encode('utf-8')).digest()
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
async def create_message(chat_id: str, request: Request):
# Повтор заново проверяет сессию и членство: сохранённый ответ не выдаётся без текущего доступа.
user_id = await current_user(request)
cid = v.uuid_value(chat_id, 'chat_id')
key = idempotency_key(request)
data = await v.read_json(request)
# sender_id в запросе не принимается — отправитель берётся из сессии
v.fields(data, {'text'}, {'attachment_ids'})
text = v.text_field(data, 'text', 0, 4000, allow_blank=True)
attachments = v.attachment_ids(data)
if not attachments and not text.strip():
raise invalid('Нужен непустой текст или хотя бы одно вложение')
fingerprint = request_fingerprint(text, attachments)
async with db.connection() as conn:
async with conn.transaction():
member = await conn.fetchval(
'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id)
if member is None:
raise not_found()
# Ключ — "замок": параллельный INSERT того же PK с другой реплики ждёт COMMIT/ROLLBACK этой
# транзакции. Просроченный (старше TTL) ключ атомарно переиспользуется как новый.
message_id = await conn.fetchval(
'INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) '
'VALUES ($1, $2, $3, $4, gen_random_uuid()) '
'ON CONFLICT (user_id, chat_id, key) DO UPDATE '
'SET request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, '
' created_at = clock_timestamp() '
'WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => $5) '
'RETURNING message_id',
user_id, cid, key, fingerprint, settings.idempotency_ttl)
if message_id is not None:
# Ключ и сообщение фиксируются одним COMMIT: при сбое не остаётся ни того, ни другого,
# поэтому транзиентная 5xx не становится "окончательным" результатом ключа.
if attachments:
# Все вложения — из ЭТОГО чата (иначе 404, как для чужого объекта). FOR SHARE
# фиксирует строки до COMMIT; составной FK (attachment_id, chat_id) страхует на уровне БД.
found = await conn.fetchval(
'SELECT count(*) FROM (SELECT 1 FROM attachments '
"WHERE chat_id = $1 AND id = ANY($2::uuid[]) AND status = 'ready' FOR SHARE) s",
cid, attachments)
if found != len(attachments):
raise not_found()
row = await conn.fetchrow(
'INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4) '
'RETURNING id, chat_id, sender_id, text, created_at', message_id, cid, user_id, text)
if attachments:
await conn.execute(
'INSERT INTO message_attachments (message_id, attachment_id, chat_id, position) '
'SELECT $1, a.id, $2, a.ord - 1 FROM unnest($3::uuid[]) WITH ORDINALITY AS a(id, ord)',
message_id, cid, attachments)
row = {**dict(row), 'attachment_ids': attachments}
else:
row = await conn.fetchrow(
'SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at, '
+ MESSAGE_ATTACHMENTS.format(m='m') + ' '
'FROM idempotency_keys k JOIN messages m ON m.id = k.message_id '
'WHERE k.user_id = $1 AND k.chat_id = $2 AND k.key = $3', user_id, cid, key)
if row is None:
raise unavailable() # не должно случаться: ключ без сообщения не коммитится
if row['request_hash'] != fingerprint:
raise ApiError(409, 'conflict', 'Idempotency-Key уже использован с другим сообщением')
return JSONResponse(message_json(row), status_code=201)
@app.get('/api/v1/chats/{chat_id}/messages')
async def list_messages(chat_id: str, request: Request):
user_id = await current_user(request)
cid = v.uuid_value(chat_id, 'chat_id')
limit = v.limit_param(request)
after = v.decode_cursor(request, 'messages:' + str(cid))
async with db.connection() as conn:
member = await conn.fetchval(
'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id)
if member is None:
raise not_found()
if after:
# Keyset: продолжаем строго после последнего элемента прошлой страницы (без OFFSET).
rows = await conn.fetch(
f'SELECT {MESSAGE_COLUMNS} FROM messages WHERE chat_id = $1 AND (created_at, id) > ($3, $4) '
'ORDER BY created_at, id LIMIT $2', cid, limit + 1, *after)
else:
rows = await conn.fetch(
f'SELECT {MESSAGE_COLUMNS} FROM messages WHERE chat_id = $1 '
'ORDER BY created_at, id LIMIT $2', cid, limit + 1)
return page(rows, limit, 'messages:' + str(cid), message_json)
# ---------- вложения (lab4) ----------
async def attachment_for_member(conn, attachment_id: UUID, user_id: UUID):
# Доступ = членство в чате вложения. Чужое и несуществующее неразличимы (404).
return await conn.fetchrow(
f'SELECT {ATTACHMENT_COLUMNS} FROM attachments a '
'JOIN chat_members m ON m.chat_id = a.chat_id AND m.user_id = $2 WHERE a.id = $1',
attachment_id, user_id)
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=201)
async def upload_attachment(chat_id: str, request: Request):
user_id = await current_user(request)
cid = v.uuid_value(chat_id, 'chat_id')
async with db.connection() as conn:
member = await conn.fetchval(
'SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2', cid, user_id)
if member is None:
raise not_found() # посторонний не загружает ни байта в чужой чат
filename = v.filename_param(request)
declared = images.declared_type(request)
data = await images.read_limited(request, settings.max_upload_bytes)
content_type, width, height = await image_checker.inspect(data, declared)
sha256 = hashlib.sha256(data).hexdigest()
attachment_id = uuid.uuid4()
object_key = f'attachments/{attachment_id}' # назначает сервер; filename в ключе не участвует
# 1) байты в S3. Сбой -> 503, метаданных нет, клиент может повторить.
await storage.put(object_key, data, content_type)
# 2) метаданные в PostgreSQL. 201 отдаётся только после COMMIT: тогда и байты, и запись уже надёжны.
try:
async with db.connection() as conn:
row = await conn.fetchrow(
'INSERT INTO attachments (id, chat_id, owner_id, filename, content_type, size_bytes, sha256, '
'width, height, object_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) '
'RETURNING id, chat_id, owner_id, filename, content_type, size_bytes, sha256, status, '
'created_at, updated_at',
attachment_id, cid, user_id, filename, content_type, len(data), sha256, width, height, object_key)
except BaseException:
# PUT прошёл, COMMIT — нет: удаляем объект, чтобы не оставить неучтённый файл.
await storage.delete_quietly(object_key)
raise
return JSONResponse(attachment_json(row), status_code=201)
@app.get('/api/v1/attachments/{attachment_id}')
async def get_attachment(attachment_id: str, request: Request):
user_id = await current_user(request)
aid = v.uuid_value(attachment_id, 'attachment_id')
async with db.connection() as conn:
row = await attachment_for_member(conn, aid, user_id)
if row is None:
raise not_found()
return attachment_json(row)
KNOWN_VARIANTS = {'thumbnail', 'crop', 'optimized'}
@app.get('/api/v1/attachments/{attachment_id}/content')
async def download_attachment(attachment_id: str, request: Request):
user_id = await current_user(request)
aid = v.uuid_value(attachment_id, 'attachment_id')
variant = request.query_params.get('variant', 'original')
if variant != 'original' and variant not in KNOWN_VARIANTS:
raise invalid('Неизвестный variant')
async with db.connection() as conn:
row = await attachment_for_member(conn, aid, user_id)
if row is None or variant != 'original':
raise not_found() # в lab4 вариантов нет: отсутствующий вариант — 404
# Байты идут через API после проверки доступа, без redirect на S3.
data = await storage.get(row['object_key'])
return Response(data, media_type=row['content_type'], headers={
'Cache-Control': 'private, no-store',
'X-Content-Type-Options': 'nosniff',
'Content-Disposition': "inline; filename*=UTF-8''" + quote(row['filename'], safe=''),
'ETag': '"' + row['sha256'] + '"',
})