635 lines
31 KiB
Python
635 lines
31 KiB
Python
"""
|
||
HTTP-слой мессенджера (lab4).
|
||
|
||
Клиент → LB (HAProxy) → эта реплика API (их ≥2, взаимозаменяемы)
|
||
→ PostgreSQL (источник истины: данные и метаданные вложений)
|
||
→ Redis (общий кэш сессий)
|
||
→ MinIO, приватный bucket (только байты вложений)
|
||
|
||
lab2: пароль Argon2id, вход Basic → Bearer-сессия в PostgreSQL, X-User-Id игнорируется.
|
||
lab3: X-Instance-Id в каждом ответе; POST сообщения требует Idempotency-Key,
|
||
идемпотентность общая для всех реплик (таблица idempotency_keys);
|
||
у реплики нет локального состояния, от которого зависит корректность.
|
||
lab4: вложения PNG/JPEG: GET /attachments/{id} и /content (только участникам чата),
|
||
attachment_ids в сообщениях.
|
||
lab5: POST /chats/{id}/attachments → 202 queued. До ответа: оригинал целиком в приватном
|
||
staging (S3), в ОДНОЙ транзакции — вложение, job и строка outbox. Затем сообщение
|
||
публикуется в брокер (Redis Streams с AOF); если не вышло — его опубликует relay.
|
||
Тяжёлую обработку (варианты) делает отдельный процесс worker (app/worker.py).
|
||
"""
|
||
from __future__ import annotations
|
||
import asyncio
|
||
import base64
|
||
import binascii
|
||
import contextlib
|
||
import logging
|
||
import os
|
||
import re
|
||
import time
|
||
import uuid
|
||
from contextlib import asynccontextmanager
|
||
|
||
import psycopg
|
||
from fastapi import Depends, FastAPI, Header, Query, Request
|
||
from fastapi.concurrency import run_in_threadpool
|
||
from fastapi.exceptions import RequestValidationError
|
||
from fastapi.responses import JSONResponse, Response, StreamingResponse
|
||
from psycopg_pool import PoolTimeout
|
||
from pydantic import BaseModel, Field, field_validator, model_validator
|
||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||
|
||
from urllib.parse import quote
|
||
|
||
from . import broker, db, images, janitor, repo, security, sessions, storage, uploads
|
||
from .config import settings
|
||
from .migrate import migrate
|
||
|
||
logger = logging.getLogger("messenger")
|
||
|
||
# Только fullmatch: в Python "$" совпадает и перед завершающим "\n",
|
||
# поэтому match("^…$") пропустил бы "review123\n" до CHECK в БД (500).
|
||
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
|
||
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||
MAX_ATTACHMENTS_PER_MESSAGE = 10
|
||
KNOWN_VARIANTS = ("thumbnail", "crop", "optimized")
|
||
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(app: FastAPI):
|
||
if settings.migrate_on_start:
|
||
try:
|
||
applied = await migrate()
|
||
logger.info("migrations: %s", applied or "up to date")
|
||
except psycopg.OperationalError as exc:
|
||
# БД ещё не готова: API всё равно стартует, /health/ready покажет 503.
|
||
logger.warning("migrations skipped, database unavailable: %s", type(exc).__name__)
|
||
await db.open_pool()
|
||
sessions.init_redis()
|
||
storage.init()
|
||
broker.init_async()
|
||
cleaner = asyncio.create_task(janitor.loop_forever()) if settings.janitor_enabled else None
|
||
yield
|
||
if cleaner is not None:
|
||
cleaner.cancel()
|
||
with contextlib.suppress(asyncio.CancelledError):
|
||
await cleaner
|
||
await sessions.close_redis()
|
||
await broker.close_async()
|
||
await db.close_pool()
|
||
|
||
|
||
app = FastAPI(title="MTUSI Messenger — lab5", lifespan=lifespan)
|
||
|
||
|
||
# ============================================================================
|
||
# 1. ЕДИНЫЙ ФОРМАТ ОШИБОК {"error":{"code","message","request_id"}}
|
||
# ============================================================================
|
||
class ApiError(Exception):
|
||
def __init__(self, status: int, code: str, message: str, headers: dict | None = None):
|
||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||
|
||
|
||
def error_response(request: Request, status: int, code: str, message: str,
|
||
headers: dict | None = None) -> JSONResponse:
|
||
rid = getattr(request.state, "request_id", str(uuid.uuid4()))
|
||
return JSONResponse(
|
||
status_code=status,
|
||
content={"error": {"code": code, "message": message, "request_id": rid}},
|
||
headers={"X-Request-Id": rid, "X-Instance-Id": settings.instance_id, **(headers or {})},
|
||
)
|
||
|
||
|
||
@app.exception_handler(ApiError)
|
||
async def handle_api_error(request: Request, exc: ApiError):
|
||
return error_response(request, exc.status, exc.code, exc.message, exc.headers)
|
||
|
||
|
||
@app.exception_handler(RequestValidationError)
|
||
async def handle_validation(request: Request, exc: RequestValidationError):
|
||
return error_response(request, 400, "invalid_request", "invalid request body or parameters")
|
||
|
||
|
||
@app.exception_handler(StarletteHTTPException)
|
||
async def handle_http(request: Request, exc: StarletteHTTPException):
|
||
# Только коды из enum контракта (contracts/openapi.json → Error.code).
|
||
code = {401: "unauthorized", 404: "not_found", 409: "conflict",
|
||
413: "payload_too_large", 415: "unsupported_media_type",
|
||
429: "rate_limited", 503: "unavailable"}.get(
|
||
exc.status_code, "internal_error" if exc.status_code >= 500 else "invalid_request")
|
||
return error_response(request, exc.status_code, code, str(exc.detail))
|
||
|
||
|
||
@app.exception_handler(psycopg.OperationalError)
|
||
@app.exception_handler(PoolTimeout)
|
||
async def handle_db_down(request: Request, exc: Exception):
|
||
# БД недоступна или не ответила за таймаут: ограниченная ошибка, а не зависание.
|
||
logger.warning("database unavailable: %s", type(exc).__name__)
|
||
return error_response(request, 503, "unavailable", "service temporarily unavailable")
|
||
|
||
|
||
@app.exception_handler(storage.StorageUnavailable)
|
||
async def handle_storage_down(request: Request, exc: Exception):
|
||
# S3 недоступен/не ответил за таймаут: 503, текстовая часть мессенджера продолжает работать.
|
||
return error_response(request, 503, "unavailable", "file storage temporarily unavailable")
|
||
|
||
|
||
@app.exception_handler(uploads.UploadError)
|
||
async def handle_upload_error(request: Request, exc: uploads.UploadError):
|
||
return error_response(request, exc.status, exc.code, exc.message)
|
||
|
||
|
||
@app.exception_handler(Exception)
|
||
async def handle_unexpected(request: Request, exc: Exception):
|
||
logger.exception("unexpected error")
|
||
return error_response(request, 500, "internal_error", "internal error")
|
||
|
||
|
||
# ============================================================================
|
||
# 2. X-Request-Id, X-Instance-Id + лог без тела запроса, токенов и паролей
|
||
# ============================================================================
|
||
@app.middleware("http")
|
||
async def add_request_id(request: Request, call_next):
|
||
rid = str(uuid.uuid4())
|
||
request.state.request_id = rid
|
||
start = time.monotonic()
|
||
response = await call_next(request)
|
||
response.headers["X-Request-Id"] = rid
|
||
response.headers["X-Instance-Id"] = settings.instance_id # какая реплика ответила
|
||
logger.info("rid=%s %s %s -> %s %sms", rid, request.method, request.url.path,
|
||
response.status_code, round((time.monotonic() - start) * 1000, 1))
|
||
return response
|
||
|
||
|
||
# ============================================================================
|
||
# 3. АУТЕНТИФИКАЦИЯ
|
||
# Bearer — для всех бизнес-запросов. Basic — только на POST /auth/sessions.
|
||
# X-User-Id больше ничего не значит и ни на что не влияет.
|
||
# ============================================================================
|
||
def unauthorized(message: str = "invalid or missing bearer token") -> ApiError:
|
||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": BEARER_CHALLENGE})
|
||
|
||
|
||
def _bearer_token(request: Request) -> str:
|
||
auth = request.headers.get("authorization", "")
|
||
scheme, _, token = auth.partition(" ")
|
||
token = token.strip()
|
||
if scheme.lower() != "bearer" or not token or len(token) > 512:
|
||
raise unauthorized()
|
||
return token
|
||
|
||
|
||
async def current_user(request: Request) -> dict:
|
||
"""Зависимость FastAPI: выполняется ДО разбора тела → сначала 401, потом 400."""
|
||
token = _bearer_token(request)
|
||
user = await sessions.resolve(db.get_pool(), token, repo.get_user)
|
||
if user is None:
|
||
raise unauthorized()
|
||
request.state.token = token
|
||
return user
|
||
|
||
|
||
def _parse_basic(request: Request) -> tuple[str, str]:
|
||
bad = ApiError(401, "unauthorized", "invalid credentials", {"WWW-Authenticate": BASIC_CHALLENGE})
|
||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||
if scheme.lower() != "basic" or not value:
|
||
raise bad
|
||
try:
|
||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||
except (binascii.Error, UnicodeDecodeError):
|
||
raise bad
|
||
username, sep, password = decoded.partition(":") # двоеточие в пароле допустимо
|
||
if not sep or not username.isascii() or not (12 <= len(password) <= 128):
|
||
raise bad
|
||
return username, password
|
||
|
||
|
||
def require_json(request: Request) -> None:
|
||
"""Тип тела проверяем ДО валидации тела: не JSON → 415, а не 400.
|
||
Подключается после current_user, поэтому без токена по-прежнему сначала 401.
|
||
Параметры вроде '; charset=utf-8' допустимы."""
|
||
media_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
|
||
if media_type != "application/json":
|
||
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
|
||
|
||
|
||
_kdf_slots: asyncio.Semaphore | None = None
|
||
|
||
|
||
async def kdf(fn, *args):
|
||
"""Argon2id по умолчанию берёт 64 МиБ RAM на один хэш. Без ограничения 20
|
||
одновременных регистраций/входов на реплике = >1 ГиБ и OOM-kill контейнера
|
||
с лимитом 256 МиБ. Поэтому одновременно считаем не больше
|
||
PASSWORD_HASH_CONCURRENCY хэшей, остальные ждут в очереди."""
|
||
global _kdf_slots
|
||
if _kdf_slots is None:
|
||
_kdf_slots = asyncio.Semaphore(settings.password_hash_concurrency)
|
||
async with _kdf_slots:
|
||
return await run_in_threadpool(fn, *args)
|
||
|
||
|
||
# ============================================================================
|
||
# 4. МОДЕЛИ ВВОДА
|
||
# ============================================================================
|
||
class CreateUser(BaseModel):
|
||
model_config = {"extra": "forbid"}
|
||
username: str
|
||
display_name: str
|
||
password: str
|
||
|
||
@field_validator("username")
|
||
@classmethod
|
||
def check_username(cls, v: str) -> str:
|
||
if not USERNAME_RE.fullmatch(v):
|
||
raise ValueError("username must match [a-z0-9_]{3,32}")
|
||
return v
|
||
|
||
@field_validator("display_name")
|
||
@classmethod
|
||
def check_display_name(cls, v: str) -> str:
|
||
if not (1 <= len(v) <= 100) or not v.strip():
|
||
raise ValueError("display_name must be 1..100 non-blank chars")
|
||
return v
|
||
|
||
@field_validator("password")
|
||
@classmethod
|
||
def check_password(cls, v: str) -> str:
|
||
if not (12 <= len(v) <= 128) or not v.strip():
|
||
raise ValueError("password must be 12..128 chars")
|
||
return v
|
||
|
||
|
||
class CreateChat(BaseModel):
|
||
model_config = {"extra": "forbid"}
|
||
title: str
|
||
member_ids: list[str]
|
||
|
||
@field_validator("title")
|
||
@classmethod
|
||
def check_title(cls, v: str) -> str:
|
||
if not (1 <= len(v) <= 100) or not v.strip():
|
||
raise ValueError("title must be 1..100 non-blank chars")
|
||
return v
|
||
|
||
|
||
class CreateMessage(BaseModel):
|
||
model_config = {"extra": "forbid"}
|
||
text: str
|
||
# lab4: 0–10 уникальных id вложений этого чата; отсутствие поля == [].
|
||
attachment_ids: list[str] = Field(default_factory=list)
|
||
|
||
@field_validator("text")
|
||
@classmethod
|
||
def check_text(cls, v: str) -> str:
|
||
if len(v) > 4000:
|
||
raise ValueError("text must be at most 4000 chars")
|
||
return v
|
||
|
||
@field_validator("attachment_ids")
|
||
@classmethod
|
||
def check_attachment_ids(cls, v: list[str]) -> list[str]:
|
||
if len(v) > MAX_ATTACHMENTS_PER_MESSAGE:
|
||
raise ValueError("at most 10 attachment_ids")
|
||
if len(set(v)) != len(v):
|
||
raise ValueError("attachment_ids must be unique")
|
||
return v
|
||
|
||
@model_validator(mode="after")
|
||
def text_or_attachments(self):
|
||
# Пустой/пробельный текст допустим только вместе с вложениями.
|
||
if not self.attachment_ids and not self.text.strip():
|
||
raise ValueError("text must be non-blank when there are no attachments")
|
||
return self
|
||
|
||
|
||
def parse_uuid(value: str, what: str) -> uuid.UUID:
|
||
try:
|
||
return uuid.UUID(value)
|
||
except ValueError:
|
||
raise ApiError(400, "invalid_request", f"{what} is not a valid uuid")
|
||
|
||
|
||
# ============================================================================
|
||
# 5. ENDPOINTS
|
||
# ============================================================================
|
||
@app.get("/health/live")
|
||
async def health_live():
|
||
return {"status": "ok"}
|
||
|
||
|
||
@app.get("/health/ready")
|
||
async def health_ready(request: Request):
|
||
# Критичная зависимость — только PostgreSQL. Redis — кэш: его отказ не делает API неготовым.
|
||
try:
|
||
async with db.get_pool().connection() as conn:
|
||
await conn.execute("SELECT 1 FROM schema_migrations LIMIT 1")
|
||
except (psycopg.Error, PoolTimeout):
|
||
return error_response(request, 503, "unavailable", "database is not ready")
|
||
return {"status": "ok"}
|
||
|
||
|
||
# -------- users и сессии --------
|
||
@app.post("/api/v1/users", status_code=201)
|
||
async def create_user(body: CreateUser, _json: None = Depends(require_json)):
|
||
# Argon2 намеренно медленный — считаем в пуле потоков, чтобы не блокировать event loop.
|
||
password_hash = await kdf(security.hash_password, body.password)
|
||
async with db.get_pool().connection() as conn:
|
||
try:
|
||
return await repo.create_user(conn, body.username, body.display_name, password_hash)
|
||
except repo.UsernameTaken:
|
||
raise ApiError(409, "conflict", "username already taken")
|
||
|
||
|
||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||
async def create_session(request: Request):
|
||
username, password = _parse_basic(request)
|
||
async with db.get_pool().connection() as conn:
|
||
row = await repo.get_credentials(conn, username)
|
||
stored_hash = row[4] if row else None
|
||
# Проверяем хэш даже для несуществующего пользователя — одинаковое время ответа.
|
||
ok = await kdf(security.verify_password, stored_hash, password)
|
||
if not ok:
|
||
raise ApiError(401, "unauthorized", "invalid credentials", {"WWW-Authenticate": BASIC_CHALLENGE})
|
||
user = repo.user_out(row)
|
||
async with db.get_pool().connection() as conn:
|
||
if security.needs_rehash(stored_hash):
|
||
await repo.update_password_hash(conn, row[0], await kdf(security.hash_password, password))
|
||
token, expires_at = await sessions.create_session(conn, row[0])
|
||
return {"token": token, "token_type": "Bearer", "expires_at": repo.iso(expires_at), "user": user}
|
||
|
||
|
||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||
async def delete_session(request: Request, user: dict = Depends(current_user)):
|
||
try:
|
||
revoked = await sessions.revoke(db.get_pool(), request.state.token)
|
||
except sessions.CacheUnavailable:
|
||
# 204 только при гарантированном отзыве; иначе ограниченная ошибка, повторить позже.
|
||
raise ApiError(503, "unavailable", "session cache unavailable, logout not confirmed")
|
||
if not revoked:
|
||
raise unauthorized()
|
||
return Response(status_code=204)
|
||
|
||
|
||
@app.get("/api/v1/users/me")
|
||
async def get_me(user: dict = Depends(current_user)):
|
||
return user
|
||
|
||
|
||
# -------- chats --------
|
||
@app.post("/api/v1/chats", status_code=201)
|
||
async def create_chat(body: CreateChat, user: dict = Depends(current_user),
|
||
_json: None = Depends(require_json)):
|
||
members = body.member_ids
|
||
if not (1 <= len(members) <= 100):
|
||
raise ApiError(400, "invalid_request", "member_ids must have 1..100 items")
|
||
if len(set(members)) != len(members):
|
||
raise ApiError(400, "invalid_request", "duplicate member_ids")
|
||
ids = [parse_uuid(m, "member id") for m in members]
|
||
if uuid.UUID(user["id"]) not in ids:
|
||
raise ApiError(400, "invalid_request", "caller must be a member")
|
||
async with db.get_pool().connection() as conn:
|
||
# Одна транзакция: чат и все участники создаются вместе или не создаются вовсе.
|
||
async with conn.transaction():
|
||
if await repo.count_existing_users(conn, ids) != len(ids):
|
||
raise ApiError(400, "invalid_request", "unknown member")
|
||
return await repo.insert_chat(conn, body.title, ids, user["id"])
|
||
|
||
|
||
@app.get("/api/v1/chats")
|
||
async def list_chats(user: dict = Depends(current_user),
|
||
limit: int = Query(default=50, ge=1, le=100),
|
||
cursor: str | None = Query(default=None, min_length=1)):
|
||
async with db.get_pool().connection() as conn:
|
||
try:
|
||
items, nxt = await repo.list_chats(conn, user["id"], limit, cursor)
|
||
except ValueError:
|
||
raise ApiError(400, "invalid_request", "bad cursor")
|
||
return {"items": items, "next_cursor": nxt}
|
||
|
||
|
||
async def accessible_chat(conn, chat_id: str, user: dict) -> dict:
|
||
"""Кривой UUID → 400. Чата нет ИЛИ ты не участник → одинаковый 404."""
|
||
cid = parse_uuid(chat_id, "chat_id")
|
||
chat = await repo.get_chat_for_member(conn, cid, user["id"])
|
||
if chat is None:
|
||
raise ApiError(404, "not_found", "chat not found")
|
||
return chat
|
||
|
||
|
||
@app.get("/api/v1/chats/{chat_id}")
|
||
async def get_chat(chat_id: str, user: dict = Depends(current_user)):
|
||
async with db.get_pool().connection() as conn:
|
||
return await accessible_chat(conn, chat_id, user)
|
||
|
||
|
||
# -------- messages --------
|
||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||
async def send_message(chat_id: str, body: CreateMessage, user: dict = Depends(current_user),
|
||
_json: None = Depends(require_json),
|
||
idempotency_key: str | None = Header(default=None, alias="Idempotency-Key")):
|
||
# Ключ обязателен: без него клиент не может безопасно повторить запрос после таймаута.
|
||
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
||
raise ApiError(400, "invalid_request", "Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}")
|
||
attachment_ids = [parse_uuid(a, "attachment_id") for a in body.attachment_ids]
|
||
if len(set(attachment_ids)) != len(attachment_ids):
|
||
raise ApiError(400, "invalid_request", "attachment_ids must be unique")
|
||
async with db.get_pool().connection() as conn:
|
||
# Повтор снова проверяет сессию (current_user) и доступ к чату.
|
||
chat = await accessible_chat(conn, chat_id, user)
|
||
try:
|
||
async with conn.transaction():
|
||
# Отправитель — из сессии, а не из тела запроса.
|
||
return await repo.send_message_idempotent(
|
||
conn, chat["id"], user["id"], idempotency_key, body.text,
|
||
settings.idempotency_ttl_seconds, attachment_ids)
|
||
except repo.IdempotencyConflict:
|
||
raise ApiError(409, "conflict", "Idempotency-Key was already used with a different request")
|
||
except repo.AttachmentNotFound:
|
||
# Нет такого вложения ИЛИ оно из другого чата — одинаковый 404.
|
||
raise ApiError(404, "not_found", "attachment not found")
|
||
except repo.AttachmentNotReady:
|
||
raise ApiError(409, "conflict", "attachment is not usable")
|
||
|
||
|
||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||
async def list_messages(chat_id: str, user: dict = Depends(current_user),
|
||
limit: int = Query(default=50, ge=1, le=100),
|
||
cursor: str | None = Query(default=None, min_length=1)):
|
||
async with db.get_pool().connection() as conn:
|
||
chat = await accessible_chat(conn, chat_id, user)
|
||
try:
|
||
items, nxt = await repo.list_messages(conn, chat["id"], limit, cursor)
|
||
except ValueError:
|
||
raise ApiError(400, "invalid_request", "bad cursor")
|
||
return {"items": items, "next_cursor": nxt}
|
||
|
||
|
||
# ============================================================================
|
||
# 6. ВЛОЖЕНИЯ (lab4): байты — в приватном S3, метаданные — в PostgreSQL
|
||
# ============================================================================
|
||
_upload_slots: asyncio.Semaphore | None = None
|
||
_image_checks: asyncio.Semaphore | None = None
|
||
|
||
|
||
def _slots() -> tuple[asyncio.Semaphore, asyncio.Semaphore]:
|
||
global _upload_slots, _image_checks
|
||
if _upload_slots is None:
|
||
_upload_slots = asyncio.Semaphore(settings.upload_max_concurrent)
|
||
_image_checks = asyncio.Semaphore(settings.image_check_concurrency)
|
||
return _upload_slots, _image_checks
|
||
|
||
|
||
async def _discard_upload(key: str) -> None:
|
||
"""Компенсация: удалить объект и строку журнала. Если не вышло —
|
||
строка журнала останется, и объект удалит janitor после grace period."""
|
||
try:
|
||
async with db.get_pool().connection() as conn:
|
||
async with conn.transaction():
|
||
cur = await conn.execute(
|
||
"DELETE FROM upload_intents WHERE object_key = %s RETURNING 1", (key,))
|
||
if await cur.fetchone() is not None:
|
||
await storage.delete(key)
|
||
except Exception as exc: # noqa: BLE001 — компенсация best effort
|
||
logger.warning("upload compensation failed, janitor will clean up: %s", type(exc).__name__)
|
||
|
||
|
||
def _fault(request: Request, point: str) -> None:
|
||
"""Внедрение отказа для тестов уровня 4 — только при FAULT_INJECTION=true
|
||
(отдельная тестовая реплика api-fault). В обычных репликах — no-op."""
|
||
if not settings.fault_injection or request.headers.get("x-fault-inject") != point:
|
||
return
|
||
logger.warning("FAULT INJECTION: %s", point)
|
||
if point in ("crash-after-s3-put", "crash-after-commit-before-publish"):
|
||
os._exit(1) # как kill -9: процесс умирает мгновенно, без finally
|
||
raise RuntimeError("injected fault: " + point)
|
||
|
||
|
||
async def _publish_now(outbox_id: int, payload: dict) -> None:
|
||
"""Быстрый путь после COMMIT: сразу отдать сообщение брокеру и отметить строку outbox.
|
||
Любая ошибка здесь НЕ влияет на ответ 202: строка outbox останется неопубликованной,
|
||
и relay (app/worker.py, поток maintenance) опубликует её позже."""
|
||
try:
|
||
async with asyncio.timeout(settings.broker_timeout):
|
||
stream_id = await broker.publish(payload) # ответ XADD = брокер принял (AOF fsync)
|
||
async with db.get_pool().connection() as conn:
|
||
await repo.mark_published(conn, outbox_id, stream_id)
|
||
except Exception as exc: # noqa: BLE001 — намерение уже в outbox
|
||
logger.warning("direct publish failed, outbox relay will retry: %s", type(exc).__name__)
|
||
|
||
|
||
@app.post("/api/v1/chats/{chat_id}/attachments", status_code=202)
|
||
async def upload_attachment(chat_id: str, request: Request, user: dict = Depends(current_user),
|
||
filename: str | None = Query(default=None)):
|
||
# 1) доступ к чату → 2) имя → 3) заявленный тип → 4) заявленный размер; всё — до чтения тела.
|
||
async with db.get_pool().connection() as conn:
|
||
chat = await accessible_chat(conn, chat_id, user)
|
||
name = uploads.validate_filename(filename)
|
||
ctype = uploads.declared_type(request)
|
||
uploads.check_declared_length(request, settings.max_upload_bytes)
|
||
|
||
slots, checks = _slots()
|
||
async with slots: # не больше N загрузок на реплику
|
||
body = await uploads.receive(request, settings.max_upload_bytes,
|
||
settings.upload_body_timeout_seconds,
|
||
settings.upload_spool_memory_bytes)
|
||
try:
|
||
async with checks: # CPU-тяжёлая проверка — ограниченно
|
||
try:
|
||
info = await run_in_threadpool(images.inspect, body.file, ctype,
|
||
settings.max_image_pixels)
|
||
except images.ImageTooLarge:
|
||
raise ApiError(413, "payload_too_large", "image has too many pixels")
|
||
except images.BadImage:
|
||
raise ApiError(415, "unsupported_media_type",
|
||
"file is not a valid image of the declared type")
|
||
|
||
storage.ensure_available() # S3 заведомо недоступен → 503 без записи в журнал
|
||
attachment_id = uuid.uuid4()
|
||
key = storage.staging_key(attachment_id) # ключ назначает сервер; сначала — staging
|
||
async with db.get_pool().connection() as conn:
|
||
await repo.add_upload_intent(conn, key, attachment_id) # журнал ДО PUT
|
||
try:
|
||
await storage.put(key, body.file, body.size, info.content_type,
|
||
body.md5_b64, body.sha256_hex)
|
||
except storage.StorageUnavailable:
|
||
await _discard_upload(key)
|
||
raise
|
||
try:
|
||
_fault(request, "crash-after-s3-put")
|
||
_fault(request, "db-error-after-s3-put")
|
||
async with db.get_pool().connection() as conn:
|
||
async with conn.transaction():
|
||
att, outbox_id, payload = await repo.commit_attachment(
|
||
conn, attachment_id=attachment_id, chat_id=chat["id"],
|
||
owner_id=user["id"], filename=name, content_type=info.content_type,
|
||
size_bytes=body.size, sha256=body.sha256_hex,
|
||
width=info.width, height=info.height,
|
||
object_key=storage.object_key(attachment_id), staging_key=key,
|
||
pipeline_version=settings.pipeline_version,
|
||
max_attempts=settings.job_max_attempts)
|
||
except Exception as exc:
|
||
# PUT прошёл, COMMIT — нет: компенсируем сразу, 202 не отдаём.
|
||
logger.warning("attachment commit failed: %s", type(exc).__name__)
|
||
await _discard_upload(key)
|
||
raise ApiError(503, "unavailable", "attachment was not saved, retry later")
|
||
# COMMIT прошёл: оригинал в staging, job и outbox в БД — работа уже не потеряется.
|
||
_fault(request, "crash-after-commit-before-publish")
|
||
await _publish_now(outbox_id, payload)
|
||
return att
|
||
finally:
|
||
body.file.close() # буфер запроса удаляется всегда
|
||
|
||
|
||
async def accessible_attachment(conn, attachment_id: str, user: dict):
|
||
aid = parse_uuid(attachment_id, "attachment_id")
|
||
row = await repo.get_attachment_for_member(conn, aid, user["id"])
|
||
if row is None:
|
||
# Нет вложения ИЛИ пользователь не участник его чата — одинаковый 404.
|
||
raise ApiError(404, "not_found", "attachment not found")
|
||
return row
|
||
|
||
|
||
@app.get("/api/v1/attachments/{attachment_id}")
|
||
async def get_attachment(attachment_id: str, user: dict = Depends(current_user)):
|
||
async with db.get_pool().connection() as conn:
|
||
row = await accessible_attachment(conn, attachment_id, user)
|
||
variants = await repo.get_variants(conn, row[0], settings.pipeline_version) \
|
||
if row[7] == "ready" else []
|
||
return repo.attachment_out(row, variants)
|
||
|
||
|
||
@app.get("/api/v1/attachments/{attachment_id}/content")
|
||
async def download_attachment(attachment_id: str, user: dict = Depends(current_user),
|
||
variant: str = Query(default="original")):
|
||
parse_uuid(attachment_id, "attachment_id")
|
||
if variant != "original" and variant not in KNOWN_VARIANTS:
|
||
raise ApiError(400, "invalid_request", "unknown variant")
|
||
async with db.get_pool().connection() as conn:
|
||
row = await accessible_attachment(conn, attachment_id, user)
|
||
att = repo.attachment_out(row)
|
||
if att["status"] != "ready":
|
||
# queued / processing / failed: байтов результата ещё (или уже) нет.
|
||
raise ApiError(409, "conflict", "attachment is not ready")
|
||
key, ctype, etag = row[11], att["content_type"], att["sha256"]
|
||
if variant != "original":
|
||
v = await repo.get_variant(conn, row[0], settings.pipeline_version, variant)
|
||
if v is None:
|
||
raise ApiError(404, "not_found", "variant not found")
|
||
key, ctype, etag = v[0], v[1], v[3]
|
||
try:
|
||
stream, length = await storage.open_object(key)
|
||
except storage.ObjectMissing:
|
||
logger.error("object for attachment %s is missing in storage", att["id"])
|
||
raise ApiError(500, "internal_error", "internal error")
|
||
headers = {
|
||
"Content-Length": str(length),
|
||
"ETag": f'"{etag}"',
|
||
"Cache-Control": "private, no-cache",
|
||
"X-Content-Type-Options": "nosniff",
|
||
# filename только для отображения; RFC 5987 — любые символы безопасно закодированы.
|
||
"Content-Disposition": "inline; filename*=UTF-8''" + quote(att["filename"], safe=""),
|
||
}
|
||
# Байты идут из S3 клиенту кусками по 64 КиБ — файл целиком в RAM не загружается.
|
||
return StreamingResponse(storage.iter_body(stream), media_type=ctype, headers=headers)
|