- S3: SeaweedFS 4.51 (MinIO-образы недоступны), приватный bucket, отдельные ограниченные ключи API, служебные порты закрыты; идемпотентный s3-init с проверкой приватности и прогревом записи - вложения: потоковая загрузка PNG/JPEG, лимиты 10 MiB / 40 Мпикс, тип по содержимому, проверка filename, скачивание только участникам - согласованность PG/S3: pending -> PUT -> ready, компенсация, sweeper с grace и dry-run; attachment_ids в сообщениях, FK по чату - back-pressure на S3 PUT после найденного OOM SeaweedFS - тесты student-tests, evidence, REPORT.md, DEMO.md
375 lines
15 KiB
Python
375 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import re
|
|
import tempfile
|
|
import uuid
|
|
from contextlib import asynccontextmanager
|
|
from typing import Optional
|
|
|
|
import asyncpg
|
|
from fastapi import Depends, FastAPI, Header, Query, Request, Response
|
|
from fastapi.exceptions import RequestValidationError
|
|
from fastapi.responses import JSONResponse, StreamingResponse
|
|
from starlette.background import BackgroundTask
|
|
from starlette.concurrency import run_in_threadpool
|
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
|
from starlette.requests import ClientDisconnect
|
|
|
|
from app import auth, config, images
|
|
from app.cache import SessionCache
|
|
from app.errors import ApiError, payload_too_large, unsupported_media_type
|
|
from app.models import (Attachment, Chat, ChatPage, CreateChat, CreateMessage, CreateUser, Health, Message, MessagePage,
|
|
Session, User)
|
|
from app.objects import ObjectStore, S3Unavailable, storage_unavailable
|
|
from app.storage import Store, attachment_row
|
|
|
|
logging.basicConfig(level=logging.INFO)
|
|
logger = logging.getLogger("messenger")
|
|
|
|
store = Store()
|
|
cache = SessionCache(config.CACHE_URL)
|
|
objects = ObjectStore()
|
|
upload_slots = asyncio.Semaphore(config.UPLOAD_CONCURRENCY)
|
|
decode_slots = asyncio.Semaphore(config.DECODE_CONCURRENCY)
|
|
s3_put_slots = asyncio.Semaphore(config.S3_PUT_CONCURRENCY)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(_app: FastAPI):
|
|
if not config.DATABASE_URL:
|
|
raise RuntimeError("DATABASE_URL or DATABASE_URL_FILE is required")
|
|
if not (config.S3_ACCESS_KEY_ID and config.S3_SECRET_ACCESS_KEY):
|
|
raise RuntimeError("S3_ACCESS_KEY_ID(_FILE) and S3_SECRET_ACCESS_KEY(_FILE) are required")
|
|
await store.open()
|
|
logger.info("session cache: %s", "valkey" if cache.enabled else "disabled")
|
|
logger.info("object storage: %s bucket=%s", config.S3_ENDPOINT, config.S3_BUCKET)
|
|
try:
|
|
with open(config.INSTANCE_ID_FILE, "w", encoding="ascii") as fh:
|
|
fh.write(config.INSTANCE_ID + "\n")
|
|
except OSError:
|
|
logger.warning("cannot write %s", config.INSTANCE_ID_FILE)
|
|
logger.info("instance id: %s", config.INSTANCE_ID)
|
|
yield
|
|
await cache.close()
|
|
await store.close()
|
|
|
|
|
|
app = FastAPI(title="MTUSI Messenger — lab4", lifespan=lifespan)
|
|
|
|
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
|
|
|
|
|
DB_UNAVAILABLE = (
|
|
OSError,
|
|
asyncpg.PostgresConnectionError,
|
|
asyncpg.InterfaceError,
|
|
asyncpg.TooManyConnectionsError,
|
|
asyncpg.exceptions.OperatorInterventionError,
|
|
asyncpg.exceptions.QueryCanceledError,
|
|
)
|
|
|
|
|
|
def error_body(code: str, message: str, request_id: str) -> dict:
|
|
return {"error": {"code": code, "message": message, "request_id": request_id}}
|
|
|
|
|
|
def error_response(request: Request, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse:
|
|
request_id = getattr(request.state, "request_id", "")
|
|
return JSONResponse(status_code=status, content=error_body(code, message, request_id), headers=headers)
|
|
|
|
|
|
@app.middleware("http")
|
|
async def request_id_middleware(request: Request, call_next):
|
|
request_id = uuid.uuid4().hex
|
|
request.state.request_id = request_id
|
|
try:
|
|
response = await call_next(request)
|
|
except Exception:
|
|
logger.exception("unhandled error for request_id=%s", request_id)
|
|
response = JSONResponse(status_code=500, content=error_body("internal_error", "internal error", request_id))
|
|
response.headers["X-Request-Id"] = request_id
|
|
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
|
return response
|
|
|
|
|
|
@app.exception_handler(ApiError)
|
|
async def api_error_handler(request: Request, exc: ApiError):
|
|
return error_response(request, exc.status_code, exc.code, exc.message, exc.headers)
|
|
|
|
|
|
@app.exception_handler(RequestValidationError)
|
|
async def validation_error_handler(request: Request, exc: RequestValidationError):
|
|
return error_response(request, 400, "invalid_request", "request failed validation")
|
|
|
|
|
|
@app.exception_handler(StarletteHTTPException)
|
|
async def http_exception_handler(request: Request, exc: StarletteHTTPException):
|
|
code = {404: "not_found", 405: "invalid_request", 413: "payload_too_large", 415: "unsupported_media_type"}.get(
|
|
exc.status_code, "invalid_request")
|
|
message = exc.detail if isinstance(exc.detail, str) else "request failed"
|
|
return error_response(request, exc.status_code, code, message)
|
|
|
|
|
|
async def db_unavailable_handler(request: Request, exc: Exception):
|
|
logger.warning("database unavailable for request_id=%s: %s", request.state.request_id, type(exc).__name__)
|
|
return error_response(request, 503, "unavailable", "service temporarily unavailable")
|
|
|
|
|
|
for _exc in DB_UNAVAILABLE:
|
|
app.add_exception_handler(_exc, db_unavailable_handler)
|
|
|
|
|
|
@app.exception_handler(S3Unavailable)
|
|
async def s3_unavailable_handler(request: Request, exc: S3Unavailable):
|
|
err = storage_unavailable()
|
|
return error_response(request, err.status_code, err.code, err.message)
|
|
|
|
|
|
async def current_user(request: Request) -> dict:
|
|
return await auth.authenticate(store, cache, request.headers.get("Authorization"))
|
|
|
|
|
|
def parse_uuid(value: str, field: str) -> uuid.UUID:
|
|
try:
|
|
return uuid.UUID(value)
|
|
except ValueError:
|
|
raise ApiError(400, "invalid_request", f"{field} is not a valid UUID") from None
|
|
|
|
|
|
def not_found() -> ApiError:
|
|
return ApiError(404, "not_found", "chat not found")
|
|
|
|
|
|
@app.get("/health/live", response_model=Health)
|
|
async def liveness():
|
|
return Health()
|
|
|
|
|
|
@app.get("/health/ready", response_model=Health)
|
|
async def readiness():
|
|
try:
|
|
await store.ping()
|
|
except DB_UNAVAILABLE:
|
|
raise ApiError(503, "unavailable", "database is not available") from None
|
|
return Health()
|
|
|
|
|
|
@app.post("/api/v1/users", response_model=User, status_code=201)
|
|
async def register_user(body: CreateUser):
|
|
if not body.display_name.strip() or not body.password.strip():
|
|
raise ApiError(400, "invalid_request", "display_name and password must not be blank")
|
|
password_hash = await auth.hash_password(body.password)
|
|
return await store.create_user(body.username, body.display_name, password_hash)
|
|
|
|
|
|
@app.get("/api/v1/users/me", response_model=User)
|
|
async def current_user_endpoint(user: dict = Depends(current_user)):
|
|
return user
|
|
|
|
|
|
@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201)
|
|
async def create_session(request: Request):
|
|
return await auth.login(store, request.headers.get("Authorization"))
|
|
|
|
|
|
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
|
async def delete_session(request: Request):
|
|
await auth.logout(store, cache, request.headers.get("Authorization"))
|
|
return Response(status_code=204)
|
|
|
|
|
|
@app.post("/api/v1/chats", response_model=Chat, status_code=201)
|
|
async def create_chat(body: CreateChat, user: dict = Depends(current_user)):
|
|
if not body.title.strip():
|
|
raise ApiError(400, "invalid_request", "title must not be blank")
|
|
if len(set(body.member_ids)) != len(body.member_ids):
|
|
raise ApiError(400, "invalid_request", "member_ids must not contain duplicates")
|
|
if uuid.UUID(user["id"]) not in body.member_ids:
|
|
raise ApiError(400, "invalid_request", "caller must be a member of the chat")
|
|
return await store.create_chat(body.title, list(body.member_ids))
|
|
|
|
|
|
@app.get("/api/v1/chats", response_model=ChatPage)
|
|
async def list_chats(
|
|
user: dict = Depends(current_user),
|
|
limit: int = Query(default=50, ge=1, le=100),
|
|
cursor: Optional[str] = Query(default=None, min_length=1),
|
|
):
|
|
return await store.list_chats(uuid.UUID(user["id"]), limit, cursor)
|
|
|
|
|
|
@app.get("/api/v1/chats/{chat_id}", response_model=Chat)
|
|
async def get_chat(chat_id: str, user: dict = Depends(current_user)):
|
|
chat = await store.get_chat_for_member(parse_uuid(chat_id, "chat_id"), uuid.UUID(user["id"]))
|
|
if chat is None:
|
|
raise not_found()
|
|
return chat
|
|
|
|
|
|
@app.post("/api/v1/chats/{chat_id}/messages", response_model=Message, status_code=201)
|
|
async def create_message(
|
|
chat_id: str,
|
|
body: CreateMessage,
|
|
user: dict = Depends(current_user),
|
|
idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"),
|
|
):
|
|
chat_uuid = parse_uuid(chat_id, "chat_id")
|
|
if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key):
|
|
raise ApiError(400, "invalid_request", "Idempotency-Key header is required: [A-Za-z0-9._:-]{1,128}")
|
|
if len(set(body.attachment_ids)) != len(body.attachment_ids):
|
|
raise ApiError(400, "invalid_request", "attachment_ids must be unique")
|
|
if not body.text.strip() and not body.attachment_ids:
|
|
if not await store.is_member(chat_uuid, uuid.UUID(user["id"])):
|
|
raise not_found()
|
|
raise ApiError(400, "invalid_request", "text must not be blank without attachments")
|
|
canonical = json.dumps(body.model_dump(mode="json"), sort_keys=True, separators=(",", ":"), ensure_ascii=False)
|
|
request_hash = hashlib.sha256(canonical.encode("utf-8")).digest()
|
|
message = await store.create_message(
|
|
chat_uuid, uuid.UUID(user["id"]), body.text, list(body.attachment_ids), idempotency_key, request_hash)
|
|
if message is None:
|
|
raise not_found()
|
|
return message
|
|
|
|
|
|
@app.get("/api/v1/chats/{chat_id}/messages", response_model=MessagePage)
|
|
async def list_messages(
|
|
chat_id: str,
|
|
user: dict = Depends(current_user),
|
|
limit: int = Query(default=50, ge=1, le=100),
|
|
cursor: Optional[str] = Query(default=None, min_length=1),
|
|
):
|
|
chat_uuid = parse_uuid(chat_id, "chat_id")
|
|
if not await store.is_member(chat_uuid, uuid.UUID(user["id"])):
|
|
raise not_found()
|
|
return await store.list_messages(chat_uuid, limit, cursor)
|
|
|
|
|
|
VARIANTS = {"original", "thumbnail", "crop", "optimized"}
|
|
|
|
|
|
async def receive_body(request: Request, spool) -> tuple:
|
|
"""Stream the body into a spooled temp file: RAM holds at most UPLOAD_SPOOL_BYTES per upload."""
|
|
digest = hashlib.sha256()
|
|
size = 0
|
|
head = b""
|
|
async for chunk in request.stream():
|
|
size += len(chunk)
|
|
if size > config.MAX_UPLOAD_BYTES:
|
|
raise payload_too_large("file exceeds %d bytes" % config.MAX_UPLOAD_BYTES)
|
|
if len(head) < images.SNIFF_BYTES:
|
|
head += chunk[:images.SNIFF_BYTES - len(head)]
|
|
digest.update(chunk)
|
|
spool.write(chunk)
|
|
return size, digest.hexdigest(), head
|
|
|
|
|
|
@app.post("/api/v1/chats/{chat_id}/attachments", response_model=Attachment, status_code=201)
|
|
async def upload_attachment(
|
|
chat_id: str,
|
|
request: Request,
|
|
user: dict = Depends(current_user),
|
|
filename: Optional[str] = Query(default=None),
|
|
):
|
|
chat_uuid = parse_uuid(chat_id, "chat_id")
|
|
owner_id = uuid.UUID(user["id"])
|
|
if not await store.is_member(chat_uuid, owner_id):
|
|
raise not_found()
|
|
filename = images.validate_filename(filename)
|
|
declared = images.declared_type(request.headers.get("Content-Type"))
|
|
length = request.headers.get("Content-Length")
|
|
if length is not None:
|
|
if not length.isdigit():
|
|
raise ApiError(400, "invalid_request", "invalid Content-Length")
|
|
if int(length) > config.MAX_UPLOAD_BYTES:
|
|
raise payload_too_large("file exceeds %d bytes" % config.MAX_UPLOAD_BYTES)
|
|
|
|
async with upload_slots:
|
|
spool = tempfile.SpooledTemporaryFile(max_size=config.UPLOAD_SPOOL_BYTES, dir=config.UPLOAD_TMP_DIR)
|
|
try:
|
|
try:
|
|
size, sha256, head = await asyncio.wait_for(receive_body(request, spool), config.UPLOAD_TIMEOUT_SECONDS)
|
|
except asyncio.TimeoutError:
|
|
raise ApiError(400, "invalid_request", "upload was not completed in time") from None
|
|
except ClientDisconnect:
|
|
logger.info("upload aborted by client after partial body; nothing stored")
|
|
raise ApiError(400, "invalid_request", "upload interrupted") from None
|
|
if size == 0:
|
|
raise unsupported_media_type("empty body is not an image")
|
|
detected = images.sniff(head)
|
|
if detected is None:
|
|
raise unsupported_media_type("content is neither PNG nor JPEG")
|
|
if detected != declared:
|
|
raise unsupported_media_type("content does not match Content-Type " + declared)
|
|
async with decode_slots:
|
|
width, height = await run_in_threadpool(images.inspect, spool, detected)
|
|
|
|
attachment_id = uuid.uuid4()
|
|
key = ObjectStore.new_key(uuid.uuid4())
|
|
# 1) intent row (invisible) -> 2) S3 PUT -> 3) mark ready. A crash leaves a pending row for the sweeper.
|
|
if not await store.create_pending_attachment(
|
|
attachment_id, chat_uuid, owner_id, filename, detected, size, sha256, width, height, key,
|
|
):
|
|
raise not_found()
|
|
try:
|
|
async with s3_put_slots:
|
|
await objects.put(key, spool, size, detected, sha256)
|
|
except S3Unavailable:
|
|
await objects.delete_quietly(key)
|
|
await store.delete_pending_attachment(attachment_id)
|
|
raise
|
|
try:
|
|
ready = await store.mark_attachment_ready(attachment_id)
|
|
except DB_UNAVAILABLE:
|
|
await objects.delete_quietly(key)
|
|
raise
|
|
if ready is None:
|
|
await objects.delete_quietly(key)
|
|
raise storage_unavailable()
|
|
return ready
|
|
finally:
|
|
spool.close()
|
|
|
|
|
|
async def readable_attachment(attachment_id: str, user: dict):
|
|
row = await store.get_attachment_for_member(parse_uuid(attachment_id, "attachment_id"), uuid.UUID(user["id"]))
|
|
if row is None:
|
|
raise ApiError(404, "not_found", "attachment not found")
|
|
return row
|
|
|
|
|
|
@app.get("/api/v1/attachments/{attachment_id}", response_model=Attachment)
|
|
async def get_attachment(attachment_id: str, user: dict = Depends(current_user)):
|
|
return attachment_row(await readable_attachment(attachment_id, user))
|
|
|
|
|
|
@app.get("/api/v1/attachments/{attachment_id}/content")
|
|
async def download_attachment(
|
|
attachment_id: str,
|
|
user: dict = Depends(current_user),
|
|
variant: str = Query(default="original"),
|
|
):
|
|
if variant not in VARIANTS:
|
|
raise ApiError(400, "invalid_request", "unknown variant")
|
|
row = await readable_attachment(attachment_id, user)
|
|
if variant != "original":
|
|
raise ApiError(404, "not_found", "variant not found")
|
|
chunks, length, close = await objects.open(row["object_key"])
|
|
if length != row["size_bytes"]:
|
|
close()
|
|
logger.error("object size differs from metadata for attachment %s", row["id"])
|
|
raise storage_unavailable()
|
|
return StreamingResponse(
|
|
chunks,
|
|
media_type=row["content_type"],
|
|
headers={
|
|
"Content-Length": str(length),
|
|
"Cache-Control": "private, no-store",
|
|
"X-Content-Type-Options": "nosniff",
|
|
},
|
|
background=BackgroundTask(close),
|
|
)
|