forked from mtusi2026/mtusi2026lab3
Гипотеза из профиля (py-spy, 2 x 1.0 CPU, 240 req/s): ~57 % CPU реплики уходит в psycopg — накладные расходы на каждый оператор; на одно сообщение ~11 обращений к PostgreSQL в двух транзакциях (проверка членства, BEGIN, ключ, часы чата, сообщение, COMMIT, проверки пула). - migrations/0003: функция post_message — та же логика и тот же SQL, что в многооператорном пути; вызывается в autocommit, поэтому вызов и есть транзакция (ключ, часы чата и сообщение — всё или ничего). - пул: autocommit-соединения; многооператорная работа по-прежнему в явной conn.transaction(); ping — один оператор без транзакции. - MESSAGE_WRITE_PATH=function (по умолчанию) | multi (прежний путь) — A/B на одном образе. Порядок ответов одинаков: постороннему всегда 404, даже при неверном ключе/теле. - Оба режима проходят одни и те же тесты идемпотентности и контракта. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
424 lines
20 KiB
Python
424 lines
20 KiB
Python
"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping."""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import contextlib
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
import uuid
|
|
from typing import Any
|
|
|
|
from starlette.applications import Starlette
|
|
from starlette.exceptions import HTTPException
|
|
from starlette.requests import Request
|
|
from starlette.responses import JSONResponse, Response
|
|
from starlette.routing import Route
|
|
from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send
|
|
|
|
from . import authhdr, cursor, docs, passwords, repo, validation
|
|
from .config import Settings
|
|
from .db import Database
|
|
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable
|
|
from .sessions import SessionStore, build_store
|
|
from .timefmt import rfc3339
|
|
|
|
log = logging.getLogger("messenger")
|
|
access_log = logging.getLogger("messenger.access")
|
|
|
|
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Responses
|
|
# --------------------------------------------------------------------------
|
|
def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse:
|
|
body = {"error": {"code": code, "message": message, "request_id": request_id}}
|
|
return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})})
|
|
|
|
|
|
async def api_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
|
assert isinstance(exc, ApiError)
|
|
return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers)
|
|
|
|
|
|
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
|
|
"""Unknown routes and wrong methods also answer in the contract error format."""
|
|
assert isinstance(exc, HTTPException)
|
|
if exc.status_code == 404:
|
|
return error_response(request.state.request_id, 404, "not_found", "Объект не найден")
|
|
if exc.status_code == 405:
|
|
return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers)
|
|
return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос")
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions)
|
|
# --------------------------------------------------------------------------
|
|
class RequestContextMiddleware:
|
|
"""Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503
|
|
included), writes one access-log line, turns crashes into 500."""
|
|
|
|
def __init__(self, app: ASGIApp, instance_id: str):
|
|
self.app = app
|
|
self.instance_id = instance_id.encode("ascii")
|
|
|
|
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
|
|
if scope["type"] != "http":
|
|
await self.app(scope, receive, send)
|
|
return
|
|
incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1")
|
|
request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
|
|
scope.setdefault("state", {})["request_id"] = request_id
|
|
status = 500
|
|
started = False
|
|
begin = time.perf_counter()
|
|
|
|
async def send_with_id(message: AsgiMessage) -> None:
|
|
nonlocal status, started
|
|
if message["type"] == "http.response.start":
|
|
started = True
|
|
status = message["status"]
|
|
headers = [(k, v) for k, v in message.get("headers", [])
|
|
if k.lower() not in (b"x-request-id", b"x-instance-id")]
|
|
headers.append((b"x-request-id", request_id.encode("latin-1")))
|
|
headers.append((b"x-instance-id", self.instance_id))
|
|
message = {**message, "headers": headers}
|
|
await send(message)
|
|
|
|
try:
|
|
await self.app(scope, receive, send_with_id)
|
|
except asyncio.CancelledError:
|
|
# uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires:
|
|
# answer 503 (retryable) instead of a generic 500, then let the cancellation proceed.
|
|
if not started:
|
|
status = 503
|
|
response = error_response(request_id, 503, "unavailable", "Сервер завершает работу")
|
|
await response(scope, receive, send_with_id)
|
|
raise
|
|
except Exception:
|
|
log.exception("unhandled_error", extra={"fields": {"request_id": request_id}})
|
|
if not started:
|
|
status = 500
|
|
response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера")
|
|
await response(scope, receive, send_with_id)
|
|
finally:
|
|
access_log.info("request", extra={"fields": {
|
|
"request_id": request_id,
|
|
"method": scope["method"],
|
|
"path": scope["path"], # ids only; query (cursor) and bodies are never logged
|
|
"status": status,
|
|
"instance_id": self.instance_id.decode("ascii"),
|
|
"duration_ms": round((time.perf_counter() - begin) * 1000, 3),
|
|
}})
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Request helpers
|
|
# --------------------------------------------------------------------------
|
|
def db_of(request: Request) -> Database:
|
|
return request.app.state.db
|
|
|
|
|
|
def sessions_of(request: Request) -> SessionStore:
|
|
return request.app.state.sessions
|
|
|
|
|
|
async def authenticate(request: Request) -> str:
|
|
"""User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2.
|
|
|
|
Store failures are 503 (Unavailable), never 401 and never anonymous access.
|
|
"""
|
|
token = authhdr.parse_bearer(request.headers.get("authorization"))
|
|
user_id = await sessions_of(request).user_id_for(token)
|
|
if user_id is None:
|
|
raise Unauthorized("Сессия недействительна, истекла или отозвана")
|
|
return user_id
|
|
|
|
|
|
def path_uuid(request: Request, name: str) -> str:
|
|
return validation.parse_uuid(request.path_params[name], name)
|
|
|
|
|
|
async def read_json(request: Request) -> dict[str, Any]:
|
|
content_type = request.headers.get("content-type", "")
|
|
media_type, _, params = content_type.partition(";")
|
|
if media_type.strip().lower() != "application/json":
|
|
raise InvalidRequest("Content-Type должен быть application/json")
|
|
charset = params.strip().lower()
|
|
if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}:
|
|
raise InvalidRequest("Поддерживается только кодировка UTF-8")
|
|
limit: int = request.app.state.settings.max_body_bytes
|
|
declared = request.headers.get("content-length")
|
|
if declared is not None and declared.isdigit() and int(declared) > limit:
|
|
raise PayloadTooLarge()
|
|
chunks: list[bytes] = []
|
|
size = 0
|
|
async for chunk in request.stream():
|
|
size += len(chunk)
|
|
if size > limit:
|
|
raise PayloadTooLarge()
|
|
chunks.append(chunk)
|
|
return validation.parse_json_object(b"".join(chunks))
|
|
|
|
|
|
def page_params(request: Request) -> tuple[int, str | None]:
|
|
query = request.query_params.multi_items()
|
|
limit = validation.parse_limit(validation.single_query_param(query, "limit"))
|
|
raw_cursor = validation.single_query_param(query, "cursor")
|
|
if raw_cursor is not None and raw_cursor == "":
|
|
raise InvalidRequest("cursor не может быть пустым")
|
|
return limit, raw_cursor
|
|
|
|
|
|
async def reject_body(request: Request) -> None:
|
|
"""POST /auth/sessions takes credentials only from the Basic header; any body is 400."""
|
|
if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0":
|
|
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
|
|
async for chunk in request.stream():
|
|
if chunk:
|
|
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
|
|
|
|
|
|
def idempotency_key(request: Request) -> str:
|
|
values = request.headers.getlist("idempotency-key")
|
|
return validation.idempotency_key(values[0] if len(values) == 1 else None)
|
|
|
|
|
|
def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes:
|
|
"""SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter."""
|
|
canonical = json.dumps({"attachment_ids": attachment_ids, "text": text},
|
|
ensure_ascii=False, sort_keys=True, separators=(",", ":"))
|
|
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
|
|
|
|
|
def ok(body: Any, status: int = 200) -> JSONResponse:
|
|
return JSONResponse(body, status_code=status)
|
|
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Handlers
|
|
# --------------------------------------------------------------------------
|
|
async def live(request: Request) -> JSONResponse:
|
|
return ok({"status": "ok"})
|
|
|
|
|
|
async def ready(request: Request) -> JSONResponse:
|
|
"""200 only if the mandatory synchronous path can be served: PostgreSQL and,
|
|
with SESSION_STORE=valkey, Valkey answer. 503 while draining after SIGTERM."""
|
|
if request.app.state.draining:
|
|
raise Unavailable("Сервер завершает работу")
|
|
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
|
|
if not db_ok:
|
|
raise Unavailable("PostgreSQL недоступен")
|
|
if not store_ok:
|
|
raise Unavailable("Хранилище сессий недоступно")
|
|
return ok({"status": "ok"})
|
|
|
|
|
|
async def register_user(request: Request) -> JSONResponse:
|
|
data = await read_json(request)
|
|
validation.check_fields(data, {"username", "display_name", "password"})
|
|
username = validation.username_field(data)
|
|
display_name = validation.text_field(data, "display_name", 100)
|
|
password = validation.password_field(data)
|
|
password_hash = await passwords.hash_password(password)
|
|
user = await repo.create_user(db_of(request), username, display_name, password_hash)
|
|
log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
|
|
return ok(user.to_json(), 201)
|
|
|
|
|
|
async def create_session(request: Request) -> JSONResponse:
|
|
"""Login: only `Authorization: Basic`; wrong user and wrong password give the same 401."""
|
|
username, password = authhdr.parse_basic(request.headers.get("authorization"))
|
|
await reject_body(request)
|
|
found = await repo.get_credentials(db_of(request), username)
|
|
user, stored_hash = found if found else (None, None)
|
|
if not await passwords.verify_password(stored_hash, password) or user is None:
|
|
log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}})
|
|
raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
|
|
ttl = request.app.state.settings.session_ttl_seconds
|
|
token, expires_at = await sessions_of(request).create(user.id, ttl)
|
|
log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
|
|
return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201)
|
|
|
|
|
|
async def revoke_session(request: Request) -> Response:
|
|
token = authhdr.parse_bearer(request.headers.get("authorization"))
|
|
if not await sessions_of(request).revoke(token):
|
|
raise Unauthorized("Сессия недействительна, истекла или отозвана")
|
|
log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}})
|
|
return Response(status_code=204)
|
|
|
|
|
|
async def me(request: Request) -> JSONResponse:
|
|
user_id = await authenticate(request)
|
|
user = await repo.get_user(db_of(request), user_id)
|
|
if user is None:
|
|
raise Unauthorized("Сессия недействительна, истекла или отозвана")
|
|
return ok(user.to_json())
|
|
|
|
|
|
async def create_chat(request: Request) -> JSONResponse:
|
|
user_id = await authenticate(request)
|
|
data = await read_json(request)
|
|
validation.check_fields(data, {"title", "member_ids"})
|
|
title = validation.text_field(data, "title", 100)
|
|
member_ids = validation.member_ids_field(data)
|
|
chat = await repo.create_chat(db_of(request), user_id, title, member_ids)
|
|
log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}})
|
|
return ok(chat.to_json(), 201)
|
|
|
|
|
|
async def list_chats(request: Request) -> JSONResponse:
|
|
user_id = await authenticate(request)
|
|
limit, raw_cursor = page_params(request)
|
|
after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None
|
|
items, has_more = await repo.list_chats(db_of(request), user_id, after, limit)
|
|
next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None
|
|
return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor})
|
|
|
|
|
|
async def get_chat(request: Request) -> JSONResponse:
|
|
user_id = await authenticate(request)
|
|
chat_id = path_uuid(request, "chat_id")
|
|
return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json())
|
|
|
|
|
|
def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]:
|
|
validation.check_fields(data, {"text"}, frozenset({"attachment_ids"}))
|
|
text = validation.text_field(data, "text", 4000)
|
|
attachment_ids = validation.attachment_ids_field(data)
|
|
if attachment_ids:
|
|
# Lab 3 has no attachments: every referenced ID is unknown (lab 4 semantics: 404).
|
|
raise NotFound("Вложение не найдено")
|
|
return text, attachment_ids
|
|
|
|
|
|
async def create_message(request: Request) -> JSONResponse:
|
|
"""Order of answers for both write paths: 401 -> 400 bad chat id -> 404 outsider -> 400 key/body
|
|
-> 409 other body with the same key -> 201 (new or replayed). sender_id comes from the session."""
|
|
user_id = await authenticate(request)
|
|
chat_id = path_uuid(request, "chat_id")
|
|
db = db_of(request)
|
|
settings = request.app.state.settings
|
|
if settings.message_write_path == "multi":
|
|
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at
|
|
key = idempotency_key(request)
|
|
text, attachment_ids = parse_message(await read_json(request))
|
|
result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text,
|
|
settings.idempotency_ttl_seconds)
|
|
else:
|
|
try:
|
|
key = idempotency_key(request)
|
|
text, attachment_ids = parse_message(await read_json(request))
|
|
except ApiError:
|
|
# Rare error path only: an outsider must still get 404, not a hint about the body.
|
|
await repo.ensure_member(db, user_id, chat_id)
|
|
raise
|
|
# Membership check, key claim, chat clock and message: one SQL call, one transaction.
|
|
result = await repo.post_message_single_call(db, user_id, chat_id, key,
|
|
request_fingerprint(text, attachment_ids), text,
|
|
settings.idempotency_ttl_seconds)
|
|
if result.replayed:
|
|
log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id,
|
|
"message_id": result.message.id}})
|
|
return ok(result.message.to_json(), 201)
|
|
|
|
|
|
async def list_messages(request: Request) -> JSONResponse:
|
|
user_id = await authenticate(request)
|
|
chat_id = path_uuid(request, "chat_id")
|
|
db = db_of(request)
|
|
await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak
|
|
limit, raw_cursor = page_params(request)
|
|
after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None
|
|
items, has_more = await repo.list_messages(db, chat_id, after, limit)
|
|
next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None
|
|
return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor})
|
|
|
|
|
|
def create_app(settings: Settings | None = None) -> Starlette:
|
|
settings = settings or Settings.from_env()
|
|
db = Database(settings)
|
|
sessions = build_store(settings, db)
|
|
|
|
async def purge_loop() -> None:
|
|
"""Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS)."""
|
|
while True:
|
|
await asyncio.sleep(600)
|
|
try:
|
|
deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds)
|
|
if deleted:
|
|
log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}})
|
|
except ApiError:
|
|
pass # database unavailable right now; next round
|
|
|
|
stats_log = logging.getLogger("messenger.stats")
|
|
|
|
async def stats_loop(every: float) -> None:
|
|
"""Experiments only: DB pool saturation (waiting requests, wait time) and process CPU."""
|
|
cpu = time.process_time()
|
|
db.pool.pop_stats()
|
|
while True:
|
|
await asyncio.sleep(every)
|
|
stats = db.pool.pop_stats()
|
|
now_cpu = time.process_time()
|
|
stats_log.info("pool_stats", extra={"fields": {
|
|
"interval_s": every, "cpu_s": round(now_cpu - cpu, 4),
|
|
**{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num",
|
|
"requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}})
|
|
cpu = now_cpu
|
|
|
|
@contextlib.asynccontextmanager
|
|
async def lifespan(app: Starlette):
|
|
passwords.configure(settings.hash_concurrency)
|
|
await db.open()
|
|
log.info("dependencies_configured", extra={"fields": {
|
|
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
|
|
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
|
|
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
|
|
"message_write_path": settings.message_write_path}})
|
|
tasks = [asyncio.create_task(purge_loop())]
|
|
if settings.stats_log_seconds > 0:
|
|
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
|
|
try:
|
|
yield
|
|
finally:
|
|
for task in tasks:
|
|
task.cancel()
|
|
await sessions.close()
|
|
await db.close()
|
|
|
|
routes = [
|
|
Route("/health/live", live, methods=["GET"]),
|
|
Route("/health/ready", ready, methods=["GET"]),
|
|
Route("/api/v1/users", register_user, methods=["POST"]),
|
|
Route("/api/v1/users/me", me, methods=["GET"]),
|
|
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
|
|
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
|
|
Route("/api/v1/chats", create_chat, methods=["POST"]),
|
|
Route("/api/v1/chats", list_chats, methods=["GET"]),
|
|
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
|
|
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
|
|
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
|
|
]
|
|
if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract
|
|
routes += [
|
|
Route("/docs", docs.swagger_page, methods=["GET"]),
|
|
Route("/openapi.json", docs.openapi_json, methods=["GET"]),
|
|
]
|
|
app = Starlette(
|
|
routes=routes,
|
|
exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler},
|
|
lifespan=lifespan,
|
|
)
|
|
app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id)
|
|
app.state.settings = settings
|
|
app.state.db = db
|
|
app.state.sessions = sessions
|
|
app.state.draining = False
|
|
return app
|