Files
mtusi2026lab3/src/messenger/app.py
T
Alexey BodanyukandClaude Opus 5.5 4830be414e perf: идемпотентная отправка сообщения одним вызовом SQL (трек 1)
Гипотеза из профиля (py-spy, 2 x 1.0 CPU, 240 req/s): ~57 % CPU реплики
уходит в psycopg — накладные расходы на каждый оператор; на одно
сообщение ~11 обращений к PostgreSQL в двух транзакциях (проверка
членства, BEGIN, ключ, часы чата, сообщение, COMMIT, проверки пула).

- migrations/0003: функция post_message — та же логика и тот же SQL,
  что в многооператорном пути; вызывается в autocommit, поэтому вызов и
  есть транзакция (ключ, часы чата и сообщение — всё или ничего).
- пул: autocommit-соединения; многооператорная работа по-прежнему в явной
  conn.transaction(); ping — один оператор без транзакции.
- MESSAGE_WRITE_PATH=function (по умолчанию) | multi (прежний путь) — A/B
  на одном образе. Порядок ответов одинаков: постороннему всегда 404,
  даже при неверном ключе/теле.
- Оба режима проходят одни и те же тесты идемпотентности и контракта.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-19 16:03:22 +03:00

424 lines
20 KiB
Python

"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping."""
from __future__ import annotations
import asyncio
import contextlib
import hashlib
import json
import logging
import re
import time
import uuid
from typing import Any
from starlette.applications import Starlette
from starlette.exceptions import HTTPException
from starlette.requests import Request
from starlette.responses import JSONResponse, Response
from starlette.routing import Route
from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send
from . import authhdr, cursor, docs, passwords, repo, validation
from .config import Settings
from .db import Database
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable
from .sessions import SessionStore, build_store
from .timefmt import rfc3339
log = logging.getLogger("messenger")
access_log = logging.getLogger("messenger.access")
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
# --------------------------------------------------------------------------
# Responses
# --------------------------------------------------------------------------
def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse:
body = {"error": {"code": code, "message": message, "request_id": request_id}}
return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})})
async def api_error_handler(request: Request, exc: Exception) -> JSONResponse:
assert isinstance(exc, ApiError)
return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers)
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Unknown routes and wrong methods also answer in the contract error format."""
assert isinstance(exc, HTTPException)
if exc.status_code == 404:
return error_response(request.state.request_id, 404, "not_found", "Объект не найден")
if exc.status_code == 405:
return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers)
return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос")
# --------------------------------------------------------------------------
# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions)
# --------------------------------------------------------------------------
class RequestContextMiddleware:
"""Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503
included), writes one access-log line, turns crashes into 500."""
def __init__(self, app: ASGIApp, instance_id: str):
self.app = app
self.instance_id = instance_id.encode("ascii")
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1")
request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
scope.setdefault("state", {})["request_id"] = request_id
status = 500
started = False
begin = time.perf_counter()
async def send_with_id(message: AsgiMessage) -> None:
nonlocal status, started
if message["type"] == "http.response.start":
started = True
status = message["status"]
headers = [(k, v) for k, v in message.get("headers", [])
if k.lower() not in (b"x-request-id", b"x-instance-id")]
headers.append((b"x-request-id", request_id.encode("latin-1")))
headers.append((b"x-instance-id", self.instance_id))
message = {**message, "headers": headers}
await send(message)
try:
await self.app(scope, receive, send_with_id)
except asyncio.CancelledError:
# uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires:
# answer 503 (retryable) instead of a generic 500, then let the cancellation proceed.
if not started:
status = 503
response = error_response(request_id, 503, "unavailable", "Сервер завершает работу")
await response(scope, receive, send_with_id)
raise
except Exception:
log.exception("unhandled_error", extra={"fields": {"request_id": request_id}})
if not started:
status = 500
response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера")
await response(scope, receive, send_with_id)
finally:
access_log.info("request", extra={"fields": {
"request_id": request_id,
"method": scope["method"],
"path": scope["path"], # ids only; query (cursor) and bodies are never logged
"status": status,
"instance_id": self.instance_id.decode("ascii"),
"duration_ms": round((time.perf_counter() - begin) * 1000, 3),
}})
# --------------------------------------------------------------------------
# Request helpers
# --------------------------------------------------------------------------
def db_of(request: Request) -> Database:
return request.app.state.db
def sessions_of(request: Request) -> SessionStore:
return request.app.state.sessions
async def authenticate(request: Request) -> str:
"""User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2.
Store failures are 503 (Unavailable), never 401 and never anonymous access.
"""
token = authhdr.parse_bearer(request.headers.get("authorization"))
user_id = await sessions_of(request).user_id_for(token)
if user_id is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return user_id
def path_uuid(request: Request, name: str) -> str:
return validation.parse_uuid(request.path_params[name], name)
async def read_json(request: Request) -> dict[str, Any]:
content_type = request.headers.get("content-type", "")
media_type, _, params = content_type.partition(";")
if media_type.strip().lower() != "application/json":
raise InvalidRequest("Content-Type должен быть application/json")
charset = params.strip().lower()
if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}:
raise InvalidRequest("Поддерживается только кодировка UTF-8")
limit: int = request.app.state.settings.max_body_bytes
declared = request.headers.get("content-length")
if declared is not None and declared.isdigit() and int(declared) > limit:
raise PayloadTooLarge()
chunks: list[bytes] = []
size = 0
async for chunk in request.stream():
size += len(chunk)
if size > limit:
raise PayloadTooLarge()
chunks.append(chunk)
return validation.parse_json_object(b"".join(chunks))
def page_params(request: Request) -> tuple[int, str | None]:
query = request.query_params.multi_items()
limit = validation.parse_limit(validation.single_query_param(query, "limit"))
raw_cursor = validation.single_query_param(query, "cursor")
if raw_cursor is not None and raw_cursor == "":
raise InvalidRequest("cursor не может быть пустым")
return limit, raw_cursor
async def reject_body(request: Request) -> None:
"""POST /auth/sessions takes credentials only from the Basic header; any body is 400."""
if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0":
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
async for chunk in request.stream():
if chunk:
raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела")
def idempotency_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
return validation.idempotency_key(values[0] if len(values) == 1 else None)
def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes:
"""SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter."""
canonical = json.dumps({"attachment_ids": attachment_ids, "text": text},
ensure_ascii=False, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).digest()
def ok(body: Any, status: int = 200) -> JSONResponse:
return JSONResponse(body, status_code=status)
# --------------------------------------------------------------------------
# Handlers
# --------------------------------------------------------------------------
async def live(request: Request) -> JSONResponse:
return ok({"status": "ok"})
async def ready(request: Request) -> JSONResponse:
"""200 only if the mandatory synchronous path can be served: PostgreSQL and,
with SESSION_STORE=valkey, Valkey answer. 503 while draining after SIGTERM."""
if request.app.state.draining:
raise Unavailable("Сервер завершает работу")
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
if not db_ok:
raise Unavailable("PostgreSQL недоступен")
if not store_ok:
raise Unavailable("Хранилище сессий недоступно")
return ok({"status": "ok"})
async def register_user(request: Request) -> JSONResponse:
data = await read_json(request)
validation.check_fields(data, {"username", "display_name", "password"})
username = validation.username_field(data)
display_name = validation.text_field(data, "display_name", 100)
password = validation.password_field(data)
password_hash = await passwords.hash_password(password)
user = await repo.create_user(db_of(request), username, display_name, password_hash)
log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok(user.to_json(), 201)
async def create_session(request: Request) -> JSONResponse:
"""Login: only `Authorization: Basic`; wrong user and wrong password give the same 401."""
username, password = authhdr.parse_basic(request.headers.get("authorization"))
await reject_body(request)
found = await repo.get_credentials(db_of(request), username)
user, stored_hash = found if found else (None, None)
if not await passwords.verify_password(stored_hash, password) or user is None:
log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}})
raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
ttl = request.app.state.settings.session_ttl_seconds
token, expires_at = await sessions_of(request).create(user.id, ttl)
log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201)
async def revoke_session(request: Request) -> Response:
token = authhdr.parse_bearer(request.headers.get("authorization"))
if not await sessions_of(request).revoke(token):
raise Unauthorized("Сессия недействительна, истекла или отозвана")
log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}})
return Response(status_code=204)
async def me(request: Request) -> JSONResponse:
user_id = await authenticate(request)
user = await repo.get_user(db_of(request), user_id)
if user is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return ok(user.to_json())
async def create_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
data = await read_json(request)
validation.check_fields(data, {"title", "member_ids"})
title = validation.text_field(data, "title", 100)
member_ids = validation.member_ids_field(data)
chat = await repo.create_chat(db_of(request), user_id, title, member_ids)
log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}})
return ok(chat.to_json(), 201)
async def list_chats(request: Request) -> JSONResponse:
user_id = await authenticate(request)
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None
items, has_more = await repo.list_chats(db_of(request), user_id, after, limit)
next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None
return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor})
async def get_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json())
def parse_message(data: dict[str, Any]) -> tuple[str, list[str]]:
validation.check_fields(data, {"text"}, frozenset({"attachment_ids"}))
text = validation.text_field(data, "text", 4000)
attachment_ids = validation.attachment_ids_field(data)
if attachment_ids:
# Lab 3 has no attachments: every referenced ID is unknown (lab 4 semantics: 404).
raise NotFound("Вложение не найдено")
return text, attachment_ids
async def create_message(request: Request) -> JSONResponse:
"""Order of answers for both write paths: 401 -> 400 bad chat id -> 404 outsider -> 400 key/body
-> 409 other body with the same key -> 201 (new or replayed). sender_id comes from the session."""
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
settings = request.app.state.settings
if settings.message_write_path == "multi":
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at
key = idempotency_key(request)
text, attachment_ids = parse_message(await read_json(request))
result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text,
settings.idempotency_ttl_seconds)
else:
try:
key = idempotency_key(request)
text, attachment_ids = parse_message(await read_json(request))
except ApiError:
# Rare error path only: an outsider must still get 404, not a hint about the body.
await repo.ensure_member(db, user_id, chat_id)
raise
# Membership check, key claim, chat clock and message: one SQL call, one transaction.
result = await repo.post_message_single_call(db, user_id, chat_id, key,
request_fingerprint(text, attachment_ids), text,
settings.idempotency_ttl_seconds)
if result.replayed:
log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id,
"message_id": result.message.id}})
return ok(result.message.to_json(), 201)
async def list_messages(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None
items, has_more = await repo.list_messages(db, chat_id, after, limit)
next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None
return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor})
def create_app(settings: Settings | None = None) -> Starlette:
settings = settings or Settings.from_env()
db = Database(settings)
sessions = build_store(settings, db)
async def purge_loop() -> None:
"""Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS)."""
while True:
await asyncio.sleep(600)
try:
deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds)
if deleted:
log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}})
except ApiError:
pass # database unavailable right now; next round
stats_log = logging.getLogger("messenger.stats")
async def stats_loop(every: float) -> None:
"""Experiments only: DB pool saturation (waiting requests, wait time) and process CPU."""
cpu = time.process_time()
db.pool.pop_stats()
while True:
await asyncio.sleep(every)
stats = db.pool.pop_stats()
now_cpu = time.process_time()
stats_log.info("pool_stats", extra={"fields": {
"interval_s": every, "cpu_s": round(now_cpu - cpu, 4),
**{k: stats.get(k, 0) for k in ("pool_size", "pool_available", "requests_waiting", "requests_num",
"requests_queued", "requests_wait_ms", "usage_ms", "requests_errors")}}})
cpu = now_cpu
@contextlib.asynccontextmanager
async def lifespan(app: Starlette):
passwords.configure(settings.hash_concurrency)
await db.open()
log.info("dependencies_configured", extra={"fields": {
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds,
"instance_id": settings.instance_id, "drain_s": settings.drain_seconds,
"message_write_path": settings.message_write_path}})
tasks = [asyncio.create_task(purge_loop())]
if settings.stats_log_seconds > 0:
tasks.append(asyncio.create_task(stats_loop(settings.stats_log_seconds)))
try:
yield
finally:
for task in tasks:
task.cancel()
await sessions.close()
await db.close()
routes = [
Route("/health/live", live, methods=["GET"]),
Route("/health/ready", ready, methods=["GET"]),
Route("/api/v1/users", register_user, methods=["POST"]),
Route("/api/v1/users/me", me, methods=["GET"]),
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
Route("/api/v1/chats", create_chat, methods=["POST"]),
Route("/api/v1/chats", list_chats, methods=["GET"]),
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
]
if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract
routes += [
Route("/docs", docs.swagger_page, methods=["GET"]),
Route("/openapi.json", docs.openapi_json, methods=["GET"]),
]
app = Starlette(
routes=routes,
exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler},
lifespan=lifespan,
)
app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id)
app.state.settings = settings
app.state.db = db
app.state.sessions = sessions
app.state.draining = False
return app