Files
mtusi2026lab5/src/main.py
T
2026-10-26 18:05:06 +03:00

634 lines
26 KiB
Python

from __future__ import annotations
import base64
import json
import logging
import os
import re
import time
import uuid
from datetime import datetime, timedelta, timezone
from contextlib import asynccontextmanager
import hashlib
import secrets
import socket
import resource
import psycopg
from psycopg.rows import dict_row
from psycopg.types.json import Jsonb
from psycopg_pool import ConnectionPool, PoolTimeout
import redis
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, InvalidHashError
from starlette.exceptions import HTTPException
from src.config import DATABASE_URL, CACHE_URL, SESSION_TTL, POOL_MAX
from src.errors import ApiProblem
from src import attachments
from botocore.exceptions import BotoCoreError, ClientError
from typing import Annotated
from fastapi import Depends, FastAPI, Header, Query, Request
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse, Response
from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
from starlette.concurrency import run_in_threadpool
logging.basicConfig(
level=os.getenv("LOG_LEVEL", "INFO").upper(),
format="%(asctime)s %(levelname)s %(message)s",
)
logger = logging.getLogger("messenger")
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def as_json(value: BaseModel) -> dict:
return value.model_dump(mode="json")
class StrictModel(BaseModel):
model_config = ConfigDict(extra="forbid")
class User(StrictModel):
id: uuid.UUID
username: str
display_name: str
created_at: datetime
class CreateUser(StrictModel):
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator("password")
@classmethod
def password_has_visible_characters(cls, value: str) -> str:
if not value.strip():
raise ValueError("password must not contain only whitespace")
return value
@field_validator("display_name")
@classmethod
def display_name_has_visible_characters(cls, value: str) -> str:
if not value.strip():
raise ValueError("display_name must not contain only whitespace")
return value
class Chat(StrictModel):
id: uuid.UUID
title: str
member_ids: list[uuid.UUID]
created_at: datetime
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
@field_validator("title")
@classmethod
def title_has_visible_characters(cls, value: str) -> str:
if not value.strip():
raise ValueError("title must not contain only whitespace")
return value
@field_validator("member_ids")
@classmethod
def member_ids_are_unique(cls, value: list[uuid.UUID]) -> list[uuid.UUID]:
if len(value) != len(set(value)):
raise ValueError("member_ids must be unique")
return value
class Message(StrictModel):
id: uuid.UUID
chat_id: uuid.UUID
sender_id: uuid.UUID
text: str
created_at: datetime
attachment_ids: list[uuid.UUID] = Field(default_factory=list)
class CreateMessage(StrictModel):
text: str = Field(max_length=4000)
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
@model_validator(mode='after')
def valid_message(self):
if not self.text.strip() and not self.attachment_ids:
raise ValueError('Нужен текст или вложение')
if len(self.attachment_ids) != len(set(self.attachment_ids)):
raise ValueError('Повторяющиеся attachment_ids')
return self
class Store:
"""Сессии и метаданные в PostgreSQL; Redis — только кеш профилей."""
def __init__(self) -> None:
self.pool = ConnectionPool(
DATABASE_URL, min_size=1, max_size=POOL_MAX, timeout=2, open=False,
kwargs={"row_factory": dict_row, "connect_timeout": 2,
"options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"},
)
self.cache = redis.Redis.from_url(
CACHE_URL, decode_responses=True, socket_connect_timeout=0.25, socket_timeout=0.25,
)
def start(self) -> None:
self.pool.open()
self.pool.wait(timeout=10)
def close(self) -> None:
self.pool.close()
self.cache.close()
def chat(self, db, chat_id: uuid.UUID) -> Chat | None:
row = db.execute("SELECT * FROM chats WHERE id=%s", (chat_id,)).fetchone()
if row is None:
return None
members = db.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id",
(chat_id,)).fetchall()
return Chat(**row, member_ids=[member["user_id"] for member in members])
def chat_messages(self, db, chat_id: uuid.UUID, limit: int, after) -> list[Message]:
# В lab1 здесь были общий список/список чата и bisect_right.
# Теперь границу ищет B-tree PostgreSQL (chat_id, created_at, id).
query = ("SELECT m.*, COALESCE((SELECT array_agg(a.attachment_id ORDER BY a.position) "
"FROM message_attachments a WHERE a.message_id=m.id),'{}'::uuid[]) AS attachment_ids "
"FROM messages m WHERE chat_id=%s")
params = [chat_id]
if after is not None:
query += " AND (created_at,id)>(%s,%s)"
params.extend(after)
rows = db.execute(query + " ORDER BY created_at,id LIMIT %s", params + [limit + 1]).fetchall()
return [Message(**row) for row in rows]
INSTANCE_ID = os.getenv("INSTANCE_ID") or f"{socket.gethostname()}:{uuid.uuid4().hex[:8]}"
store = Store()
passwords = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
dummy_hash = passwords.hash(secrets.token_urlsafe(32))
@asynccontextmanager
async def lifespan(app):
store.start()
yield
store.close()
app = FastAPI(title="MTUCI PostgreSQL messenger", version="5.0.0", lifespan=lifespan,
docs_url=None, redoc_url=None, openapi_url=None)
def request_id(request: Request) -> str:
return request.state.request_id
def error_response(request: Request, status: int, code: str, message: str, challenge: str | None = None) -> JSONResponse:
rid = getattr(request.state, "request_id", str(uuid.uuid4()))
headers = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID}
if challenge:
headers["WWW-Authenticate"] = challenge
return JSONResponse(
status_code=status,
content={"error": {"code": code, "message": message, "request_id": rid}},
headers=headers,
)
@app.middleware("http")
async def request_context(request: Request, call_next):
request.state.request_id = str(uuid.uuid4())
started = time.perf_counter()
status = 500
try:
# Без токена отказываем ДО обращения к БД и проверки JSON.
if request.url.path.startswith("/api/v1/") and request.url.path not in {
"/api/v1/users", "/api/v1/auth/sessions"
}:
scheme, _, token = request.headers.get("authorization", "").partition(" ")
if scheme.lower() != "bearer" or not token or len(token) > 256:
status = 401
return error_response(request, 401, "unauthorized", "Требуется Bearer",
'Bearer realm="messenger"')
json_endpoints = {
"/api/v1/users",
"/api/v1/chats",
}
requires_json = request.method == "POST" and (
request.url.path in json_endpoints
or re.fullmatch(r"/api/v1/chats/[^/]+/messages", request.url.path) is not None
)
media_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower()
if requires_json and media_type != "application/json":
status = 415
return error_response(
request,
415,
"unsupported_media_type",
"Требуется Content-Type: application/json",
)
if requires_json:
# Ограниченный JSON; после body() Starlette повторно отдаёт его маршруту.
raw = await request.body()
if len(raw) > 65536:
status = 413
return error_response(request, 413, "payload_too_large", "Запрос слишком большой")
response = await call_next(request)
status = response.status_code
response.headers["X-Request-Id"] = request.state.request_id
response.headers["X-Instance-Id"] = INSTANCE_ID
return response
finally:
logger.info(
"request method=%s path=%s status=%s duration_ms=%.3f request_id=%s",
request.method,
request.url.path,
status,
(time.perf_counter() - started) * 1000,
request.state.request_id,
)
@app.exception_handler(ApiProblem)
async def api_problem_handler(request: Request, exc: ApiProblem):
return error_response(request, exc.status, exc.code, exc.message, exc.challenge)
@app.exception_handler(RequestValidationError)
async def validation_handler(request: Request, exc: RequestValidationError):
return error_response(request, 400, "invalid_request", "Запрос не соответствует контракту")
@app.exception_handler(Exception)
async def unexpected_error_handler(request: Request, exc: Exception):
logger.error("unexpected_error request_id=%s", request_id(request))
return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера")
@app.exception_handler(psycopg.Error)
@app.exception_handler(PoolTimeout)
async def database_error_handler(request: Request, exc: Exception):
return error_response(request, 503, "unavailable", "Хранилище временно недоступно")
@app.exception_handler(BotoCoreError)
@app.exception_handler(ClientError)
async def object_store_error_handler(request: Request, exc: Exception):
return error_response(request, 503, 'unavailable', 'Объектное хранилище временно недоступно')
@app.exception_handler(HTTPException)
async def http_error_handler(request: Request, exc: HTTPException):
code = "not_found" if exc.status_code == 404 else "invalid_request"
return error_response(request, exc.status_code, code, "Неверный маршрут или метод")
def unauthorized(basic=False):
challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
raise ApiProblem(401, "unauthorized", "Нужна действительная идентификация", challenge)
def token_hash(token: str) -> str:
return hashlib.sha256(token.encode()).hexdigest()
def current_user(authorization: Annotated[str | None, Header()] = None) -> User:
scheme, _, token = (authorization or "").partition(" ")
if scheme.lower() != "bearer" or not token or len(token) > 256:
unauthorized()
with store.pool.connection() as db:
session = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>now()",
(token_hash(token),)).fetchone()
if session is None:
unauthorized()
key = "user:" + str(session["user_id"])
cache_available = True
try:
cached = store.cache.get(key)
if cached:
user = User.model_validate_json(cached)
if user.id == session["user_id"]:
return user
except (redis.RedisError, ValueError):
cache_available = False
row = db.execute("SELECT id,username,display_name,created_at FROM users WHERE id=%s",
(session["user_id"],)).fetchone()
user = User(**row)
if cache_available:
try:
store.cache.setex(key, 60, user.model_dump_json())
except redis.RedisError:
pass
return user
def decode_cursor(cursor: str, expected_scope: str) -> tuple[datetime, uuid.UUID]:
try:
if not cursor or len(cursor) > 1024:
raise ValueError
padded = cursor + "=" * (-len(cursor) % 4)
payload = json.loads(base64.b64decode(padded.encode(), altchars=b"-_", validate=True).decode())
if not isinstance(payload, dict) or set(payload) != {"scope", "created_at", "id"}:
raise ValueError
if payload["scope"] != expected_scope:
raise ValueError
if not isinstance(payload["created_at"], str) or not isinstance(payload["id"], str):
raise ValueError
created_at = datetime.fromisoformat(payload["created_at"].replace("Z", "+00:00"))
item_id = uuid.UUID(payload["id"])
if created_at.tzinfo is None:
raise ValueError
return created_at, item_id
except (ValueError, TypeError, KeyError, json.JSONDecodeError, UnicodeError):
raise ApiProblem(400, "invalid_request", "Некорректный cursor") from None
def encode_cursor(scope: str, created_at: datetime, item_id: uuid.UUID) -> str:
payload = {
"scope": scope,
"created_at": created_at.isoformat().replace("+00:00", "Z"),
"id": str(item_id),
}
return base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).decode().rstrip("=")
def page(items: list, limit: int, scope: str) -> dict:
# SQL уже выбрал limit+1 строк после курсора; весь список не загружаем.
selected = items[:limit]
next_cursor = None
if len(items) > limit and selected:
last = selected[-1]
next_cursor = encode_cursor(scope, last.created_at, last.id)
return {"items": [as_json(item) for item in selected], "next_cursor": next_cursor}
def visible_chat(chat_id: uuid.UUID, user: User) -> Chat:
with store.pool.connection() as db:
member = db.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",
(chat_id, user.id)).fetchone()
if member is None:
raise ApiProblem(404, "not_found", "Объект не найден")
return store.chat(db, chat_id)
def page_parameters(request: Request, scope: str, limit: int, cursor: str | None):
# Query(ge/le) сохранён из lab1, добавлена проверка дублей и формы limit.
if set(request.query_params) - {"limit", "cursor"} or any(
len(request.query_params.getlist(k)) > 1 for k in request.query_params
):
raise ApiProblem(400, "invalid_request", "Некорректные параметры страницы")
if not re.fullmatch(r"[0-9]{1,3}", request.query_params.get("limit", "50")):
raise ApiProblem(400, "invalid_request", "Некорректный limit")
return decode_cursor(cursor, scope) if cursor is not None else None
@app.get("/health/live")
def liveness():
return {"status": "ok"}
@app.get("/health/ready")
def readiness():
with store.pool.connection() as db:
db.execute("SELECT 1")
attachments.storage.ready()
return {"status": "ok"}
@app.get("/internal/pool")
def pool_statistics():
# Диагностика доступна только внутри Docker network: LB закрывает этот путь.
return store.pool.get_stats()
@app.get('/internal/memory')
def memory_statistics():
result = {'process_peak_rss_bytes': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024}
from pathlib import Path
for name in ['/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory/memory.max_usage_in_bytes']:
path = Path(name)
if path.is_file():
result['container_peak_bytes'] = int(path.read_text().strip())
result['container_peak_source'] = name
break
return result
@app.post("/api/v1/users", status_code=201)
def register_user(body: CreateUser):
# Синхронные маршруты FastAPI исполняет в worker thread, не в event loop.
hashed = passwords.hash(body.password)
try:
with store.pool.connection() as db:
row = db.execute(
"INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) "
"RETURNING id,username,display_name,created_at",
(uuid.uuid4(), body.username, body.display_name, hashed),
).fetchone()
except psycopg.errors.UniqueViolation:
raise ApiProblem(409, "conflict", "Username уже занят") from None
return as_json(User(**row))
@app.post("/api/v1/auth/sessions", status_code=201)
def create_session(request: Request, authorization: Annotated[str | None, Header()] = None):
if request.headers.get("content-length", "0") != "0" or "transfer-encoding" in request.headers:
raise ApiProblem(400, "invalid_request", "Вход не принимает тело запроса")
scheme, _, encoded = (authorization or "").partition(" ")
try:
if scheme.lower() != "basic" or len(encoded) > 2048:
raise ValueError
username, password = base64.b64decode(encoded, validate=True).decode("utf-8").split(":", 1)
if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128:
raise ValueError
except (ValueError, UnicodeError):
unauthorized(basic=True)
with store.pool.connection() as db:
row = db.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone()
try:
passwords.verify(row["password_hash"] if row else dummy_hash, password)
except (VerificationError, InvalidHashError):
unauthorized(basic=True)
if row is None:
unauthorized(basic=True)
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL)
with store.pool.connection() as db:
db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)",
(token_hash(token), row["id"], expires_at))
user = User(**{k: row[k] for k in ("id", "username", "display_name", "created_at")})
return {"token": token, "token_type": "Bearer", "user": as_json(user),
"expires_at": expires_at.isoformat().replace("+00:00", "Z")}
@app.delete("/api/v1/auth/sessions/current", status_code=204)
def delete_session(user: Annotated[User, Depends(current_user)],
authorization: Annotated[str, Header()]):
token = authorization.partition(" ")[2]
with store.pool.connection() as db:
db.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash(token),))
return Response(status_code=204)
@app.get("/api/v1/users/me")
def get_current_user(user: Annotated[User, Depends(current_user)]):
return as_json(user)
@app.post("/api/v1/chats", status_code=201)
def create_chat(body: CreateChat, user: Annotated[User, Depends(current_user)]):
if user.id not in body.member_ids:
raise ApiProblem(400, "invalid_request", "Текущий пользователь должен быть участником")
with store.pool.connection() as db:
found = db.execute("SELECT id FROM users WHERE id=ANY(%s)", (body.member_ids,)).fetchall()
if len(found) != len(body.member_ids):
raise ApiProblem(400, "invalid_request", "Указан неизвестный участник")
row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *",
(uuid.uuid4(), body.title)).fetchone()
with db.cursor() as cur:
cur.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)",
[(row["id"], member) for member in body.member_ids])
chat = store.chat(db, row["id"])
return as_json(chat) # COMMIT уже выполнен для чата и всех участников.
@app.get("/api/v1/chats")
def list_chats(
request: Request,
user: Annotated[User, Depends(current_user)],
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
):
scope = f"chats:{user.id}"
after = page_parameters(request, scope, limit, cursor)
query = "SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s"
params = [user.id]
if after is not None:
query += " AND (c.created_at,c.id)>(%s,%s)"
params.extend(after)
with store.pool.connection() as db:
rows = db.execute(query + " ORDER BY c.created_at,c.id LIMIT %s", params + [limit + 1]).fetchall()
chats = [store.chat(db, row["id"]) for row in rows]
return page(chats, limit, scope)
@app.get("/api/v1/chats/{chat_id}")
def get_chat(chat_id: uuid.UUID, user: Annotated[User, Depends(current_user)]):
return as_json(visible_chat(chat_id, user))
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
def create_message(
request: Request,
chat_id: uuid.UUID,
body: CreateMessage,
user: Annotated[User, Depends(current_user)],
idempotency_key: Annotated[str | None, Header()] = None,
):
if (not idempotency_key or
not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) or
len(request.headers.getlist("idempotency-key")) != 1):
raise ApiProblem(400, "invalid_request", "Нужен корректный Idempotency-Key")
visible_chat(chat_id, user) # Повтор тоже проверяет сессию и участие.
canonical = {'text': body.text}
if body.attachment_ids:
canonical['attachment_ids'] = [str(value) for value in body.attachment_ids]
payload = json.dumps(canonical, ensure_ascii=False, sort_keys=True,
separators=(",", ":"))
digest = hashlib.sha256(payload.encode()).hexdigest()
lock_scope = f"{user.id}:POST:{chat_id}:{idempotency_key}"
with store.pool.connection() as db:
# Общая блокировка в PostgreSQL, а не локальный Lock одной реплики.
# Коллизия хеша только сериализует независимые запросы; ключи сравниваются полностью.
db.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (lock_scope,))
previous = db.execute(
"SELECT * FROM message_requests WHERE user_id=%s AND chat_id=%s AND request_key=%s",
(user.id, chat_id, idempotency_key),
).fetchone()
if previous and previous["expires_at"] > utc_now():
if previous["body_hash"] != digest:
raise ApiProblem(409, "conflict", "Ключ уже использован с другим сообщением")
result = previous["response"]
else:
for attachment_id in body.attachment_ids:
found = db.execute('SELECT chat_id,status FROM attachments WHERE id=%s FOR KEY SHARE',
(attachment_id,)).fetchone()
if not found or found['chat_id'] != chat_id:
raise ApiProblem(404, 'not_found', 'Вложение недоступно')
if found['status'] not in {'queued','processing','ready'}:
raise ApiProblem(409, 'conflict', 'Вложение ещё недоступно')
if previous:
db.execute("DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s AND request_key=%s",
(user.id, chat_id, idempotency_key))
row = db.execute(
"INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *",
(uuid.uuid4(), chat_id, user.id, body.text),
).fetchone()
for position, attachment_id in enumerate(body.attachment_ids):
db.execute('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)',
(row['id'], attachment_id, position))
result = as_json(Message(**row, attachment_ids=body.attachment_ids))
db.execute(
"INSERT INTO message_requests(user_id,chat_id,request_key,body_hash,response,expires_at) "
"VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')",
(user.id, chat_id, idempotency_key, digest, Jsonb(result)),
)
# Сообщение и результат повтора зафиксированы одной транзакцией до ответа.
return result
@app.get("/api/v1/chats/{chat_id}/messages")
def list_messages(
request: Request,
chat_id: uuid.UUID,
user: Annotated[User, Depends(current_user)],
limit: Annotated[int, Query(ge=1, le=100)] = 50,
cursor: Annotated[str | None, Query(min_length=1)] = None,
):
visible_chat(chat_id, user)
scope = f"messages:{chat_id}"
after = page_parameters(request, scope, limit, cursor)
with store.pool.connection() as db:
messages = store.chat_messages(db, chat_id, limit, after)
return page(messages, limit, scope)
@app.post('/api/v1/chats/{chat_id}/attachments', status_code=202)
async def upload_attachment(request: Request, chat_id: uuid.UUID,
user: Annotated[User, Depends(current_user)], filename: str = Query()):
await run_in_threadpool(visible_chat, chat_id, user)
return await attachments.receive(request, store.pool, chat_id, user.id, filename)
def accessible_attachment(attachment_id, user):
with store.pool.connection() as db:
row = db.execute('SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id '
'WHERE a.id=%s AND m.user_id=%s', (attachment_id, user.id)).fetchone()
if not row:
raise ApiProblem(404, 'not_found', 'Вложение недоступно')
return row
@app.get('/api/v1/attachments/{attachment_id}')
def attachment_metadata(attachment_id: uuid.UUID, user: Annotated[User, Depends(current_user)]):
row = accessible_attachment(attachment_id, user)
return attachments.public_attachment(row)
@app.get('/api/v1/attachments/{attachment_id}/content')
def attachment_content(attachment_id: uuid.UUID, user: Annotated[User, Depends(current_user)],
variant: str = Query('original')):
row = accessible_attachment(attachment_id, user)
if row['status'] != 'ready':
raise ApiProblem(409, 'conflict', 'Вложение ещё не готово или обработка завершилась ошибкой')
return attachments.download(row,variant)