from __future__ import annotations import base64 import json import logging import os import re import time import uuid from datetime import datetime, timedelta, timezone from contextlib import asynccontextmanager import hashlib import secrets import socket import resource import psycopg from psycopg.rows import dict_row from psycopg.types.json import Jsonb from psycopg_pool import ConnectionPool, PoolTimeout import redis from argon2 import PasswordHasher from argon2.exceptions import VerificationError, InvalidHashError from starlette.exceptions import HTTPException from src.config import DATABASE_URL, CACHE_URL, SESSION_TTL, POOL_MAX from src.errors import ApiProblem from src import attachments from botocore.exceptions import BotoCoreError, ClientError from typing import Annotated from fastapi import Depends, FastAPI, Header, Query, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse, Response from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from starlette.concurrency import run_in_threadpool logging.basicConfig( level=os.getenv("LOG_LEVEL", "INFO").upper(), format="%(asctime)s %(levelname)s %(message)s", ) logger = logging.getLogger("messenger") def utc_now() -> datetime: return datetime.now(timezone.utc) def as_json(value: BaseModel) -> dict: return value.model_dump(mode="json") class StrictModel(BaseModel): model_config = ConfigDict(extra="forbid") class User(StrictModel): id: uuid.UUID username: str display_name: str created_at: datetime class CreateUser(StrictModel): username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") display_name: str = Field(min_length=1, max_length=100) password: str = Field(min_length=12, max_length=128) @field_validator("password") @classmethod def password_has_visible_characters(cls, value: str) -> str: if not value.strip(): raise ValueError("password must not contain only whitespace") return value @field_validator("display_name") @classmethod def display_name_has_visible_characters(cls, value: str) -> str: if not value.strip(): raise ValueError("display_name must not contain only whitespace") return value class Chat(StrictModel): id: uuid.UUID title: str member_ids: list[uuid.UUID] created_at: datetime class CreateChat(StrictModel): title: str = Field(min_length=1, max_length=100) member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100) @field_validator("title") @classmethod def title_has_visible_characters(cls, value: str) -> str: if not value.strip(): raise ValueError("title must not contain only whitespace") return value @field_validator("member_ids") @classmethod def member_ids_are_unique(cls, value: list[uuid.UUID]) -> list[uuid.UUID]: if len(value) != len(set(value)): raise ValueError("member_ids must be unique") return value class Message(StrictModel): id: uuid.UUID chat_id: uuid.UUID sender_id: uuid.UUID text: str created_at: datetime attachment_ids: list[uuid.UUID] = Field(default_factory=list) class CreateMessage(StrictModel): text: str = Field(max_length=4000) attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10) @model_validator(mode='after') def valid_message(self): if not self.text.strip() and not self.attachment_ids: raise ValueError('Нужен текст или вложение') if len(self.attachment_ids) != len(set(self.attachment_ids)): raise ValueError('Повторяющиеся attachment_ids') return self class Store: """Сессии и метаданные в PostgreSQL; Redis — только кеш профилей.""" def __init__(self) -> None: self.pool = ConnectionPool( DATABASE_URL, min_size=1, max_size=POOL_MAX, timeout=2, open=False, kwargs={"row_factory": dict_row, "connect_timeout": 2, "options": "-c statement_timeout=3000 -c lock_timeout=2000 -c timezone=UTC"}, ) self.cache = redis.Redis.from_url( CACHE_URL, decode_responses=True, socket_connect_timeout=0.25, socket_timeout=0.25, ) def start(self) -> None: self.pool.open() self.pool.wait(timeout=10) def close(self) -> None: self.pool.close() self.cache.close() def chat(self, db, chat_id: uuid.UUID) -> Chat | None: row = db.execute("SELECT * FROM chats WHERE id=%s", (chat_id,)).fetchone() if row is None: return None members = db.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id", (chat_id,)).fetchall() return Chat(**row, member_ids=[member["user_id"] for member in members]) def chat_messages(self, db, chat_id: uuid.UUID, limit: int, after) -> list[Message]: # В lab1 здесь были общий список/список чата и bisect_right. # Теперь границу ищет B-tree PostgreSQL (chat_id, created_at, id). query = ("SELECT m.*, COALESCE((SELECT array_agg(a.attachment_id ORDER BY a.position) " "FROM message_attachments a WHERE a.message_id=m.id),'{}'::uuid[]) AS attachment_ids " "FROM messages m WHERE chat_id=%s") params = [chat_id] if after is not None: query += " AND (created_at,id)>(%s,%s)" params.extend(after) rows = db.execute(query + " ORDER BY created_at,id LIMIT %s", params + [limit + 1]).fetchall() return [Message(**row) for row in rows] INSTANCE_ID = os.getenv("INSTANCE_ID") or f"{socket.gethostname()}:{uuid.uuid4().hex[:8]}" store = Store() passwords = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) dummy_hash = passwords.hash(secrets.token_urlsafe(32)) @asynccontextmanager async def lifespan(app): store.start() yield store.close() app = FastAPI(title="MTUCI PostgreSQL messenger", version="5.0.0", lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) def request_id(request: Request) -> str: return request.state.request_id def error_response(request: Request, status: int, code: str, message: str, challenge: str | None = None) -> JSONResponse: rid = getattr(request.state, "request_id", str(uuid.uuid4())) headers = {"X-Request-Id": rid, "X-Instance-Id": INSTANCE_ID} if challenge: headers["WWW-Authenticate"] = challenge return JSONResponse( status_code=status, content={"error": {"code": code, "message": message, "request_id": rid}}, headers=headers, ) @app.middleware("http") async def request_context(request: Request, call_next): request.state.request_id = str(uuid.uuid4()) started = time.perf_counter() status = 500 try: # Без токена отказываем ДО обращения к БД и проверки JSON. if request.url.path.startswith("/api/v1/") and request.url.path not in { "/api/v1/users", "/api/v1/auth/sessions" }: scheme, _, token = request.headers.get("authorization", "").partition(" ") if scheme.lower() != "bearer" or not token or len(token) > 256: status = 401 return error_response(request, 401, "unauthorized", "Требуется Bearer", 'Bearer realm="messenger"') json_endpoints = { "/api/v1/users", "/api/v1/chats", } requires_json = request.method == "POST" and ( request.url.path in json_endpoints or re.fullmatch(r"/api/v1/chats/[^/]+/messages", request.url.path) is not None ) media_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() if requires_json and media_type != "application/json": status = 415 return error_response( request, 415, "unsupported_media_type", "Требуется Content-Type: application/json", ) if requires_json: # Ограниченный JSON; после body() Starlette повторно отдаёт его маршруту. raw = await request.body() if len(raw) > 65536: status = 413 return error_response(request, 413, "payload_too_large", "Запрос слишком большой") response = await call_next(request) status = response.status_code response.headers["X-Request-Id"] = request.state.request_id response.headers["X-Instance-Id"] = INSTANCE_ID return response finally: logger.info( "request method=%s path=%s status=%s duration_ms=%.3f request_id=%s", request.method, request.url.path, status, (time.perf_counter() - started) * 1000, request.state.request_id, ) @app.exception_handler(ApiProblem) async def api_problem_handler(request: Request, exc: ApiProblem): return error_response(request, exc.status, exc.code, exc.message, exc.challenge) @app.exception_handler(RequestValidationError) async def validation_handler(request: Request, exc: RequestValidationError): return error_response(request, 400, "invalid_request", "Запрос не соответствует контракту") @app.exception_handler(Exception) async def unexpected_error_handler(request: Request, exc: Exception): logger.error("unexpected_error request_id=%s", request_id(request)) return error_response(request, 500, "internal_error", "Внутренняя ошибка сервера") @app.exception_handler(psycopg.Error) @app.exception_handler(PoolTimeout) async def database_error_handler(request: Request, exc: Exception): return error_response(request, 503, "unavailable", "Хранилище временно недоступно") @app.exception_handler(BotoCoreError) @app.exception_handler(ClientError) async def object_store_error_handler(request: Request, exc: Exception): return error_response(request, 503, 'unavailable', 'Объектное хранилище временно недоступно') @app.exception_handler(HTTPException) async def http_error_handler(request: Request, exc: HTTPException): code = "not_found" if exc.status_code == 404 else "invalid_request" return error_response(request, exc.status_code, code, "Неверный маршрут или метод") def unauthorized(basic=False): challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' raise ApiProblem(401, "unauthorized", "Нужна действительная идентификация", challenge) def token_hash(token: str) -> str: return hashlib.sha256(token.encode()).hexdigest() def current_user(authorization: Annotated[str | None, Header()] = None) -> User: scheme, _, token = (authorization or "").partition(" ") if scheme.lower() != "bearer" or not token or len(token) > 256: unauthorized() with store.pool.connection() as db: session = db.execute("SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at>now()", (token_hash(token),)).fetchone() if session is None: unauthorized() key = "user:" + str(session["user_id"]) cache_available = True try: cached = store.cache.get(key) if cached: user = User.model_validate_json(cached) if user.id == session["user_id"]: return user except (redis.RedisError, ValueError): cache_available = False row = db.execute("SELECT id,username,display_name,created_at FROM users WHERE id=%s", (session["user_id"],)).fetchone() user = User(**row) if cache_available: try: store.cache.setex(key, 60, user.model_dump_json()) except redis.RedisError: pass return user def decode_cursor(cursor: str, expected_scope: str) -> tuple[datetime, uuid.UUID]: try: if not cursor or len(cursor) > 1024: raise ValueError padded = cursor + "=" * (-len(cursor) % 4) payload = json.loads(base64.b64decode(padded.encode(), altchars=b"-_", validate=True).decode()) if not isinstance(payload, dict) or set(payload) != {"scope", "created_at", "id"}: raise ValueError if payload["scope"] != expected_scope: raise ValueError if not isinstance(payload["created_at"], str) or not isinstance(payload["id"], str): raise ValueError created_at = datetime.fromisoformat(payload["created_at"].replace("Z", "+00:00")) item_id = uuid.UUID(payload["id"]) if created_at.tzinfo is None: raise ValueError return created_at, item_id except (ValueError, TypeError, KeyError, json.JSONDecodeError, UnicodeError): raise ApiProblem(400, "invalid_request", "Некорректный cursor") from None def encode_cursor(scope: str, created_at: datetime, item_id: uuid.UUID) -> str: payload = { "scope": scope, "created_at": created_at.isoformat().replace("+00:00", "Z"), "id": str(item_id), } return base64.urlsafe_b64encode(json.dumps(payload, separators=(",", ":")).encode()).decode().rstrip("=") def page(items: list, limit: int, scope: str) -> dict: # SQL уже выбрал limit+1 строк после курсора; весь список не загружаем. selected = items[:limit] next_cursor = None if len(items) > limit and selected: last = selected[-1] next_cursor = encode_cursor(scope, last.created_at, last.id) return {"items": [as_json(item) for item in selected], "next_cursor": next_cursor} def visible_chat(chat_id: uuid.UUID, user: User) -> Chat: with store.pool.connection() as db: member = db.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s", (chat_id, user.id)).fetchone() if member is None: raise ApiProblem(404, "not_found", "Объект не найден") return store.chat(db, chat_id) def page_parameters(request: Request, scope: str, limit: int, cursor: str | None): # Query(ge/le) сохранён из lab1, добавлена проверка дублей и формы limit. if set(request.query_params) - {"limit", "cursor"} or any( len(request.query_params.getlist(k)) > 1 for k in request.query_params ): raise ApiProblem(400, "invalid_request", "Некорректные параметры страницы") if not re.fullmatch(r"[0-9]{1,3}", request.query_params.get("limit", "50")): raise ApiProblem(400, "invalid_request", "Некорректный limit") return decode_cursor(cursor, scope) if cursor is not None else None @app.get("/health/live") def liveness(): return {"status": "ok"} @app.get("/health/ready") def readiness(): with store.pool.connection() as db: db.execute("SELECT 1") attachments.storage.ready() return {"status": "ok"} @app.get("/internal/pool") def pool_statistics(): # Диагностика доступна только внутри Docker network: LB закрывает этот путь. return store.pool.get_stats() @app.get('/internal/memory') def memory_statistics(): result = {'process_peak_rss_bytes': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss * 1024} from pathlib import Path for name in ['/sys/fs/cgroup/memory.peak', '/sys/fs/cgroup/memory/memory.max_usage_in_bytes']: path = Path(name) if path.is_file(): result['container_peak_bytes'] = int(path.read_text().strip()) result['container_peak_source'] = name break return result @app.post("/api/v1/users", status_code=201) def register_user(body: CreateUser): # Синхронные маршруты FastAPI исполняет в worker thread, не в event loop. hashed = passwords.hash(body.password) try: with store.pool.connection() as db: row = db.execute( "INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) " "RETURNING id,username,display_name,created_at", (uuid.uuid4(), body.username, body.display_name, hashed), ).fetchone() except psycopg.errors.UniqueViolation: raise ApiProblem(409, "conflict", "Username уже занят") from None return as_json(User(**row)) @app.post("/api/v1/auth/sessions", status_code=201) def create_session(request: Request, authorization: Annotated[str | None, Header()] = None): if request.headers.get("content-length", "0") != "0" or "transfer-encoding" in request.headers: raise ApiProblem(400, "invalid_request", "Вход не принимает тело запроса") scheme, _, encoded = (authorization or "").partition(" ") try: if scheme.lower() != "basic" or len(encoded) > 2048: raise ValueError username, password = base64.b64decode(encoded, validate=True).decode("utf-8").split(":", 1) if not re.fullmatch(r"[a-z0-9_]{3,32}", username) or not 12 <= len(password) <= 128: raise ValueError except (ValueError, UnicodeError): unauthorized(basic=True) with store.pool.connection() as db: row = db.execute("SELECT * FROM users WHERE username=%s", (username,)).fetchone() try: passwords.verify(row["password_hash"] if row else dummy_hash, password) except (VerificationError, InvalidHashError): unauthorized(basic=True) if row is None: unauthorized(basic=True) token = secrets.token_urlsafe(32) expires_at = utc_now() + timedelta(seconds=SESSION_TTL) with store.pool.connection() as db: db.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)", (token_hash(token), row["id"], expires_at)) user = User(**{k: row[k] for k in ("id", "username", "display_name", "created_at")}) return {"token": token, "token_type": "Bearer", "user": as_json(user), "expires_at": expires_at.isoformat().replace("+00:00", "Z")} @app.delete("/api/v1/auth/sessions/current", status_code=204) def delete_session(user: Annotated[User, Depends(current_user)], authorization: Annotated[str, Header()]): token = authorization.partition(" ")[2] with store.pool.connection() as db: db.execute("DELETE FROM sessions WHERE token_hash=%s", (token_hash(token),)) return Response(status_code=204) @app.get("/api/v1/users/me") def get_current_user(user: Annotated[User, Depends(current_user)]): return as_json(user) @app.post("/api/v1/chats", status_code=201) def create_chat(body: CreateChat, user: Annotated[User, Depends(current_user)]): if user.id not in body.member_ids: raise ApiProblem(400, "invalid_request", "Текущий пользователь должен быть участником") with store.pool.connection() as db: found = db.execute("SELECT id FROM users WHERE id=ANY(%s)", (body.member_ids,)).fetchall() if len(found) != len(body.member_ids): raise ApiProblem(400, "invalid_request", "Указан неизвестный участник") row = db.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *", (uuid.uuid4(), body.title)).fetchone() with db.cursor() as cur: cur.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", [(row["id"], member) for member in body.member_ids]) chat = store.chat(db, row["id"]) return as_json(chat) # COMMIT уже выполнен для чата и всех участников. @app.get("/api/v1/chats") def list_chats( request: Request, user: Annotated[User, Depends(current_user)], limit: Annotated[int, Query(ge=1, le=100)] = 50, cursor: Annotated[str | None, Query(min_length=1)] = None, ): scope = f"chats:{user.id}" after = page_parameters(request, scope, limit, cursor) query = "SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s" params = [user.id] if after is not None: query += " AND (c.created_at,c.id)>(%s,%s)" params.extend(after) with store.pool.connection() as db: rows = db.execute(query + " ORDER BY c.created_at,c.id LIMIT %s", params + [limit + 1]).fetchall() chats = [store.chat(db, row["id"]) for row in rows] return page(chats, limit, scope) @app.get("/api/v1/chats/{chat_id}") def get_chat(chat_id: uuid.UUID, user: Annotated[User, Depends(current_user)]): return as_json(visible_chat(chat_id, user)) @app.post("/api/v1/chats/{chat_id}/messages", status_code=201) def create_message( request: Request, chat_id: uuid.UUID, body: CreateMessage, user: Annotated[User, Depends(current_user)], idempotency_key: Annotated[str | None, Header()] = None, ): if (not idempotency_key or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key) or len(request.headers.getlist("idempotency-key")) != 1): raise ApiProblem(400, "invalid_request", "Нужен корректный Idempotency-Key") visible_chat(chat_id, user) # Повтор тоже проверяет сессию и участие. canonical = {'text': body.text} if body.attachment_ids: canonical['attachment_ids'] = [str(value) for value in body.attachment_ids] payload = json.dumps(canonical, ensure_ascii=False, sort_keys=True, separators=(",", ":")) digest = hashlib.sha256(payload.encode()).hexdigest() lock_scope = f"{user.id}:POST:{chat_id}:{idempotency_key}" with store.pool.connection() as db: # Общая блокировка в PostgreSQL, а не локальный Lock одной реплики. # Коллизия хеша только сериализует независимые запросы; ключи сравниваются полностью. db.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))", (lock_scope,)) previous = db.execute( "SELECT * FROM message_requests WHERE user_id=%s AND chat_id=%s AND request_key=%s", (user.id, chat_id, idempotency_key), ).fetchone() if previous and previous["expires_at"] > utc_now(): if previous["body_hash"] != digest: raise ApiProblem(409, "conflict", "Ключ уже использован с другим сообщением") result = previous["response"] else: for attachment_id in body.attachment_ids: found = db.execute('SELECT chat_id,status FROM attachments WHERE id=%s FOR KEY SHARE', (attachment_id,)).fetchone() if not found or found['chat_id'] != chat_id: raise ApiProblem(404, 'not_found', 'Вложение недоступно') if found['status'] not in {'queued','processing','ready'}: raise ApiProblem(409, 'conflict', 'Вложение ещё недоступно') if previous: db.execute("DELETE FROM message_requests WHERE user_id=%s AND chat_id=%s AND request_key=%s", (user.id, chat_id, idempotency_key)) row = db.execute( "INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING *", (uuid.uuid4(), chat_id, user.id, body.text), ).fetchone() for position, attachment_id in enumerate(body.attachment_ids): db.execute('INSERT INTO message_attachments(message_id,attachment_id,position) VALUES (%s,%s,%s)', (row['id'], attachment_id, position)) result = as_json(Message(**row, attachment_ids=body.attachment_ids)) db.execute( "INSERT INTO message_requests(user_id,chat_id,request_key,body_hash,response,expires_at) " "VALUES (%s,%s,%s,%s,%s,clock_timestamp()+interval '24 hours')", (user.id, chat_id, idempotency_key, digest, Jsonb(result)), ) # Сообщение и результат повтора зафиксированы одной транзакцией до ответа. return result @app.get("/api/v1/chats/{chat_id}/messages") def list_messages( request: Request, chat_id: uuid.UUID, user: Annotated[User, Depends(current_user)], limit: Annotated[int, Query(ge=1, le=100)] = 50, cursor: Annotated[str | None, Query(min_length=1)] = None, ): visible_chat(chat_id, user) scope = f"messages:{chat_id}" after = page_parameters(request, scope, limit, cursor) with store.pool.connection() as db: messages = store.chat_messages(db, chat_id, limit, after) return page(messages, limit, scope) @app.post('/api/v1/chats/{chat_id}/attachments', status_code=202) async def upload_attachment(request: Request, chat_id: uuid.UUID, user: Annotated[User, Depends(current_user)], filename: str = Query()): await run_in_threadpool(visible_chat, chat_id, user) return await attachments.receive(request, store.pool, chat_id, user.id, filename) def accessible_attachment(attachment_id, user): with store.pool.connection() as db: row = db.execute('SELECT a.* FROM attachments a JOIN chat_members m ON m.chat_id=a.chat_id ' 'WHERE a.id=%s AND m.user_id=%s', (attachment_id, user.id)).fetchone() if not row: raise ApiProblem(404, 'not_found', 'Вложение недоступно') return row @app.get('/api/v1/attachments/{attachment_id}') def attachment_metadata(attachment_id: uuid.UUID, user: Annotated[User, Depends(current_user)]): row = accessible_attachment(attachment_id, user) return attachments.public_attachment(row) @app.get('/api/v1/attachments/{attachment_id}/content') def attachment_content(attachment_id: uuid.UUID, user: Annotated[User, Depends(current_user)], variant: str = Query('original')): row = accessible_attachment(attachment_id, user) if row['status'] != 'ready': raise ApiProblem(409, 'conflict', 'Вложение ещё не готово или обработка завершилась ошибкой') return attachments.download(row,variant)