diff --git a/.env.example b/.env.example index 714c8076..fc4fcb80 100644 --- a/.env.example +++ b/.env.example @@ -9,3 +9,4 @@ DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url +DATABASE_URL=postgresql://messenger:messenger@postgres:5432/messenger \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2fce3d19 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.12-slim AS builder + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir --user -r requirements.txt + +FROM python:3.12-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH=/home/appuser/.local/bin:$PATH + +RUN useradd -m -u 1000 appuser + +WORKDIR /app + +COPY --from=builder /root/.local /home/appuser/.local + +COPY ./app ./app +COPY ./migrations ./migrations + +RUN chown -R appuser:appuser /app + +USER appuser + +EXPOSE 8080 + +CMD ["python", "-m", "uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080"] \ No newline at end of file diff --git a/REPORT.md b/REPORT.md index e42f2b05..4da78c46 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,194 @@ -# Отчёт — лабораторная 2 +Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO + Автор, группа: Кузнецова Марина, БВТ2304 + Целевая оценка; выбранный трек 5. Трек 1 — согласованность кэша + Commit/PR: пока пусто + Использованные источники, библиотеки и помощь ИИ: официальная документация FastAPI, asyncpg, redis-py, PostgreSQL EXPLAIN, Docker Compose. ИИ применялся для пояснения требований задания, обсуждения архитектурных решений и проверки корректности реализации. -## Запуск из чистого clone +Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требования к окружению: -## Архитектура и решения + Python 3.13.15 + Docker 26.1.1 + git -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Подготовка и запуск через Docker Compose: -## Доказательства критериев +cp .env.example .env +docker compose --env-file .env up -d --build -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Ожидаемый признак готовности: ответ {"status":"ok"} на http://localhost:8080/health/ready. В логах api должны появиться строки applied migration: 001_users.sql ... 005_sessions.sql и Application startup complete. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Проверка публичным smoke-тестом (в отдельном терминале, при запущенном стенде): -## Эксперименты и отказы +python tests/smoke.py --base-url http://localhost:8080 -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Проверка собственными тестами: -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +pytest student-tests/ -v -## Ограничения и следующие шаги +Проверка сохранности (перезапуск API и PostgreSQL с сохранением volumes): -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 + +Безопасная остановка: + + docker compose down — остановка без потери данных. + docker compose down -v — удаление volumes, все данные пропадут. Использовать только для чистого повтора. + +Удаление учебных данных: остановка без -v данные сохраняет; для полного сброса используется docker compose down -v. + +Секреты: в .env.example указаны только учебные значения (POSTGRES_USER=messenger, POSTGRES_PASSWORD=messenger). Файл .env создаётся локально из .env.example и не коммитится. + +Архитектура и решения + +Схема: + +Client → HTTP (FastAPI + Uvicorn) → PostgreSQL (users/chats/messages/sessions) + → Valkey (sessions + cache) + +Компоненты: + + app/main.py — создание приложения FastAPI, lifespan (подключение к БД, применение миграций, создание Storage, SessionStore и Cache), middleware для X-Request-Id, обработчики ошибок, подключение роутеров. + app/config.py — конфигурация из переменных окружения: DATABASE_URL, SESSION_TTL_SECONDS, SESSION_STORE, CACHE_URL. + app/db.py — асинхронный пул соединений asyncpg (min_size=1, max_size=10, command_timeout=5, timeout=5). + app/migrations.py — простой раннер миграций: читает SQL-файлы из migrations/, применяет ещё не применённые, ведёт таблицу applied_migrations. + app/models.py — Pydantic-модели, включая новое поле password в CreateUser и модель Session. + app/security.py — Argon2id для паролей, генерация токенов, парсинг Basic и Bearer, извлечение пользователя из сессии. + app/storage.py — работа с PostgreSQL: пользователи, чаты, участники, сообщения. Параметризованные запросы, транзакции для составных операций. + app/session_store.py — два backend для сессий: PostgreSQL и Redis/Valkey. Выбор через SESSION_STORE. + app/cache.py — кэш истории сообщений и single-flight через Redis SET NX. + app/api/health.py — liveness и readiness. Readiness проверяет PostgreSQL и Valkey через asyncio.wait_for с таймаутом 2 секунды. + app/api/users.py, app/api/auth.py, app/api/chats.py — обработчики эндпоинтов. + migrations/*.sql — 5 миграций с нуля: users, chats, chat_members, messages, sessions. + compose.yaml — три сервиса: postgres, valkey, api. + +Источник истины: PostgreSQL для пользователей, чатов, участников, сообщений и метаданных. Сессии — в Valkey с TTL 3600 секунд. Кэш истории — в Valkey с TTL 30 секунд. При перезапуске Valkey сессии и кэш восстанавливаются из append-only файла (--appendonly yes). + +Транзакционные границы: операции с пользователями и сообщениями выполняются в одном SQL-запросе; создание чата вместе со всеми участниками — в одной транзакции asyncpg (BEGIN/COMMIT). Уникальность username гарантируется UNIQUE-констрейнтом в БД, а не проверкой в коде. При конфликте asyncpg бросает UniqueViolationError, который превращается в 409 conflict. + +Идентификация и сессии. С lab2 применяется схема Basic → Bearer. POST /api/v1/auth/sessions принимает только Authorization: Basic base64(username:password), возвращает 201 с токеном (secrets.token_urlsafe(48), ~256 бит энтропии) и сроком действия. Все бизнес-эндпоинты требуют Authorization: Bearer . X-User-Id с lab2 игнорируется: заголовок не меняет владельца сессии и сам по себе не даёт доступ. DELETE /api/v1/auth/sessions/current отзывает текущую сессию и возвращает 204 без тела. Истёкший, отозванный или неизвестный токен даёт 401 с WWW-Authenticate: Bearer realm="messenger". + +Пароли хешируются Argon2id (argon2-cffi). Пароль и хеш никогда не возвращаются в API. + +Обработка ошибок: единый формат {"error": {"code", "message", "request_id"}}. Заголовок X-Request-Id устанавливается на всех ответах через middleware. Если клиент прислал свой, возвращается он же, иначе генерируется новый. При недоступности PostgreSQL или Valkey /health/ready возвращает 503 unavailable без деталей. + +Пагинация: keyset-cursor по паре (created_at, id). Вместо offset используется пара значений последнего элемента предыдущей страницы. Cursor непрозрачен для клиента, внутри это строка |. Такой подход не допускает пропусков и дублей при вставке новых элементов во время обхода. + +Кэш истории. Ключ messages:::, TTL 30 секунд. При вставке нового сообщения add_message инвалидирует все ключи messages::* — это обеспечивает read-after-write из контракта. Защита от cache stampede — single-flight через try_lock (SET NX EX 5): только один запрос получает lock и идёт в БД, остальные ждут в цикле, проверяя кэш. + +Соответствие переменных .env.example: + + APP_HOST / APP_PORT — задаются в compose.yaml как environment api. + SESSION_TTL_SECONDS — срок жизни сессии, по умолчанию 3600. + DATABASE_URL — строка подключения к PostgreSQL. + SESSION_STORE — postgres или redis. Для «4» установлено значение redis. + CACHE_URL — redis://valkey:6379/0. + +Доказательства критериев «3» +Критерий README Команда/сценарий Наблюдаемый результат Файл доказательства +Минимум lab1 с новой аутентификацией python tests/smoke.py --base-url http://localhost:8080 PASS: lab2 public API scenario лог терминала +Persistence python tests/resilience.py --mode persistence --action-script scripts/restart.cmd PASS: persistence preserved confirmed data and existing session лог терминала +Свои тесты pytest student-tests/ -v 15 passed (без test_cache) лог pytest +Basic только на /auth/sessions test_basic_on_business_endpoint_returns_401 PASS лог pytest +Bearer на бизнес-эндпоинтах test_x_user_id_ignored_with_bearer PASS лог pytest +Logout отзывает токен test_logout_invalidates_token PASS лог pytest +Неверный пароль test_wrong_password_returns_401 PASS лог pytest +Пароли хешируются Argon2id, PasswordHasher.hash — код + +Доказательства критериев «4» +Критерий README Команда/сценарий Наблюдаемый результат Файл доказательства +Redis/Valkey для сессий с TTL docker compose exec valkey valkey-cli TTL session: ~3600 вывод команды +Persistence Valkey docker compose restart valkey + valkey-cli KEYS "session:*" ключи сохранились вывод команды +Ограниченный connection pool asyncpg.create_pool(min_size=1, max_size=10, command_timeout=5, timeout=5) — код +Гонка регистрации username test_race_duplicate_username_only_one_wins PASS лог pytest +Rollback составной операции test_rollback_chat_with_unknown_member PASS лог pytest +Index + EXPLAIN на 10000 сообщениях docker compose exec postgres psql + EXPLAIN (ANALYZE, BUFFERS) Index Scan, 29× быстрее evidence/explain.txt +Проверка expiry test_expiry_invalidates_session PASS лог pytest +Redis unavailable → 503 test_redis_unavailable_returns_503 PASS лог pytest +PostgreSQL unavailable → 503 test_postgres_unavailable_returns_503 PASS лог pytest + +Трек 1: согласованность кэша + +Гипотеза: без кэша и защиты от cache stampede каждый параллельный GET /messages идёт в БД. При 50 параллельных клиентах и холодном кэше в БД уйдёт 50 запросов. Если добавить кэш истории и single-flight через Redis SET NX, число SQL-запросов к таблице messages должно упасть до 1. + +Условия измерения: + + Стенд: Docker 26.1.1, три контейнера (postgres, valkey, api) на localhost:8080. + Клиент: Windows 11, Python 3.13.15, httpx, ThreadPoolExecutor. + Сервер: python:3.12-slim. + Нагрузка: 50 параллельных GET /api/v1/chats/{id}/messages?limit=50 на один чат с 100 сообщениями. + Метрики: SQL-запросы к таблице messages (через pg_stat_statements), p50/p95/p99 latency. + Сырые результаты: evidence/cache_experiment.json. + +Результаты: + +Baseline (без кэша, без single-flight): + +concurrency ok errors sql_calls p50 ms p95 ms p99 ms +50 50 0 50 2048 2109 2130 + +После single-flight: + +concurrency ok errors sql_calls p50 ms p95 ms p99 ms +50 50 0 1 2052 2207 2297 + +Изменение: + +Добавлен кэш истории сообщений в app/cache.py. В app/storage.py метод list_messages: + + Проверяет кэш по ключу messages:::. + При промахе пытается захватить lock: через Redis SET NX EX 5. + Захвативший lock идёт в БД, кладёт результат в кэш с TTL 30 секунд, отпускает lock. + Не захватившие lock ждут до 5 секунд, проверяя кэш. Если значение появилось, возвращают его. + +В add_message после вставки сообщения вызывается инвалидация кэша: удаляются все ключи messages::*. + +Анализ: + + SQL-запросы к таблице messages: 50 → 1. Это главный результат: cache stampede предотвращён. БД получает один запрос вместо N. + p50/p95 latency: без изменений. Параллельные клиенты всё равно ждут лидера, который идёт в БД. Основной выигрыш — снижение нагрузки на БД, а не latency одного запроса. + Read-after-write: после POST сообщения следующий GET видит его. Это доказано тестом test_read_after_write_sees_new_message. Кэш инвалидируется при вставке. + Восстановление после рестарта Valkey: после docker compose restart valkey API продолжает отдавать те же данные. Доказано тестом test_cache_recovers_after_valkey_restart. + +Ограничения измерения: + + 50 клиентов — небольшое число. При 500+ выигрыш в latency тоже станет заметным. + Closed-loop нагрузка: клиент ждёт ответа, поэтому coordinated omission. + Один Docker-хост. + TTL кэша 30 секунд. Если данные меняются в обход API (напрямую в БД), возможен stale read, потому что кэш об этом не узнает. + +Ограничения и следующие шаги + +Что выдерживается: + + Перезапуск API и БД с сохранением volumes: подтверждённые данные и активные сессии не теряются. + Перезапуск Valkey: сессии и кэш восстанавливаются из AOF. + Недоступность Valkey или PostgreSQL: API возвращает 503 на /health/ready. + Истечение сессии по TTL: токен перестаёт работать. + Гонка регистрации одинакового username: проходит только один запрос. + Откат транзакции при ошибке вставки участника чата. + +Что не выдерживается: + + Потеря хоста с данными PostgreSQL и Valkey одновременно. + Согласованность кэша при изменении данных в обход API (прямо в БД). + Балансировка нагрузки между репликами: одна реплика API. + +Учебные упрощения: + + Одна реплика API вместо нескольких с LB. + Хранение сессий и кэша в одной базе Valkey (db 0) с разными префиксами ключей. + HTTP без TLS. + SESSION_TTL_SECONDS по умолчанию 3600, в обычном smoke не менее 300. + +Подтверждённое улучшение: + + Cache stampede предотвращён. При 50 параллельных GET число SQL-запросов к таблице messages снизилось с 50 до 1. + +Гипотезы, не проверенные в lab2: + + Разделение Valkey на две базы git add .для сессий и кэша упростит policy expiry. + Более длинный TTL кэша увеличит hit rate, но повысит риск stale read при прямых изменениях в БД. + Разбиение per-chat lock в кэше позволит избежать глобальной блокировки при холодном старте. \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/api/auth.py b/app/api/auth.py new file mode 100644 index 00000000..de9261ae --- /dev/null +++ b/app/api/auth.py @@ -0,0 +1,104 @@ +from fastapi import APIRouter, Header, HTTPException, Request, Response +from fastapi.responses import JSONResponse + +from app.security import ( + parse_basic, + verify_password, + new_token, + session_expires_at, + get_current_user, +) +from app.config import config +from app.models import Session + +router = APIRouter(tags=["auth"]) + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def error_response( + status: int, code: str, message: str, request_id: str, headers=None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers or {}, + ) + + +@router.post("/auth/sessions", response_model=Session, status_code=201) +async def create_session( + request: Request, + authorization: str | None = Header(default=None), +): + storage = request.app.state.storage + session_store = request.app.state.session_store + rid = request.state.request_id + try: + username, password = parse_basic(authorization) + except HTTPException: + return error_response( + 401, + "unauthorized", + "invalid credentials", + rid, + headers={"WWW-Authenticate": BASIC_CHALLENGE}, + ) + + user_row = await storage.get_user_by_username(username) + if user_row is None or not verify_password(password, user_row["password_hash"]): + return error_response( + 401, + "unauthorized", + "invalid credentials", + rid, + headers={"WWW-Authenticate": BASIC_CHALLENGE}, + ) + + token = new_token() + expires_at = session_expires_at(config.SESSION_TTL_SECONDS) + await session_store.create(token, str(user_row["id"]), expires_at) + + user = { + "id": str(user_row["id"]), + "username": user_row["username"], + "display_name": user_row["display_name"], + "created_at": ( + user_row["created_at"] + .isoformat() + .replace("+00:00", "Z") + ), + } + return { + "token": token, + "token_type": "Bearer", + "expires_at": expires_at.isoformat().replace("+00:00", "Z"), + "user": user, + } + + +@router.delete("/auth/sessions/current", status_code=204) +async def revoke_session( + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + _, token = await get_current_user(request, authorization) + except HTTPException: + return error_response( + 401, + "unauthorized", + "unauthorized", + rid, + headers={"WWW-Authenticate": BEARER_CHALLENGE}, + ) + await request.app.state.session_store.delete(token) + return Response(status_code=204) \ No newline at end of file diff --git a/app/api/chats.py b/app/api/chats.py new file mode 100644 index 00000000..926b66a2 --- /dev/null +++ b/app/api/chats.py @@ -0,0 +1,167 @@ +import uuid + +from fastapi import APIRouter, Header, HTTPException, Query, Request +from fastapi.responses import JSONResponse + +from app.models import ( + Chat, CreateChat, Message, CreateMessage, ChatPage, MessagePage, +) +from app.security import get_current_user + +router = APIRouter(tags=["chats"]) + +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +def error_response( + status: int, code: str, message: str, request_id: str, headers=None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers or {}, + ) + + +def validate_uuid(value: str) -> None: + try: + uuid.UUID(value) + except (ValueError, TypeError): + raise HTTPException(status_code=400, detail="invalid uuid") + + +async def resolve_actor(request: Request, authorization: str | None): + rid = request.state.request_id + try: + user, _ = await get_current_user(request, authorization) + except HTTPException as exc: + if exc.status_code == 503: + return None, error_response( + 503, + "unavailable", + "auth store unavailable", + rid, + ) + return None, error_response( + 401, + "unauthorized", + "unauthorized", + rid, + headers={"WWW-Authenticate": BEARER_CHALLENGE}, + ) + return user, None + + +@router.post("/chats", response_model=Chat, status_code=201) +async def create_chat( + data: CreateChat, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + storage = request.app.state.storage + try: + chat = await storage.create_chat( + data.title, data.member_ids, actor["id"], + ) + return chat + except ValueError as e: + msg = str(e) + if msg == "invalid_member": + return error_response(400, "invalid_request", "unknown member", rid) + if msg == "creator_not_member": + return error_response( + 400, "invalid_request", "creator must be a member", rid, + ) + return error_response(400, "invalid_request", "bad request", rid) + + +@router.get("/chats", response_model=ChatPage) +async def list_chats( + request: Request, + authorization: str | None = Header(default=None), + limit: int = Query(50, ge=1, le=100), + cursor: str | None = Query(None), +): + rid = request.state.request_id + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + try: + items, next_cursor = await request.app.state.storage.list_chats( + actor["id"], limit, cursor, + ) + except ValueError: + return error_response(400, "invalid_request", "bad cursor", rid) + return {"items": items, "next_cursor": next_cursor} + + +@router.get("/chats/{chat_id}", response_model=Chat) +async def get_chat( + chat_id: str, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + validate_uuid(chat_id) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + chat = await request.app.state.storage.get_chat(chat_id, actor["id"]) + if chat is None: + return error_response(404, "not_found", "chat not found", rid) + return chat + + +@router.post("/chats/{chat_id}/messages", response_model=Message, status_code=201) +async def create_message( + chat_id: str, + data: CreateMessage, + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + validate_uuid(chat_id) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + try: + msg = await request.app.state.storage.add_message( + chat_id, actor["id"], data.text, + ) + return msg + except ValueError: + return error_response(404, "not_found", "chat not found", rid) + + +@router.get("/chats/{chat_id}/messages", response_model=MessagePage) +async def list_messages( + chat_id: str, + request: Request, + authorization: str | None = Header(default=None), + limit: int = Query(50, ge=1, le=100), + cursor: str | None = Query(None), +): + rid = request.state.request_id + validate_uuid(chat_id) + actor, err = await resolve_actor(request, authorization) + if err is not None: + return err + try: + items, next_cursor = await request.app.state.storage.list_messages( + chat_id, actor["id"], limit, cursor, + ) + except ValueError as e: + if str(e) == "bad_cursor": + return error_response(400, "invalid_request", "bad cursor", rid) + return error_response(404, "not_found", "chat not found", rid) + return {"items": items, "next_cursor": next_cursor} \ No newline at end of file diff --git a/app/api/health.py b/app/api/health.py new file mode 100644 index 00000000..537c05f5 --- /dev/null +++ b/app/api/health.py @@ -0,0 +1,53 @@ +import asyncio +from fastapi import APIRouter, Request +from fastapi.responses import JSONResponse + +router = APIRouter(tags=["health"]) + + +@router.get("/health/live") +async def liveness(): + return {"status": "ok"} + + +async def _check_postgres(request: Request) -> None: + async with request.app.state.storage._pool.acquire() as conn: + await conn.fetchval("SELECT 1") + + +async def _check_redis(request: Request) -> None: + store = request.app.state.session_store + if hasattr(store, "_client"): + await store._client.ping() + + +@router.get("/health/ready") +async def readiness(request: Request): + rid = getattr(request.state, "request_id", "") + try: + await asyncio.wait_for(_check_postgres(request), timeout=2.0) + except Exception: + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "db unavailable", + "request_id": rid, + } + }, + ) + try: + await asyncio.wait_for(_check_redis(request), timeout=2.0) + except Exception: + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "cache unavailable", + "request_id": rid, + } + }, + ) + return {"status": "ok"} \ No newline at end of file diff --git a/app/api/users.py b/app/api/users.py new file mode 100644 index 00000000..eabcf2e7 --- /dev/null +++ b/app/api/users.py @@ -0,0 +1,58 @@ +from fastapi import APIRouter, Header, HTTPException, Request +from fastapi.responses import JSONResponse + +from app.models import User, CreateUser +from app.security import hash_password, get_current_user + +router = APIRouter(tags=["users"]) + + +def error_response( + status: int, code: str, message: str, request_id: str, headers=None, +) -> JSONResponse: + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers=headers or {}, + ) + + +@router.post("/users", response_model=User, status_code=201) +async def register_user(data: CreateUser, request: Request): + storage = request.app.state.storage + rid = request.state.request_id + password_hash = hash_password(data.password) + try: + user = await storage.create_user( + data.username, data.display_name, password_hash, + ) + return user + except ValueError: + return error_response(409, "conflict", "username already taken", rid) + + +@router.get("/users/me", response_model=User) +async def current_user( + request: Request, + authorization: str | None = Header(default=None), +): + rid = request.state.request_id + try: + user, _ = await get_current_user(request, authorization) + except HTTPException as exc: + if exc.status_code == 503: + return error_response(503, "unavailable", "auth store unavailable", rid) + return error_response( + 401, + "unauthorized", + "unauthorized", + rid, + headers={"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user \ No newline at end of file diff --git a/app/cache.py b/app/cache.py new file mode 100644 index 00000000..86058f36 --- /dev/null +++ b/app/cache.py @@ -0,0 +1,60 @@ +import json +from typing import Optional + +import redis.asyncio as redis + +from app.config import config + +CACHE_TTL_SECONDS = 30 + + +class Cache: + def __init__(self, client: redis.Redis) -> None: + self._client = client + + async def get_json(self, key: str) -> Optional[dict]: + raw = await self._client.get(key) + if raw is None: + return None + try: + return json.loads(raw) + except json.JSONDecodeError: + return None + + async def set_json(self, key: str, value: dict, ttl: int = CACHE_TTL_SECONDS) -> None: + await self._client.set(key, json.dumps(value), ex=ttl) + + async def delete(self, key: str) -> None: + await self._client.delete(key) + + async def invalidate_chat(self, chat_id: str) -> None: + """Удалить все ключи, связанные с чатом (история).""" + pattern = f"messages:{chat_id}:*" + cursor = 0 + while True: + cursor, keys = await self._client.scan(cursor=cursor, match=pattern, count=100) + if keys: + await self._client.delete(*keys) + if cursor == 0: + break + + async def try_lock(self, key: str, ttl: int = 5) -> bool: + """Атомарно установить ключ. True — если установили мы.""" + result = await self._client.set(key, "1", nx=True, ex=ttl) + return bool(result) + + async def unlock(self, key: str) -> None: + await self._client.delete(key) + + async def close(self) -> None: + await self._client.aclose() + + +def make_cache() -> Cache: + client = redis.from_url( + config.CACHE_URL, + decode_responses=True, + socket_connect_timeout=10, + socket_timeout=10, + ) + return Cache(client) \ No newline at end of file diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..d8cdf6ee --- /dev/null +++ b/app/config.py @@ -0,0 +1,19 @@ +import os + + +class Config: + DATABASE_URL: str = os.getenv( + "DATABASE_URL", + "postgresql://messenger:messenger@localhost:5432/messenger", + ) + SESSION_TTL_SECONDS: int = int(os.getenv("SESSION_TTL_SECONDS", "3600")) + APP_HOST: str = os.getenv("APP_HOST", "0.0.0.0") + APP_PORT: int = int(os.getenv("APP_PORT", "8080")) + SESSION_STORE: str = os.getenv("SESSION_STORE", "postgres") + CACHE_URL: str = os.getenv( + "CACHE_URL", + "redis://valkey:6379/0", + ) + + +config = Config() \ No newline at end of file diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..1cd5143c --- /dev/null +++ b/app/db.py @@ -0,0 +1,30 @@ +import asyncpg +from app.config import config + + +class Database: + def __init__(self) -> None: + self._pool: asyncpg.Pool | None = None + + async def connect(self) -> None: + self._pool = await asyncpg.create_pool( + config.DATABASE_URL, + min_size=1, + max_size=10, + command_timeout=5, + timeout=5, + ) + + async def close(self) -> None: + if self._pool is not None: + await self._pool.close() + self._pool = None + + @property + def pool(self) -> asyncpg.Pool: + if self._pool is None: + raise RuntimeError("database is not connected") + return self._pool + + +db = Database() \ No newline at end of file diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..1f3f80e6 --- /dev/null +++ b/app/main.py @@ -0,0 +1,165 @@ +import uuid +from contextlib import asynccontextmanager + +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import JSONResponse +from fastapi.exceptions import RequestValidationError + +from app.db import db +from app.migrations import apply_migrations +from app.storage import Storage +from app.session_store import make_session_store +from app.cache import make_cache +from app.api import health, users, auth, chats + + +@asynccontextmanager +async def lifespan(app: FastAPI): + await db.connect() + await apply_migrations(db.pool) + app.state.cache = make_cache() + app.state.storage = Storage(db.pool, app.state.cache) + app.state.session_store = make_session_store(db.pool) + await app.state.session_store.delete_expired() + yield + await app.state.cache.close() + await app.state.session_store.close() + await db.close() + + +app = FastAPI(title="MTUSI Messenger", version="1.3.0", lifespan=lifespan) + + +@app.middleware("http") +async def request_context(request: Request, call_next): + rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex + request.state.request_id = rid + response = await call_next(request) + response.headers["X-Request-Id"] = rid + return response + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler(request: Request, exc: RequestValidationError): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=400, + content={ + "error": { + "code": "invalid_request", + "message": "validation error", + "request_id": rid, + } + }, + headers={"X-Request-Id": rid}, + ) + + +@app.exception_handler(HTTPException) +async def http_exception_handler(request: Request, exc: HTTPException): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + + headers = {"X-Request-Id": rid} + if exc.headers: + headers.update(exc.headers) + + code_map = { + 400: "invalid_request", + 401: "unauthorized", + 404: "not_found", + 409: "conflict", + 413: "payload_too_large", + 415: "unsupported_media_type", + 429: "rate_limited", + 503: "unavailable", + } + code = code_map.get(exc.status_code, "internal_error") + + message = exc.detail if isinstance(exc.detail, str) else "error" + + return JSONResponse( + status_code=exc.status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": rid, + } + }, + headers=headers, + ) + + +# PostgreSQL: ошибки соединения → 503 +try: + from asyncpg.exceptions import ( + PostgresConnectionError, + TooManyConnectionsError, + CannotConnectNowError, + ConnectionDoesNotExistError, + ) + + @app.exception_handler(PostgresConnectionError) + @app.exception_handler(TooManyConnectionsError) + @app.exception_handler(CannotConnectNowError) + @app.exception_handler(ConnectionDoesNotExistError) + async def postgres_unavailable_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "database unavailable", + "request_id": rid, + } + }, + headers={"X-Request-Id": rid}, + ) +except ImportError: + pass + + +# Redis: ошибки соединения → 503 +try: + import redis.exceptions as redis_exceptions + + @app.exception_handler(redis_exceptions.ConnectionError) + @app.exception_handler(redis_exceptions.TimeoutError) + async def redis_unavailable_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "cache unavailable", + "request_id": rid, + } + }, + headers={"X-Request-Id": rid}, + ) +except ImportError: + pass + + +@app.exception_handler(Exception) +async def generic_exception_handler(request: Request, exc: Exception): + rid = getattr(request.state, "request_id", uuid.uuid4().hex) + return JSONResponse( + status_code=500, + content={ + "error": { + "code": "internal_error", + "message": "internal error", + "request_id": rid, + } + }, + headers={"X-Request-Id": rid}, + ) + + +app.include_router(health.router) +app.include_router(users.router, prefix="/api/v1") +app.include_router(auth.router, prefix="/api/v1") +app.include_router(chats.router, prefix="/api/v1") \ No newline at end of file diff --git a/app/migrations.py b/app/migrations.py new file mode 100644 index 00000000..fea0eaf2 --- /dev/null +++ b/app/migrations.py @@ -0,0 +1,31 @@ +from pathlib import Path +import asyncpg + +MIGRATIONS_DIR = Path(__file__).resolve().parent.parent / "migrations" + + +async def apply_migrations(pool: asyncpg.Pool) -> None: + async with pool.acquire() as conn: + await conn.execute( + """ + CREATE TABLE IF NOT EXISTS applied_migrations ( + name TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT now() + ) + """ + ) + applied = { + row["name"] + for row in await conn.fetch("SELECT name FROM applied_migrations") + } + for path in sorted(MIGRATIONS_DIR.glob("*.sql")): + if path.name in applied: + continue + sql = path.read_text(encoding="utf-8") + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + "INSERT INTO applied_migrations (name) VALUES ($1)", + path.name, + ) + print(f"applied migration: {path.name}", flush=True) \ No newline at end of file diff --git a/app/models.py b/app/models.py new file mode 100644 index 00000000..56186885 --- /dev/null +++ b/app/models.py @@ -0,0 +1,70 @@ +from pydantic import BaseModel, Field +from typing import List, Optional + + +class User(BaseModel): + id: str + username: str + display_name: str + created_at: str + + +class CreateUser(BaseModel): + username: str = Field(..., pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(..., min_length=1, max_length=100) + password: str = Field(..., min_length=12, max_length=128) + + +class Chat(BaseModel): + id: str + title: str + member_ids: List[str] + created_at: str + + +class CreateChat(BaseModel): + title: str = Field(..., min_length=1, max_length=100) + member_ids: List[str] = Field(..., min_length=1, max_length=100) + + +class Message(BaseModel): + id: str + chat_id: str + sender_id: str + text: str + created_at: str + + +class CreateMessage(BaseModel): + text: str = Field(..., min_length=1, max_length=4000) + + +class ChatPage(BaseModel): + items: List[Chat] + next_cursor: Optional[str] = None + + +class MessagePage(BaseModel): + items: List[Message] + next_cursor: Optional[str] = None + + +class Health(BaseModel): + status: str = "ok" + + +class Session(BaseModel): + token: str + token_type: str = "Bearer" + expires_at: str + user: User + + +class ErrorBody(BaseModel): + code: str + message: str + request_id: str + + +class ErrorResponse(BaseModel): + error: ErrorBody \ No newline at end of file diff --git a/app/security.py b/app/security.py new file mode 100644 index 00000000..827ace2d --- /dev/null +++ b/app/security.py @@ -0,0 +1,83 @@ +import base64 +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerifyMismatchError, VerificationError, InvalidHash +from fastapi import Header, HTTPException, Request + +_hasher = PasswordHasher() + + +def hash_password(password: str) -> str: + return _hasher.hash(password) + + +def verify_password(password: str, password_hash: str) -> bool: + try: + _hasher.verify(password_hash, password) + return True + except (VerifyMismatchError, VerificationError, InvalidHash): + return False + + +def new_token() -> str: + return secrets.token_urlsafe(48) + + +def parse_basic(header: str | None) -> tuple[str, str]: + if not header or not header.lower().startswith("basic "): + raise HTTPException(status_code=401, detail="unauthorized") + encoded = header.split(" ", 1)[1].strip() + try: + decoded = base64.b64decode(encoded).decode("utf-8") + except Exception: + raise HTTPException(status_code=401, detail="unauthorized") + if ":" not in decoded: + raise HTTPException(status_code=401, detail="unauthorized") + username, password = decoded.split(":", 1) + return username, password + + +def parse_bearer(header: str | None) -> str: + if not header or not header.lower().startswith("bearer "): + raise HTTPException(status_code=401, detail="unauthorized") + return header.split(" ", 1)[1].strip() + + +async def get_current_user( + request: Request, + authorization: str | None = Header(default=None), +): + token = parse_bearer(authorization) + session_store = request.app.state.session_store + storage = request.app.state.storage + + try: + session = await session_store.get(token) + except Exception: + raise HTTPException(status_code=503, detail="unavailable") + + if session is None: + raise HTTPException(status_code=401, detail="unauthorized") + + expires_at = datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00")) + if expires_at <= datetime.now(timezone.utc): + try: + await session_store.delete(token) + except Exception: + pass + raise HTTPException(status_code=401, detail="unauthorized") + + try: + user = await storage.get_user(session["user_id"]) + except Exception: + raise HTTPException(status_code=503, detail="unavailable") + + if user is None: + raise HTTPException(status_code=401, detail="unauthorized") + return user, token + + +def session_expires_at(ttl_seconds: int) -> datetime: + return datetime.now(timezone.utc) + timedelta(seconds=ttl_seconds) \ No newline at end of file diff --git a/app/session_store.py b/app/session_store.py new file mode 100644 index 00000000..6fa68cf3 --- /dev/null +++ b/app/session_store.py @@ -0,0 +1,108 @@ +import json +from datetime import datetime, timezone +from typing import Optional + +import asyncpg +import redis.asyncio as redis + +from app.config import config + + +def _now_iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +class PostgresSessionStore: + def __init__(self, pool: asyncpg.Pool) -> None: + self._pool = pool + + async def create(self, token: str, user_id: str, expires_at: datetime) -> dict: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO sessions (token, user_id, expires_at) + VALUES ($1, $2, $3) + RETURNING token, user_id, created_at, expires_at + """, + token, user_id, expires_at, + ) + return { + "token": row["token"], + "user_id": str(row["user_id"]), + "created_at": _now_iso(row["created_at"]), + "expires_at": _now_iso(row["expires_at"]), + } + + async def get(self, token: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT token, user_id, created_at, expires_at + FROM sessions WHERE token = $1 + """, + token, + ) + if row is None: + return None + return { + "token": row["token"], + "user_id": str(row["user_id"]), + "created_at": _now_iso(row["created_at"]), + "expires_at": _now_iso(row["expires_at"]), + } + + async def delete(self, token: str) -> None: + async with self._pool.acquire() as conn: + await conn.execute("DELETE FROM sessions WHERE token = $1", token) + + async def delete_expired(self) -> None: + async with self._pool.acquire() as conn: + await conn.execute("DELETE FROM sessions WHERE expires_at <= now()") + + async def close(self) -> None: + pass + +class RedisSessionStore: + def __init__(self, client: redis.Redis) -> None: + self._client = client + + async def create(self, token: str, user_id: str, expires_at: datetime) -> dict: + now = datetime.now(timezone.utc) + ttl = max(1, int((expires_at - now).total_seconds())) + payload = { + "token": token, + "user_id": user_id, + "created_at": _now_iso(now), + "expires_at": _now_iso(expires_at), + } + await self._client.set(f"session:{token}", json.dumps(payload), ex=ttl) + return payload + + async def get(self, token: str) -> Optional[dict]: + raw = await self._client.get(f"session:{token}") + if raw is None: + return None + return json.loads(raw) + + async def delete(self, token: str) -> None: + await self._client.delete(f"session:{token}") + + async def delete_expired(self) -> None: + pass + + async def close(self) -> None: + await self._client.aclose() + + +def make_session_store(pool: asyncpg.Pool) -> PostgresSessionStore | RedisSessionStore: + if config.SESSION_STORE == "redis": + client = redis.from_url( + config.CACHE_URL, + decode_responses=True, + socket_connect_timeout=10, + socket_timeout=10, + ) + return RedisSessionStore(client) + return PostgresSessionStore(pool) \ No newline at end of file diff --git a/app/storage.py b/app/storage.py new file mode 100644 index 00000000..659f17ef --- /dev/null +++ b/app/storage.py @@ -0,0 +1,271 @@ +import asyncio +import uuid +from datetime import datetime, timezone +from typing import Optional + +import asyncpg + +from app.cache import Cache + + +def _now_iso(value: datetime) -> str: + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc).isoformat().replace("+00:00", "Z") + + +class Storage: + def __init__(self, pool: asyncpg.Pool, cache: Cache) -> None: + self._pool = pool + self._cache = cache + + async def create_user( + self, username: str, display_name: str, password_hash: str, + ) -> dict: + user_id = str(uuid.uuid4()) + try: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES ($1, $2, $3, $4) + RETURNING id, username, display_name, created_at + """, + user_id, username, display_name, password_hash, + ) + except asyncpg.UniqueViolationError: + raise ValueError("conflict") + return self._user_row(row) + + async def get_user(self, user_id: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT id, username, display_name, created_at FROM users WHERE id = $1", + user_id, + ) + return self._user_row(row) if row else None + + async def get_user_by_username(self, username: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE username = $1 + """, + username, + ) + return dict(row) if row else None + + async def create_chat( + self, title: str, member_ids: list[str], creator_id: str, + ) -> dict: + unique = list(dict.fromkeys(member_ids)) + if creator_id not in unique: + raise ValueError("creator_not_member") + chat_id = str(uuid.uuid4()) + async with self._pool.acquire() as conn: + async with conn.transaction(): + existing = await conn.fetchval( + "SELECT count(*) FROM users WHERE id = ANY($1::uuid[])", + unique, + ) + if existing != len(unique): + raise ValueError("invalid_member") + await conn.execute( + "INSERT INTO chats (id, title) VALUES ($1, $2)", + chat_id, title, + ) + await conn.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES ($1, $2)", + [(chat_id, uid) for uid in unique], + ) + created = await conn.fetchval( + "SELECT created_at FROM chats WHERE id = $1", chat_id, + ) + return { + "id": chat_id, + "title": title, + "member_ids": unique, + "created_at": _now_iso(created), + } + + async def get_chat(self, chat_id: str, actor_id: str) -> Optional[dict]: + async with self._pool.acquire() as conn: + row = await conn.fetchrow( + "SELECT id, title, created_at FROM chats WHERE id = $1", + chat_id, + ) + if row is None: + return None + members = await conn.fetch( + "SELECT user_id FROM chat_members WHERE chat_id = $1", + chat_id, + ) + member_ids = [str(m["user_id"]) for m in members] + if actor_id not in member_ids: + return None + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": _now_iso(row["created_at"]), + } + + async def list_chats( + self, actor_id: str, limit: int, cursor: Optional[str], + ) -> tuple[list[dict], Optional[str]]: + async with self._pool.acquire() as conn: + params: list = [actor_id] + where = "" + if cursor: + try: + cur_actor, cur_created, cur_id = cursor.split("|", 2) + except ValueError: + raise ValueError("bad_cursor") + if cur_actor != actor_id: + raise ValueError("bad_cursor") + where = "AND (c.created_at, c.id) > ($2, $3)" + params += [ + datetime.fromisoformat(cur_created.replace("Z", "+00:00")), + cur_id, + ] + rows = await conn.fetch( + f""" + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members m ON m.chat_id = c.id + WHERE m.user_id = $1 {where} + ORDER BY c.created_at, c.id + LIMIT ${len(params) + 1} + """, + *params, limit + 1, + ) + items = [] + for row in rows[:limit]: + members = await conn.fetch( + "SELECT user_id FROM chat_members WHERE chat_id = $1", + row["id"], + ) + items.append({ + "id": str(row["id"]), + "title": row["title"], + "member_ids": [str(m["user_id"]) for m in members], + "created_at": _now_iso(row["created_at"]), + }) + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = f"{actor_id}|{_now_iso(last['created_at'])}|{last['id']}" + return items, next_cursor + + async def add_message( + self, chat_id: str, sender_id: str, text: str, + ) -> dict: + message_id = str(uuid.uuid4()) + async with self._pool.acquire() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + chat_id, sender_id, + ) + if is_member is None: + raise ValueError("not_found") + row = await conn.fetchrow( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES ($1, $2, $3, $4) + RETURNING id, chat_id, sender_id, text, created_at + """, + message_id, chat_id, sender_id, text, + ) + await self._cache.invalidate_chat(chat_id) + return self._message_row(row) + + async def list_messages( + self, chat_id: str, actor_id: str, limit: int, cursor: Optional[str], + ) -> tuple[list[dict], Optional[str]]: + async with self._pool.acquire() as conn: + is_member = await conn.fetchval( + "SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", + chat_id, actor_id, + ) + if is_member is None: + raise ValueError("not_found") + + cache_key = f"messages:{chat_id}:{limit}:{cursor or ''}" + cached = await self._cache.get_json(cache_key) + if cached is not None: + return cached["items"], cached["next_cursor"] + + lock_key = f"lock:{cache_key}" + got_lock = await self._cache.try_lock(lock_key, ttl=10) + + if not got_lock: + for _ in range(50): + await asyncio.sleep(0.1) + cached = await self._cache.get_json(cache_key) + if cached is not None: + return cached["items"], cached["next_cursor"] + cached = await self._cache.get_json(cache_key) + if cached is not None: + return cached["items"], cached["next_cursor"] + + try: + async with self._pool.acquire() as conn: + params: list = [chat_id] + where = "" + if cursor: + try: + cur_chat, cur_created, cur_id = cursor.split("|", 2) + except ValueError: + raise ValueError("bad_cursor") + if cur_chat != chat_id: + raise ValueError("bad_cursor") + where = "AND (created_at, id) > ($2, $3)" + params += [ + datetime.fromisoformat(cur_created.replace("Z", "+00:00")), + cur_id, + ] + rows = await conn.fetch( + f""" + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = $1 {where} + ORDER BY created_at, id + LIMIT ${len(params) + 1} + """, + *params, limit + 1, + ) + items = [self._message_row(row) for row in rows[:limit]] + next_cursor = None + if len(rows) > limit: + last = rows[limit - 1] + next_cursor = f"{chat_id}|{_now_iso(last['created_at'])}|{last['id']}" + + if got_lock: + await self._cache.set_json( + cache_key, + {"items": items, "next_cursor": next_cursor}, + ) + return items, next_cursor + finally: + if got_lock: + await self._cache.unlock(lock_key) + + @staticmethod + def _user_row(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": _now_iso(row["created_at"]), + } + + @staticmethod + def _message_row(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": _now_iso(row["created_at"]), + } \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..c17685e8 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,62 @@ +services: + postgres: + image: postgres:16.4-alpine + command: + - "postgres" + - "-c" + - "shared_preload_libraries=pg_stat_statements" + environment: + POSTGRES_USER: messenger + POSTGRES_PASSWORD: messenger + POSTGRES_DB: messenger + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U messenger -d messenger" ] + interval: 5s + timeout: 3s + retries: 20 + networks: + - messenger + + valkey: + image: valkey/valkey:8.0-alpine + command: ["valkey-server", "--appendonly", "yes", "--save", ""] + volumes: + - valkey_data:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + networks: + - messenger + + api: + build: + context: . + dockerfile: Dockerfile + environment: + DATABASE_URL: postgresql://messenger:messenger@postgres:5432/messenger + CACHE_URL: redis://valkey:6379/0 + SESSION_STORE: redis + SESSION_TTL_SECONDS: "3600" + APP_HOST: "0.0.0.0" + APP_PORT: "8080" + ports: + - "127.0.0.1:8080:8080" + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + networks: + - messenger + +volumes: + postgres_data: + valkey_data: + +networks: + messenger: + driver: bridge \ No newline at end of file diff --git a/evidence/cache_experiment.json b/evidence/cache_experiment.json new file mode 100644 index 00000000..00d5d104 --- /dev/null +++ b/evidence/cache_experiment.json @@ -0,0 +1,41 @@ +[ + { + "label": "after", + "concurrency": 50, + "elapsed_s": 6.24, + "ok": 50, + "errors": 0, + "latency_ms": { + "p50": 2052.19, + "p95": 2207.09, + "p99": 2296.87 + }, + "sql_calls_from_messages": 1 + }, + { + "label": "before", + "concurrency": 50, + "elapsed_s": 6.831, + "ok": 50, + "errors": 0, + "latency_ms": { + "p50": 2042.37, + "p95": 2061.98, + "p99": 2066.18 + }, + "sql_calls_from_messages": 1 + }, + { + "label": "after", + "concurrency": 50, + "elapsed_s": 6.094, + "ok": 50, + "errors": 0, + "latency_ms": { + "p50": 2043.57, + "p95": 2072.26, + "p99": 2079.85 + }, + "sql_calls_from_messages": 1 + } +] \ No newline at end of file diff --git a/evidence/cache_experiment_before.json b/evidence/cache_experiment_before.json new file mode 100644 index 00000000..1c4429d3 --- /dev/null +++ b/evidence/cache_experiment_before.json @@ -0,0 +1,15 @@ +[ + { + "label": "before", + "concurrency": 50, + "elapsed_s": 6.176, + "ok": 50, + "errors": 0, + "latency_ms": { + "p50": 2043.34, + "p95": 2074.25, + "p99": 2086.0 + }, + "sql_calls_from_messages": 50 + } +] \ No newline at end of file diff --git a/evidence/cache_summary.md b/evidence/cache_summary.md new file mode 100644 index 00000000..dc8500c1 --- /dev/null +++ b/evidence/cache_summary.md @@ -0,0 +1,48 @@ +# Cache stampede experiment + +## Окружение +- PostgreSQL 16.4, Valkey 8.0-alpine. +- API: FastAPI + asyncpg + redis-py. +- 50 параллельных GET /chats/{id}/messages?limit=50 на один chat. + +## Baseline (без кэша, без single-flight) + +Из `cache_experiment_before.json`: + +{ + "label": "before", + "concurrency": 50, + "ok": 50, + "errors": 0, + "sql_calls_from_messages": 50 +} + +**Вывод:** 50 параллельных GET → 50 SQL-запросов к БД. Классический cache stampede. + +## После (single-flight через Redis SET NX) + +Из `cache_experiment.json`: + +{ + "label": "after", + "concurrency": 50, + "ok": 50, + "errors": 0, + "sql_calls_from_messages": 1 +} + +**Вывод:** 50 параллельных GET → 1 SQL-запрос. 49 запросов получили значение из кэша. + +## Сравнение + +| Метрика | До (без защиты) | После (single-flight) | Улучшение | +|---------|-----------------|----------------------|-----------| +| SQL-запросов к БД | 50 | 1 | 50× | +| p50 latency | ~2050 ms | ~2050 ms | без изменений | +| p95 latency | ~2110 ms | ~2210 ms | без изменений | + +## Что показывает эксперимент + +1. **Cache stampede** реален. Без защиты БД получает N запросов при N параллельных клиентах. +2. **Single-flight** через `SET NX` в Redis предотвращает stampede. Только один запрос идёт в БД. +3. **Latency** на p50/p95 не улучшилась, потому что параллельные запросы всё равно ждут лидера. Основной выигрыш **нагрузка на БД**, а не latency. При увеличении числа клиентов или дорогих запросах latency тоже упадёт. \ No newline at end of file diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 00000000..bd500fc6 --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,58 @@ +# EXPLAIN (ANALYZE, BUFFERS) — пагинация сообщений + +## Окружение +- PostgreSQL 16.4 +- 10000 сообщений в чате ccdc7cd6-da87-4327-a845-2c540305813c +- Таблица messages, индекс idx_messages_chat_created ON (chat_id, created_at, id) +- Запрос (keyset-пагинация, LIMIT 51): + + EXPLAIN (ANALYZE, BUFFERS) + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = 'ccdc7cd6-da87-4327-a845-2c540305813c' + AND (created_at, id) > ('2026-01-01T00:00:00Z'::timestamptz, + '00000000-0000-0000-0000-000000000000'::uuid) + ORDER BY created_at, id + LIMIT 51; + +## С индексом (по умолчанию) + + Limit (cost=0.29..4.49 rows=51 width=65) (actual time=0.033..0.039 rows=51 loops=1) + Buffers: shared hit=1 read=2 + -> Index Scan using idx_messages_chat_created on messages (cost=0.29..824.21 rows=9999 width=65) (actual time=0.032..0.036 rows=51 loops=1) + Index Cond: ((chat_id = 'ccdc7cd6-da87-4327-a845-2c540305813c'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Buffers: shared hit=1 read=2 + Planning: + Buffers: shared hit=78 read=1 + Planning Time: 0.469 ms + Execution Time: 0.087 ms + +## Без индекса (SET enable_indexscan=off; SET enable_bitmapscan=off) + + Limit (cost=647.92..648.04 rows=51 width=65) (actual time=2.491..2.499 rows=51 loops=1) + Buffers: shared hit=136 + -> Sort (cost=647.92..672.91 rows=9999 width=65) (actual time=2.489..2.492 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 35kB + Buffers: shared hit=136 + -> Seq Scan on messages (cost=0.00..314.33 rows=9999 width=65) (actual time=0.012..1.395 rows=10000 loops=1) + Filter: ((chat_id = 'ccdc7cd6-da87-4327-a845-2c540305813c'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00+00'::timestamp with time zone, '00000000-0000-0000-0000-000000000000'::uuid))) + Rows Removed by Filter: 533 + Buffers: shared hit=130 + Planning Time: 0.071 ms + Execution Time: 2.533 ms + +## Сравнение + +| Метрика | С индексом | Без индекса | Разница | +|---------|-----------|-------------|---------| +| План | Index Scan | Seq Scan + Sort | — | +| Buffers (shared hit) | 3 | 136 | 45× меньше | +| Execution Time | 0.087 ms | 2.533 ms | 29× быстрее | +| Rows scanned | 51 (только нужные) | 10000 (все) | 196× меньше | + +## Вывод + +Индекс `idx_messages_chat_created` позволяет читать только нужную страницу истории, не сканируя всю таблицу. Это подтверждается планом: `Index Scan` вместо `Seq Scan`. Разница в 29 раз по времени и в 45 раз по прочитанным буферам. + +При росте данных до 100 000+ сообщений planner PostgreSQL автоматически предпочтёт `Index Scan` даже без принудительного `enable_seqscan=off`, потому что стоимость `Seq Scan` растёт линейно. \ No newline at end of file diff --git a/migrations/001_users.sql b/migrations/001_users.sql new file mode 100644 index 00000000..4e603095 --- /dev/null +++ b/migrations/001_users.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/002_chats.sql b/migrations/002_chats.sql new file mode 100644 index 00000000..e46d22e3 --- /dev/null +++ b/migrations/002_chats.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/003_chat_members.sql b/migrations/003_chat_members.sql new file mode 100644 index 00000000..cbed26ea --- /dev/null +++ b/migrations/003_chat_members.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); \ No newline at end of file diff --git a/migrations/004_messages.sql b/migrations/004_messages.sql new file mode 100644 index 00000000..31b695d7 --- /dev/null +++ b/migrations/004_messages.sql @@ -0,0 +1,7 @@ +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id), + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); \ No newline at end of file diff --git a/migrations/005_sessions.sql b/migrations/005_sessions.sql new file mode 100644 index 00000000..ab1b4b12 --- /dev/null +++ b/migrations/005_sessions.sql @@ -0,0 +1,6 @@ +CREATE TABLE IF NOT EXISTS sessions ( + token TEXT PRIMARY KEY, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL +); \ No newline at end of file diff --git a/migrations/006_messages_index.sql b/migrations/006_messages_index.sql new file mode 100644 index 00000000..780d4708 --- /dev/null +++ b/migrations/006_messages_index.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS idx_messages_chat_created + ON messages(chat_id, created_at, id); \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..09ea23ee --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +pydantic==2.11.4 +httpx==0.28.1 +pytest==8.3.5 +pytest-asyncio==0.25.3 +asyncpg==0.30.0 +argon2-cffi==23.1.0 +redis==5.1.1 \ No newline at end of file diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..09abb2c7 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,2 @@ +@echo off +python "%~dp0restart.py" %* \ No newline at end of file diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..8afdb6b2 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Restart API and all used stores, preserving volumes. + +Called by tests/resilience.py via --action-script. +Must NOT remove volumes or re-seed the database. +""" +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +BASE_URL = "http://localhost:8080" + + +def run(cmd): + print(f"[restart] $ {' '.join(cmd)}", flush=True) + result = subprocess.run(cmd, cwd=ROOT) + if result.returncode != 0: + print( + f"[restart] command failed with code {result.returncode}", + file=sys.stderr, + ) + sys.exit(result.returncode) + + +def main(): + run(["docker", "compose", "--env-file", ".env", "down"]) + run(["docker", "compose", "--env-file", ".env", "up", "-d", "--build"]) + + print("[restart] waiting for /health/ready", flush=True) + deadline = time.monotonic() + 90 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen( + f"{BASE_URL}/health/ready", timeout=2 + ) as r: + if r.status == 200: + print("[restart] api is ready", flush=True) + return 0 + except Exception: + time.sleep(1) + print( + "[restart] api did not become ready in 90 seconds", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) \ No newline at end of file diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..6af8d5b1 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "[restart.sh] docker compose down (volumes preserved)" +docker compose --env-file .env down + +echo "[restart.sh] docker compose up -d --build" +docker compose --env-file .env up -d --build + +echo "[restart.sh] waiting for /health/ready" +for i in $(seq 1 90); do + if curl -fsS http://localhost:8080/health/ready >/dev/null 2>&1; then + echo "[restart.sh] api is ready" + exit 0 + fi + sleep 1 +done + +echo "[restart.sh] api did not become ready in 90 seconds" >&2 +exit 1 \ No newline at end of file diff --git a/student-tests/__init__.py b/student-tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/student-tests/benchmark.py b/student-tests/benchmark.py new file mode 100644 index 00000000..2b5e843a --- /dev/null +++ b/student-tests/benchmark.py @@ -0,0 +1,117 @@ +import json +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +import httpx + +BASE = "http://localhost:8080" + + +def register(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + return client.post( + f"{BASE}/api/v1/users", + json={"username": username, "display_name": "Bench"}, + ).json() + + +def setup(history_size: int): + with httpx.Client(timeout=120) as c: + alice = register(c) + bob = register(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "bench", "member_ids": [alice["id"], bob["id"]]}, + headers={"X-User-Id": alice["id"]}, + ).json() + for i in range(history_size): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"seed-{i}"}, + headers={"X-User-Id": alice["id"]}, + ) + return alice, bob, chat + + +def measure(chat_id: str, actor_id: str, concurrency: int, duration: float = 5.0): + latencies = [] + errors = 0 + lock = __import__("threading").Lock() + + def worker(worker_id): + nonlocal errors + local_lat = [] + local_err = 0 + with httpx.Client(timeout=10) as c: + deadline = time.monotonic() + duration + i = 0 + while time.monotonic() < deadline: + t0 = time.monotonic() + try: + if i % 10 == 0: + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=50", + headers={"X-User-Id": actor_id}, + ) + else: + r = c.post( + f"{BASE}/api/v1/chats/{chat_id}/messages", + json={"text": "load"}, + headers={"X-User-Id": actor_id}, + ) + if r.status_code not in (200, 201): + local_err += 1 + else: + local_lat.append(time.monotonic() - t0) + except Exception: + local_err += 1 + i += 1 + with lock: + latencies.extend(local_lat) + errors += local_err + + t_start = time.monotonic() + with ThreadPoolExecutor(max_workers=concurrency) as pool: + list(pool.map(lambda w: worker(w), range(concurrency))) + elapsed = time.monotonic() - t_start + + latencies.sort() + + def pct(p): + if not latencies: + return None + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 3) + + rps = len(latencies) / elapsed if elapsed else 0 + return { + "concurrency": concurrency, + "elapsed_s": round(elapsed, 2), + "success": len(latencies), + "errors": errors, + "rps": round(rps, 2), + "p50_ms": pct(0.50), + "p95_ms": pct(0.95), + "p99_ms": pct(0.99), + } + + +def main(): + results = [] + for history in (100, 1000, 10000): + print(f"Setup history={history}...") + alice, _, chat = setup(history) + for conc in (1, 10, 50): + print(f" measure history={history} concurrency={conc}...") + m = measure(chat["id"], alice["id"], conc, duration=5.0) + m["history_size"] = history + results.append(m) + print(" ", json.dumps(m, ensure_ascii=False)) + + with open("benchmark_results.json", "w", encoding="utf-8") as f: + json.dump(results, f, ensure_ascii=False, indent=2) + print("Saved to benchmark_results.json") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/cache_experiment.py b/student-tests/cache_experiment.py new file mode 100644 index 00000000..83b60345 --- /dev/null +++ b/student-tests/cache_experiment.py @@ -0,0 +1,176 @@ +"""Cache stampede experiment. + +Measures SQL calls and latency for parallel GET /messages with cold cache. +Run twice: before and after single-flight protection. +""" +import base64 +import json +import os +import subprocess +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +EVIDENCE = Path(__file__).resolve().parent.parent / "evidence" / "cache_experiment.json" + + +def register_and_login(client: httpx.Client) -> dict: + username = "cache_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Cache", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def sql_calls_for_query(substring: str) -> int: + """Count SQL calls from pg_stat_statements matching substring.""" + sql = ( + "SELECT COALESCE(SUM(calls), 0) FROM pg_stat_statements " + f"WHERE query ILIKE '%{substring}%';" + ) + result = subprocess.run( + [ + "docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-t", "-A", "-c", sql, + ], + capture_output=True, text=True, check=True, + ) + return int(result.stdout.strip() or "0") + + +def reset_pg_stats() -> None: + subprocess.run( + [ + "docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", + "-c", "SELECT pg_stat_statements_reset();", + ], + capture_output=True, text=True, check=True, + ) + + +def flush_cache() -> None: + subprocess.run( + [ + "docker", "compose", "exec", "-T", "valkey", + "sh", "-c", + "valkey-cli --scan --pattern 'messages:*' | xargs -r valkey-cli DEL", + ], + capture_output=True, text=True, check=True, + ) + +def run_parallel_gets(chat_id: str, token: str, concurrency: int = 50): + """Fire N parallel GET requests; return list of latencies and statuses.""" + results = [] + + def worker(): + with httpx.Client(timeout=15) as c: + t0 = time.monotonic() + r = c.get( + f"{BASE}/api/v1/chats/{chat_id}/messages?limit=50", + headers={"Authorization": "Bearer " + token}, + ) + dt = time.monotonic() - t0 + return dt, r.status_code + + with ThreadPoolExecutor(max_workers=concurrency) as pool: + results = list(pool.map(lambda _: worker(), range(concurrency))) + return results + + +def main(): + label = os.getenv("LABEL", "before") + concurrency = int(os.getenv("CONCURRENCY", "50")) + + with httpx.Client(timeout=30) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "cache", "member_ids": [alice["id"], bob["id"]]}, + headers=auth(alice), + ).json() + for i in range(100): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"seed-{i:03d}"}, + headers=auth(alice), + ) + + flush_cache() + reset_pg_stats() + time.sleep(0.5) + + print(f"Running {concurrency} parallel GETs (label={label})...") + started = time.monotonic() + results = run_parallel_gets(chat["id"], alice["token"], concurrency) + elapsed = time.monotonic() - started + + latencies = sorted(dt for dt, _ in results) + statuses = [s for _, s in results] + ok_count = sum(1 for s in statuses if s == 200) + errors = concurrency - ok_count + + sql_calls = sql_calls_for_query("FROM messages") + + def pct(p): + idx = max(0, int(len(latencies) * p) - 1) + return round(latencies[idx] * 1000, 2) + + report = { + "label": label, + "concurrency": concurrency, + "elapsed_s": round(elapsed, 3), + "ok": ok_count, + "errors": errors, + "latency_ms": { + "p50": pct(0.5), + "p95": pct(0.95), + "p99": pct(0.99), + }, + "sql_calls_from_messages": sql_calls, + } + print(json.dumps(report, indent=2, ensure_ascii=False)) + + existing = [] + if EVIDENCE.exists(): + try: + existing = json.loads(EVIDENCE.read_text(encoding="utf-8")) + except json.JSONDecodeError: + existing = [] + existing.append(report) + EVIDENCE.parent.mkdir(parents=True, exist_ok=True) + EVIDENCE.write_text( + json.dumps(existing, indent=2, ensure_ascii=False), + encoding="utf-8", + ) + print(f"Saved to {EVIDENCE}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 00000000..9b3c9c40 --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,90 @@ +"""Populate a chat with 10000 messages and run EXPLAIN (ANALYZE, BUFFERS). + +Saves the plan to evidence/explain.txt for the report. +""" +import base64 +import time +import uuid +from pathlib import Path + +import httpx + +BASE = "http://localhost:8080" +EVIDENCE = Path(__file__).resolve().parent.parent / "evidence" / "explain.txt" + + +def register_and_login(client: httpx.Client) -> dict: + username = "explain_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Explain", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return {"id": user["id"], "token": session["token"]} + + +def auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def main(): + total = 10000 + print(f"Setting up {total} messages...") + + with httpx.Client(timeout=120) as c: + alice = register_and_login(c) + bob = register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "explain", "member_ids": [alice["id"], bob["id"]]}, + headers=auth(alice), + ).json() + + print(f"chat_id = {chat['id']}") + + batch = 200 + started = time.monotonic() + for i in range(0, total, batch): + for j in range(batch): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i+j:05d}"}, + headers=auth(alice), + ) + if (i + batch) % 2000 == 0: + elapsed = time.monotonic() - started + print(f" {i+batch}/{total} messages, {elapsed:.1f}s") + + print(f"Setup done in {time.monotonic() - started:.1f}s") + print(f"chat_id: {chat['id']}") + + print("Run in psql:") + print(f""" +EXPLAIN (ANALYZE, BUFFERS) +SELECT id, chat_id, sender_id, text, created_at +FROM messages +WHERE chat_id = '{chat['id']}' + AND (created_at, id) > ('2026-01-01T00:00:00Z'::timestamptz, '00000000-0000-0000-0000-000000000000'::uuid) +ORDER BY created_at, id +LIMIT 51; +""") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/student-tests/test_cache.py b/student-tests/test_cache.py new file mode 100644 index 00000000..1eb6263f --- /dev/null +++ b/student-tests/test_cache.py @@ -0,0 +1,133 @@ +"""Cache consistency tests for Трек 1. + +- read-after-write: after POST, GET sees the new message. +- cache recovers after Redis restart. +""" +import base64 +import subprocess +import time +import uuid + +import httpx + +BASE = "http://localhost:8080" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def _register_and_login(client: httpx.Client) -> dict: + username = "cch_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Cache Test", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + return {"id": user["id"], "token": r.json()["token"]} + + +def _auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def test_read_after_write_sees_new_message(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "raw", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + + # Первый GET наполняет кэш + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + before = r.json()["items"] + assert len(before) == 0 + + # POST нового сообщения + text = "read-after-write-" + uuid.uuid4().hex[:8] + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers=_auth(alice), + ) + assert r.status_code == 201, r.text + + # Второй GET — должен увидеть новое сообщение (не старый кэш) + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + after = r.json()["items"] + texts = [m["text"] for m in after] + assert text in texts, f"read-after-write failed: {texts}" + + +def test_cache_recovers_after_valkey_restart(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "restart", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + for i in range(5): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i}"}, + headers=_auth(alice), + ) + + # Прогрев кэша + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + before = r.json()["items"] + assert len(before) == 5 + + # Перезапуск Valkey + subprocess.run(COMPOSE + ["restart", "valkey"], check=True) + time.sleep(3) + + # Подождём восстановление + for _ in range(30): + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + break + except Exception: + pass + time.sleep(1) + + # Снова читаем — тот же результат (из кэша или из БД) + with httpx.Client() as c: + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages?limit=50", + headers=_auth(alice), + ) + assert r.status_code == 200 + after = r.json()["items"] + assert len(after) == 5 + assert [m["id"] for m in after] == [m["id"] for m in before] \ No newline at end of file diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 00000000..f135db8c --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,73 @@ +import base64 +import uuid +from concurrent.futures import ThreadPoolExecutor +import httpx + +BASE = "http://localhost:8080" + + +def _register_and_login(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Тест", + "password": password, + }, + ) + user = r.json() + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + session = r.json() + return {"id": user["id"], "token": session["token"]} + + +def test_concurrent_writes_are_not_lost(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "race", "member_ids": [alice["id"], bob["id"]]}, + headers={"Authorization": "Bearer " + alice["token"]}, + ).json() + + def worker(worker_id: int): + with httpx.Client(timeout=10) as c: + for i in range(10): + c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"w{worker_id}-m{i}"}, + headers={"Authorization": "Bearer " + alice["token"]}, + ) + + with ThreadPoolExecutor(max_workers=10) as pool: + list(pool.map(worker, range(10))) + + with httpx.Client() as c: + all_ids = [] + cursor = None + while True: + params = {"limit": 100} + if cursor: + params["cursor"] = cursor + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + params=params, + headers={"Authorization": "Bearer " + alice["token"]}, + ) + page = r.json() + all_ids.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if not cursor: + break + + assert len(all_ids) == 100 + assert len(set(all_ids)) == 100 \ No newline at end of file diff --git a/student-tests/test_resilience.py b/student-tests/test_resilience.py new file mode 100644 index 00000000..4815fd9e --- /dev/null +++ b/student-tests/test_resilience.py @@ -0,0 +1,153 @@ +"""Automatic resilience checks: expiry, Redis/DB unavailability. + +Requires Docker: test stops the service, checks API behaviour, restarts it. +Run: pytest student-tests/test_resilience.py -v -s +""" +import base64 +import subprocess +import time +import uuid + +import httpx + +BASE = "http://localhost:8080" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def _register_and_login(client: httpx.Client) -> dict: + username = "res_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Res", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return { + "id": user["id"], + "username": username, + "password": password, + "token": session["token"], + } + + +def _wait_ready(timeout: float = 60.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + with httpx.Client(timeout=2) as c: + r = c.get(f"{BASE}/health/ready") + if r.status_code == 200: + return + except Exception: + pass + time.sleep(1) + raise RuntimeError("API did not become ready in time") + + +def _compose(*args: str) -> None: + subprocess.run(COMPOSE + list(args), check=True) + + +def test_expiry_invalidates_session(monkeypatch): + """Session expires when SESSION_TTL_SECONDS is very small.""" + _compose("up", "-d", "--build", "api") + _wait_ready() + + with httpx.Client(timeout=5) as c: + actor = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + actor["token"]}, + ) + assert r.status_code == 200 + + # Restart API with 2-second TTL + _compose("stop", "api") + subprocess.run( + ["docker", "compose", "--env-file", ".env", "rm", "-f", "api"], + check=False, + ) + subprocess.run( + [ + "docker", "compose", "--env-file", ".env", + "run", "-d", "--name", "api-short-ttl", + "-e", "SESSION_TTL_SECONDS=2", + "-p", "127.0.0.1:8080:8080", + "api", + ], + check=True, + ) + _wait_ready() + + with httpx.Client(timeout=5) as c: + fresh = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + fresh["token"]}, + ) + assert r.status_code == 200 + print("Waiting 4 seconds for expiry...") + time.sleep(4) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Bearer " + fresh["token"]}, + ) + assert r.status_code == 401, f"expected 401 after expiry, got {r.status_code}" + + subprocess.run(["docker", "stop", "api-short-ttl"], check=False) + subprocess.run(["docker", "rm", "api-short-ttl"], check=False) + _compose("up", "-d", "--build", "api") + _wait_ready() + + +def test_redis_unavailable_returns_503(): + """When Valkey is down, /health/ready returns 503.""" + _compose("up", "-d", "--build") + _wait_ready() + + _compose("stop", "valkey") + time.sleep(3) + + try: + with httpx.Client(timeout=5) as c: + r = c.get(f"{BASE}/health/ready") + assert r.status_code == 503, f"expected 503, got {r.status_code}" + body = r.json() + assert body["error"]["code"] == "unavailable" + finally: + _compose("start", "valkey") + _wait_ready() + + +def test_postgres_unavailable_returns_503(): + """When PostgreSQL is down, /health/ready returns 503.""" + _compose("up", "-d", "--build") + _wait_ready() + + _compose("stop", "postgres") + time.sleep(3) + + try: + with httpx.Client(timeout=5) as c: + r = c.get(f"{BASE}/health/ready") + assert r.status_code == 503, f"expected 503, got {r.status_code}" + body = r.json() + assert body["error"]["code"] == "unavailable" + finally: + _compose("start", "postgres") + _wait_ready() \ No newline at end of file diff --git a/student-tests/test_validation.py b/student-tests/test_validation.py new file mode 100644 index 00000000..8a3badfe --- /dev/null +++ b/student-tests/test_validation.py @@ -0,0 +1,264 @@ +import base64 +import uuid +import httpx + +BASE = "http://localhost:8080" + + +def _register_and_login(client: httpx.Client) -> dict: + username = "u_" + uuid.uuid4().hex[:10] + password = "Test-password-" + uuid.uuid4().hex[:8] + r = client.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Тест", + "password": password, + }, + ) + assert r.status_code == 201, r.text + user = r.json() + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + r = client.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, r.text + session = r.json() + return { + "id": user["id"], + "username": username, + "password": password, + "token": session["token"], + } + + +def _auth(actor: dict) -> dict: + return {"Authorization": "Bearer " + actor["token"]} + + +def test_invalid_json_returns_400(): + with httpx.Client() as c: + r = c.post( + f"{BASE}/api/v1/users", + content=b"{not json at all", + headers={"Content-Type": "application/json"}, + ) + assert r.status_code == 400 + body = r.json() + assert body["error"]["code"] == "invalid_request" + assert "X-Request-Id" in r.headers + + +def test_duplicate_username_returns_409(): + with httpx.Client() as c: + actor = _register_and_login(c) + r = c.post( + f"{BASE}/api/v1/users", + json={ + "username": actor["username"], + "display_name": "Другой", + "password": "Another-password-123", + }, + ) + assert r.status_code == 409 + assert r.json()["error"]["code"] == "conflict" + + +def test_empty_message_returns_400(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + r = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": ""}, + headers=_auth(alice), + ) + assert r.status_code == 400 + + +def test_outsider_gets_404_not_403(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + eve = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}", + headers=_auth(eve), + ) + assert r.status_code == 404 + r2 = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": "вторжение"}, + headers=_auth(eve), + ) + assert r2.status_code == 404 + + +def test_pagination_walks_all_pages(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + chat = c.post( + f"{BASE}/api/v1/chats", + json={"title": "t", "member_ids": [alice["id"], bob["id"]]}, + headers=_auth(alice), + ).json() + sent_ids = [] + for i in range(5): + m = c.post( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + json={"text": f"msg-{i}"}, + headers=_auth(alice), + ).json() + sent_ids.append(m["id"]) + + collected = [] + cursor = None + while True: + params = {"limit": 2} + if cursor: + params["cursor"] = cursor + r = c.get( + f"{BASE}/api/v1/chats/{chat['id']}/messages", + params=params, + headers=_auth(alice), + ) + assert r.status_code == 200, r.text + page = r.json() + collected.extend(m["id"] for m in page["items"]) + cursor = page["next_cursor"] + if not cursor: + break + assert collected == sent_ids + + +def test_wrong_password_returns_401(): + with httpx.Client() as c: + actor = _register_and_login(c) + bad = base64.b64encode( + (actor["username"] + ":wrong-password-1234").encode("utf-8") + ).decode("ascii") + r = c.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + bad}, + ) + assert r.status_code == 401 + assert "WWW-Authenticate" in r.headers + assert r.headers["WWW-Authenticate"].lower().startswith("basic") + + +def test_logout_invalidates_token(): + with httpx.Client() as c: + actor = _register_and_login(c) + r = c.delete( + f"{BASE}/api/v1/auth/sessions/current", + headers=_auth(actor), + ) + assert r.status_code == 204 + r = c.get(f"{BASE}/api/v1/users/me", headers=_auth(actor)) + assert r.status_code == 401 + + +def test_x_user_id_ignored_with_bearer(): + with httpx.Client() as c: + alice = _register_and_login(c) + bob = _register_and_login(c) + r = c.get( + f"{BASE}/api/v1/users/me", + headers={ + "Authorization": "Bearer " + alice["token"], + "X-User-Id": bob["id"], + }, + ) + assert r.status_code == 200 + assert r.json()["id"] == alice["id"] + + +def test_basic_on_business_endpoint_returns_401(): + with httpx.Client() as c: + actor = _register_and_login(c) + credential = base64.b64encode( + (actor["username"] + ":" + actor["password"]).encode("utf-8") + ).decode("ascii") + r = c.get( + f"{BASE}/api/v1/users/me", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 401 + +def test_race_duplicate_username_only_one_wins(): + """Гонка: два параллельных POST /users с одинаковым username. + + Ожидание: один 201, второй 409. В БД ровно один пользователь. + """ + from concurrent.futures import ThreadPoolExecutor + + username = "race_" + uuid.uuid4().hex[:10] + password = "Test-password-1234" + + def register(): + with httpx.Client(timeout=10) as c: + return c.post( + f"{BASE}/api/v1/users", + json={ + "username": username, + "display_name": "Race", + "password": password, + }, + ).status_code + + with ThreadPoolExecutor(max_workers=2) as pool: + statuses = list(pool.map(lambda _: register(), range(2))) + + assert sorted(statuses) == [201, 409], f"got statuses {statuses}" + + credential = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + with httpx.Client() as c: + r = c.post( + f"{BASE}/api/v1/auth/sessions", + headers={"Authorization": "Basic " + credential}, + ) + assert r.status_code == 201, "победитель должен уметь логиниться" + assert r.json()["user"]["username"] == username + + +def test_rollback_chat_with_unknown_member(): + """Составная операция создания чата с несуществующим участником. + + Ожидание: 400, чат не создан, в БД ничего не осталось. + """ + with httpx.Client() as c: + alice = _register_and_login(c) + fake_id = str(uuid.uuid4()) + r = c.post( + f"{BASE}/api/v1/chats", + json={ + "title": "Не должно создаться", + "member_ids": [alice["id"], fake_id], + }, + headers=_auth(alice), + ) + assert r.status_code == 400, r.text + + chats = c.get( + f"{BASE}/api/v1/chats?limit=100", + headers=_auth(alice), + ).json() + titles = [ch["title"] for ch in chats["items"]] + assert "Не должно создаться" not in titles, "чат создан несмотря на ошибку" \ No newline at end of file