diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b0e6fd4d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.11-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + DATABASE_URL=postgresql://messenger:messenger@db:5432/messenger \ + SESSION_TTL_SECONDS=3600 + +WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt +COPY app.py /app/app.py +COPY migrations /app/migrations +EXPOSE 8080 +CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/Makefile b/Makefile index 9ecb223e..d25bf0ae 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,10 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence cache-consistency test-storage research-cache help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes; cache-consistency: cache TTL/stale-session proof" check: $(PYTHON) scripts/check_contract.py @@ -18,6 +18,15 @@ validate: test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) +cache-consistency: + $(PYTHON) tests/cache_consistency.py + +test-storage: + $(PYTHON) tests/storage_regression.py + +research-cache: + $(PYTHON) scripts/research_cache.py + load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index e42f2b05..c6ba04a6 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,130 @@ -# Отчёт — лабораторная 2 +# Лабораторная работа 2. -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +## Автор и состояние -## Запуск из чистого clone +- Автор: Савин Александр, БВТ2303. +- Целевая оценка: 5, трек 1 — согласованность кэша. +- Среда: macOS, Colima/Docker, docker-compose; API в образе Python 3.11, локальные исследовательские скрипты — Python 3.13. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +## Исправление замечания -## Архитектура и решения +Замечание: действительный токен пользователя A при конфликтующей, ещё не истёкшей записи session:T в Valkey возвращал пользователя B, в том числе во всех восьми параллельных запросах. [Исходный комментарий](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab2/pulls/63#issuecomment-436). -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Причина: прежний код сверял действительность сессии в PostgreSQL, но возвращал user_id и user из кэша без сравнения с БД. Исправление в Store.get_session_by_token/get_cached_session: PostgreSQL определяет возвращаемую идентичность; несовпадающий кэш удаляется и восстанавливается из авторитетной записи. Повторный SELECT сессии устранён. -## Доказательства критериев +В tests/cache_consistency.py добавлены проверки конфликтующего payload с expires_at=2100 год, восьми параллельных GET /api/v1/users/me, повреждённой записи и повторного заполнения кэша после logout. В последнем запуске на исправленном контейнере все восемь запросов вернули владельца токена A: concurrent_identity_checks=8/8. Отдельно проверены forged_identity_rejected=True, malformed_cache_recovers=True, repopulated_revoked_cache_rejected=True. Это дополняет прежнюю проверку истёкшего payload, которая не покрывала замечание №436. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +## Архитектура и схема -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +FastAPI/Uvicorn обслуживает REST API. PostgreSQL 16 — источник истины для пользователей, чатов, участников, сообщений и сессий. Valkey 8 хранит JSON-кэш сессий под session:. История сообщений читается непосредственно из PostgreSQL. -## Эксперименты и отказы +Миграция migrations/001_initial_schema.sql задаёт PK, FK, UNIQUE username/token, составной PK членства и индексы. Store.init ведёт schema_migrations и пропускает применённые версии. Миграции включены в Docker-образ. Создание чата и всех участников — одна транзакция; исключение приводит к rollback. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +SQL параметризован. psycopg_pool.ConnectionPool ограничивает количество соединений: min_size=1, DB_POOL_MAX_SIZE=10 по умолчанию, timeout=5 секунд, max_waiting=20. На соединениях заданы connect_timeout=5 секунд, statement_timeout=5000 мс и lock_timeout=500 мс. Пул открывается при старте API и закрывается при shutdown. Операции не удерживают соединение при повторном получении другого соединения; smoke отдельно прошёл при DB_POOL_MAX_SIZE=1. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +## Аутентификация и сессии -## Ограничения и следующие шаги +Регистрация использует библиотечный Argon2id: time_cost=3, memory_cost=65536 KiB, parallelism=2, hash_len=32, salt_len=16. Basic принимается только при создании сессии. Токен генерируется secrets.token_urlsafe(32); защищённые операции требуют Bearer и проверяют членство в чате. -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Сессия хранит expires_at и revoked_at в PostgreSQL. При каждом запросе выполняется один SELECT сессии с JOIN пользователя. Если сессия истекла или отозвана, кэш не даёт доступ. Если сессия действительна, кэш сравнивается с авторитетными данными целиком; повреждённое или несовпадающее значение заменяется. Возвращаются данные из БД, поэтому чужой user_id из кэша не используется. + +TTL кэша рассчитывается из оставшегося времени сессии; он не продлевает expires_at. Logout фиксирует revoked_at, затем удаляет кэш. RedisError приводит к обходу кэша, а недоступность PostgreSQL не позволяет подтвердить сессию. Для OperationalError, включая ошибки пула, API формирует 503. Health/live проверяет процесс, health/ready — PostgreSQL; Valkey не является обязательным для доступности. + +Важно: кэш не устраняет обязательный SELECT сессии. Преимущество кэша относительно решения вообще без Valkey не доказано. + +## Воспроизводимый запуск + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements.txt -r requirements-dev.txt +# Подготовить локальный .env по .env.example. +docker-compose up -d --build +# Дождаться 200 от http://localhost:8080/health/ready. +make check +make validate PYTHON=.venv/bin/python +make test PYTHON=.venv/bin/python +make cache-consistency PYTHON=.venv/bin/python +make test-storage PYTHON=.venv/bin/python +make research-cache PYTHON=.venv/bin/python +.venv/bin/python scripts/explain_pagination.py +make test-persistence PYTHON=.venv/bin/python ACTION_SCRIPT=scripts/restart.sh +``` + +Если установлен плагин Compose v2, эквивалентная команда — docker compose. Скрипт restart.sh поддерживает обе формы. Прямые SQL/Valkey-проверки по умолчанию используют опубликованные localhost:5432 и localhost:6379; при иных настройках задать DATABASE_URL и CACHE_URL. Для API-тестов можно задать BASE_URL. + +## Результаты после исправлений + +| Проверка | Результат | +| --- | --- | +| make check | PASS | +| make validate | PASS | +| Smoke на локальном API :8081 с pool max_size=1 | PASS | +| Smoke на пересобранном контейнере :8080 | PASS | +| Cache consistency на пересобранном контейнере | PASS | +| tests/storage_regression.py | 5/5 PASS | +| tests/session_expiry.py на API с SESSION_TTL_SECONDS=2 | PASS: неверный пароль, действующая сессия, затем 401 после expiry | +| EXPLAIN на 10 000 временных сообщениях | Index Scan, 3 непустые страницы | +| Полный persistence через scripts/restart.sh | PASS | + +Регрессионные тесты проверили откат составной операции при несуществующем участнике, две конкурентные регистрации одного username, обход семи сообщений с одинаковой временной меткой страницами по два, истечение сессии в БД при наличии кэша и fallback при RedisError. Созданные ими пользователи/чаты удаляются после проверки. + +Расширенная проверка cache_consistency подтвердила: stale_cache_rejected, forged_identity_rejected, malformed_cache_recovers, repopulated_revoked_cache_rejected, short_ttl_recovers_from_db, logout_revokes_cache. Пример последнего запуска на контейнере: avg_latency_ms=2.36, p95_latency_ms=2.49 (10 запросов; это не надёжный benchmark). + +Короткий TTL кэша проверяет восстановление, а не expiry сессии. Регрессионный тест отдельно сдвигает expires_at в прошлое. Дополнительно tests/session_expiry.py выполнен на временном API :8082 с SESSION_TTL_SECONDS=2: неверный пароль дал 401, действующая сессия — 200, истёкшая — 401. Полная матрица реальных остановок зависимостей пока не выполнена; имитация RedisError не заменяет её. + +## Индекс и пагинация + +Индекс: messages(chat_id, created_at, id). Запрос использует WHERE chat_id = %s AND (created_at, id) > (%s, %s::uuid), ORDER BY created_at ASC, id ASC, LIMIT limit+1. Это позволяет продолжать чтение после составного ключа без OFFSET в API. Составной ключ устраняет неоднозначность одинакового created_at. + +scripts/explain_pagination.py создаёт 10 000 строк в транзакции, измеряет страницы после позиций 0, 5000 и 9900 с LIMIT 51 и откатывает fixtures. OFFSET используется только для выбора экспериментальных границ, не в запросе истории. + +| Позиция | Возвращено строк | План | shared hit | Execution Time | +| --- | --- | --- | --- | --- | +| 0 | 51 | Index Scan idx_messages_chat_created | 5 | 0.052 мс | +| 5000 | 51 | Index Scan idx_messages_chat_created | 5 | 0.025 мс | +| 9900 | 51 | Index Scan idx_messages_chat_created | 6 | 0.027 мс | + +Это локальное измерение на прогретом небольшом наборе. Оно подтверждает план конкретных страниц, но не универсальные задержки под нагрузкой. Курсор не поддерживает единый снимок между HTTP-запросами; вставки с ключом до уже пройденной границы могут быть пропущены. + +## Трек 1: конкурентная подмена данных кэша + +Гипотеза: проверка действительности токена в БД без сверки идентичности из кэша допускает возврат чужого пользователя. Исправление должно возвращать авторитетную идентичность и исключить повторный SELECT сессии. + +Методика scripts/research_cache.py: + +1. Создать двух тестовых пользователей и действующую сессию первого. +2. В отдельной изолированной функции воспроизвести прежний алгоритм: SELECT сессии, чтение кэша, повторный SELECT действительности, возврат кэша. +3. Измерить 100 прогретых последовательных чтений каждого алгоритма, подсчитать вызовы SELECT сессии. +4. Использовать два потока и Event: после первого чтения БД, до GET кэша, записать в кэш идентичность второго пользователя с действующим expiry. +5. Убедиться, что прежний алгоритм вернул чужую идентичность, исправленный вернул владельца токена. +6. Удалить собственный ключ и проверить восстановление из БД. В конце удалить fixtures. + +| Показатель | Прежний алгоритм | Исправленный | +| --- | --- | --- | +| Чтений | 100 | 100 | +| SELECT сессии | 200 | 100 | +| Среднее | 5.154 мс | 6.867 мс | +| p95 | 6.739 мс | 8.805 мс | +| Чужая идентичность при конкурентной подмене | Да | Нет | + +Вывод: исправлена подмена и устранён лишний SELECT. Улучшение задержки не подтверждено: в этом запуске исправленный вариант медленнее. Алгоритмы измерены последовательно в локальной среде; влияние нагрузки и порядка не исключено. Baseline существует только внутри исследовательского скрипта и не публикует небезопасный API. + +Граница эксперимента: детерминированное чередование чтения и повреждения кэша, а не случайный stress-тест. Возможность записать чужое значение в кэш предполагается. Защита от cache stampede и кэш истории не реализованы. Read-after-write сообщений обеспечивается фиксацией до ответа и чтением из основной БД; smoke и persistence проверяют подтверждённые сообщения. Удаление собственного ключа моделирует cache miss, а реальный рестарт проверяется отдельно. + +## Сохранность и ограничения + +scripts/restart.sh пересоздаёт app, db, valkey с сохранением именованных volumes. Persistence-тест подтвердил, что прежний токен читает прежние подтверждённые данные после этого действия. PostgreSQL хранит основное состояние; Valkey использует AOF и RDB-настройку save 60 1. TTL не продлевается при рестарте. + +Проверка не моделирует потерю хоста, диска, volumes или аварийное отключение питания. Volume не является резервной копией. Нет координации одновременных миграторов нескольких API-реплик. + +## Повтор проверки короткой сессии + +В отдельном терминале запустить временный API, затем выполнить тест и остановить временный процесс: + +```sh +SESSION_TTL_SECONDS=2 .venv/bin/uvicorn app:app --host 127.0.0.1 --port 8082 +# В другом терминале: +BASE_URL=http://127.0.0.1:8082 .venv/bin/python tests/session_expiry.py +``` + +Обычный Compose-стенд сохраняет TTL=3600. Как и публичный smoke, HTTP-тест оставляет созданного синтетического пользователя в БД. diff --git a/app.py b/app.py new file mode 100644 index 00000000..d2a24ed1 --- /dev/null +++ b/app.py @@ -0,0 +1,842 @@ +from __future__ import annotations + +import base64 +import json +import os +import secrets +import uuid +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import psycopg +from psycopg_pool import ConnectionPool +import redis +from argon2 import PasswordHasher +from fastapi import FastAPI, Header, HTTPException, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, Response +from pydantic import BaseModel, ConfigDict, StringConstraints, ValidationError +from typing_extensions import Annotated + + +APP_HOST = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT = int(os.getenv("APP_PORT", "8080")) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +INSTANCE_ID = os.getenv("INSTANCE_ID", uuid.uuid4().hex[:12]) + +USERNAME_RE = r"^[a-z0-9_]{3,32}$" + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, *, headers: Optional[Dict[str, str]] = None): + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +class UserOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + username: str + display_name: str + created_at: str + + +class SessionOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + token: str + token_type: str + expires_at: str + user: UserOut + + +class ChatOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + title: str + member_ids: List[str] + created_at: str + + +class MessageOut(BaseModel): + model_config = ConfigDict(extra="forbid") + + id: str + chat_id: str + sender_id: str + text: str + created_at: str + + +class CreateUserIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: Annotated[str, StringConstraints(pattern=USERNAME_RE, min_length=3, max_length=32)] + display_name: Annotated[str, StringConstraints(min_length=1, max_length=100)] + password: Annotated[str, StringConstraints(min_length=12, max_length=128)] + + +class CreateChatIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: Annotated[str, StringConstraints(min_length=1, max_length=100)] + member_ids: List[str] + + +class CreateMessageIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: Annotated[str, StringConstraints(min_length=1, max_length=4000)] + + +class Store: + def __init__(self) -> None: + self.connection_url = self._resolve_database_url() + self.cache_url = self._resolve_cache_url() + self.session_store = os.getenv("SESSION_STORE", "valkey") + self.password_hasher = PasswordHasher(time_cost=3, memory_cost=65536, parallelism=2, hash_len=32, salt_len=16) + self.pool = ConnectionPool( + self.connection_url, min_size=1, + max_size=int(os.getenv("DB_POOL_MAX_SIZE", "10")), + timeout=5, max_waiting=20, open=False, + kwargs={"connect_timeout": 5, "autocommit": False, + "options": "-c statement_timeout=5000 -c lock_timeout=500"}, + ) + + def _resolve_database_url(self) -> str: + value = os.getenv("DATABASE_URL") + if value: + return value + secret_path = os.getenv("DATABASE_URL_FILE") + if secret_path and os.path.exists(secret_path): + with open(secret_path, "r", encoding="utf-8") as handle: + return handle.read().strip() + return os.getenv("DEFAULT_DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger") + + def _resolve_cache_url(self) -> str: + value = os.getenv("CACHE_URL") + if value: + return value + secret_path = os.getenv("CACHE_URL_FILE") + if secret_path and os.path.exists(secret_path): + with open(secret_path, "r", encoding="utf-8") as handle: + return handle.read().strip() + return os.getenv("DEFAULT_CACHE_URL", "redis://localhost:6379/0") + + def connect(self): + return self.pool.connection() + + def cache_client(self): + return redis.Redis.from_url(self.cache_url, decode_responses=True, socket_connect_timeout=2, socket_timeout=2) + + def cache_session(self, session: Dict[str, Any]) -> None: + if self.session_store != "valkey": + return + try: + client = self.cache_client() + ttl_seconds = max(1, int((self.parse_session_expiry(session["expires_at"]) - self.utc_now()).total_seconds())) + payload = json.dumps({ + "token": session["token"], + "user_id": session["user_id"], + "expires_at": session["expires_at"], + "revoked_at": session.get("revoked_at"), + "user": session["user"], + }, separators=(",", ":"), ensure_ascii=False) + client.setex(f"session:{session['token']}", ttl_seconds, payload) + except redis.RedisError: + return + + def get_cached_session(self, token: str, *, db_session) -> Optional[Dict[str, Any]]: + if self.session_store != "valkey": + return None + try: + client = self.cache_client() + payload = client.get(f"session:{token}") + if not payload: + return None + data = json.loads(payload) + session = { + "token": data["token"], + "user_id": data["user_id"], + "expires_at": data["expires_at"], + "revoked_at": data.get("revoked_at"), + "user": data["user"], + } + if not self.is_session_valid(session): + client.delete(f"session:{token}") + return None + if session != db_session: + client.delete(f"session:{token}") + return None + return db_session + except (redis.RedisError, KeyError, TypeError, ValueError): + return None + + def delete_cached_session(self, token: str) -> None: + if self.session_store != "valkey": + return + try: + self.cache_client().delete(f"session:{token}") + except redis.RedisError: + return + + @staticmethod + def parse_session_expiry(value: Any) -> datetime: + if isinstance(value, datetime): + return value.astimezone(timezone.utc) + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc) + raise ValueError("invalid session expiry") + + def init(self) -> None: + migration_dir = Path(__file__).resolve().parent / "migrations" + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW() + ); + """ + ) + + applied = set() + cur.execute("SELECT version FROM schema_migrations") + for (version,) in cur.fetchall(): + applied.add(version) + + for migration_path in sorted(migration_dir.glob("*.sql")): + version = migration_path.stem + if version in applied: + continue + sql = migration_path.read_text(encoding="utf-8") + cur.execute(sql) + cur.execute( + "INSERT INTO schema_migrations(version, applied_at) VALUES (%s, %s)", + (version, self.utc_now()), + ) + conn.commit() + + def utc_now(self) -> datetime: + return datetime.now(timezone.utc) + + def isoformat_utc(self, value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + def hash_password(self, password: str) -> str: + return self.password_hasher.hash(password) + + def verify_password(self, password_hash: str, password: str) -> bool: + try: + self.password_hasher.verify(password_hash, password) + return True + except Exception: + return False + + def require_user_by_username(self, username: str, conn): + with conn.cursor() as cur: + cur.execute("SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", (username,)) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row[0]), + "username": row[1], + "display_name": row[2], + "created_at": self.isoformat_utc(row[3]), + "password_hash": row[4], + } + + def fetch_user(self, user_id: str, conn): + with conn.cursor() as cur: + cur.execute("SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,)) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row[0]), + "username": row[1], + "display_name": row[2], + "created_at": self.isoformat_utc(row[3]), + } + + def fetch_session_by_token(self, token: str, conn): + with conn.cursor() as cur: + cur.execute( + "SELECT s.token, s.user_id, s.expires_at, s.revoked_at, u.id, u.username, u.display_name, u.created_at FROM sessions s JOIN users u ON u.id = s.user_id WHERE s.token = %s", + (token,), + ) + row = cur.fetchone() + if row is None: + return None + return { + "token": row[0], + "user_id": str(row[1]), + "expires_at": self.isoformat_utc(row[2]), + "revoked_at": row[3], + "user": { + "id": str(row[4]), + "username": row[5], + "display_name": row[6], + "created_at": self.isoformat_utc(row[7]), + }, + } + + def list_member_ids(self, chat_id: str, conn) -> List[str]: + with conn.cursor() as cur: + cur.execute("SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", (chat_id,)) + rows = cur.fetchall() + return [str(row[0]) for row in rows] + + def fetch_chat(self, chat_id: str, conn): + with conn.cursor() as cur: + cur.execute("SELECT id, title, created_at FROM chats WHERE id = %s", (chat_id,)) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row[0]), + "title": row[1], + "created_at": self.isoformat_utc(row[2]), + "member_ids": self.list_member_ids(str(row[0]), conn), + } + + def fetch_message(self, message_id: str, conn): + with conn.cursor() as cur: + cur.execute("SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE id = %s", (message_id,)) + row = cur.fetchone() + if row is None: + return None + return { + "id": str(row[0]), + "chat_id": str(row[1]), + "sender_id": str(row[2]), + "text": row[3], + "created_at": self.isoformat_utc(row[4]), + } + + def fetch_chat_membership(self, chat_id: str, user_id: str, conn) -> bool: + with conn.cursor() as cur: + cur.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)) + return cur.fetchone() is not None + + def insert_user(self, username: str, display_name: str, password: str): + now = self.utc_now() + user_id = uuid.uuid4() + password_hash = self.hash_password(password) + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO users(id, username, display_name, password_hash, created_at) VALUES (%s,%s,%s,%s,%s)", + (user_id, username, display_name, password_hash, now), + ) + conn.commit() + return { + "id": str(user_id), + "username": username, + "display_name": display_name, + "created_at": self.isoformat_utc(now), + } + + def insert_chat(self, title: str, member_ids: List[str], actor_id: str): + now = self.utc_now() + chat_id = uuid.uuid4() + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute("INSERT INTO chats(id,title,created_at) VALUES (%s,%s,%s)", (chat_id, title, now)) + for member_id in sorted(set(member_ids)): + cur.execute("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)", (chat_id, member_id)) + conn.commit() + return { + "id": str(chat_id), + "title": title, + "member_ids": sorted(set(member_ids)), + "created_at": self.isoformat_utc(now), + } + + def insert_message(self, chat_id: str, sender_id: str, text: str): + now = self.utc_now() + message_id = uuid.uuid4() + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO messages(id, chat_id, sender_id, text, created_at) VALUES (%s,%s,%s,%s,%s)", + (message_id, chat_id, sender_id, text, now), + ) + conn.commit() + return { + "id": str(message_id), + "chat_id": chat_id, + "sender_id": sender_id, + "text": text, + "created_at": self.isoformat_utc(now), + } + + @staticmethod + def cursor_uuid_key(value: str) -> int: + try: + return uuid.UUID(value).int + except (ValueError, TypeError): + return 0 + + def list_chats_for_user(self, user_id: str, limit: int, cursor: Optional[str]): + with self.connect() as conn: + with conn.cursor() as cur: + query = """ + SELECT c.id, c.title, c.created_at + FROM chats c + JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = %s + """ + params: List[Any] = [user_id] + if cursor: + prefix = f"user:{user_id}|" + if not cursor.startswith(prefix): + raise ApiError(400, "invalid_request", "cursor does not match user chat list") + remainder = cursor[len(prefix):] + if not remainder: + raise ApiError(400, "invalid_request", "invalid cursor") + try: + cursor_ts_raw, cursor_id = remainder.split("|", 1) + cursor_dt = datetime.fromisoformat(cursor_ts_raw.replace("Z", "+00:00")).astimezone(timezone.utc) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid cursor") from exc + query += " AND (c.created_at > %s OR (c.created_at = %s AND c.id > %s::uuid))" + params.extend([cursor_dt, cursor_dt, cursor_id]) + query += " ORDER BY c.created_at ASC, c.id ASC LIMIT %s" + params.append(limit + 1) + cur.execute(query, tuple(params)) + rows = cur.fetchall() + items = [ + { + "id": str(row[0]), + "title": row[1], + "created_at": self.isoformat_utc(row[2]), + "member_ids": self.list_member_ids(str(row[0]), conn), + } + for row in rows + ] + + next_cursor = None + if len(items) > limit: + last_item = items[limit - 1] + next_cursor = f"user:{user_id}|{last_item['created_at']}|{last_item['id']}" + items = items[:limit] + + return {"items": items, "next_cursor": next_cursor} + + def list_messages_for_chat(self, chat_id: str, limit: int, cursor: Optional[str]): + with self.connect() as conn: + with conn.cursor() as cur: + query = "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s" + params: List[Any] = [chat_id] + if cursor: + prefix = f"chat:{chat_id}|" + if not cursor.startswith(prefix): + raise ApiError(400, "invalid_request", "cursor does not match chat") + remainder = cursor[len(prefix):] + if not remainder: + raise ApiError(400, "invalid_request", "invalid cursor") + try: + cursor_ts_raw, cursor_id = remainder.split("|", 1) + cursor_dt = datetime.fromisoformat(cursor_ts_raw.replace("Z", "+00:00")).astimezone(timezone.utc) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid cursor") from exc + query += " AND (created_at, id) > (%s, %s::uuid)" + params.extend([cursor_dt, cursor_id]) + query += " ORDER BY created_at ASC, id ASC LIMIT %s" + params.append(limit + 1) + cur.execute(query, tuple(params)) + rows = cur.fetchall() + + items = [{ + "id": str(row[0]), + "chat_id": str(row[1]), + "sender_id": str(row[2]), + "text": row[3], + "created_at": self.isoformat_utc(row[4]), + } for row in rows] + + next_cursor = None + if len(items) > limit: + last_item = items[limit - 1] + next_cursor = f"chat:{chat_id}|{last_item['created_at']}|{last_item['id']}" + items = items[:limit] + + return {"items": items, "next_cursor": next_cursor} + + def create_session(self, user_id: str, username: str): + token = secrets.token_urlsafe(32) + expires_at = self.utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute( + "INSERT INTO sessions(id, token, user_id, expires_at, created_at) VALUES (%s,%s,%s,%s,%s)", + (uuid.uuid4(), token, user_id, expires_at, self.utc_now()), + ) + conn.commit() + with self.connect() as conn: + user = self.fetch_user(user_id, conn) + if user is None: + raise ApiError(500, "internal_error", "session user missing") + session = { + "token": token, + "user_id": user_id, + "expires_at": self.isoformat_utc(expires_at), + "revoked_at": None, + "user": user, + } + self.cache_session(session) + return { + "token": token, + "token_type": "Bearer", + "expires_at": self.isoformat_utc(expires_at), + "user": user, + } + + def get_session_by_token(self, token: str, conn=None): + if conn is None: + with self.connect() as db_conn: + return self.get_session_by_token(token, db_conn) + db_session = self.fetch_session_by_token(token, conn) + if db_session is None or not self.is_session_valid(db_session): + self.delete_cached_session(token) + return None + cached = self.get_cached_session(token, db_session=db_session) + if cached is None: + self.cache_session(db_session) + return db_session + + def revoke_current_session(self, token: str): + with self.connect() as conn: + with conn.cursor() as cur: + cur.execute("UPDATE sessions SET revoked_at = %s WHERE token = %s AND revoked_at IS NULL", (self.utc_now(), token)) + conn.commit() + self.delete_cached_session(token) + + def is_session_valid(self, session) -> bool: + if session is None: + return False + if session.get("revoked_at") is not None: + return False + try: + expiry = self.parse_session_expiry(session["expires_at"]) + except ValueError: + return False + if expiry <= self.utc_now(): + return False + return True + + +store = Store() +app = FastAPI(title="MTUSI Messenger") + + +@app.on_event("startup") +def startup_event() -> None: + store.pool.open(wait=True, timeout=10) + store.init() + + +@app.on_event("shutdown") +def shutdown_event() -> None: + store.pool.close() + + +@app.exception_handler(psycopg.OperationalError) +async def database_error_handler(request: Request, exc): + return await api_error_handler(request, ApiError(503, "unavailable", "database unavailable")) + + +@app.middleware("http") +async def request_id_middleware(request: Request, call_next): + request_id = uuid.uuid4().hex + request.state.request_id = request_id + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + body = {"error": {"code": exc.code, "message": exc.message, "request_id": getattr(request.state, "request_id", "")}} + return JSONResponse(status_code=exc.status_code, headers={**exc.headers, "X-Request-Id": getattr(request.state, "request_id", ""), "X-Instance-Id": INSTANCE_ID}, content=body) + + +@app.exception_handler(RequestValidationError) +async def validation_error_handler(request: Request, exc: RequestValidationError): + request_id = getattr(request.state, "request_id", "") + return JSONResponse( + status_code=400, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, + content={"error": {"code": "invalid_request", "message": "invalid request", "request_id": request_id}}, + ) + + +@app.exception_handler(Exception) +async def generic_error_handler(request: Request, exc: Exception): + request_id = getattr(request.state, "request_id", "") + return JSONResponse( + status_code=500, + headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID}, + content={"error": {"code": "internal_error", "message": "internal error", "request_id": request_id}}, + ) + + +def get_auth_header(request: Request): + auth = request.headers.get("Authorization") + if not auth: + raise ApiError(401, "unauthorized", "missing authorization", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + scheme, _, token = auth.partition(" ") + if scheme.lower() != "bearer" or not token: + raise ApiError(401, "unauthorized", "invalid bearer token", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + return token.strip() + + +def require_bearer_session(request: Request): + token = get_auth_header(request) + with store.connect() as conn: + session = store.get_session_by_token(token, conn) + if session is None or not store.is_session_valid(session): + raise ApiError(401, "unauthorized", "invalid or expired session", headers={"WWW-Authenticate": 'Bearer realm="messenger"'}) + return session + + +async def parse_json_body(request: Request) -> Dict[str, Any]: + try: + raw = await request.body() + if not raw: + return {} + return __import__("json").loads(raw) + except Exception as exc: + raise ApiError(400, "invalid_request", "malformed JSON") from exc + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + try: + with store.connect() as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + return {"status": "ok"} + except Exception as exc: + raise ApiError(503, "unavailable", "database unavailable") from exc + + +@app.post("/api/v1/users", status_code=201) +async def create_user_endpoint(request: Request): + try: + body = await request.json() + except Exception as exc: + raise ApiError(400, "invalid_request", "malformed JSON") from exc + if not isinstance(body, dict): + raise ApiError(400, "invalid_request", "invalid user payload") + try: + payload = CreateUserIn.model_validate(body) + except ValidationError as exc: + raise ApiError(400, "invalid_request", "invalid user payload") from exc + if not payload.username or not payload.display_name.strip(): + raise ApiError(400, "invalid_request", "invalid request") + payload.display_name = payload.display_name.strip() + try: + user = store.insert_user(payload.username, payload.display_name, payload.password) + return UserOut.model_validate(user) + except Exception as exc: + if "duplicate key" in str(exc).lower() or "unique" in str(exc).lower(): + raise ApiError(409, "conflict", "username already exists") from exc + raise ApiError(503, "unavailable", "database unavailable") from exc + + +@app.get("/api/v1/users/me") +async def current_user(request: Request): + session = require_bearer_session(request) + user = session["user"] + return UserOut.model_validate(user) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def create_session(request: Request): + auth = request.headers.get("Authorization", "") + if not auth.lower().startswith("basic "): + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + encoded = auth.split(" ", 1)[1].strip() + try: + decoded = base64.b64decode(encoded.encode("ascii"), validate=True) + raw = decoded.decode("utf-8") + except Exception: + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + if ":" not in raw: + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + username, password = raw.split(":", 1) + if not re_match(username): + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + if not (12 <= len(password) <= 128): + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + try: + with store.connect() as conn: + user = store.require_user_by_username(username, conn) + if user is None or not store.verify_password(user["password_hash"], password): + raise ApiError(401, "unauthorized", "invalid credentials", headers={"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}) + session = store.create_session(user["id"], username) + return SessionOut.model_validate(session) + except ApiError: + raise + except Exception as exc: + raise ApiError(503, "unavailable", "database unavailable") from exc + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def revoke_session(request: Request): + session = require_bearer_session(request) + token = session["token"] + store.revoke_current_session(token) + return Response(status_code=204) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat_endpoint(request: Request): + session = require_bearer_session(request) + try: + body = await request.json() + except Exception as exc: + raise ApiError(400, "invalid_request", "malformed JSON") from exc + if not isinstance(body, dict): + raise ApiError(400, "invalid_request", "invalid chat payload") + try: + payload = CreateChatIn.model_validate(body) + except ValidationError as exc: + raise ApiError(400, "invalid_request", "invalid chat payload") from exc + if len(payload.member_ids) < 1 or len(payload.member_ids) > 100: + raise ApiError(400, "invalid_request", "invalid member_ids") + title = payload.title.strip() + if not title: + raise ApiError(400, "invalid_request", "invalid title") + normalized = [] + seen = set() + for member_id in payload.member_ids: + try: + member_id = str(uuid.UUID(member_id)) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid member_ids") from exc + if member_id in seen: + raise ApiError(400, "invalid_request", "duplicate member_ids") + seen.add(member_id) + normalized.append(member_id) + if session["user_id"] not in normalized: + raise ApiError(400, "invalid_request", "current user must be a member") + try: + with store.connect() as conn: + for member_id in normalized: + with conn.cursor() as cur: + cur.execute("SELECT id FROM users WHERE id = %s", (member_id,)) + if cur.fetchone() is None: + raise ApiError(400, "invalid_request", "unknown member") + chat = store.insert_chat(title, normalized, session["user_id"]) + return ChatOut.model_validate(chat) + except ApiError: + raise + except Exception as exc: + raise ApiError(503, "unavailable", "database unavailable") from exc + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + session = require_bearer_session(request) + limit = request.query_params.get("limit", "50") + cursor = request.query_params.get("cursor") + try: + limit_int = int(limit) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid limit") from exc + if not (1 <= limit_int <= 100): + raise ApiError(400, "invalid_request", "invalid limit") + result = store.list_chats_for_user(session["user_id"], limit_int, cursor) + return {"items": [ChatOut.model_validate(item) for item in result["items"]], "next_cursor": result["next_cursor"]} + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(request: Request, chat_id: str): + session = require_bearer_session(request) + try: + uuid.UUID(chat_id) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid chat_id") from exc + with store.connect() as conn: + chat = store.fetch_chat(chat_id, conn) + if chat is None or not store.fetch_chat_membership(chat_id, session["user_id"], conn): + raise ApiError(404, "not_found", "chat not found") + return ChatOut.model_validate(chat) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(request: Request, chat_id: str): + session = require_bearer_session(request) + try: + uuid.UUID(chat_id) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid chat_id") from exc + try: + body = await request.json() + except Exception as exc: + raise ApiError(400, "invalid_request", "malformed JSON") from exc + if not isinstance(body, dict): + raise ApiError(400, "invalid_request", "invalid message payload") + try: + payload = CreateMessageIn.model_validate(body) + except ValidationError as exc: + raise ApiError(400, "invalid_request", "invalid message payload") from exc + if not payload.text.strip(): + raise ApiError(400, "invalid_request", "invalid text") + payload.text = payload.text.strip() + with store.connect() as conn: + chat = store.fetch_chat(chat_id, conn) + if chat is None or not store.fetch_chat_membership(chat_id, session["user_id"], conn): + raise ApiError(404, "not_found", "chat not found") + message = store.insert_message(chat_id, session["user_id"], payload.text) + return MessageOut.model_validate(message) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(request: Request, chat_id: str): + session = require_bearer_session(request) + try: + uuid.UUID(chat_id) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid chat_id") from exc + limit = request.query_params.get("limit", "50") + cursor = request.query_params.get("cursor") + try: + limit_int = int(limit) + except ValueError as exc: + raise ApiError(400, "invalid_request", "invalid limit") from exc + if not (1 <= limit_int <= 100): + raise ApiError(400, "invalid_request", "invalid limit") + with store.connect() as conn: + if not store.fetch_chat_membership(chat_id, session["user_id"], conn): + raise ApiError(404, "not_found", "chat not found") + result = store.list_messages_for_chat(chat_id, limit_int, cursor) + return {"items": [MessageOut.model_validate(item) for item in result["items"]], "next_cursor": result["next_cursor"]} + + +def re_match(value: str) -> bool: + import re + + return bool(re.fullmatch(USERNAME_RE, value)) + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run("app:app", host=APP_HOST, port=APP_PORT, reload=False) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..62bdc0d2 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,50 @@ +services: + db: + image: postgres:16-alpine + container_name: mtusi-lab2-db + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: messenger + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 5s + retries: 10 + start_period: 10s + + valkey: + image: valkey/valkey:8-alpine + container_name: mtusi-lab2-valkey + command: ["--save", "60", "1", "--appendonly", "yes"] + ports: + - "6379:6379" + volumes: + - valkey_data:/data + + app: + build: . + container_name: mtusi-lab2-app + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: postgresql://messenger:messenger@db:5432/messenger + CACHE_URL: redis://valkey:6379/0 + SESSION_STORE: valkey + SESSION_TTL_SECONDS: 3600 + ports: + - "8080:8080" + depends_on: + db: + condition: service_healthy + valkey: + condition: service_started + restart: unless-stopped + +volumes: + postgres_data: + valkey_data: diff --git a/migrations/001_initial_schema.sql b/migrations/001_initial_schema.sql new file mode 100644 index 00000000..74371785 --- /dev/null +++ b/migrations/001_initial_schema.sql @@ -0,0 +1,41 @@ +CREATE TABLE IF NOT EXISTS users ( + id UUID PRIMARY KEY, + username TEXT NOT NULL UNIQUE, + display_name TEXT NOT NULL, + password_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chats ( + id UUID PRIMARY KEY, + title TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id UUID PRIMARY KEY, + chat_id UUID NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE IF NOT EXISTS sessions ( + id UUID PRIMARY KEY, + token TEXT NOT NULL UNIQUE, + user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at TIMESTAMPTZ NOT NULL, + revoked_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_created ON messages(chat_id, created_at, id); +CREATE INDEX IF NOT EXISTS idx_chat_members_user ON chat_members(user_id, chat_id); +CREATE INDEX IF NOT EXISTS idx_chats_created ON chats(created_at, id); +CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id, expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..af6803db --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.0 +uvicorn==0.30.6 +psycopg[binary]==3.3.6 +psycopg-pool==3.2.6 +argon2-cffi==23.1.0 +python-dotenv==1.0.1 +redis==5.1.0 diff --git a/scripts/explain_pagination.py b/scripts/explain_pagination.py new file mode 100644 index 00000000..a5d9a46e --- /dev/null +++ b/scripts/explain_pagination.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Produce an EXPLAIN (ANALYZE, BUFFERS) proof for the chat-history cursor query.""" +import os +import uuid + +import psycopg + +DB_URL = os.getenv("DATABASE_URL", "postgresql://messenger:messenger@localhost:5432/messenger") + + +def main() -> None: + with psycopg.connect(DB_URL, connect_timeout=5) as conn: + with conn.cursor() as cur: + chat_id = uuid.uuid4() + user_id = uuid.uuid4() + cur.execute("INSERT INTO users(id, username, display_name, password_hash, created_at) VALUES (%s, %s, %s, %s, NOW())", (user_id, f"exp_{uuid.uuid4().hex[:12]}", "exp", "dummyhash")) + cur.execute("INSERT INTO chats(id, title, created_at) VALUES (%s, %s, NOW())", (chat_id, "exp")) + cur.execute("INSERT INTO chat_members(chat_id, user_id) VALUES (%s, %s)", (chat_id, user_id)) + for idx in range(10000): + cur.execute( + "INSERT INTO messages(id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, NOW() - %s * INTERVAL '1 millisecond')", + (uuid.uuid4(), chat_id, user_id, f"msg-{idx}", idx), + ) + explain_sql = """ + EXPLAIN (ANALYZE, BUFFERS) + SELECT id, chat_id, sender_id, text, created_at + FROM messages + WHERE chat_id = %s + AND (created_at, id) > (%s::timestamptz, %s::uuid) + ORDER BY created_at ASC, id ASC + LIMIT 51 + """ + for offset in (0, 5000, 9900): + cur.execute("SELECT created_at, id FROM messages WHERE chat_id = %s ORDER BY created_at, id OFFSET %s LIMIT 1", (chat_id, offset)) + cursor_ts, cursor_id = cur.fetchone() + cur.execute(explain_sql, (chat_id, cursor_ts, cursor_id)) + print(f"Page after position {offset}:") + print("\n".join(row[0] for row in cur.fetchall())) + # Keep the measurement repeatable without accumulating fixtures. + conn.rollback() + +if __name__ == "__main__": + main() diff --git a/scripts/research_cache.py b/scripts/research_cache.py new file mode 100644 index 00000000..cf062d63 --- /dev/null +++ b/scripts/research_cache.py @@ -0,0 +1,118 @@ +"""Isolated before/after session-cache experiment; never used by the API. + +Run against a local test database and Valkey. Creates and removes its own users. +The baseline reproduces the former two-read algorithm accepting cached identity. +""" +import json +import statistics +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from app import Store + + +def main(): + store = Store() + store.pool.open(wait=True) + store.init() + cache = store.cache_client() + users, tokens = [], [] + try: + for _ in range(2): + users.append(store.insert_user("research_" + uuid.uuid4().hex[:16], "Research", "research-password")) + result = store.create_session(users[0]["id"], users[0]["username"]) + token = result["token"] + tokens.append(token) + with store.connect() as conn: + canonical = store.fetch_session_by_token(token, conn) + forged = dict(canonical, user_id=users[1]["id"], user=users[1]) + + def baseline(): + # Former algorithm: DB validity does not authenticate cached fields. + with store.connect() as conn: + db_session = store.fetch_session_by_token(token, conn) + if not store.is_session_valid(db_session): + return None + payload = cache.get(f"session:{token}") + if payload: + cached = json.loads(payload) + if store.is_session_valid(cached): + again = store.fetch_session_by_token(token, conn) + if store.is_session_valid(again): + return cached + store.cache_session(db_session) + return db_session + + outputs = {} + for name, read in (("before", baseline), ("after", lambda: store.get_session_by_token(token))): + store.cache_session(canonical) + count = 0 + original = store.fetch_session_by_token + def counted(*args, **kwargs): + nonlocal count + count += 1 + return original(*args, **kwargs) + latencies = [] + with patch.object(store, "fetch_session_by_token", side_effect=counted): + for _ in range(100): + start = time.perf_counter() + assert read()["user_id"] == users[0]["id"] + latencies.append((time.perf_counter() - start) * 1000) + + # Force a cache writer between the DB read and cache read. + db_read, writer_done = threading.Event(), threading.Event() + first = True + def interleaved(*args, **kwargs): + nonlocal first + session = original(*args, **kwargs) + if first: + first = False + db_read.set() + if not writer_done.wait(5): + raise AssertionError("cache writer timed out") + return session + def writer(): + if not db_read.wait(5): + raise AssertionError("DB reader timed out") + try: + cache.setex(f"session:{token}", 60, json.dumps(forged)) + finally: + writer_done.set() + with ThreadPoolExecutor(max_workers=1) as executor: + future = executor.submit(writer) + with patch.object(store, "fetch_session_by_token", side_effect=interleaved): + observed = read() + future.result() + outputs[name] = { + "requests": 100, "session_selects": count, + "mean_ms": round(statistics.mean(latencies), 3), + "p95_ms": round(sorted(latencies)[94], 3), + "concurrent_wrong_identity": observed["user_id"] != users[0]["id"], + } + + # A cache restart/flush is a miss; delete only our key on shared fixtures. + cache.delete(f"session:{token}") + assert store.get_session_by_token(token)["user_id"] == users[0]["id"] + assert cache.exists(f"session:{token}") + assert outputs["before"]["concurrent_wrong_identity"] + assert not outputs["after"]["concurrent_wrong_identity"] + assert outputs["after"]["session_selects"] == 100 + print(json.dumps({"results": outputs, "cache_miss_recovered": True, + "limitations": "Local sequential Store benchmark, warm DB; deterministic interleaving; baseline is former algorithm, not a deployed insecure API. No host-failure or HTTP throughput claim."}, indent=2)) + finally: + for token in tokens: + store.delete_cached_session(token) + with store.connect() as conn: + for user in users: + conn.execute("DELETE FROM users WHERE id = %s", (user["id"],)) + store.pool.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..a9cb0be8 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +if docker compose version >/dev/null 2>&1; then + docker compose --env-file .env up -d --no-deps --force-recreate app db valkey >/dev/null 2>&1 || { + docker compose up -d --no-deps --force-recreate app db valkey + } +elif command -v docker-compose >/dev/null 2>&1; then + docker-compose --env-file .env up -d --no-deps --force-recreate app db valkey >/dev/null 2>&1 || { + docker-compose up -d --no-deps --force-recreate app db valkey + } +else + echo "Neither docker compose nor docker-compose is available." >&2 + exit 127 +fi + +echo "Restarted app, postgres and valkey with persisted volumes." diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..576c12d9 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +default_instance="mtusi-lab2-app" +if [[ -n "${TARGET_INSTANCE_ID:-}" ]]; then + docker stop "${TARGET_INSTANCE_ID}" >/dev/null 2>&1 || true + echo "Stopped ${TARGET_INSTANCE_ID}" + exit 0 +fi + +docker stop "${default_instance}" >/dev/null 2>&1 || true +printf '%s\n' "Stopped ${default_instance}" diff --git a/tests/cache_consistency.py b/tests/cache_consistency.py new file mode 100644 index 00000000..2179b553 --- /dev/null +++ b/tests/cache_consistency.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Research track: cache consistency and stale session protection for Valkey-backed sessions.""" +import base64 +import json +import os +import statistics +import time +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor + + +BASE_URL = os.environ.get("BASE_URL", "http://localhost:8080") +CACHER_URL = os.environ.get("CACHE_URL", "redis://localhost:6379/0") + + +def http_json(method, path, *, expected=200, headers=None, body=None): + req_headers = {"Accept": "application/json"} + if headers: + req_headers.update(headers) + data = None if body is None else json.dumps(body, ensure_ascii=False).encode("utf-8") + if data is not None: + req_headers.setdefault("Content-Type", "application/json") + request = urllib.request.Request(BASE_URL + path, data=data, headers=req_headers, method=method) + try: + with urllib.request.urlopen(request, timeout=10) as resp: + payload = resp.read() + if not payload: + return {"status": resp.status, "headers": dict(resp.headers), "body": None} + if resp.status not in (expected if isinstance(expected, (list, tuple, set)) else [expected]): + raise AssertionError(f"Expected {expected}, got {resp.status}") + return {"status": resp.status, "headers": dict(resp.headers), "body": json.loads(payload.decode("utf-8"))} + except urllib.error.HTTPError as exc: + payload = exc.read() + if payload: + try: + payload = json.loads(payload.decode("utf-8")) + except Exception: + payload = payload.decode("utf-8", errors="replace") + if exc.code not in (expected if isinstance(expected, (list, tuple, set)) else [expected]): + raise AssertionError(f"Expected status {expected}, got {exc.code} for {method} {path}: {payload}") + return {"status": exc.code, "headers": dict(exc.headers), "body": payload} + + +def call_with_latency(method, path, *, expected=200, headers=None, body=None): + start = time.perf_counter() + result = http_json(method, path, expected=expected, headers=headers, body=body) + elapsed_ms = (time.perf_counter() - start) * 1000.0 + return result, elapsed_ms + + +def main(): + import redis + + r = redis.Redis.from_url(CACHER_URL, decode_responses=True, socket_connect_timeout=2, socket_timeout=2) + + username = f"cache_{uuid.uuid4().hex[:12]}" + password = "123456789012" + body = { + "username": username, + "display_name": "Cache Consistency", + "password": password, + } + registration = http_json("POST", "/api/v1/users", expected=201, body=body) + assert registration["status"] == 201, registration + + basic = "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode() + session = http_json("POST", "/api/v1/auth/sessions", expected=201, headers={"Authorization": basic}) + token = session["body"]["token"] + assert token, session + + key = f"session:{token}" + assert r.get(key), "session was not populated in Valkey" + ttl = r.ttl(key) + assert ttl and ttl > 0, f"session TTL must be positive, got {ttl}" + + latencies = [] + for _ in range(10): + _, elapsed_ms = call_with_latency("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token}) + latencies.append(elapsed_ms) + avg = sum(latencies) / len(latencies) + p95 = sorted(latencies)[int(len(latencies) * 0.95) - 1] + + stale_payload = { + "token": token, + "user_id": str(uuid.uuid4()), + "expires_at": "2000-01-01T00:00:00Z", + "revoked_at": None, + "user": {"id": str(uuid.uuid4()), "username": "stale", "display_name": "stale", "created_at": "2020-01-01T00:00:00Z"}, + } + r.setex(key, 120, json.dumps(stale_payload, separators=(",", ":"), ensure_ascii=False)) + stale_result = http_json("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token}) + assert stale_result["status"] == 200, stale_result + assert stale_result["body"]["username"] == username, stale_result + + # An unexpired payload must not be able to substitute another identity. + stale_payload["expires_at"] = "2100-01-01T00:00:00Z" + r.setex(key, 120, json.dumps(stale_payload)) + forged = http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert forged["body"]["id"] == session["body"]["user"]["id"], forged + assert forged["body"]["username"] == username, forged + + r.setex(key, 120, json.dumps(stale_payload)) + def read_identity(_): + return http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token})["body"]["id"] + with ThreadPoolExecutor(max_workers=8) as executor: + identities = list(executor.map(read_identity, range(8))) + assert identities == [session["body"]["user"]["id"]] * 8, identities + + # Syntactically valid JSON with missing fields must be treated as a miss. + r.setex(key, 120, "{}") + malformed = http_json("GET", "/api/v1/users/me", headers={"Authorization": "Bearer " + token}) + assert malformed["body"]["username"] == username, malformed + + r.setex(key, 1, json.dumps({ + "token": token, + "user_id": str(uuid.uuid4()), + "expires_at": "2100-01-01T00:00:00Z", + "revoked_at": None, + "user": {"id": str(uuid.uuid4()), "username": "ttl-recovery", "display_name": "ttl-recovery", "created_at": "2020-01-01T00:00:00Z"}, + }, separators=(",", ":"), ensure_ascii=False)) + time.sleep(2) + short_ttl = http_json("GET", "/api/v1/users/me", expected=200, headers={"Authorization": "Bearer " + token}) + assert short_ttl["status"] == 200, short_ttl + assert short_ttl["body"]["username"] == username, short_ttl + + revoke = http_json("DELETE", "/api/v1/auth/sessions/current", expected=204, headers={"Authorization": "Bearer " + token}) + assert revoke["status"] == 204, revoke + assert not r.get(key), "revoked session key should be deleted from cache" + + after_logout = http_json("GET", "/api/v1/users/me", expected=401, headers={"Authorization": "Bearer " + token}) + assert after_logout["status"] == 401, after_logout + + # Simulate a writer repopulating the cache after logout's DELETE. + stale_payload.update(session["body"]) + stale_payload["user_id"] = session["body"]["user"]["id"] + stale_payload["revoked_at"] = None + r.setex(key, 120, json.dumps(stale_payload)) + http_json("GET", "/api/v1/users/me", expected=401, headers={"Authorization": "Bearer " + token}) + assert not r.exists(key), "stale key after logout must be removed" + + print("cache_key_exists=True") + print(f"ttl_seconds={ttl}") + print(f"avg_latency_ms={avg:.2f}") + print(f"p95_latency_ms={p95:.2f}") + print("stale_cache_rejected=True") + print("forged_identity_rejected=True") + print("concurrent_identity_checks=8/8") + print("malformed_cache_recovers=True") + print("repopulated_revoked_cache_rejected=True") + print("short_ttl_recovers_from_db=True") + print("logout_revokes_cache=True") + + +if __name__ == "__main__": + main() diff --git a/tests/session_expiry.py b/tests/session_expiry.py new file mode 100644 index 00000000..53bf3496 --- /dev/null +++ b/tests/session_expiry.py @@ -0,0 +1,34 @@ +"""Run against an isolated API started with SESSION_TTL_SECONDS=2. + +BASE_URL defaults to http://localhost:8080; do not use a normal one-hour server. +""" +import base64 +import time +import uuid +from datetime import datetime, timezone + +from cache_consistency import http_json + + +def main(): + username = "expiry_" + uuid.uuid4().hex[:16] + password = "expiry-password-123" + http_json("POST", "/api/v1/users", expected=201, + body={"username": username, "display_name": "Expiry", "password": password}) + credentials = base64.b64encode(f"{username}:{password}".encode()).decode() + http_json("POST", "/api/v1/auth/sessions", expected=401, + headers={"Authorization": "Basic " + base64.b64encode(f"{username}:wrong-password".encode()).decode()}) + result = http_json("POST", "/api/v1/auth/sessions", expected=201, + headers={"Authorization": "Basic " + credentials}) + session = result["body"] + remaining = (datetime.fromisoformat(session["expires_at"].replace("Z", "+00:00")) - datetime.now(timezone.utc)).total_seconds() + assert 0 < remaining <= 3, "Use an isolated server with SESSION_TTL_SECONDS=2" + headers = {"Authorization": "Bearer " + session["token"]} + http_json("GET", "/api/v1/users/me", headers=headers) + time.sleep(max(0, remaining) + 0.2) + http_json("GET", "/api/v1/users/me", expected=401, headers=headers) + print("PASS: wrong password rejected; live session accepted; expired session rejected") + + +if __name__ == "__main__": + main() diff --git a/tests/storage_regression.py b/tests/storage_regression.py new file mode 100644 index 00000000..fc6740b7 --- /dev/null +++ b/tests/storage_regression.py @@ -0,0 +1,91 @@ +"""Run with .venv/bin/python tests/storage_regression.py against local test DB.""" +import sys +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from datetime import timedelta +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import app +import psycopg +import redis + + +class StorageRegression(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.store = app.Store() + cls.store.pool.open(wait=True) + cls.store.init() + + @classmethod + def tearDownClass(cls): + cls.store.pool.close() + + def setUp(self): + self.user = self.store.insert_user("reg_" + uuid.uuid4().hex[:20], "Regression", "test-password-123") + + def tearDown(self): + with self.store.connect() as conn: + conn.execute("DELETE FROM chats WHERE id IN (SELECT chat_id FROM chat_members WHERE user_id = %s)", (self.user["id"],)) + conn.execute("DELETE FROM users WHERE id = %s", (self.user["id"],)) + + def test_atomic_chat_rollback(self): + title = "rollback_" + uuid.uuid4().hex + with self.assertRaises(psycopg.errors.ForeignKeyViolation): + self.store.insert_chat(title, [self.user["id"], str(uuid.uuid4())], self.user["id"]) + with self.store.connect() as conn: + self.assertEqual(conn.execute("SELECT count(*) FROM chats WHERE title = %s", (title,)).fetchone()[0], 0) + + def test_registration_race(self): + username = "race_" + uuid.uuid4().hex[:20] + def register(_): + try: + return self.store.insert_user(username, "Race", "test-password-123")["id"] + except psycopg.errors.UniqueViolation: + return None + with ThreadPoolExecutor(max_workers=2) as executor: + results = list(executor.map(register, range(2))) + try: + self.assertEqual(sum(result is not None for result in results), 1) + finally: + with self.store.connect() as conn: + conn.execute("DELETE FROM users WHERE username = %s", (username,)) + + def test_equal_timestamp_pagination(self): + chat = self.store.insert_chat("Paging", [self.user["id"]], self.user["id"]) + ids = sorted(str(uuid.uuid4()) for _ in range(7)) + with self.store.connect() as conn: + for message_id in ids: + conn.execute("INSERT INTO messages(id, chat_id, sender_id, text, created_at) VALUES (%s,%s,%s,'same time',%s)", + (message_id, chat["id"], self.user["id"], "2026-01-01T00:00:00Z")) + cursor, actual = None, [] + while True: + page = self.store.list_messages_for_chat(chat["id"], 2, cursor) + actual.extend(item["id"] for item in page["items"]) + cursor = page["next_cursor"] + if cursor is None: + break + self.assertEqual(actual, ids) + + def test_expired_db_session_overrides_live_cache(self): + session = self.store.create_session(self.user["id"], self.user["username"]) + with self.store.connect() as conn: + conn.execute("UPDATE sessions SET expires_at = %s WHERE token = %s", + (self.store.utc_now() - timedelta(seconds=1), session["token"])) + self.assertIsNone(self.store.get_session_by_token(session["token"])) + + def test_cache_unavailable_falls_back_to_db(self): + session = self.store.create_session(self.user["id"], self.user["username"]) + try: + with patch.object(self.store, "cache_client", side_effect=redis.ConnectionError("offline")): + actual = self.store.get_session_by_token(session["token"]) + self.assertEqual(actual["user_id"], self.user["id"]) + finally: + self.store.delete_cached_session(session["token"]) + + +if __name__ == "__main__": + unittest.main(verbosity=2)