diff --git a/.env.example b/.env.example index 714c8076..10c382de 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,14 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +POSTGRES_USER=messenger +POSTGRES_DB=messenger +# Replace with a generated local-only value (hex is safe to embed in DATABASE_URL). +POSTGRES_PASSWORD=replace-with-local-password +DB_POOL_MAX_SIZE=16 +DB_POOL_TIMEOUT_SECONDS=5 +DB_STATEMENT_TIMEOUT_MS=5000 +DB_LOCK_TIMEOUT_MS=2000 +# Compose injects CACHE_URL for the API; Redis is a disposable session lookup cache. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..83823c86 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.12.7-slim + +WORKDIR /app + +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r /app/requirements.txt + +COPY . /app +RUN chown -R app:app /app + +USER app + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).read()" || exit 1 + +CMD ["python3", "src/server.py"] diff --git a/Makefile b/Makefile index 9ecb223e..82a58ce8 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://127.0.0.1:8080 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load grade4 track5 test-persistence migrate up down logs help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: contract structure; validate: OpenAPI; test: public smoke; grade4: dependency/rollback/expiry checks; track5: query/index/restore experiment" check: $(PYTHON) scripts/check_contract.py @@ -24,3 +24,23 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +grade4: + $(PYTHON) student-tests/test_level4.py --base-url "$(BASE_URL)" + docker compose --env-file .env exec -T api python3 student-tests/test_session_cache_race.py + ./scripts/test_expiry.sh + +track5: + $(PYTHON) student-tests/track5_query_restore.py + +migrate: + $(PYTHON) src/migrate.py + +up: + docker compose --env-file .env up -d --build + +down: + docker compose --env-file .env down + +logs: + docker compose --env-file .env logs -f diff --git a/README.md b/README.md index 0e073ff7..8321061e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,9 @@ Перенести состояние в постоянное хранилище и связать запросы с аутентифицированным пользователем. Подтверждённые сообщения и действующие сессии переживают перезапуск. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +В этой версии перенесён REST API из lab1 и реализованы PostgreSQL, миграции, хеширование паролей scrypt и постоянные Bearer-сессии. PostgreSQL остаётся источником истины; Redis используется как TTL-кэш для чтения сессий, с fallback на PostgreSQL при недоступности кэша. Web UI не требуется. + +Код приложения разделён в `src/`: `handlers.py` обрабатывает HTTP и маршруты, `store.py` отвечает за PostgreSQL/Redis, `security.py` — хеширование паролей и общие проверки, `config.py` содержит настройки, `errors.py` — ошибки API, `app.py` собирает и запускает сервер. `server.py` остаётся точкой входа. [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) @@ -44,27 +46,43 @@ **Трек 2. План запроса и восстановление.** Исследуйте ≥100 000 сообщений с неравномерными размерами чатов, сравните ≥2 индекса/запроса и стоимость записи. Сделайте backup → восстановление в новое хранилище → сверку количества и выбранных сообщений. Обсудите время восстановления и границы потери данных. -## Как начать и проверить +## Запуск и проверка + +Создайте локальный `.env` и замените пароль PostgreSQL на новый локальный пароль (для строки подключения используйте hex-значение): ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: -make check - -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. -make test -``` - -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. - -```sh -# Compose-вариант после реализации: -docker compose --env-file .env config --quiet +cp .env.example .env +# Отредактируйте POSTGRES_PASSWORD в .env; не добавляйте .env в Git. docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. ``` +API ждёт готовности PostgreSQL, применяет новые SQL-миграции из `migrations/` и слушает `http://127.0.0.1:8080`. Схема создаётся при первом старте; повторный запуск не применяет миграции заново. + +```sh +make check +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +# Остановить контейнеры, сохранив данные в именованном volume: +docker compose --env-file .env down +``` + +Данные сохраняются в `postgres_data`. Для намеренного удаления всего учебного состояния, включая пользователей, сообщения и действующие сессии, выполните отдельно и только после проверки нужного compose-проекта: + +```sh +docker compose --env-file .env down --volumes +``` + +`.env.example`, `compose.yaml` и `contracts/infrastructure.json` описывают локальные настройки и роли компонентов. Секреты, `.env` и Docker volumes не должны попадать в Git. + +Проверки уровня 4 и исследовательский трек уровня 5 запускаются явно на учебном стенде: + +```sh +make grade4 +make track5 +``` + +`make grade4` временно перезапускает API с двухсекундным TTL для проверки expiry, запускает интеграционные проверки Redis/PostgreSQL, включая конкурентное заполнение кэша сессии одновременно с logout, и возвращает штатный TTL. `make track5` создаёт отдельные случайно именованные временные БД, измеряет два варианта индекса на skewed-наборе по 100 000 сообщений на вариант, затем проверяет custom-format backup/restore и удаляет только созданные им БД. Численные результаты и планы сохраняются в `evidence/track5_query_restore.*`. + Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически): diff --git a/REPORT.md b/REPORT.md index e42f2b05..fce16234 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,78 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: не указаны в исходных данных. +- Целевая оценка: 5, исследовательский трек 2 — «План запроса и восстановление». +- Commit/PR: не создавались. +- Источники, библиотеки и помощь ИИ: перенесён и адаптирован собственный API-каркас из lab1; Python standard library; psycopg 3.2.10, psycopg-pool 3.3.3, redis-py 5.2.1; PostgreSQL 16.4, Redis 7.4.2; использовалась помощь ИИ. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Desktop с Compose v2 и Python 3.9+ для локальных проверок. Образы закреплены версиями `python:3.12.7-slim`, `postgres:16.4-alpine`, `redis:7.4.2-alpine`. Проведённые проверки — macOS arm64, Docker 28.0.4, Docker Compose 2.34.0, 10 логических CPU; фактическая память хоста и лимиты контейнеров зафиксированы в [track5_query_restore.json](evidence/track5_query_restore.json). + +```sh +cp .env.example .env +# Заменить POSTGRES_PASSWORD в .env на hex-значение, например: +python3 -c 'import secrets; print(secrets.token_hex(24))' +# Не добавлять .env в Git. + +docker compose --env-file .env up -d --build +curl --fail http://127.0.0.1:8080/health/ready +make check +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make grade4 +make track5 +``` + +API применяет новые миграции из `migrations/` при старте; таблица `schema_migrations` и транзакционная advisory lock обеспечивают однократное применение. `scripts/restart.sh` перезапускает PostgreSQL, Redis и API с сохранением БД volume. Redis является кэшем, поэтому утрата его содержимого не удаляет учётные записи или сессии. + +Обычная остановка сохраняет состояние: + +```sh +docker compose --env-file .env down +``` + +Только для намеренного удаления всех учебных данных, включая пользователей, сообщения и сессии: + +```sh +docker compose --env-file .env down --volumes +``` + +`make grade4` выполняет проверки зависимостей и временно перезапускает API с двухсекундным TTL для сценария expiry, затем возвращает обычную конфигурацию. `make track5` использует случайно именованные временные базы, создаёт benchmark-данные, делает custom-format backup, восстанавливает его в новую пустую БД, сверяет counts/checksums и удаляет только созданные им базы. Не запускайте эти команды на production-окружении. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +- Перенесены и адаптированы REST-поведение, Dockerfile и ignore-правила из lab1. Сервер разделён по обязанностям: `config.py` — настройки, `errors.py` — API-ошибки, `security.py` — password KDF/UUID/time helpers, `store.py` — PostgreSQL и Redis, `handlers.py` — HTTP endpoints, `app.py` — сборка/жизненный цикл сервера, `server.py` — совместимая точка входа. Публичные `tests/` и `contracts/` текущей лабораторной сохранены без подмены. +- PostgreSQL — источник истины для пользователей, чатов, членства, сообщений и сессий. Именованный volume `postgres_data` сохраняет их при перезапуске. +- Redis кэширует результат проверки opaque session token по SHA-256 digest ключа и хранит только публичный объект пользователя. Redis key имеет TTL, рассчитанный по оставшемуся `sessions.expires_at`; пароль, сырой токен и его текст не попадают в кэш. +- Создание и отзыв сессии записываются в PostgreSQL. Чтение сессии при промахе кэша и logout сериализуются транзакционной advisory lock на digest токена: конкурентное заполнение не может записать сессию в Redis уже после logout. Logout удаляет Redis-ключ до удаления записи в БД; при недоступном Redis logout завершается ограниченным `503`, а не сообщает об успехе без инвалидации кэша. При чтении недоступность Redis логируется, запрос безопасно переходит на PostgreSQL; после восстановления промах кэша заполняется из БД. При недоступной БД readiness и операции, требующие её, возвращают ограниченную ошибку. +- Список чатов использует keyset-пагинацию по `(created_at, id)`: курсор привязан к пользователю, страницы не повторяют и не пропускают чаты при обходе неизменённого списка. +- Индекс истории в рабочей схеме — `(chat_id, created_at, id)`, соответствующий keyset-пагинации. Все SQL-запросы параметризованы; пул ограничен 16 соединениями, timeout ожидания пула/подключения — 5 секунд, `statement_timeout` — 5 секунд, `lock_timeout` — 2 секунды. +- Миграция создаёт ключи и ограничения таблиц. Создание чата и всех участников — одна транзакция; уникальный индекс `username` сериализует гонку регистрации. +- Пароли хранятся как scrypt-хеши с индивидуальной солью (`N=32768`, `r=8`, `p=1`). Сессии имеют expiry и отзываются logout-запросом. `X-User-Id` не используется для авторизации. +- Лимит Redis — 128 MB, политика вытеснения `allkeys-lru`; постоянство сессий не зависит от Redis, поэтому Redis AOF/RDB не включён. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Минимум lab1 и lab2; публичный API | `make test` | PASS: регистрация, Basic login, Bearer auth, чаты, сообщения, пагинация, ошибочные credentials, отказ анониму, logout и отказ отозванного токена | `tests/smoke.py`, `tests/client.py` | +| PostgreSQL, ограничения, миграции и сохранность | Чистый старт, повторный старт, `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | Миграция записана один раз; прежний token и подтверждённое сообщение переживают рестарт PostgreSQL, Redis и API | `migrations/001_initial.sql`, `scripts/restart.sh`, `tests/resilience.py` | +| Redis/Valkey используется с TTL | `make grade4` | Session cache entry обнаружена в Redis с положительным TTL; при остановленном Redis текущая сессия проверяется через PostgreSQL; после запуска Redis доступ восстанавливается | `student-tests/test_level4.py`, `src/server.py` | +| Конкурентная регистрация и rollback | `make grade4` | 4 одновременных запроса к одному username дали ровно один `201`, три `409`, в БД осталась одна запись; попытка чата с отсутствующим участником дала `400` и ноль частичных строк | `student-tests/test_level4.py` | +| Запросы, индекс истории и ошибки зависимостей | `make grade4`; `make track5` | Проверены параметризованные запросы, ограниченные timeout; при остановке PostgreSQL readiness и операция чтения вернули ограниченный `503`; 100k-набор использован для `EXPLAIN (ANALYZE, BUFFERS)` | `src/server.py`, `evidence/track5_query_restore.json` | +| Короткий TTL, неверный пароль и отзыв | `make test`; `make grade4` | Неверный пароль и отозванная/истёкшая сессия дают 401; logout выдаёт 204; короткая сессия проверена при TTL=2 секунды | `student-tests/test_level4.py`, `scripts/test_expiry.sh`, `tests/smoke.py` | +| Трек 5.2: планы двух индексов и write-cost | `make track5` | 100,000 синтетических строк на каждый индексный вариант, неравномерное распределение; одинаковый keyset SELECT измерен пять раз; сравнен INSERT cost | [track5_query_restore.md](evidence/track5_query_restore.md), [raw plans/results](evidence/track5_query_restore.json) | +| Трек 5.2: backup → новая БД → проверка | `make track5` | Custom-format backup восстановлен в отдельную пустую БД; количество строк и контрольные суммы выбранных сообщений совпали; тест сообщил PASS | `evidence/track5_query_restore.json` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Исследование: план запроса и восстановление -## Эксперименты и отказы +**Гипотеза.** Индекс только по `chat_id` должен обслужить фильтрацию истории, но keyset-фильтр/сортировка по `(created_at, id)` потребуют отсеивания и top-N сортировки. Составной `(chat_id, created_at, id)` должен уменьшить чтение/время страницы ценой более дорогой записи. -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Методика.** Скрипт `student-tests/track5_query_restore.py` использует изолированные случайно именованные PostgreSQL database, не рабочую БД API. В двух вариантах создано по 100 000 synthetic rows: 80 000 в одном чате, оставшиеся 20 000 распределены поровну между 100 чатами. Выполняется один и тот же keyset SELECT страницы из небольшого чата с лимитом 25. После ANALYZE собирается пять `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` планов на вариант. Стоимость записи измеряется пятью EXPLAIN ANALYZE вставками по 2 000 строк; каждая вставка откатывается, поэтому результат отражает execution/index maintenance, но не durable commit/fsync. Затем custom-format `pg_dump` восстанавливается в новую пустую базу, сверяются 100 000 строк каждого варианта и MD5 агрегаты выбранных строк. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +**Результат текущего запуска.** Сводная таблица медиан, планы, buffer counts, параметры PostgreSQL, restore time и контрольные суммы записаны в [track5_query_restore.md](evidence/track5_query_restore.md) и JSON-файле рядом. В этом запуске составной индекс уменьшил медианное время выбранной страницы примерно с 0.297 ms до 0.097 ms (около 67%); вставка 2 000 строк заняла примерно 5.428 ms против 7.218 ms для составного индекса (около 33% дороже). Restore в новую БД и сверка количества/контрольных сумм завершились PASS. Результаты относятся к этому single-host synthetic run; для повторных запусков числа могут отличаться. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Redis — ускоряющий кэш, а не самостоятельное долговременное хранилище; источником истины остаётся PostgreSQL. Проверены контролируемые остановки Redis и PostgreSQL, но не потеря host, повреждение диска, HA/failover БД, реальная нагрузка API/RPS, commit fsync или backup по расписанию. Benchmark использует две таблицы с одинаковыми данными и синтетическую нагрузку; он не доказывает пропускную способность production-системы. Docker-контейнеры запускаются без явных CPU/RAM limits (нулевые лимиты указаны в evidence). Сравнение планов и разовый restore PASS не заменяют регулярный проверенный backup/DR-процесс. diff --git a/codes.md b/codes.md new file mode 100644 index 00000000..10437895 --- /dev/null +++ b/codes.md @@ -0,0 +1,135 @@ +# Запуск и проверка лабораторной 2 + +Команды рассчитаны на локальный тестовый стенд: Docker с Compose v2, `make` и Python 3.9+. В этой лабораторной проверяются уровни 3, 4 и 5. + +## Запуск стенда + +Из каталога `mtusi2026lab2`: + +```sh +cp .env.example .env +``` + +Задайте в `.env` собственный локальный `POSTGRES_PASSWORD` (hex-строка), затем запустите API и зависимости: + +```sh +docker compose --env-file .env up -d --build +``` + +## Быстрая проверка через curl + +```sh +BASE_URL=http://127.0.0.1:8080 + +# Процесс API работает: ожидается HTTP 200 и {"status":"ok"} +curl --fail --show-error "$BASE_URL/health/live" + +# API и PostgreSQL готовы принимать запросы: ожидается HTTP 200 +curl --fail --show-error "$BASE_URL/health/ready" + +# Без Bearer-токена доступ запрещён: ожидается HTTP 401 +curl -sS -o /dev/null -w 'HTTP %{http_code}\n' "$BASE_URL/api/v1/users/me" +``` + +Следующий пример регистрирует двух тестовых пользователей, получает Bearer-токен Алисы через Basic-аутентификацию, создаёт чат и отправляет в него сообщение. Нужен `jq` для чтения JSON-ответов; используйте только тестовые пароли: + +```sh +ALICE_USERNAME="codes_alice_$(date +%s)" +ALICE_PASSWORD='Example-password-alice-42' +BOB_USERNAME="codes_bob_$(date +%s)" +BOB_PASSWORD='Example-password-bob-42' + +# Создать пользователей и сохранить их ID +ALICE=$(curl --fail --show-error "$BASE_URL/api/v1/users" \ + -H 'Content-Type: application/json' \ + -d "$(jq -n --arg username "$ALICE_USERNAME" --arg password "$ALICE_PASSWORD" \ + '{username:$username, display_name:"Алиса (тест)", password:$password}')") +ALICE_ID=$(printf '%s' "$ALICE" | jq -r '.id') + +BOB=$(curl --fail --show-error "$BASE_URL/api/v1/users" \ + -H 'Content-Type: application/json' \ + -d "$(jq -n --arg username "$BOB_USERNAME" --arg password "$BOB_PASSWORD" \ + '{username:$username, display_name:"Боб (тест)", password:$password}')") +BOB_ID=$(printf '%s' "$BOB" | jq -r '.id') + +# Войти как Алиса с Basic-аутентификацией и получить Bearer-токен +SESSION=$(curl --fail --show-error "$BASE_URL/api/v1/auth/sessions" \ + -u "$ALICE_USERNAME:$ALICE_PASSWORD" \ + -X POST) +TOKEN=$(printf '%s' "$SESSION" | jq -r '.token') + +# Проверить авторизованный запрос +curl --fail --show-error "$BASE_URL/api/v1/users/me" \ + -H "Authorization: Bearer $TOKEN" + +# Создать чат от имени Алисы и включить обоих пользователей +CHAT=$(curl --fail --show-error "$BASE_URL/api/v1/chats" \ + -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d "$(jq -n --arg alice "$ALICE_ID" --arg bob "$BOB_ID" \ + '{title:"Тестовый чат", member_ids:[$alice, $bob]}')") +CHAT_ID=$(printf '%s' "$CHAT" | jq -r '.id') +printf 'Создан чат: %s\n' "$CHAT_ID" + +# Отправить сообщение; ожидается HTTP 201 и JSON нового сообщения +curl --fail --show-error "$BASE_URL/api/v1/chats/$CHAT_ID/messages" \ + -H "Authorization: Bearer $TOKEN" \ + -H 'Content-Type: application/json' \ + -d '{"text":"Проверка отправки сообщения через curl"}' +``` + +Для полной проверки API используйте smoke-тест ниже: он сам регистрирует синтетических пользователей, создаёт чат и сообщения. Эти данные остаются в тестовой базе. + +## Проверки по уровням + +Сначала выполните общую проверку контракта и OpenAPI: + +```sh +make check + +# Один раз, если ещё не установлен валидатор OpenAPI: +python3 -m venv .venv +. .venv/bin/activate +pip install -r requirements-dev.txt +make validate +``` + +Далее команды проверки уровней: + +```sh +# Уровень 3: публичный API и сохранность данных/сессии после перезапуска +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh + +# Уровень 4: проверки Redis TTL/fallback, конкурентной регистрации, +# rollback транзакции и истечения срока сессии +make grade4 + +# Уровень 5, трек 2: сравнение планов/индексов и backup/restore +make track5 +``` + +Уровень 4 предполагает прохождение проверок уровня 3; уровень 5 — уровней 3 и 4, поэтому запускайте команды по порядку. `make track5` создаёт и удаляет собственные временные базы для эксперимента. Используйте только локальный учебный стенд, не production. + +## Остановка + +Останавливает контейнеры, сохраняя PostgreSQL volume и данные: + +```sh +docker compose --env-file .env down +``` + + +docker compose --env-file .env up -d +curl --fail http://127.0.0.1:8080/health/ready +Затем публичные тесты из tests/ можно запустить напрямую: +python3 tests/smoke.py +Он проверяет основной сценарий API: регистрацию и вход, чаты, отправку и чтение сообщений, доступы и пагинацию. В конце должен вывести PASS. Тест создаёт записи в базе, они автоматически не удаляются. +python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh +Он проверяет, что данные и существующая сессия доступны после перезапуска. Скрипт перезапуска должен быть исполняемым; если нужно: +python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh +Он проверяет, что данные и существующая сессия доступны после перезапуска. Скрипт перезапуска должен быть исполняемым; если нужно: +chmod +x scripts/restart.sh +python3 tests/load.py --duration 30 --concurrency 4 +Это тест нагрузки: отправляет сообщения несколько секунд и показывает успешные запросы в секунду, ошибки и задержки. Он тоже создаёт тестовые сообщения. +tests/client.py — вспомогательный модуль для этих проверок, отдельно его запускать не нужно. То же самое можно запускать короче командами Make: make test, make test-persistence ACTION_SCRIPT=scripts/restart.sh. С описанием публичных проверок можно свериться в tests/README.md. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..dc406b7d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,62 @@ +services: + postgres: + image: postgres:16.4-alpine + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Copy .env.example to .env and set POSTGRES_PASSWORD} + POSTGRES_DB: ${POSTGRES_DB:-messenger} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\""] + interval: 5s + timeout: 3s + retries: 20 + start_period: 5s + restart: unless-stopped + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"] + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 20 + restart: unless-stopped + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger} + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-16} + DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-5} + DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000} + DB_LOCK_TIMEOUT_MS: ${DB_LOCK_TIMEOUT_MS:-2000} + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: + - CMD + - python3 + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + restart: unless-stopped + +volumes: + postgres_data: diff --git a/contracts/README.md b/contracts/README.md index 8c58b32f..8d527e20 100644 --- a/contracts/README.md +++ b/contracts/README.md @@ -14,7 +14,7 @@ ## Пользователи, чаты и сообщения - `POST /users` публичен; возвращает `201 User`. Пароль/его hash и сессионные данные никогда не входят в User. -- `POST /chats` принимает title и явный список существующих `member_ids`, включая вызывающего. Дубликаты, неизвестные участники и отсутствие вызывающего — 400. Чат создаётся атомарно. GET списка чатов сортируется по `(created_at, id)` по возрастанию. +- `POST /chats` принимает title и явный список существующих `member_ids`, включая вызывающего. Дубликаты, неизвестные участники и отсутствие вызывающего — 400. Чат создаётся атомарно. GET списка чатов использует keyset-пагинацию по `(created_at, id)` по возрастанию: `limit` — целое 1–100 (по умолчанию 50), `cursor` привязан к пользователю и последнему чату предыдущей страницы. Последний элемент не повторяется; `next_cursor` равен null, если следующей страницы нет. Непустой cursor с неверным форматом или другого пользователя — 400. - Отправитель сообщения выводится из текущей идентификации, поле `sender_id` в запросе не принимается. Текст — до 4000 Unicode code points. Нельзя отправить пустой/пробельный текст без вложений. - История использует keyset cursor: `(created_at, id)`, по возрастанию; `limit` — целое 1–100, по умолчанию 50. Ответ `{"items":[...],"next_cursor":null}`. Непустой cursor — непрозрачная для клиента строка; последний элемент предыдущей страницы не повторяется. Cursor привязан к ресурсу/запросу: от другого чата — 400. Изменять `limit` при продолжении можно. - `next_cursor` равен null, если на момент чтения следующей записи нет. Гарантируется отсутствие повторов при обходе неизменной истории; snapshot изоляция между несколькими HTTP-запросами не требуется. При конкурентных вставках опишите выбранную семантику и ограничения часов. diff --git a/evidence/track5_query_restore.json b/evidence/track5_query_restore.json new file mode 100644 index 00000000..f79f9d51 --- /dev/null +++ b/evidence/track5_query_restore.json @@ -0,0 +1,916 @@ +{ + "run_utc": "2026-10-09T13:21:05.722546+00:00", + "environment": { + "host_platform": "macOS-27.0-arm64-arm-64bit-Mach-O", + "host_architecture": "arm64", + "host_logical_cpus": 10, + "host_memory_bytes": 17179869184, + "postgres_image": "postgres:16.4-alpine", + "postgres_version": "16.4", + "postgres_container_memory_limit_bytes": 0, + "postgres_container_cpu_limit_nanos": 0, + "warmup": "initial data load followed by ANALYZE; plan query is executed 5 times", + "postgres_memory_setting": "128MB", + "postgres_work_mem": "4MB" + }, + "method": { + "rows_per_index_variant": 100000, + "distribution": "80,000 messages in one chat; 20,000 distributed evenly across 100 chats", + "warmup": "ANALYZE followed by five executed EXPLAIN ANALYZE samples per query", + "query": "SELECT id, created_at, text FROM messages WHERE chat_id = 2 AND (created_at, id) > (timestamptz '2025-01-01 23:36:40+00', 0) ORDER BY created_at, id LIMIT 25" + }, + "query_plans": { + "chat_id_only": { + "index": "(chat_id)", + "samples": [ + { + "execution_ms": 0.265, + "planning_ms": 0.384, + "actual_rows": 25, + "shared_hit_blocks": 206, + "shared_read_blocks": 2, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.67, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.217, + "Actual Total Time": 0.22, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 206, + "Shared Read Blocks": 2, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.68, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.216, + "Actual Total Time": 0.217, + "Actual Rows": 25, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 26, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 206, + "Shared Read Blocks": 2, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_only_idx", + "Relation Name": "messages_chat_only", + "Alias": "messages_chat_only", + "Startup Cost": 0.29, + "Total Cost": 25.87, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.073, + "Actual Total Time": 0.182, + "Actual Rows": 150, + "Actual Loops": 1, + "Index Cond": "(chat_id = 2)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))", + "Rows Removed by Filter": 50, + "Shared Hit Blocks": 200, + "Shared Read Blocks": 2, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + }, + { + "execution_ms": 0.233, + "planning_ms": 0.353, + "actual_rows": 25, + "shared_hit_blocks": 208, + "shared_read_blocks": 0, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.67, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.195, + "Actual Total Time": 0.198, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 208, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.68, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.194, + "Actual Total Time": 0.195, + "Actual Rows": 25, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 26, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 208, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_only_idx", + "Relation Name": "messages_chat_only", + "Alias": "messages_chat_only", + "Startup Cost": 0.29, + "Total Cost": 25.87, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.055, + "Actual Total Time": 0.166, + "Actual Rows": 150, + "Actual Loops": 1, + "Index Cond": "(chat_id = 2)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))", + "Rows Removed by Filter": 50, + "Shared Hit Blocks": 202, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + }, + { + "execution_ms": 0.245, + "planning_ms": 0.363, + "actual_rows": 25, + "shared_hit_blocks": 208, + "shared_read_blocks": 0, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.67, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.198, + "Actual Total Time": 0.2, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 208, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.68, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.197, + "Actual Total Time": 0.198, + "Actual Rows": 25, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 26, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 208, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_only_idx", + "Relation Name": "messages_chat_only", + "Alias": "messages_chat_only", + "Startup Cost": 0.29, + "Total Cost": 25.87, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.051, + "Actual Total Time": 0.158, + "Actual Rows": 150, + "Actual Loops": 1, + "Index Cond": "(chat_id = 2)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))", + "Rows Removed by Filter": 50, + "Shared Hit Blocks": 202, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + }, + { + "execution_ms": 0.218, + "planning_ms": 0.285, + "actual_rows": 25, + "shared_hit_blocks": 208, + "shared_read_blocks": 0, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.67, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.184, + "Actual Total Time": 0.186, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 208, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.68, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.183, + "Actual Total Time": 0.184, + "Actual Rows": 25, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 26, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 208, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_only_idx", + "Relation Name": "messages_chat_only", + "Alias": "messages_chat_only", + "Startup Cost": 0.29, + "Total Cost": 25.87, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.048, + "Actual Total Time": 0.155, + "Actual Rows": 150, + "Actual Loops": 1, + "Index Cond": "(chat_id = 2)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))", + "Rows Removed by Filter": 50, + "Shared Hit Blocks": 202, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + }, + { + "execution_ms": 0.268, + "planning_ms": 0.365, + "actual_rows": 25, + "shared_hit_blocks": 208, + "shared_read_blocks": 0, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.67, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.228, + "Actual Total Time": 0.23, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 208, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Sort", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 26.61, + "Total Cost": 26.68, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.227, + "Actual Total Time": 0.228, + "Actual Rows": 25, + "Actual Loops": 1, + "Sort Key": [ + "created_at", + "id" + ], + "Sort Method": "top-N heapsort", + "Sort Space Used": 26, + "Sort Space Type": "Memory", + "Shared Hit Blocks": 208, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_only_idx", + "Relation Name": "messages_chat_only", + "Alias": "messages_chat_only", + "Startup Cost": 0.29, + "Total Cost": 25.87, + "Plan Rows": 30, + "Plan Width": 41, + "Actual Startup Time": 0.049, + "Actual Total Time": 0.186, + "Actual Rows": 150, + "Actual Loops": 1, + "Index Cond": "(chat_id = 2)", + "Rows Removed by Index Recheck": 0, + "Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))", + "Rows Removed by Filter": 50, + "Shared Hit Blocks": 202, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + ] + } + } + ] + }, + "composite_history": { + "index": "(chat_id, created_at, id)", + "samples": [ + { + "execution_ms": 0.085, + "planning_ms": 0.331, + "actual_rows": 25, + "shared_hit_blocks": 25, + "shared_read_blocks": 3, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 50.6, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.034, + "Actual Total Time": 0.055, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 25, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_history_idx", + "Relation Name": "messages_composite", + "Alias": "messages_composite", + "Startup Cost": 0.42, + "Total Cost": 62.64, + "Plan Rows": 31, + "Plan Width": 41, + "Actual Startup Time": 0.033, + "Actual Total Time": 0.053, + "Actual Rows": 25, + "Actual Loops": 1, + "Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 25, + "Shared Read Blocks": 3, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + }, + { + "execution_ms": 0.087, + "planning_ms": 0.362, + "actual_rows": 25, + "shared_hit_blocks": 28, + "shared_read_blocks": 0, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 50.6, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.022, + "Actual Total Time": 0.043, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 28, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_history_idx", + "Relation Name": "messages_composite", + "Alias": "messages_composite", + "Startup Cost": 0.42, + "Total Cost": 62.64, + "Plan Rows": 31, + "Plan Width": 41, + "Actual Startup Time": 0.021, + "Actual Total Time": 0.041, + "Actual Rows": 25, + "Actual Loops": 1, + "Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 28, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + }, + { + "execution_ms": 0.065, + "planning_ms": 0.307, + "actual_rows": 25, + "shared_hit_blocks": 28, + "shared_read_blocks": 0, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 50.6, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.015, + "Actual Total Time": 0.035, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 28, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_history_idx", + "Relation Name": "messages_composite", + "Alias": "messages_composite", + "Startup Cost": 0.42, + "Total Cost": 62.64, + "Plan Rows": 31, + "Plan Width": 41, + "Actual Startup Time": 0.014, + "Actual Total Time": 0.032, + "Actual Rows": 25, + "Actual Loops": 1, + "Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 28, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + }, + { + "execution_ms": 0.079, + "planning_ms": 0.365, + "actual_rows": 25, + "shared_hit_blocks": 28, + "shared_read_blocks": 0, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 50.6, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.021, + "Actual Total Time": 0.042, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 28, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_history_idx", + "Relation Name": "messages_composite", + "Alias": "messages_composite", + "Startup Cost": 0.42, + "Total Cost": 62.64, + "Plan Rows": 31, + "Plan Width": 41, + "Actual Startup Time": 0.02, + "Actual Total Time": 0.039, + "Actual Rows": 25, + "Actual Loops": 1, + "Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 28, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + }, + { + "execution_ms": 0.066, + "planning_ms": 0.299, + "actual_rows": 25, + "shared_hit_blocks": 28, + "shared_read_blocks": 0, + "plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.42, + "Total Cost": 50.6, + "Plan Rows": 25, + "Plan Width": 41, + "Actual Startup Time": 0.015, + "Actual Total Time": 0.036, + "Actual Rows": 25, + "Actual Loops": 1, + "Shared Hit Blocks": 28, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_chat_history_idx", + "Relation Name": "messages_composite", + "Alias": "messages_composite", + "Startup Cost": 0.42, + "Total Cost": 62.64, + "Plan Rows": 31, + "Plan Width": 41, + "Actual Startup Time": 0.014, + "Actual Total Time": 0.033, + "Actual Rows": 25, + "Actual Loops": 1, + "Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 28, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + } + } + ] + } + }, + "write_cost": { + "chat_id_only": { + "samples": [ + { + "execution_ms": 4.163, + "planning_ms": 0.062, + "shared_write_blocks": 25 + }, + { + "execution_ms": 3.936, + "planning_ms": 0.05, + "shared_write_blocks": 40 + }, + { + "execution_ms": 3.97, + "planning_ms": 0.051, + "shared_write_blocks": 25 + }, + { + "execution_ms": 3.722, + "planning_ms": 0.05, + "shared_write_blocks": 24 + }, + { + "execution_ms": 4.037, + "planning_ms": 0.059, + "shared_write_blocks": 25 + } + ], + "median_execution_ms": 3.97, + "median_shared_write_blocks": 25, + "rows_per_sample": 2000, + "rolled_back_after_each_measurement": true + }, + "composite_history": { + "samples": [ + { + "execution_ms": 5.166, + "planning_ms": 0.062, + "shared_write_blocks": 26 + }, + { + "execution_ms": 7.163, + "planning_ms": 0.051, + "shared_write_blocks": 120 + }, + { + "execution_ms": 4.777, + "planning_ms": 0.05, + "shared_write_blocks": 23 + }, + { + "execution_ms": 4.501, + "planning_ms": 0.049, + "shared_write_blocks": 24 + }, + { + "execution_ms": 5.049, + "planning_ms": 0.05, + "shared_write_blocks": 25 + } + ], + "median_execution_ms": 5.049, + "median_shared_write_blocks": 25, + "rows_per_sample": 2000, + "rolled_back_after_each_measurement": true + } + }, + "backup_restore": { + "format": "pg_dump custom format", + "dump_bytes": 1596489, + "dump_seconds": 0.25986058299895376, + "restore_seconds": 0.31837804202223197, + "restored_counts": { + "messages_chat_only": 100000, + "messages_composite": 100000 + }, + "source_selected_checksums": { + "messages_chat_only": "59b4843022f2d85760e29d81bb6ca73c", + "messages_composite": "59b4843022f2d85760e29d81bb6ca73c" + }, + "restored_selected_checksums": { + "messages_chat_only": "59b4843022f2d85760e29d81bb6ca73c", + "messages_composite": "59b4843022f2d85760e29d81bb6ca73c" + }, + "verification": "PASS" + } +} diff --git a/evidence/track5_query_restore.md b/evidence/track5_query_restore.md new file mode 100644 index 00000000..36df7f0a --- /dev/null +++ b/evidence/track5_query_restore.md @@ -0,0 +1,24 @@ +# Lab2 level-5 track 2 — query plans and recovery + +- Run UTC: 2026-10-09T13:21:05.722546+00:00 +- Host: macOS-27.0-arm64-arm-64bit-Mach-O (arm64), logical CPUs: 10 +- Host physical memory: 17179869184 bytes. +- PostgreSQL: 16.4 in `postgres:16.4-alpine`; container memory limit: 0 bytes; CPU limit: 0 nanocores (0 means unlimited). +- Dataset: 100,000 synthetic rows in each comparison table; one chat has 80,000 rows and 100 small chats share the remaining 20,000. Only generated text is used. +- Query: keyset page from chat 2, identical SELECT and cursor for both indexes; each plan was executed five times after ANALYZE. + +| Index | Median execution (ms) | Shared hits | Shared reads | Rows | +|---|---:|---:|---:|---:| +| `(chat_id)` | 0.245 | 206 | 2 | 25 | +| `(chat_id, created_at, id)` | 0.079 | 25 | 3 | 25 | + +| Index | Median time for 2,000 inserted rows (ms) | Median shared write blocks | +|---|---:|---:| +| `(chat_id)` | 3.970 | 25 | +| `(chat_id, created_at, id)` | 5.049 | 25 | + +Each write-cost query was measured five times inside a transaction and rolled back. This isolates execution and index-maintenance work but does not measure durable commit/fsync latency. + +Backup/restore: custom-format dump 1596489 bytes; dump 0.260s; restore into a separately created empty database 0.318s. Restored row counts and deterministic checksums for the selected messages were verified: **PASS**. + +Raw JSON plans, samples, environment and restore checksums are in `track5_query_restore.json`. This single-host synthetic run is a reproducible comparison, not a production capacity or HA claim. diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..f5431a5f --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,43 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + joined_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id); + +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); + +CREATE INDEX sessions_expiry_idx ON sessions (expires_at); diff --git a/requirements-dev.txt b/requirements-dev.txt index 6b103ddb..be165673 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ +-r requirements.txt # Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib. openapi-spec-validator==0.7.2 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..177a9775 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +psycopg[binary]==3.2.10 +psycopg-pool==3.3.3 +redis==5.2.1 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..9e758076 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu + +docker compose --env-file .env restart postgres redis + +deadline=90 +elapsed=0 +until docker compose --env-file .env exec -T postgres pg_isready -q \ + && docker compose --env-file .env exec -T redis redis-cli ping | grep -q PONG +do + if [ "$elapsed" -ge "$deadline" ]; then + echo "PostgreSQL and Redis did not become ready within ${deadline}s" >&2 + exit 1 + fi + sleep 2 + elapsed=$((elapsed + 2)) +done + +docker compose --env-file .env restart api +elapsed=0 +until docker compose --env-file .env exec -T api \ + python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()" +do + if [ "$elapsed" -ge "$deadline" ]; then + echo "API did not become ready within ${deadline}s" >&2 + exit 1 + fi + sleep 2 + elapsed=$((elapsed + 2)) +done diff --git a/scripts/test_expiry.sh b/scripts/test_expiry.sh new file mode 100755 index 00000000..41ec0180 --- /dev/null +++ b/scripts/test_expiry.sh @@ -0,0 +1,43 @@ +#!/bin/sh +set -eu + +cd "$(dirname "$0")/.." +python_cmd=${PYTHON:-python3} +base_url=${BASE_URL:-http://127.0.0.1:8080} +restored=0 + +restore_api() { + if [ "$restored" -eq 0 ]; then + restored=1 + docker compose --env-file .env up -d --force-recreate api >/dev/null + restore_elapsed=0 + until curl -fsS "$base_url/health/ready" >/dev/null 2>&1 + do + if [ "$restore_elapsed" -ge 45 ]; then + echo "API did not return to normal readiness after expiry test" >&2 + return 1 + fi + sleep 1 + restore_elapsed=$((restore_elapsed + 1)) + done + fi +} + +trap restore_api EXIT HUP INT TERM +SESSION_TTL_SECONDS=2 docker compose --env-file .env up -d --force-recreate api >/dev/null + +elapsed=0 +until curl -fsS "$base_url/health/ready" >/dev/null 2>&1 +do + if [ "$elapsed" -ge 45 ]; then + echo "API did not become ready for expiry test" >&2 + exit 1 + fi + sleep 1 + elapsed=$((elapsed + 1)) +done + +"$python_cmd" student-tests/test_level4.py \ + --expiry-only --ttl-seconds 2 --base-url "$base_url" +restore_api +trap - EXIT HUP INT TERM diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..5bc8da31 --- /dev/null +++ b/src/__init__.py @@ -0,0 +1 @@ +"""Messenger API application package.""" diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..f14dd321 --- /dev/null +++ b/src/app.py @@ -0,0 +1,40 @@ +import logging +import os +import signal +import threading +from functools import partial +from http.server import ThreadingHTTPServer + +from .config import APP_HOST, APP_PORT +from .handlers import MessengerHandler +from .store import Store + +LOGGER = logging.getLogger("messenger") + + +def run_server(): + store = Store() + store.open() + try: + store.migrate() + handler = partial(MessengerHandler, store=store) + server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler) + + def handle_shutdown(signum, frame): + LOGGER.info("received signal %s, shutting down gracefully", signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, handle_shutdown) + signal.signal(signal.SIGINT, handle_shutdown) + LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT) + try: + server.serve_forever() + finally: + server.server_close() + finally: + store.close() + + +def main(): + logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO")) + run_server() diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..b6fb8fc0 --- /dev/null +++ b/src/config.py @@ -0,0 +1,24 @@ +import os +import re +import uuid + +APP_HOST = os.getenv("APP_HOST", "0.0.0.0") +APP_PORT = int(os.getenv("APP_PORT", "8080")) +INSTANCE_ID = os.getenv("INSTANCE_ID", uuid.uuid4().hex[:12]) +SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600")) +DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000")) +DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000")) +DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16")) +DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5")) +MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128")) +MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576")) +USER_RE = re.compile(r"^[a-z0-9_]{3,32}$") + +if SESSION_TTL_SECONDS < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be positive") +if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1: + raise RuntimeError("Database timeouts must be positive") +if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0: + raise RuntimeError("Database pool settings must be positive") +if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1: + raise RuntimeError("HTTP limits must be positive") diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..6c80239d --- /dev/null +++ b/src/errors.py @@ -0,0 +1,14 @@ +from typing import Dict, Optional + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None): + super().__init__(message) + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + + +class CacheUnavailableError(Exception): + pass diff --git a/src/handlers.py b/src/handlers.py new file mode 100644 index 00000000..ac96b003 --- /dev/null +++ b/src/handlers.py @@ -0,0 +1,326 @@ +import base64 +import json +import logging +import threading +import uuid +from http.server import BaseHTTPRequestHandler +from typing import Any, Dict, List, Optional, Tuple +from urllib.parse import parse_qs, urlsplit + +import psycopg +from psycopg_pool import PoolTimeout + +from .config import INSTANCE_ID, MAX_INFLIGHT_REQUESTS, MAX_REQUEST_BODY_BYTES, USER_RE +from .errors import ApiError, CacheUnavailableError +from .security import hash_password, isoformat, valid_uuid +from .store import Store + +LOGGER = logging.getLogger("messenger.http") + + +class RequestGuard: + def __init__(self): + self.lock = threading.Lock() + self.inflight = 0 + + def acquire(self): + with self.lock: + if self.inflight >= MAX_INFLIGHT_REQUESTS: + raise ApiError(503, "unavailable", "Server is busy") + self.inflight += 1 + + def release(self): + with self.lock: + self.inflight = max(0, self.inflight - 1) + + +REQUEST_GUARD = RequestGuard() + + +class MessengerHandler(BaseHTTPRequestHandler): + def __init__(self, request, client_address, server, store: Store): + self.store = store + super().__init__(request, client_address, server) + + def log_message(self, format: str, *args): + return + + def do_GET(self): + self._dispatch("GET") + + def do_POST(self): + self._dispatch("POST") + + def do_DELETE(self): + self._dispatch("DELETE") + + def _dispatch(self, method: str): + request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4()) + acquired = False + try: + REQUEST_GUARD.acquire() + acquired = True + parsed = urlsplit(self.path) + path = parsed.path + query = parse_qs(parsed.query, keep_blank_values=True) + if path == "/health/live" and method == "GET": + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/health/ready" and method == "GET": + try: + self.store.ready() + except (psycopg.Error, PoolTimeout, OSError): + raise ApiError(503, "unavailable", "Database is unavailable") + self._send_json(200, {"status": "ok"}, request_id) + elif path == "/api/v1/users" and method == "POST": + self._handle_register(request_id) + elif path == "/api/v1/auth/sessions" and method == "POST": + self._handle_create_session(request_id) + elif path == "/api/v1/auth/sessions/current" and method == "DELETE": + self._handle_revoke_session(request_id) + elif path == "/api/v1/users/me" and method == "GET": + self._send_json(200, self._get_actor(request_id), request_id) + elif path == "/api/v1/chats" and method == "POST": + self._handle_create_chat(request_id) + elif path == "/api/v1/chats" and method == "GET": + self._handle_list_chats(query, request_id) + elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"): + chat_id = path.rsplit("/", 2)[1] + if method == "POST": + self._handle_create_message(chat_id, request_id) + else: + self._handle_list_messages(chat_id, query, request_id) + elif path.startswith("/api/v1/chats/") and method == "GET": + self._handle_get_chat(path.rsplit("/", 1)[1], request_id) + else: + raise ApiError(404, "not_found", "Object not found") + except ApiError as exc: + self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers) + except CacheUnavailableError: + self._send_error(503, "unavailable", "Session cache is unavailable", request_id) + except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError): + self._send_error(503, "unavailable", "Database is unavailable", request_id) + except psycopg.errors.UniqueViolation: + self._send_error(409, "conflict", "Username already exists", request_id) + except psycopg.errors.ForeignKeyViolation: + self._send_error(400, "invalid_request", "Unknown member", request_id) + except Exception as exc: + LOGGER.error("request %s failed with %s", request_id, type(exc).__name__) + self._send_error(500, "internal_error", "Internal server error", request_id) + finally: + if acquired: + REQUEST_GUARD.release() + + def _read_json(self, request_id: str) -> Any: + if self.headers.get_content_type() != "application/json": + raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json") + length = self.headers.get("Content-Length") + if length is None: + raise ApiError(400, "invalid_request", "Missing Content-Length") + try: + size = int(length) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size < 0: + raise ApiError(400, "invalid_request", "Invalid Content-Length") + if size > MAX_REQUEST_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Request body too large") + raw = self.rfile.read(size) + if not raw: + raise ApiError(400, "invalid_request", "Empty body") + try: + return json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + raise ApiError(400, "invalid_request", "Malformed JSON") + + def _get_actor(self, request_id: str) -> Dict[str, Any]: + header = self.headers.get("Authorization", "") + scheme, separator, token = header.partition(" ") + if scheme.lower() != "bearer" or not separator or not token or " " in token: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + user = self.store.session_user(token) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + return user + + def _handle_register(self, request_id: str): + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}: + raise ApiError(400, "invalid_request", "Expected username, display_name and password") + username = body["username"] + display_name = body["display_name"] + password = body["password"] + if not isinstance(username, str) or not USER_RE.fullmatch(username): + raise ApiError(400, "invalid_request", "Invalid username") + if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100: + raise ApiError(400, "invalid_request", "Invalid display_name") + if not isinstance(password, str) or not (12 <= len(password) <= 128): + raise ApiError(400, "invalid_request", "Invalid password") + user = self.store.create_user(username, display_name, hash_password(password)) + self._send_json(201, user, request_id) + + def _basic_credentials(self, request_id: str) -> Tuple[str, str]: + value = self.headers.get("Authorization", "") + scheme, separator, encoded = value.partition(" ") + challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'} + if scheme.lower() != "basic" or not separator or not encoded: + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + try: + decoded = base64.b64decode(encoded, validate=True).decode("utf-8") + username, password = decoded.split(":", 1) + except (ValueError, UnicodeError): + raise ApiError(401, "unauthorized", "Invalid credentials", challenge) + return username, password + + def _handle_create_session(self, request_id: str): + username, password = self._basic_credentials(request_id) + user = self.store.authenticate(username, password) + if user is None: + raise ApiError( + 401, + "unauthorized", + "Invalid credentials", + {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}, + ) + token, expires_at = self.store.create_session(user["id"]) + self._send_json( + 201, + {"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user}, + request_id, + ) + + def _handle_revoke_session(self, request_id: str): + self._get_actor(request_id) + token = self.headers["Authorization"].partition(" ")[2] + if not self.store.revoke_session(token): + raise ApiError( + 401, + "unauthorized", + "Authentication required", + {"WWW-Authenticate": 'Bearer realm="messenger"'}, + ) + self.send_response(204) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + self.end_headers() + + def _handle_create_chat(self, request_id: str): + actor = self._get_actor(request_id) + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"title", "member_ids"}: + raise ApiError(400, "invalid_request", "Expected title and member_ids") + title, member_ids = body["title"], body["member_ids"] + if not isinstance(title, str) or not title.strip() or len(title) > 100: + raise ApiError(400, "invalid_request", "Invalid title") + if ( + not isinstance(member_ids, list) + or not 1 <= len(member_ids) <= 100 + or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids) + or len(member_ids) != len(set(member_ids)) + or actor["id"] not in member_ids + ): + raise ApiError(400, "invalid_request", "Invalid member_ids") + chat = self.store.create_chat(title, member_ids, actor["id"]) + self._send_json(201, chat, request_id) + + def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id) + + def _handle_get_chat(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + chat = self.store.get_chat(chat_id, actor["id"]) + if chat is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(200, chat, request_id) + + def _handle_create_message(self, chat_id: str, request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + body = self._read_json(request_id) + if not isinstance(body, dict) or set(body) != {"text"}: + raise ApiError(400, "invalid_request", "Expected text") + text = body["text"] + if not isinstance(text, str) or not text.strip() or len(text) > 4000: + raise ApiError(400, "invalid_request", "Invalid message text") + message = self.store.create_message(chat_id, actor["id"], text) + if message is None: + raise ApiError(404, "not_found", "Chat not found") + self._send_json(201, message, request_id) + + def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str): + actor = self._get_actor(request_id) + if not valid_uuid(chat_id): + raise ApiError(400, "invalid_request", "Invalid chat id") + try: + limit = int(query.get("limit", ["50"])[0]) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid limit") + if not 1 <= limit <= 100: + raise ApiError(400, "invalid_request", "Invalid limit") + cursor = query.get("cursor", [None])[0] + try: + result = self.store.list_messages(chat_id, actor["id"], limit, cursor) + except ValueError: + raise ApiError(400, "invalid_request", "Invalid cursor") + if result is None: + raise ApiError(404, "not_found", "Chat not found") + items, next_cursor = result + self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id) + + def _send_json( + self, + status: int, + payload: Dict[str, Any], + request_id: str, + headers: Optional[Dict[str, str]] = None, + ): + body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("X-Request-Id", request_id) + self.send_header("X-Instance-Id", INSTANCE_ID) + if headers: + for key, value in headers.items(): + self.send_header(key, value) + self.end_headers() + self.wfile.write(body) + + def _send_error( + self, + status: int, + code: str, + message: str, + request_id: str, + extra_headers: Optional[Dict[str, str]] = None, + ): + self._send_json( + status, + {"error": {"code": code, "message": message, "request_id": request_id}}, + request_id, + extra_headers, + ) diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..1ae27009 --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +import logging +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.store import Store +else: + from .store import Store + + +def main(): + logging.basicConfig(level="INFO") + store = Store() + store.open() + try: + store.migrate() + finally: + store.close() + + +if __name__ == "__main__": + main() diff --git a/src/security.py b/src/security.py new file mode 100644 index 00000000..6ec821f7 --- /dev/null +++ b/src/security.py @@ -0,0 +1,53 @@ +import base64 +import hashlib +import hmac +import secrets +import uuid +from datetime import datetime, timezone + + +def isoformat(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def valid_uuid(value: str) -> bool: + try: + uuid.UUID(value) + return True + except (ValueError, TypeError, AttributeError): + return False + + +def hash_password(password: str) -> str: + salt = secrets.token_bytes(16) + derived = hashlib.scrypt( + password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024 + ) + salt_text = base64.urlsafe_b64encode(salt).decode("ascii") + hash_text = base64.urlsafe_b64encode(derived).decode("ascii") + return f"scrypt$32768$8$1${salt_text}${hash_text}" + + +def verify_password(password: str, stored: str) -> bool: + try: + scheme, n, r, p, salt_text, hash_text = stored.split("$") + if scheme != "scrypt": + return False + salt = base64.urlsafe_b64decode(salt_text.encode("ascii")) + expected = base64.urlsafe_b64decode(hash_text.encode("ascii")) + actual = hashlib.scrypt( + password.encode("utf-8"), + salt=salt, + n=int(n), + r=int(r), + p=int(p), + dklen=len(expected), + maxmem=64 * 1024 * 1024, + ) + return hmac.compare_digest(actual, expected) + except (ValueError, TypeError, UnicodeError): + return False diff --git a/src/server.py b/src/server.py new file mode 100644 index 00000000..60fd3d56 --- /dev/null +++ b/src/server.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import sys +from pathlib import Path + +if __package__ in (None, ""): + sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from src.app import main +else: + from .app import main + + +if __name__ == "__main__": + main() diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..90f17209 --- /dev/null +++ b/src/store.py @@ -0,0 +1,348 @@ +import base64 +import hashlib +import json +import logging +import math +import os +import secrets +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +import redis +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +from .config import ( + DB_LOCK_TIMEOUT_MS, + DB_POOL_MAX_SIZE, + DB_POOL_TIMEOUT_SECONDS, + DB_STATEMENT_TIMEOUT_MS, + SESSION_TTL_SECONDS, +) +from .errors import CacheUnavailableError +from .security import isoformat, utc_now, valid_uuid, verify_password + +LOGGER = logging.getLogger("messenger.store") + + +def database_url() -> str: + value = os.getenv("DATABASE_URL") + if value: + return value + file_path = os.getenv("DATABASE_URL_FILE") + if file_path: + return Path(file_path).read_text(encoding="utf-8").strip() + raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE") + + +class Store: + """PostgreSQL source of truth and optional Redis session lookup cache.""" + + def __init__(self): + self.pool = ConnectionPool( + conninfo=database_url(), + min_size=1, + max_size=DB_POOL_MAX_SIZE, + timeout=DB_POOL_TIMEOUT_SECONDS, + kwargs={ + "row_factory": dict_row, + "connect_timeout": 5, + "options": ( + f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} " + f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} " + "-c idle_in_transaction_session_timeout=10000" + ), + }, + open=False, + ) + cache_url = os.getenv("CACHE_URL") + self.cache = ( + redis.Redis.from_url( + cache_url, + decode_responses=True, + socket_connect_timeout=0.25, + socket_timeout=0.25, + retry_on_timeout=False, + max_connections=16, + ) + if cache_url + else None + ) + + def open(self): + self.pool.open(wait=True, timeout=10) + + def close(self): + if self.cache is not None: + self.cache.close() + self.pool.close() + + def ready(self) -> bool: + with self.pool.connection(timeout=2) as conn: + conn.execute("SELECT 1") + return True + + def migrate(self): + migrations = Path(__file__).resolve().parents[1] / "migrations" + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations " + "(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())" + ) + conn.execute("SELECT pg_advisory_xact_lock(2741662026)") + applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")} + for migration in sorted(migrations.glob("*.sql")): + if migration.name in applied: + continue + conn.execute(migration.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,)) + + @staticmethod + def user(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": isoformat(row["created_at"]), + } + + @staticmethod + def message(row: Dict[str, Any]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": isoformat(row["created_at"]), + } + + def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]: + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO users (username, display_name, password_hash) " + "VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at", + (username, display_name, password_hash), + ).fetchone() + return self.user(row) + + def get_user(self, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", + (user_id,), + ).fetchone() + return self.user(row) if row else None + + def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT id, username, display_name, created_at, password_hash " + "FROM users WHERE username = %s", + (username,), + ).fetchone() + if not row or not verify_password(password, row["password_hash"]): + return None + return self.user(row) + + def create_session(self, user_id: str) -> Tuple[str, datetime]: + token = secrets.token_urlsafe(32) + expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS) + token_hash = hashlib.sha256(token.encode("ascii")).digest() + with self.pool.connection() as conn: + conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash, user_id, expires_at), + ) + return token, expires_at + + @staticmethod + def _session_lock_key(token_hash: bytes) -> int: + return int.from_bytes(token_hash[:8], byteorder="big", signed=True) + + def session_user(self, token: str) -> Optional[Dict[str, Any]]: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + cache_key = "messenger:session:" + token_hash.hex() + if self.cache is not None: + try: + cached = self.cache.get(cache_key) + if cached: + return json.loads(cached) + except (redis.RedisError, ValueError, TypeError): + LOGGER.warning("session cache read failed; falling back to PostgreSQL") + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + row = conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at " + "FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE s.token_hash = %s AND s.expires_at > now()", + (token_hash,), + ).fetchone() + if not row: + return None + user = self.user(row) + if self.cache is not None: + ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000) + if ttl_ms > 0: + try: + self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms) + except redis.RedisError: + LOGGER.warning("session cache write failed; PostgreSQL remains authoritative") + return user + + def revoke_session(self, token: str) -> bool: + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + with self.pool.connection() as conn: + with conn.transaction(): + conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),)) + if self.cache is not None: + try: + self.cache.delete("messenger:session:" + token_hash.hex()) + except redis.RedisError as exc: + raise CacheUnavailableError from exc + result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,)) + return result.rowcount == 1 + + def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]: + if actor_id not in member_ids or len(member_ids) != len(set(member_ids)): + raise ValueError("invalid members") + with self.pool.connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at", + (title,), + ).fetchone() + for member_id in member_ids: + conn.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (row["id"], member_id), + ) + return self.chat(row, member_ids) + + def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + row = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id " + "WHERE c.id = %s AND m.user_id = %s", + (chat_id, user_id), + ).fetchone() + if not row: + return None + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (chat_id,), + ).fetchall() + return self.chat(row, [str(item["user_id"]) for item in members]) + + def list_chats( + self, user_id: str, limit: int, cursor: Optional[str] + ) -> Tuple[List[Dict[str, Any]], Optional[str]]: + where = "own.user_id = %s" + params: List[Any] = [user_id] + if cursor is not None: + try: + raw = base64.b64decode( + cursor.encode("ascii") + b"=" * (-len(cursor) % 4), + altchars=b"-_", + validate=True, + ) + cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2) + if cursor_user != user_id or not valid_uuid(chat_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + if created_at.tzinfo is None or created_at.utcoffset() is None: + raise ValueError("cursor timestamp must include a timezone") + where += " AND (c.created_at, c.id) > (%s, %s)" + params.extend((created_at, chat_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + with self.pool.connection() as conn: + rows = conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members own ON own.chat_id = c.id " + f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s", + params, + ).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + result = [] + for row in page: + members = conn.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id", + (row["id"],), + ).fetchall() + result.append(self.chat(row, [str(item["user_id"]) for item in members])) + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return result, next_cursor + + def create_message(self, chat_id: str, sender_id: str, text: str) -> Optional[Dict[str, Any]]: + with self.pool.connection() as conn: + with conn.transaction(): + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ).fetchone() + if not member: + return None + row = conn.execute( + "INSERT INTO messages (chat_id, sender_id, text) " + "VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at", + (chat_id, sender_id, text), + ).fetchone() + return self.message(row) + + def list_messages( + self, chat_id: str, user_id: str, limit: int, cursor: Optional[str] + ) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]: + with self.pool.connection() as conn: + is_member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id), + ).fetchone() + if not is_member: + return None + where = "chat_id = %s" + params: List[Any] = [chat_id] + if cursor: + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4)) + decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2) + if decoded_chat != chat_id or not valid_uuid(message_id): + raise ValueError("cursor scope") + created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00")) + where += " AND (created_at, id) > (%s, %s)" + params.extend((created_at, message_id)) + except (ValueError, UnicodeError, TypeError) as exc: + raise ValueError("invalid cursor") from exc + params.append(limit + 1) + rows = conn.execute( + "SELECT id, chat_id, sender_id, text, created_at FROM messages " + f"WHERE {where} ORDER BY created_at, id LIMIT %s", + params, + ).fetchall() + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = None + if has_more and page: + last = page[-1] + cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8") + next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=") + return [self.message(row) for row in page], next_cursor diff --git a/student-tests/test_level4.py b/student-tests/test_level4.py new file mode 100644 index 00000000..a4ec31f8 --- /dev/null +++ b/student-tests/test_level4.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Integration checks for the lab2 level-4 rubric; requires the local Compose stack.""" +import base64 +from concurrent.futures import ThreadPoolExecutor +import hashlib +import os +import subprocess +import sys +import threading +import time +import uuid + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure, check # noqa: E402 + + +def compose(*args): + result = subprocess.run( + ["docker", "compose", "--env-file", ".env", *args], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode: + raise Failure("Docker Compose operation failed: " + args[0]) + return result.stdout.strip() + + +def psql_scalar(sql): + result = subprocess.run( + [ + "docker", + "compose", + "--env-file", + ".env", + "exec", + "-T", + "postgres", + "sh", + "-c", + 'psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"', + "level4-test", + sql, + ], + cwd=ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=False, + ) + if result.returncode: + raise Failure("PostgreSQL verification query failed") + return result.stdout.strip() + + +def assert_ready(client, expected=200, deadline_seconds=45): + deadline = time.monotonic() + deadline_seconds + while time.monotonic() < deadline: + try: + client.call("GET", "/health/ready", expected, schema="Health" if expected == 200 else None) + return + except Failure: + if expected != 200: + raise + time.sleep(0.5) + raise Failure("API did not become ready") + + +def expire_only(args): + client = Client(args.base_url) + actor = client.user("level4expiry") + time.sleep(args.ttl_seconds + 1) + client.call("GET", "/api/v1/users/me", 401, actor=actor) + print("PASS: expired session rejected") + + +def main(): + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--expiry-only", action="store_true") + parser.add_argument("--ttl-seconds", type=int, default=2) + args = parser.parse_args() + if args.expiry_only: + check(args.ttl_seconds >= 1, "TTL must be positive") + expire_only(args) + return + + client = Client(args.base_url) + user = client.user("level4") + username = user["registration"]["username"] + wrong = base64.b64encode((username + ":not-the-correct-password").encode()).decode() + client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + wrong}) + + client.call("GET", "/api/v1/users/me", actor=user, schema="User") + token_hash = hashlib.sha256(user["token"].encode("utf-8")).hexdigest() + redis_key = "messenger:session:" + token_hash + ttl_ms = int( + compose( + "exec", + "-T", + "redis", + "redis-cli", + "--raw", + "PTTL", + redis_key, + ) + ) + check(ttl_ms > 0, "session cache entry must have a positive Redis TTL") + + unique_name = "race_" + uuid.uuid4().hex[:20] + registration = { + "username": unique_name, + "display_name": "Concurrent registration", + "password": "Concurrent-test-password-42", + } + barrier = threading.Barrier(4) + + def register_once(_): + barrier.wait(timeout=10) + return Client(args.base_url).call( + "POST", "/api/v1/users", expected=[201, 409], body=registration + )["status"] + + with ThreadPoolExecutor(max_workers=4) as executor: + statuses = list(executor.map(register_once, range(4))) + check(statuses.count(201) == 1 and statuses.count(409) == 3, "username race was not serialized") + check( + psql_scalar("SELECT count(*) FROM users WHERE username = '" + unique_name + "'") == "1", + "username race persisted other than one user", + ) + + chat_title = "rollback_" + uuid.uuid4().hex + missing_user_id = str(uuid.uuid4()) + client.call( + "POST", + "/api/v1/chats", + 400, + actor=user, + body={"title": chat_title, "member_ids": [user["id"], missing_user_id]}, + ) + check( + psql_scalar("SELECT count(*) FROM chats WHERE title = '" + chat_title + "'") == "0", + "failed chat creation left a partial chat row", + ) + + compose("stop", "redis") + try: + client.call("GET", "/api/v1/users/me", actor=user, schema="User") + finally: + compose("start", "redis") + deadline = time.monotonic() + 30 + while True: + try: + if compose("exec", "-T", "redis", "redis-cli", "ping") == "PONG": + break + except Failure: + if time.monotonic() >= deadline: + raise Failure("Redis did not recover") + time.sleep(0.5) + client.call("GET", "/api/v1/users/me", actor=user, schema="User") + + compose("stop", "postgres") + try: + client.call("GET", "/health/ready", 503) + client.call("GET", "/api/v1/chats?limit=10", 503, actor=user) + finally: + compose("start", "postgres") + assert_ready(client) + client.call("GET", "/api/v1/users/me", actor=user, schema="User") + print( + "PASS: level-4 checks (password failure, username race, atomic rollback, " + "Redis TTL/fallback, bounded PostgreSQL outage)" + ) + + +if __name__ == "__main__": + try: + main() + except (Failure, OSError, subprocess.SubprocessError) as exc: + print("FAIL:", str(exc)) + raise SystemExit(1) diff --git a/student-tests/test_session_cache_race.py b/student-tests/test_session_cache_race.py new file mode 100644 index 00000000..594c9e3f --- /dev/null +++ b/student-tests/test_session_cache_race.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Deterministically verify that a cache miss cannot repopulate a logged-out session.""" +import hashlib +import secrets +import threading + +from src.store import Store + + +class BlockingCache: + def __init__(self, cache): + self.cache = cache + self.set_started = threading.Event() + self.allow_set = threading.Event() + + def __getattr__(self, name): + return getattr(self.cache, name) + + def set(self, key, value, **kwargs): + self.set_started.set() + if not self.allow_set.wait(timeout=10): + raise RuntimeError("timed out waiting to release the paused cache write") + return self.cache.set(key, value, **kwargs) + + +def main(): + store = Store() + store.open() + underlying_cache = store.cache + if underlying_cache is None: + store.close() + raise RuntimeError("Redis cache is required for this test") + username = "sessionrace_" + secrets.token_hex(8) + user = store.create_user(username, "Session race test", "unused-test-hash") + token, _ = store.create_session(user["id"]) + token_hash = hashlib.sha256(token.encode("ascii")).digest() + cache_key = "messenger:session:" + token_hash.hex() + + blocking_cache = BlockingCache(underlying_cache) + store.cache = blocking_cache + outcomes = {} + failures = [] + revoker_done = threading.Event() + + def read_session(): + try: + outcomes["reader"] = store.session_user(token) + except Exception as exc: + failures.append(exc) + + def revoke_session(): + try: + outcomes["revoker"] = store.revoke_session(token) + except Exception as exc: + failures.append(exc) + finally: + revoker_done.set() + + reader = threading.Thread(target=read_session) + revoker = None + try: + underlying_cache.delete(cache_key) + reader.start() + if not blocking_cache.set_started.wait(timeout=10): + raise RuntimeError("session read did not reach the cache write") + + revoker_started = threading.Event() + + def start_revocation(): + revoker_started.set() + revoke_session() + + revoker = threading.Thread(target=start_revocation) + revoker.start() + if not revoker_started.wait(timeout=5): + raise RuntimeError("logout thread did not start") + if revoker_done.wait(timeout=0.2): + raise RuntimeError("logout passed the read's session lock before cache write completed") + + blocking_cache.allow_set.set() + reader.join(timeout=10) + revoker.join(timeout=10) + if reader.is_alive() or revoker.is_alive(): + raise RuntimeError("session race test threads did not finish") + if failures: + raise RuntimeError("session race worker failed") from failures[0] + if outcomes.get("reader") != user or outcomes.get("revoker") is not True: + raise RuntimeError("session read/logout returned unexpected results") + if underlying_cache.get(cache_key) is not None: + raise RuntimeError("logged-out session remains in Redis") + if store.session_user(token) is not None: + raise RuntimeError("logged-out session is still accepted") + print("PASS: concurrent session cache fill and logout leave no cached or valid session") + finally: + blocking_cache.allow_set.set() + reader.join(timeout=10) + if revoker is not None: + revoker.join(timeout=10) + store.cache = underlying_cache + underlying_cache.delete(cache_key) + with store.pool.connection() as conn: + conn.execute("DELETE FROM users WHERE id = %s", (user["id"],)) + store.close() + + +if __name__ == "__main__": + main() diff --git a/student-tests/track5_query_restore.py b/student-tests/track5_query_restore.py new file mode 100644 index 00000000..4fe94009 --- /dev/null +++ b/student-tests/track5_query_restore.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Reproducible 100k-message plan/index/write-cost and backup-restore experiment.""" +import json +import os +import platform +import secrets +import statistics +import subprocess +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" / "track5_query_restore.json" +MARKDOWN = ROOT / "evidence" / "track5_query_restore.md" +COMPOSE = ["docker", "compose", "--env-file", ".env"] + + +def run_container(*args, input_text=None, stdin=None, stdout=None, check=True): + result = subprocess.run( + [*COMPOSE, "exec", "-T", "postgres", *args], + cwd=ROOT, + input=input_text, + stdin=stdin, + stdout=stdout if stdout is not None else subprocess.PIPE, + stderr=subprocess.PIPE, + text=input_text is not None, + check=False, + ) + if check and result.returncode: + detail = result.stderr.decode("utf-8", "replace") if isinstance(result.stderr, bytes) else result.stderr + raise RuntimeError("PostgreSQL benchmark command failed: " + detail[-1000:].strip()) + return result + + +def psql(database, sql): + result = run_container( + "sh", + "-c", + 'psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$1"', + "track5", + database, + input_text=sql, + ) + return result.stdout.strip() + + +def stdout_text(result): + if isinstance(result.stdout, bytes): + return result.stdout.decode("utf-8", "replace").strip() + return result.stdout.strip() + + +def admin(command, database): + return run_container( + "sh", + "-c", + f'{command} -U "$POSTGRES_USER" "$1"', + "track5", + database, + check=command != "dropdb", + ) + + +def plan(database, table, query): + raw = psql(database, f"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) {query};") + document = json.loads(raw)[0] + root = document["Plan"] + return { + "execution_ms": document["Execution Time"], + "planning_ms": document["Planning Time"], + "actual_rows": root.get("Actual Rows"), + "shared_hit_blocks": root.get("Shared Hit Blocks", 0), + "shared_read_blocks": root.get("Shared Read Blocks", 0), + "plan": root, + } + + +def container_metadata(): + container_id = subprocess.run( + [*COMPOSE, "ps", "-q", "postgres"], + cwd=ROOT, + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + inspected = subprocess.run( + ["docker", "inspect", "--format", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", container_id], + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + memory, nano_cpus = (int(value) for value in inspected.split()) + try: + host_memory = int( + subprocess.run( + ["sysctl", "-n", "hw.memsize"], + check=True, + stdout=subprocess.PIPE, + text=True, + ).stdout.strip() + ) + except (OSError, subprocess.CalledProcessError, ValueError): + host_memory = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") + pg_version = stdout_text(run_container( + "sh", + "-c", + 'psql -X -qAt -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT current_setting(\'server_version\')"', + "track5", + )) + return { + "host_platform": platform.platform(), + "host_architecture": platform.machine(), + "host_logical_cpus": os.cpu_count(), + "host_memory_bytes": host_memory, + "postgres_image": "postgres:16.4-alpine", + "postgres_version": pg_version, + "postgres_container_memory_limit_bytes": memory, + "postgres_container_cpu_limit_nanos": nano_cpus, + "warmup": "initial data load followed by ANALYZE; plan query is executed 5 times", + } + + +def dataset_sql(): + return """ +CREATE TABLE messages_chat_only ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + chat_id integer NOT NULL, + sender_id integer NOT NULL, + text varchar(200) NOT NULL, + created_at timestamptz NOT NULL +); +CREATE TABLE messages_composite (LIKE messages_chat_only INCLUDING ALL); +INSERT INTO messages_chat_only (chat_id, sender_id, text, created_at) +SELECT CASE WHEN n <= 80000 THEN 1 ELSE 2 + ((n - 80001) % 100) END, + 1 + (n % 3), + 'synthetic-message-' || lpad(n::text, 6, '0'), + timestamptz '2025-01-01 00:00:00+00' + n * interval '1 second' +FROM generate_series(1, 100000) AS n; +INSERT INTO messages_composite (chat_id, sender_id, text, created_at) +SELECT chat_id, sender_id, text, created_at FROM messages_chat_only ORDER BY id; +CREATE INDEX messages_chat_only_idx ON messages_chat_only (chat_id); +CREATE INDEX messages_chat_history_idx ON messages_composite (chat_id, created_at, id); +ANALYZE messages_chat_only; +ANALYZE messages_composite; +""" + + +def history_query(table): + return ( + f"SELECT id, created_at, text FROM {table} " + "WHERE chat_id = 2 " + "AND (created_at, id) > (timestamptz '2025-01-01 23:36:40+00', 0) " + "ORDER BY created_at, id LIMIT 25" + ) + + +def write_samples(database, table): + sql = ( + "BEGIN; " + "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + f"INSERT INTO {table} (chat_id, sender_id, text, created_at) " + "SELECT 2 + (n % 100), 1, 'synthetic-write-' || n::text, " + "timestamptz '2025-01-01 00:00:00+00' + n * interval '1 second' " + "FROM generate_series(100001, 102000) AS n; " + "ROLLBACK;" + ) + samples = [] + for _ in range(5): + raw = psql(database, sql) + document = json.loads(raw)[0] + samples.append( + { + "execution_ms": document["Execution Time"], + "planning_ms": document["Planning Time"], + "shared_write_blocks": document["Plan"].get("Shared Written Blocks", 0), + } + ) + return { + "samples": samples, + "median_execution_ms": statistics.median(item["execution_ms"] for item in samples), + "median_shared_write_blocks": statistics.median(item["shared_write_blocks"] for item in samples), + "rows_per_sample": 2000, + "rolled_back_after_each_measurement": True, + } + + +def render_markdown(result): + small = result["query_plans"] + writes = result["write_cost"] + restore = result["backup_restore"] + return f"""# Lab2 level-5 track 2 — query plans and recovery + +- Run UTC: {result['run_utc']} +- Host: {result['environment']['host_platform']} ({result['environment']['host_architecture']}), logical CPUs: {result['environment']['host_logical_cpus']} +- Host physical memory: {result['environment']['host_memory_bytes']} bytes. +- PostgreSQL: {result['environment']['postgres_version']} in `{result['environment']['postgres_image']}`; container memory limit: {result['environment']['postgres_container_memory_limit_bytes']} bytes; CPU limit: {result['environment']['postgres_container_cpu_limit_nanos']} nanocores (0 means unlimited). +- Dataset: 100,000 synthetic rows in each comparison table; one chat has 80,000 rows and 100 small chats share the remaining 20,000. Only generated text is used. +- Query: keyset page from chat 2, identical SELECT and cursor for both indexes; each plan was executed five times after ANALYZE. + +| Index | Median execution (ms) | Shared hits | Shared reads | Rows | +|---|---:|---:|---:|---:| +| `(chat_id)` | {statistics.median(item['execution_ms'] for item in small['chat_id_only']['samples']):.3f} | {small['chat_id_only']['samples'][0]['shared_hit_blocks']} | {small['chat_id_only']['samples'][0]['shared_read_blocks']} | {small['chat_id_only']['samples'][0]['actual_rows']} | +| `(chat_id, created_at, id)` | {statistics.median(item['execution_ms'] for item in small['composite_history']['samples']):.3f} | {small['composite_history']['samples'][0]['shared_hit_blocks']} | {small['composite_history']['samples'][0]['shared_read_blocks']} | {small['composite_history']['samples'][0]['actual_rows']} | + +| Index | Median time for 2,000 inserted rows (ms) | Median shared write blocks | +|---|---:|---:| +| `(chat_id)` | {writes['chat_id_only']['median_execution_ms']:.3f} | {writes['chat_id_only']['median_shared_write_blocks']} | +| `(chat_id, created_at, id)` | {writes['composite_history']['median_execution_ms']:.3f} | {writes['composite_history']['median_shared_write_blocks']} | + +Each write-cost query was measured five times inside a transaction and rolled back. This isolates execution and index-maintenance work but does not measure durable commit/fsync latency. + +Backup/restore: custom-format dump {restore['dump_bytes']} bytes; dump {restore['dump_seconds']:.3f}s; restore into a separately created empty database {restore['restore_seconds']:.3f}s. Restored row counts and deterministic checksums for the selected messages were verified: **PASS**. + +Raw JSON plans, samples, environment and restore checksums are in `track5_query_restore.json`. This single-host synthetic run is a reproducible comparison, not a production capacity or HA claim. +""" + + +def main(): + tag = secrets.token_hex(5) + source = "lab2_track5_source_" + tag + target = "lab2_track5_restore_" + tag + created = [] + try: + for database in (source, target): + admin("createdb", database) + created.append(database) + psql(source, dataset_sql()) + select_chat = "SELECT id, chat_id, sender_id, text, created_at FROM {table} WHERE chat_id IN (1, 2) ORDER BY chat_id, created_at, id" + plans = { + "chat_id_only": [ + plan(source, "messages_chat_only", history_query("messages_chat_only")) + for _ in range(5) + ], + "composite_history": [ + plan(source, "messages_composite", history_query("messages_composite")) + for _ in range(5) + ], + } + writes = { + "chat_id_only": write_samples(source, "messages_chat_only"), + "composite_history": write_samples(source, "messages_composite"), + } + selected_hash = {} + for table in ("messages_chat_only", "messages_composite"): + selected_hash[table] = psql( + source, + "SELECT md5(string_agg(md5(row_to_json(r)::text), '' ORDER BY r.chat_id, r.created_at, r.id)) " + f"FROM ({select_chat.format(table=table)}) AS r;", + ) + + with tempfile.NamedTemporaryFile(prefix="lab2-track5-", suffix=".dump") as dump_file: + started = time.monotonic() + dump = run_container( + "sh", + "-c", + 'pg_dump -U "$POSTGRES_USER" -Fc "$1"', + "track5", + source, + stdout=dump_file, + ) + if dump.returncode: + raise RuntimeError("PostgreSQL benchmark backup failed") + dump_seconds = time.monotonic() - started + dump_file.flush() + dump_size = os.fstat(dump_file.fileno()).st_size + started = time.monotonic() + dump_file.seek(0) + restore = run_container( + "sh", + "-c", + 'pg_restore -U "$POSTGRES_USER" --exit-on-error -d "$1"', + "track5", + target, + stdin=dump_file, + stdout=subprocess.DEVNULL, + ) + if restore.returncode: + raise RuntimeError("PostgreSQL benchmark restore failed") + restore_seconds = time.monotonic() - started + + restored_counts = { + table: int(psql(target, f"SELECT count(*) FROM {table};")) + for table in ("messages_chat_only", "messages_composite") + } + restored_hashes = { + table: psql( + target, + "SELECT md5(string_agg(md5(row_to_json(r)::text), '' ORDER BY r.chat_id, r.created_at, r.id)) " + f"FROM ({select_chat.format(table=table)}) AS r;", + ) + for table in ("messages_chat_only", "messages_composite") + } + if any(count != 100000 for count in restored_counts.values()): + raise RuntimeError("Restored message counts do not match the 100,000-row source tables") + if restored_hashes != selected_hash: + raise RuntimeError("Restored selected-message checksums differ from the source") + + env = container_metadata() + env["postgres_memory_setting"] = psql(source, "SHOW shared_buffers;") + env["postgres_work_mem"] = psql(source, "SHOW work_mem;") + result = { + "run_utc": datetime.now(timezone.utc).isoformat(), + "environment": env, + "method": { + "rows_per_index_variant": 100000, + "distribution": "80,000 messages in one chat; 20,000 distributed evenly across 100 chats", + "warmup": "ANALYZE followed by five executed EXPLAIN ANALYZE samples per query", + "query": history_query("messages"), + }, + "query_plans": { + "chat_id_only": {"index": "(chat_id)", "samples": plans["chat_id_only"]}, + "composite_history": { + "index": "(chat_id, created_at, id)", + "samples": plans["composite_history"], + }, + }, + "write_cost": writes, + "backup_restore": { + "format": "pg_dump custom format", + "dump_bytes": dump_size, + "dump_seconds": dump_seconds, + "restore_seconds": restore_seconds, + "restored_counts": restored_counts, + "source_selected_checksums": selected_hash, + "restored_selected_checksums": restored_hashes, + "verification": "PASS", + }, + } + EVIDENCE.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + MARKDOWN.write_text(render_markdown(result), encoding="utf-8") + print("PASS: level-5 track 2; evidence written to evidence/track5_query_restore.md") + finally: + for database in reversed(created): + admin("dropdb", database) + + +if __name__ == "__main__": + try: + main() + except (OSError, subprocess.SubprocessError, ValueError, KeyError, RuntimeError) as exc: + print("FAIL:", str(exc)) + raise SystemExit(1) diff --git a/tests/README.md b/tests/README.md index 0dfa3f89..4e86d6fe 100644 --- a/tests/README.md +++ b/tests/README.md @@ -9,7 +9,7 @@ make test BASE_URL=http://localhost:8080 python3 tests/smoke.py --base-url http://localhost:8080 ``` -`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite. +`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → несколько чатов и их keyset-пагинация → сообщения и пагинация истории → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, отсутствие повторов/пропусков при пагинации, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite. Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются. diff --git a/tests/smoke.py b/tests/smoke.py index 4cbfa558..5054440f 100644 --- a/tests/smoke.py +++ b/tests/smoke.py @@ -20,6 +20,54 @@ def main(): check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat') hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json'] check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider') + page_chats = [ + c.call( + 'POST', + '/api/v1/chats', + 201, + actor=alice, + body={'title':'Pagination test %d' % index,'member_ids':[alice['id'],bob['id']]}, + schema='Chat', + )['json'] + for index in range(4) + ] + from urllib.parse import quote + chat_pages, seen_cursors = [], set() + cursor = None + while True: + query = '/api/v1/chats?limit=2' + if cursor is not None: + query += '&cursor=' + quote(cursor, safe='') + page = c.call('GET',query,actor=alice,schema='ChatPage')['json'] + check(1 <= len(page['items']) <= 2,'chat page must contain at most the requested limit') + chat_pages.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + break + check(cursor not in seen_cursors,'chat pagination cursor repeated') + seen_cursors.add(cursor) + check(len(seen_cursors) <= 3,'chat pagination did not finish within expected pages') + chat_ids = [item['id'] for item in chat_pages] + check(len(chat_ids)==len(set(chat_ids)),'chat pagination duplicated an item') + check({item['id'] for item in page_chats + [chat]} <= set(chat_ids),'chat pagination lost a chat') + ordered_chats = sorted( + chat_pages, + key=lambda item:(timestamp(item['created_at']),uuid.UUID(item['id']).int), + ) + check(chat_pages==ordered_chats,'chat pagination is not ordered by (created_at, id)') + check( + chat_pages == c.call('GET','/api/v1/chats?limit=100',actor=alice,schema='ChatPage')['json']['items'], + 'chat pages differ from the complete chat list', + ) + c.call('GET','/api/v1/chats?limit=2&cursor=not-a-valid-cursor',400,actor=alice) + if page['next_cursor'] is None and seen_cursors: + foreign_cursor = next(iter(seen_cursors)) + c.call( + 'GET', + '/api/v1/chats?limit=2&cursor=' + quote(foreign_cursor,safe=''), + 400, + actor=bob, + ) key = uuid.uuid4().hex first = c.message(alice,chat,'Привет, мир 👋',key=key)['json'] second = c.message(bob,chat,'Сообщение Боба')['json'] @@ -28,7 +76,6 @@ def main(): check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed') page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json'] check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor') - from urllib.parse import quote page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json'] check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history') ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int))