ЛР 2 — Группа БВТ2304 — Аккуратов — на 5 #18

Closed
kaktuz2000 wants to merge 2 commits from kaktuz2000/mtusi2026lab2:master into master
29 changed files with 2940 additions and 47 deletions
+9 -6
View File
@@ -1,11 +1,14 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
POSTGRES_USER=messenger
POSTGRES_DB=messenger
# Replace with a generated local-only value (hex is safe to embed in DATABASE_URL).
POSTGRES_PASSWORD=replace-with-local-password
DB_POOL_MAX_SIZE=16
DB_POOL_TIMEOUT_SECONDS=5
DB_STATEMENT_TIMEOUT_MS=5000
DB_LOCK_TIMEOUT_MS=2000
# Compose injects CACHE_URL for the API; Redis is a disposable session lookup cache.
+25
View File
@@ -0,0 +1,25 @@
FROM python:3.12.7-slim
WORKDIR /app
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
RUN groupadd --system app && useradd --system --gid app --create-home --home-dir /home/app app
COPY requirements.txt /app/requirements.txt
RUN pip install --no-cache-dir -r /app/requirements.txt
COPY . /app
RUN chown -R app:app /app
USER app
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).read()" || exit 1
CMD ["python3", "src/server.py"]
+23 -3
View File
@@ -1,13 +1,13 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://127.0.0.1:8080
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence
.PHONY: help check validate test load grade4 track5 test-persistence migrate up down logs
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check: contract structure; validate: OpenAPI; test: public smoke; grade4: dependency/rollback/expiry checks; track5: query/index/restore experiment"
check:
$(PYTHON) scripts/check_contract.py
@@ -24,3 +24,23 @@ load:
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
grade4:
$(PYTHON) student-tests/test_level4.py --base-url "$(BASE_URL)"
docker compose --env-file .env exec -T api python3 student-tests/test_session_cache_race.py
./scripts/test_expiry.sh
track5:
$(PYTHON) student-tests/track5_query_restore.py
migrate:
$(PYTHON) src/migrate.py
up:
docker compose --env-file .env up -d --build
down:
docker compose --env-file .env down
logs:
docker compose --env-file .env logs -f
+35 -17
View File
@@ -2,7 +2,9 @@
Перенести состояние в постоянное хранилище и связать запросы с аутентифицированным пользователем. Подтверждённые сообщения и действующие сессии переживают перезапуск.
Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен.
В этой версии перенесён REST API из lab1 и реализованы PostgreSQL, миграции, хеширование паролей scrypt и постоянные Bearer-сессии. PostgreSQL остаётся источником истины; Redis используется как TTL-кэш для чтения сессий, с fallback на PostgreSQL при недоступности кэша. Web UI не требуется.
Код приложения разделён в `src/`: `handlers.py` обрабатывает HTTP и маршруты, `store.py` отвечает за PostgreSQL/Redis, `security.py` — хеширование паролей и общие проверки, `config.py` содержит настройки, `errors.py` — ошибки API, `app.py` собирает и запускает сервер. `server.py` остаётся точкой входа.
[Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md)
@@ -44,27 +46,43 @@
**Трек 2. План запроса и восстановление.** Исследуйте ≥100 000 сообщений с неравномерными размерами чатов, сравните ≥2 индекса/запроса и стоимость записи. Сделайте backup → восстановление в новое хранилище → сверку количества и выбранных сообщений. Обсудите время восстановления и границы потери данных.
## Как начать и проверить
## Запуск и проверка
Создайте локальный `.env` и замените пароль PostgreSQL на новый локальный пароль (для строки подключения используйте hex-значение):
```sh
# Работает прямо в skeleton, Python 3.9+; приложение не запускается:
make check
# После реализации запустите свой сервер/стенд по REPORT.md.
# По умолчанию тест использует http://localhost:8080.
make test
```
`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость.
```sh
# Compose-вариант после реализации:
docker compose --env-file .env config --quiet
cp .env.example .env
# Отредактируйте POSTGRES_PASSWORD в .env; не добавляйте .env в Git.
docker compose --env-file .env up -d --build
# Дождитесь /health/ready; затем make test.
# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md.
```
API ждёт готовности PostgreSQL, применяет новые SQL-миграции из `migrations/` и слушает `http://127.0.0.1:8080`. Схема создаётся при первом старте; повторный запуск не применяет миграции заново.
```sh
make check
make test
make test-persistence ACTION_SCRIPT=scripts/restart.sh
# Остановить контейнеры, сохранив данные в именованном volume:
docker compose --env-file .env down
```
Данные сохраняются в `postgres_data`. Для намеренного удаления всего учебного состояния, включая пользователей, сообщения и действующие сессии, выполните отдельно и только после проверки нужного compose-проекта:
```sh
docker compose --env-file .env down --volumes
```
`.env.example`, `compose.yaml` и `contracts/infrastructure.json` описывают локальные настройки и роли компонентов. Секреты, `.env` и Docker volumes не должны попадать в Git.
Проверки уровня 4 и исследовательский трек уровня 5 запускаются явно на учебном стенде:
```sh
make grade4
make track5
```
`make grade4` временно перезапускает API с двухсекундным TTL для проверки expiry, запускает интеграционные проверки Redis/PostgreSQL, включая конкурентное заполнение кэша сессии одновременно с logout, и возвращает штатный TTL. `make track5` создаёт отдельные случайно именованные временные БД, измеряет два варианта индекса на skewed-наборе по 100 000 сообщений на вариант, затем проверяет custom-format backup/restore и удаляет только созданные им БД. Численные результаты и планы сохраняются в `evidence/track5_query_restore.*`.
Примеры запросов доступны в `examples/requests.http`, примеры JSON — в `contracts/examples.json`. Для полной проверки схемы (по желанию локально; CI делает её автоматически):
+59 -18
View File
@@ -1,37 +1,78 @@
# Отчёт — лабораторная 2
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: не указаны в исходных данных.
- Целевая оценка: 5, исследовательский трек 2 — «План запроса и восстановление».
- Commit/PR: не создавались.
- Источники, библиотеки и помощь ИИ: перенесён и адаптирован собственный API-каркас из lab1; Python standard library; psycopg 3.2.10, psycopg-pool 3.3.3, redis-py 5.2.1; PostgreSQL 16.4, Redis 7.4.2; использовалась помощь ИИ.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Требуются Docker Desktop с Compose v2 и Python 3.9+ для локальных проверок. Образы закреплены версиями `python:3.12.7-slim`, `postgres:16.4-alpine`, `redis:7.4.2-alpine`. Проведённые проверки — macOS arm64, Docker 28.0.4, Docker Compose 2.34.0, 10 логических CPU; фактическая память хоста и лимиты контейнеров зафиксированы в [track5_query_restore.json](evidence/track5_query_restore.json).
```sh
cp .env.example .env
# Заменить POSTGRES_PASSWORD в .env на hex-значение, например:
python3 -c 'import secrets; print(secrets.token_hex(24))'
# Не добавлять .env в Git.
docker compose --env-file .env up -d --build
curl --fail http://127.0.0.1:8080/health/ready
make check
make test
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make grade4
make track5
```
API применяет новые миграции из `migrations/` при старте; таблица `schema_migrations` и транзакционная advisory lock обеспечивают однократное применение. `scripts/restart.sh` перезапускает PostgreSQL, Redis и API с сохранением БД volume. Redis является кэшем, поэтому утрата его содержимого не удаляет учётные записи или сессии.
Обычная остановка сохраняет состояние:
```sh
docker compose --env-file .env down
```
Только для намеренного удаления всех учебных данных, включая пользователей, сообщения и сессии:
```sh
docker compose --env-file .env down --volumes
```
`make grade4` выполняет проверки зависимостей и временно перезапускает API с двухсекундным TTL для сценария expiry, затем возвращает обычную конфигурацию. `make track5` использует случайно именованные временные базы, создаёт benchmark-данные, делает custom-format backup, восстанавливает его в новую пустую БД, сверяет counts/checksums и удаляет только созданные им базы. Не запускайте эти команды на production-окружении.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
- Перенесены и адаптированы REST-поведение, Dockerfile и ignore-правила из lab1. Сервер разделён по обязанностям: `config.py` — настройки, `errors.py` — API-ошибки, `security.py` — password KDF/UUID/time helpers, `store.py` — PostgreSQL и Redis, `handlers.py` — HTTP endpoints, `app.py` — сборка/жизненный цикл сервера, `server.py` — совместимая точка входа. Публичные `tests/` и `contracts/` текущей лабораторной сохранены без подмены.
- PostgreSQL — источник истины для пользователей, чатов, членства, сообщений и сессий. Именованный volume `postgres_data` сохраняет их при перезапуске.
- Redis кэширует результат проверки opaque session token по SHA-256 digest ключа и хранит только публичный объект пользователя. Redis key имеет TTL, рассчитанный по оставшемуся `sessions.expires_at`; пароль, сырой токен и его текст не попадают в кэш.
- Создание и отзыв сессии записываются в PostgreSQL. Чтение сессии при промахе кэша и logout сериализуются транзакционной advisory lock на digest токена: конкурентное заполнение не может записать сессию в Redis уже после logout. Logout удаляет Redis-ключ до удаления записи в БД; при недоступном Redis logout завершается ограниченным `503`, а не сообщает об успехе без инвалидации кэша. При чтении недоступность Redis логируется, запрос безопасно переходит на PostgreSQL; после восстановления промах кэша заполняется из БД. При недоступной БД readiness и операции, требующие её, возвращают ограниченную ошибку.
- Список чатов использует keyset-пагинацию по `(created_at, id)`: курсор привязан к пользователю, страницы не повторяют и не пропускают чаты при обходе неизменённого списка.
- Индекс истории в рабочей схеме — `(chat_id, created_at, id)`, соответствующий keyset-пагинации. Все SQL-запросы параметризованы; пул ограничен 16 соединениями, timeout ожидания пула/подключения — 5 секунд, `statement_timeout` — 5 секунд, `lock_timeout` — 2 секунды.
- Миграция создаёт ключи и ограничения таблиц. Создание чата и всех участников — одна транзакция; уникальный индекс `username` сериализует гонку регистрации.
- Пароли хранятся как scrypt-хеши с индивидуальной солью (`N=32768`, `r=8`, `p=1`). Сессии имеют expiry и отзываются logout-запросом. `X-User-Id` не используется для авторизации.
- Лимит Redis — 128 MB, политика вытеснения `allkeys-lru`; постоянство сессий не зависит от Redis, поэтому Redis AOF/RDB не включён.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| Критерий | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Минимум lab1 и lab2; публичный API | `make test` | PASS: регистрация, Basic login, Bearer auth, чаты, сообщения, пагинация, ошибочные credentials, отказ анониму, logout и отказ отозванного токена | `tests/smoke.py`, `tests/client.py` |
| PostgreSQL, ограничения, миграции и сохранность | Чистый старт, повторный старт, `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | Миграция записана один раз; прежний token и подтверждённое сообщение переживают рестарт PostgreSQL, Redis и API | `migrations/001_initial.sql`, `scripts/restart.sh`, `tests/resilience.py` |
| Redis/Valkey используется с TTL | `make grade4` | Session cache entry обнаружена в Redis с положительным TTL; при остановленном Redis текущая сессия проверяется через PostgreSQL; после запуска Redis доступ восстанавливается | `student-tests/test_level4.py`, `src/server.py` |
| Конкурентная регистрация и rollback | `make grade4` | 4 одновременных запроса к одному username дали ровно один `201`, три `409`, в БД осталась одна запись; попытка чата с отсутствующим участником дала `400` и ноль частичных строк | `student-tests/test_level4.py` |
| Запросы, индекс истории и ошибки зависимостей | `make grade4`; `make track5` | Проверены параметризованные запросы, ограниченные timeout; при остановке PostgreSQL readiness и операция чтения вернули ограниченный `503`; 100k-набор использован для `EXPLAIN (ANALYZE, BUFFERS)` | `src/server.py`, `evidence/track5_query_restore.json` |
| Короткий TTL, неверный пароль и отзыв | `make test`; `make grade4` | Неверный пароль и отозванная/истёкшая сессия дают 401; logout выдаёт 204; короткая сессия проверена при TTL=2 секунды | `student-tests/test_level4.py`, `scripts/test_expiry.sh`, `tests/smoke.py` |
| Трек 5.2: планы двух индексов и write-cost | `make track5` | 100,000 синтетических строк на каждый индексный вариант, неравномерное распределение; одинаковый keyset SELECT измерен пять раз; сравнен INSERT cost | [track5_query_restore.md](evidence/track5_query_restore.md), [raw plans/results](evidence/track5_query_restore.json) |
| Трек 5.2: backup → новая БД → проверка | `make track5` | Custom-format backup восстановлен в отдельную пустую БД; количество строк и контрольные суммы выбранных сообщений совпали; тест сообщил PASS | `evidence/track5_query_restore.json` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
## Исследование: план запроса и восстановление
## Эксперименты и отказы
**Гипотеза.** Индекс только по `chat_id` должен обслужить фильтрацию истории, но keyset-фильтр/сортировка по `(created_at, id)` потребуют отсеивания и top-N сортировки. Составной `(chat_id, created_at, id)` должен уменьшить чтение/время страницы ценой более дорогой записи.
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Методика.** Скрипт `student-tests/track5_query_restore.py` использует изолированные случайно именованные PostgreSQL database, не рабочую БД API. В двух вариантах создано по 100 000 synthetic rows: 80 000 в одном чате, оставшиеся 20 000 распределены поровну между 100 чатами. Выполняется один и тот же keyset SELECT страницы из небольшого чата с лимитом 25. После ANALYZE собирается пять `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` планов на вариант. Стоимость записи измеряется пятью EXPLAIN ANALYZE вставками по 2 000 строк; каждая вставка откатывается, поэтому результат отражает execution/index maintenance, но не durable commit/fsync. Затем custom-format `pg_dump` восстанавливается в новую пустую базу, сверяются 100 000 строк каждого варианта и MD5 агрегаты выбранных строк.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
**Результат текущего запуска.** Сводная таблица медиан, планы, buffer counts, параметры PostgreSQL, restore time и контрольные суммы записаны в [track5_query_restore.md](evidence/track5_query_restore.md) и JSON-файле рядом. В этом запуске составной индекс уменьшил медианное время выбранной страницы примерно с 0.297 ms до 0.097 ms (около 67%); вставка 2 000 строк заняла примерно 5.428 ms против 7.218 ms для составного индекса (около 33% дороже). Restore в новую БД и сверка количества/контрольных сумм завершились PASS. Результаты относятся к этому single-host synthetic run; для повторных запусков числа могут отличаться.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Redis — ускоряющий кэш, а не самостоятельное долговременное хранилище; источником истины остаётся PostgreSQL. Проверены контролируемые остановки Redis и PostgreSQL, но не потеря host, повреждение диска, HA/failover БД, реальная нагрузка API/RPS, commit fsync или backup по расписанию. Benchmark использует две таблицы с одинаковыми данными и синтетическую нагрузку; он не доказывает пропускную способность production-системы. Docker-контейнеры запускаются без явных CPU/RAM limits (нулевые лимиты указаны в evidence). Сравнение планов и разовый restore PASS не заменяют регулярный проверенный backup/DR-процесс.
+135
View File
@@ -0,0 +1,135 @@
# Запуск и проверка лабораторной 2
Команды рассчитаны на локальный тестовый стенд: Docker с Compose v2, `make` и Python 3.9+. В этой лабораторной проверяются уровни 3, 4 и 5.
## Запуск стенда
Из каталога `mtusi2026lab2`:
```sh
cp .env.example .env
```
Задайте в `.env` собственный локальный `POSTGRES_PASSWORD` (hex-строка), затем запустите API и зависимости:
```sh
docker compose --env-file .env up -d --build
```
## Быстрая проверка через curl
```sh
BASE_URL=http://127.0.0.1:8080
# Процесс API работает: ожидается HTTP 200 и {"status":"ok"}
curl --fail --show-error "$BASE_URL/health/live"
# API и PostgreSQL готовы принимать запросы: ожидается HTTP 200
curl --fail --show-error "$BASE_URL/health/ready"
# Без Bearer-токена доступ запрещён: ожидается HTTP 401
curl -sS -o /dev/null -w 'HTTP %{http_code}\n' "$BASE_URL/api/v1/users/me"
```
Следующий пример регистрирует двух тестовых пользователей, получает Bearer-токен Алисы через Basic-аутентификацию, создаёт чат и отправляет в него сообщение. Нужен `jq` для чтения JSON-ответов; используйте только тестовые пароли:
```sh
ALICE_USERNAME="codes_alice_$(date +%s)"
ALICE_PASSWORD='Example-password-alice-42'
BOB_USERNAME="codes_bob_$(date +%s)"
BOB_PASSWORD='Example-password-bob-42'
# Создать пользователей и сохранить их ID
ALICE=$(curl --fail --show-error "$BASE_URL/api/v1/users" \
-H 'Content-Type: application/json' \
-d "$(jq -n --arg username "$ALICE_USERNAME" --arg password "$ALICE_PASSWORD" \
'{username:$username, display_name:"Алиса (тест)", password:$password}')")
ALICE_ID=$(printf '%s' "$ALICE" | jq -r '.id')
BOB=$(curl --fail --show-error "$BASE_URL/api/v1/users" \
-H 'Content-Type: application/json' \
-d "$(jq -n --arg username "$BOB_USERNAME" --arg password "$BOB_PASSWORD" \
'{username:$username, display_name:"Боб (тест)", password:$password}')")
BOB_ID=$(printf '%s' "$BOB" | jq -r '.id')
# Войти как Алиса с Basic-аутентификацией и получить Bearer-токен
SESSION=$(curl --fail --show-error "$BASE_URL/api/v1/auth/sessions" \
-u "$ALICE_USERNAME:$ALICE_PASSWORD" \
-X POST)
TOKEN=$(printf '%s' "$SESSION" | jq -r '.token')
# Проверить авторизованный запрос
curl --fail --show-error "$BASE_URL/api/v1/users/me" \
-H "Authorization: Bearer $TOKEN"
# Создать чат от имени Алисы и включить обоих пользователей
CHAT=$(curl --fail --show-error "$BASE_URL/api/v1/chats" \
-H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: application/json' \
-d "$(jq -n --arg alice "$ALICE_ID" --arg bob "$BOB_ID" \
'{title:"Тестовый чат", member_ids:[$alice, $bob]}')")
CHAT_ID=$(printf '%s' "$CHAT" | jq -r '.id')
printf 'Создан чат: %s\n' "$CHAT_ID"
# Отправить сообщение; ожидается HTTP 201 и JSON нового сообщения
curl --fail --show-error "$BASE_URL/api/v1/chats/$CHAT_ID/messages" \
-H "Authorization: Bearer $TOKEN" \
-H 'Content-Type: application/json' \
-d '{"text":"Проверка отправки сообщения через curl"}'
```
Для полной проверки API используйте smoke-тест ниже: он сам регистрирует синтетических пользователей, создаёт чат и сообщения. Эти данные остаются в тестовой базе.
## Проверки по уровням
Сначала выполните общую проверку контракта и OpenAPI:
```sh
make check
# Один раз, если ещё не установлен валидатор OpenAPI:
python3 -m venv .venv
. .venv/bin/activate
pip install -r requirements-dev.txt
make validate
```
Далее команды проверки уровней:
```sh
# Уровень 3: публичный API и сохранность данных/сессии после перезапуска
make test
make test-persistence ACTION_SCRIPT=scripts/restart.sh
# Уровень 4: проверки Redis TTL/fallback, конкурентной регистрации,
# rollback транзакции и истечения срока сессии
make grade4
# Уровень 5, трек 2: сравнение планов/индексов и backup/restore
make track5
```
Уровень 4 предполагает прохождение проверок уровня 3; уровень 5 — уровней 3 и 4, поэтому запускайте команды по порядку. `make track5` создаёт и удаляет собственные временные базы для эксперимента. Используйте только локальный учебный стенд, не production.
## Остановка
Останавливает контейнеры, сохраняя PostgreSQL volume и данные:
```sh
docker compose --env-file .env down
```
docker compose --env-file .env up -d
curl --fail http://127.0.0.1:8080/health/ready
Затем публичные тесты из tests/ можно запустить напрямую:
python3 tests/smoke.py
Он проверяет основной сценарий API: регистрацию и вход, чаты, отправку и чтение сообщений, доступы и пагинацию. В конце должен вывести PASS. Тест создаёт записи в базе, они автоматически не удаляются.
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh
Он проверяет, что данные и существующая сессия доступны после перезапуска. Скрипт перезапуска должен быть исполняемым; если нужно:
python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh
Он проверяет, что данные и существующая сессия доступны после перезапуска. Скрипт перезапуска должен быть исполняемым; если нужно:
chmod +x scripts/restart.sh
python3 tests/load.py --duration 30 --concurrency 4
Это тест нагрузки: отправляет сообщения несколько секунд и показывает успешные запросы в секунду, ошибки и задержки. Он тоже создаёт тестовые сообщения.
tests/client.py — вспомогательный модуль для этих проверок, отдельно его запускать не нужно. То же самое можно запускать короче командами Make: make test, make test-persistence ACTION_SCRIPT=scripts/restart.sh. С описанием публичных проверок можно свериться в tests/README.md.
+62
View File
@@ -0,0 +1,62 @@
services:
postgres:
image: postgres:16.4-alpine
environment:
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Copy .env.example to .env and set POSTGRES_PASSWORD}
POSTGRES_DB: ${POSTGRES_DB:-messenger}
volumes:
- postgres_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U \"$${POSTGRES_USER}\" -d \"$${POSTGRES_DB}\""]
interval: 5s
timeout: 3s
retries: 20
start_period: 5s
restart: unless-stopped
redis:
image: redis:7.4.2-alpine
command: ["redis-server", "--save", "", "--appendonly", "no", "--maxmemory", "128mb", "--maxmemory-policy", "allkeys-lru"]
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 20
restart: unless-stopped
api:
build:
context: .
dockerfile: Dockerfile
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-messenger}
CACHE_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-16}
DB_POOL_TIMEOUT_SECONDS: ${DB_POOL_TIMEOUT_SECONDS:-5}
DB_STATEMENT_TIMEOUT_MS: ${DB_STATEMENT_TIMEOUT_MS:-5000}
DB_LOCK_TIMEOUT_MS: ${DB_LOCK_TIMEOUT_MS:-2000}
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
healthcheck:
test:
- CMD
- python3
- -c
- "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"
interval: 5s
timeout: 3s
retries: 20
start_period: 10s
restart: unless-stopped
volumes:
postgres_data:
+1 -1
View File
@@ -14,7 +14,7 @@
## Пользователи, чаты и сообщения
- `POST /users` публичен; возвращает `201 User`. Пароль/его hash и сессионные данные никогда не входят в User.
- `POST /chats` принимает title и явный список существующих `member_ids`, включая вызывающего. Дубликаты, неизвестные участники и отсутствие вызывающего — 400. Чат создаётся атомарно. GET списка чатов сортируется по `(created_at, id)` по возрастанию.
- `POST /chats` принимает title и явный список существующих `member_ids`, включая вызывающего. Дубликаты, неизвестные участники и отсутствие вызывающего — 400. Чат создаётся атомарно. GET списка чатов использует keyset-пагинацию по `(created_at, id)` по возрастанию: `limit` — целое 1–100 (по умолчанию 50), `cursor` привязан к пользователю и последнему чату предыдущей страницы. Последний элемент не повторяется; `next_cursor` равен null, если следующей страницы нет. Непустой cursor с неверным форматом или другого пользователя — 400.
- Отправитель сообщения выводится из текущей идентификации, поле `sender_id` в запросе не принимается. Текст — до 4000 Unicode code points. Нельзя отправить пустой/пробельный текст без вложений.
- История использует keyset cursor: `(created_at, id)`, по возрастанию; `limit` — целое 1–100, по умолчанию 50. Ответ `{"items":[...],"next_cursor":null}`. Непустой cursor — непрозрачная для клиента строка; последний элемент предыдущей страницы не повторяется. Cursor привязан к ресурсу/запросу: от другого чата — 400. Изменять `limit` при продолжении можно.
- `next_cursor` равен null, если на момент чтения следующей записи нет. Гарантируется отсутствие повторов при обходе неизменной истории; snapshot изоляция между несколькими HTTP-запросами не требуется. При конкурентных вставках опишите выбранную семантику и ограничения часов.
+916
View File
@@ -0,0 +1,916 @@
{
"run_utc": "2026-10-09T13:21:05.722546+00:00",
"environment": {
"host_platform": "macOS-27.0-arm64-arm-64bit-Mach-O",
"host_architecture": "arm64",
"host_logical_cpus": 10,
"host_memory_bytes": 17179869184,
"postgres_image": "postgres:16.4-alpine",
"postgres_version": "16.4",
"postgres_container_memory_limit_bytes": 0,
"postgres_container_cpu_limit_nanos": 0,
"warmup": "initial data load followed by ANALYZE; plan query is executed 5 times",
"postgres_memory_setting": "128MB",
"postgres_work_mem": "4MB"
},
"method": {
"rows_per_index_variant": 100000,
"distribution": "80,000 messages in one chat; 20,000 distributed evenly across 100 chats",
"warmup": "ANALYZE followed by five executed EXPLAIN ANALYZE samples per query",
"query": "SELECT id, created_at, text FROM messages WHERE chat_id = 2 AND (created_at, id) > (timestamptz '2025-01-01 23:36:40+00', 0) ORDER BY created_at, id LIMIT 25"
},
"query_plans": {
"chat_id_only": {
"index": "(chat_id)",
"samples": [
{
"execution_ms": 0.265,
"planning_ms": 0.384,
"actual_rows": 25,
"shared_hit_blocks": 206,
"shared_read_blocks": 2,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.67,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.217,
"Actual Total Time": 0.22,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 206,
"Shared Read Blocks": 2,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Sort",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.68,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.216,
"Actual Total Time": 0.217,
"Actual Rows": 25,
"Actual Loops": 1,
"Sort Key": [
"created_at",
"id"
],
"Sort Method": "top-N heapsort",
"Sort Space Used": 26,
"Sort Space Type": "Memory",
"Shared Hit Blocks": 206,
"Shared Read Blocks": 2,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_only_idx",
"Relation Name": "messages_chat_only",
"Alias": "messages_chat_only",
"Startup Cost": 0.29,
"Total Cost": 25.87,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.073,
"Actual Total Time": 0.182,
"Actual Rows": 150,
"Actual Loops": 1,
"Index Cond": "(chat_id = 2)",
"Rows Removed by Index Recheck": 0,
"Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))",
"Rows Removed by Filter": 50,
"Shared Hit Blocks": 200,
"Shared Read Blocks": 2,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
]
}
},
{
"execution_ms": 0.233,
"planning_ms": 0.353,
"actual_rows": 25,
"shared_hit_blocks": 208,
"shared_read_blocks": 0,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.67,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.195,
"Actual Total Time": 0.198,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 208,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Sort",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.68,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.194,
"Actual Total Time": 0.195,
"Actual Rows": 25,
"Actual Loops": 1,
"Sort Key": [
"created_at",
"id"
],
"Sort Method": "top-N heapsort",
"Sort Space Used": 26,
"Sort Space Type": "Memory",
"Shared Hit Blocks": 208,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_only_idx",
"Relation Name": "messages_chat_only",
"Alias": "messages_chat_only",
"Startup Cost": 0.29,
"Total Cost": 25.87,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.055,
"Actual Total Time": 0.166,
"Actual Rows": 150,
"Actual Loops": 1,
"Index Cond": "(chat_id = 2)",
"Rows Removed by Index Recheck": 0,
"Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))",
"Rows Removed by Filter": 50,
"Shared Hit Blocks": 202,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
]
}
},
{
"execution_ms": 0.245,
"planning_ms": 0.363,
"actual_rows": 25,
"shared_hit_blocks": 208,
"shared_read_blocks": 0,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.67,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.198,
"Actual Total Time": 0.2,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 208,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Sort",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.68,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.197,
"Actual Total Time": 0.198,
"Actual Rows": 25,
"Actual Loops": 1,
"Sort Key": [
"created_at",
"id"
],
"Sort Method": "top-N heapsort",
"Sort Space Used": 26,
"Sort Space Type": "Memory",
"Shared Hit Blocks": 208,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_only_idx",
"Relation Name": "messages_chat_only",
"Alias": "messages_chat_only",
"Startup Cost": 0.29,
"Total Cost": 25.87,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.051,
"Actual Total Time": 0.158,
"Actual Rows": 150,
"Actual Loops": 1,
"Index Cond": "(chat_id = 2)",
"Rows Removed by Index Recheck": 0,
"Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))",
"Rows Removed by Filter": 50,
"Shared Hit Blocks": 202,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
]
}
},
{
"execution_ms": 0.218,
"planning_ms": 0.285,
"actual_rows": 25,
"shared_hit_blocks": 208,
"shared_read_blocks": 0,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.67,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.184,
"Actual Total Time": 0.186,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 208,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Sort",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.68,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.183,
"Actual Total Time": 0.184,
"Actual Rows": 25,
"Actual Loops": 1,
"Sort Key": [
"created_at",
"id"
],
"Sort Method": "top-N heapsort",
"Sort Space Used": 26,
"Sort Space Type": "Memory",
"Shared Hit Blocks": 208,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_only_idx",
"Relation Name": "messages_chat_only",
"Alias": "messages_chat_only",
"Startup Cost": 0.29,
"Total Cost": 25.87,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.048,
"Actual Total Time": 0.155,
"Actual Rows": 150,
"Actual Loops": 1,
"Index Cond": "(chat_id = 2)",
"Rows Removed by Index Recheck": 0,
"Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))",
"Rows Removed by Filter": 50,
"Shared Hit Blocks": 202,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
]
}
},
{
"execution_ms": 0.268,
"planning_ms": 0.365,
"actual_rows": 25,
"shared_hit_blocks": 208,
"shared_read_blocks": 0,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.67,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.228,
"Actual Total Time": 0.23,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 208,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Sort",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 26.61,
"Total Cost": 26.68,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.227,
"Actual Total Time": 0.228,
"Actual Rows": 25,
"Actual Loops": 1,
"Sort Key": [
"created_at",
"id"
],
"Sort Method": "top-N heapsort",
"Sort Space Used": 26,
"Sort Space Type": "Memory",
"Shared Hit Blocks": 208,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_only_idx",
"Relation Name": "messages_chat_only",
"Alias": "messages_chat_only",
"Startup Cost": 0.29,
"Total Cost": 25.87,
"Plan Rows": 30,
"Plan Width": 41,
"Actual Startup Time": 0.049,
"Actual Total Time": 0.186,
"Actual Rows": 150,
"Actual Loops": 1,
"Index Cond": "(chat_id = 2)",
"Rows Removed by Index Recheck": 0,
"Filter": "(ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0))",
"Rows Removed by Filter": 50,
"Shared Hit Blocks": 202,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
]
}
}
]
},
"composite_history": {
"index": "(chat_id, created_at, id)",
"samples": [
{
"execution_ms": 0.085,
"planning_ms": 0.331,
"actual_rows": 25,
"shared_hit_blocks": 25,
"shared_read_blocks": 3,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.42,
"Total Cost": 50.6,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.034,
"Actual Total Time": 0.055,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 25,
"Shared Read Blocks": 3,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_history_idx",
"Relation Name": "messages_composite",
"Alias": "messages_composite",
"Startup Cost": 0.42,
"Total Cost": 62.64,
"Plan Rows": 31,
"Plan Width": 41,
"Actual Startup Time": 0.033,
"Actual Total Time": 0.053,
"Actual Rows": 25,
"Actual Loops": 1,
"Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 25,
"Shared Read Blocks": 3,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"execution_ms": 0.087,
"planning_ms": 0.362,
"actual_rows": 25,
"shared_hit_blocks": 28,
"shared_read_blocks": 0,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.42,
"Total Cost": 50.6,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.022,
"Actual Total Time": 0.043,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 28,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_history_idx",
"Relation Name": "messages_composite",
"Alias": "messages_composite",
"Startup Cost": 0.42,
"Total Cost": 62.64,
"Plan Rows": 31,
"Plan Width": 41,
"Actual Startup Time": 0.021,
"Actual Total Time": 0.041,
"Actual Rows": 25,
"Actual Loops": 1,
"Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 28,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"execution_ms": 0.065,
"planning_ms": 0.307,
"actual_rows": 25,
"shared_hit_blocks": 28,
"shared_read_blocks": 0,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.42,
"Total Cost": 50.6,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.015,
"Actual Total Time": 0.035,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 28,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_history_idx",
"Relation Name": "messages_composite",
"Alias": "messages_composite",
"Startup Cost": 0.42,
"Total Cost": 62.64,
"Plan Rows": 31,
"Plan Width": 41,
"Actual Startup Time": 0.014,
"Actual Total Time": 0.032,
"Actual Rows": 25,
"Actual Loops": 1,
"Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 28,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"execution_ms": 0.079,
"planning_ms": 0.365,
"actual_rows": 25,
"shared_hit_blocks": 28,
"shared_read_blocks": 0,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.42,
"Total Cost": 50.6,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.021,
"Actual Total Time": 0.042,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 28,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_history_idx",
"Relation Name": "messages_composite",
"Alias": "messages_composite",
"Startup Cost": 0.42,
"Total Cost": 62.64,
"Plan Rows": 31,
"Plan Width": 41,
"Actual Startup Time": 0.02,
"Actual Total Time": 0.039,
"Actual Rows": 25,
"Actual Loops": 1,
"Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 28,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
},
{
"execution_ms": 0.066,
"planning_ms": 0.299,
"actual_rows": 25,
"shared_hit_blocks": 28,
"shared_read_blocks": 0,
"plan": {
"Node Type": "Limit",
"Parallel Aware": false,
"Async Capable": false,
"Startup Cost": 0.42,
"Total Cost": 50.6,
"Plan Rows": 25,
"Plan Width": 41,
"Actual Startup Time": 0.015,
"Actual Total Time": 0.036,
"Actual Rows": 25,
"Actual Loops": 1,
"Shared Hit Blocks": 28,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0,
"Plans": [
{
"Node Type": "Index Scan",
"Parent Relationship": "Outer",
"Parallel Aware": false,
"Async Capable": false,
"Scan Direction": "Forward",
"Index Name": "messages_chat_history_idx",
"Relation Name": "messages_composite",
"Alias": "messages_composite",
"Startup Cost": 0.42,
"Total Cost": 62.64,
"Plan Rows": 31,
"Plan Width": 41,
"Actual Startup Time": 0.014,
"Actual Total Time": 0.033,
"Actual Rows": 25,
"Actual Loops": 1,
"Index Cond": "((chat_id = 2) AND (ROW(created_at, id) > ROW('2025-01-01 23:36:40+00'::timestamp with time zone, 0)))",
"Rows Removed by Index Recheck": 0,
"Shared Hit Blocks": 28,
"Shared Read Blocks": 0,
"Shared Dirtied Blocks": 0,
"Shared Written Blocks": 0,
"Local Hit Blocks": 0,
"Local Read Blocks": 0,
"Local Dirtied Blocks": 0,
"Local Written Blocks": 0,
"Temp Read Blocks": 0,
"Temp Written Blocks": 0
}
]
}
}
]
}
},
"write_cost": {
"chat_id_only": {
"samples": [
{
"execution_ms": 4.163,
"planning_ms": 0.062,
"shared_write_blocks": 25
},
{
"execution_ms": 3.936,
"planning_ms": 0.05,
"shared_write_blocks": 40
},
{
"execution_ms": 3.97,
"planning_ms": 0.051,
"shared_write_blocks": 25
},
{
"execution_ms": 3.722,
"planning_ms": 0.05,
"shared_write_blocks": 24
},
{
"execution_ms": 4.037,
"planning_ms": 0.059,
"shared_write_blocks": 25
}
],
"median_execution_ms": 3.97,
"median_shared_write_blocks": 25,
"rows_per_sample": 2000,
"rolled_back_after_each_measurement": true
},
"composite_history": {
"samples": [
{
"execution_ms": 5.166,
"planning_ms": 0.062,
"shared_write_blocks": 26
},
{
"execution_ms": 7.163,
"planning_ms": 0.051,
"shared_write_blocks": 120
},
{
"execution_ms": 4.777,
"planning_ms": 0.05,
"shared_write_blocks": 23
},
{
"execution_ms": 4.501,
"planning_ms": 0.049,
"shared_write_blocks": 24
},
{
"execution_ms": 5.049,
"planning_ms": 0.05,
"shared_write_blocks": 25
}
],
"median_execution_ms": 5.049,
"median_shared_write_blocks": 25,
"rows_per_sample": 2000,
"rolled_back_after_each_measurement": true
}
},
"backup_restore": {
"format": "pg_dump custom format",
"dump_bytes": 1596489,
"dump_seconds": 0.25986058299895376,
"restore_seconds": 0.31837804202223197,
"restored_counts": {
"messages_chat_only": 100000,
"messages_composite": 100000
},
"source_selected_checksums": {
"messages_chat_only": "59b4843022f2d85760e29d81bb6ca73c",
"messages_composite": "59b4843022f2d85760e29d81bb6ca73c"
},
"restored_selected_checksums": {
"messages_chat_only": "59b4843022f2d85760e29d81bb6ca73c",
"messages_composite": "59b4843022f2d85760e29d81bb6ca73c"
},
"verification": "PASS"
}
}
+24
View File
@@ -0,0 +1,24 @@
# Lab2 level-5 track 2 — query plans and recovery
- Run UTC: 2026-10-09T13:21:05.722546+00:00
- Host: macOS-27.0-arm64-arm-64bit-Mach-O (arm64), logical CPUs: 10
- Host physical memory: 17179869184 bytes.
- PostgreSQL: 16.4 in `postgres:16.4-alpine`; container memory limit: 0 bytes; CPU limit: 0 nanocores (0 means unlimited).
- Dataset: 100,000 synthetic rows in each comparison table; one chat has 80,000 rows and 100 small chats share the remaining 20,000. Only generated text is used.
- Query: keyset page from chat 2, identical SELECT and cursor for both indexes; each plan was executed five times after ANALYZE.
| Index | Median execution (ms) | Shared hits | Shared reads | Rows |
|---|---:|---:|---:|---:|
| `(chat_id)` | 0.245 | 206 | 2 | 25 |
| `(chat_id, created_at, id)` | 0.079 | 25 | 3 | 25 |
| Index | Median time for 2,000 inserted rows (ms) | Median shared write blocks |
|---|---:|---:|
| `(chat_id)` | 3.970 | 25 |
| `(chat_id, created_at, id)` | 5.049 | 25 |
Each write-cost query was measured five times inside a transaction and rolled back. This isolates execution and index-maintenance work but does not measure durable commit/fsync latency.
Backup/restore: custom-format dump 1596489 bytes; dump 0.260s; restore into a separately created empty database 0.318s. Restored row counts and deterministic checksums for the selected messages were verified: **PASS**.
Raw JSON plans, samples, environment and restore checksums are in `track5_query_restore.json`. This single-host synthetic run is a reproducible comparison, not a production capacity or HA claim.
+43
View File
@@ -0,0 +1,43 @@
CREATE TABLE users (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
username varchar(32) NOT NULL UNIQUE,
display_name varchar(100) NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
title varchar(100) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
joined_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_chat_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text varchar(4000) NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT messages_text_not_blank CHECK (length(btrim(text)) > 0)
);
CREATE INDEX messages_chat_history_idx ON messages (chat_id, created_at, id);
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX sessions_expiry_idx ON sessions (expires_at);
+1
View File
@@ -1,2 +1,3 @@
-r requirements.txt
# Only for full OpenAPI/JSON Schema validation; HTTP tests use stdlib.
openapi-spec-validator==0.7.2
+3
View File
@@ -0,0 +1,3 @@
psycopg[binary]==3.2.10
psycopg-pool==3.3.3
redis==5.2.1
+30
View File
@@ -0,0 +1,30 @@
#!/bin/sh
set -eu
docker compose --env-file .env restart postgres redis
deadline=90
elapsed=0
until docker compose --env-file .env exec -T postgres pg_isready -q \
&& docker compose --env-file .env exec -T redis redis-cli ping | grep -q PONG
do
if [ "$elapsed" -ge "$deadline" ]; then
echo "PostgreSQL and Redis did not become ready within ${deadline}s" >&2
exit 1
fi
sleep 2
elapsed=$((elapsed + 2))
done
docker compose --env-file .env restart api
elapsed=0
until docker compose --env-file .env exec -T api \
python3 -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).read()"
do
if [ "$elapsed" -ge "$deadline" ]; then
echo "API did not become ready within ${deadline}s" >&2
exit 1
fi
sleep 2
elapsed=$((elapsed + 2))
done
+43
View File
@@ -0,0 +1,43 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
python_cmd=${PYTHON:-python3}
base_url=${BASE_URL:-http://127.0.0.1:8080}
restored=0
restore_api() {
if [ "$restored" -eq 0 ]; then
restored=1
docker compose --env-file .env up -d --force-recreate api >/dev/null
restore_elapsed=0
until curl -fsS "$base_url/health/ready" >/dev/null 2>&1
do
if [ "$restore_elapsed" -ge 45 ]; then
echo "API did not return to normal readiness after expiry test" >&2
return 1
fi
sleep 1
restore_elapsed=$((restore_elapsed + 1))
done
fi
}
trap restore_api EXIT HUP INT TERM
SESSION_TTL_SECONDS=2 docker compose --env-file .env up -d --force-recreate api >/dev/null
elapsed=0
until curl -fsS "$base_url/health/ready" >/dev/null 2>&1
do
if [ "$elapsed" -ge 45 ]; then
echo "API did not become ready for expiry test" >&2
exit 1
fi
sleep 1
elapsed=$((elapsed + 1))
done
"$python_cmd" student-tests/test_level4.py \
--expiry-only --ttl-seconds 2 --base-url "$base_url"
restore_api
trap - EXIT HUP INT TERM
+1
View File
@@ -0,0 +1 @@
"""Messenger API application package."""
+40
View File
@@ -0,0 +1,40 @@
import logging
import os
import signal
import threading
from functools import partial
from http.server import ThreadingHTTPServer
from .config import APP_HOST, APP_PORT
from .handlers import MessengerHandler
from .store import Store
LOGGER = logging.getLogger("messenger")
def run_server():
store = Store()
store.open()
try:
store.migrate()
handler = partial(MessengerHandler, store=store)
server = ThreadingHTTPServer((APP_HOST, APP_PORT), handler)
def handle_shutdown(signum, frame):
LOGGER.info("received signal %s, shutting down gracefully", signum)
threading.Thread(target=server.shutdown, daemon=True).start()
signal.signal(signal.SIGTERM, handle_shutdown)
signal.signal(signal.SIGINT, handle_shutdown)
LOGGER.info("listening on %s:%s", APP_HOST, APP_PORT)
try:
server.serve_forever()
finally:
server.server_close()
finally:
store.close()
def main():
logging.basicConfig(level=os.getenv("LOG_LEVEL", "INFO"))
run_server()
+24
View File
@@ -0,0 +1,24 @@
import os
import re
import uuid
APP_HOST = os.getenv("APP_HOST", "0.0.0.0")
APP_PORT = int(os.getenv("APP_PORT", "8080"))
INSTANCE_ID = os.getenv("INSTANCE_ID", uuid.uuid4().hex[:12])
SESSION_TTL_SECONDS = int(os.getenv("SESSION_TTL_SECONDS", "3600"))
DB_STATEMENT_TIMEOUT_MS = int(os.getenv("DB_STATEMENT_TIMEOUT_MS", "5000"))
DB_LOCK_TIMEOUT_MS = int(os.getenv("DB_LOCK_TIMEOUT_MS", "2000"))
DB_POOL_MAX_SIZE = int(os.getenv("DB_POOL_MAX_SIZE", "16"))
DB_POOL_TIMEOUT_SECONDS = float(os.getenv("DB_POOL_TIMEOUT_SECONDS", "5"))
MAX_INFLIGHT_REQUESTS = int(os.getenv("MAX_INFLIGHT_REQUESTS", "128"))
MAX_REQUEST_BODY_BYTES = int(os.getenv("MAX_REQUEST_BODY_BYTES", "1048576"))
USER_RE = re.compile(r"^[a-z0-9_]{3,32}$")
if SESSION_TTL_SECONDS < 1:
raise RuntimeError("SESSION_TTL_SECONDS must be positive")
if DB_STATEMENT_TIMEOUT_MS < 1 or DB_LOCK_TIMEOUT_MS < 1:
raise RuntimeError("Database timeouts must be positive")
if DB_POOL_MAX_SIZE < 1 or DB_POOL_TIMEOUT_SECONDS <= 0:
raise RuntimeError("Database pool settings must be positive")
if MAX_INFLIGHT_REQUESTS < 1 or MAX_REQUEST_BODY_BYTES < 1:
raise RuntimeError("HTTP limits must be positive")
+14
View File
@@ -0,0 +1,14 @@
from typing import Dict, Optional
class ApiError(Exception):
def __init__(self, status: int, code: str, message: str, headers: Optional[Dict[str, str]] = None):
super().__init__(message)
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
class CacheUnavailableError(Exception):
pass
+326
View File
@@ -0,0 +1,326 @@
import base64
import json
import logging
import threading
import uuid
from http.server import BaseHTTPRequestHandler
from typing import Any, Dict, List, Optional, Tuple
from urllib.parse import parse_qs, urlsplit
import psycopg
from psycopg_pool import PoolTimeout
from .config import INSTANCE_ID, MAX_INFLIGHT_REQUESTS, MAX_REQUEST_BODY_BYTES, USER_RE
from .errors import ApiError, CacheUnavailableError
from .security import hash_password, isoformat, valid_uuid
from .store import Store
LOGGER = logging.getLogger("messenger.http")
class RequestGuard:
def __init__(self):
self.lock = threading.Lock()
self.inflight = 0
def acquire(self):
with self.lock:
if self.inflight >= MAX_INFLIGHT_REQUESTS:
raise ApiError(503, "unavailable", "Server is busy")
self.inflight += 1
def release(self):
with self.lock:
self.inflight = max(0, self.inflight - 1)
REQUEST_GUARD = RequestGuard()
class MessengerHandler(BaseHTTPRequestHandler):
def __init__(self, request, client_address, server, store: Store):
self.store = store
super().__init__(request, client_address, server)
def log_message(self, format: str, *args):
return
def do_GET(self):
self._dispatch("GET")
def do_POST(self):
self._dispatch("POST")
def do_DELETE(self):
self._dispatch("DELETE")
def _dispatch(self, method: str):
request_id = self.headers.get("X-Request-Id") or str(uuid.uuid4())
acquired = False
try:
REQUEST_GUARD.acquire()
acquired = True
parsed = urlsplit(self.path)
path = parsed.path
query = parse_qs(parsed.query, keep_blank_values=True)
if path == "/health/live" and method == "GET":
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/health/ready" and method == "GET":
try:
self.store.ready()
except (psycopg.Error, PoolTimeout, OSError):
raise ApiError(503, "unavailable", "Database is unavailable")
self._send_json(200, {"status": "ok"}, request_id)
elif path == "/api/v1/users" and method == "POST":
self._handle_register(request_id)
elif path == "/api/v1/auth/sessions" and method == "POST":
self._handle_create_session(request_id)
elif path == "/api/v1/auth/sessions/current" and method == "DELETE":
self._handle_revoke_session(request_id)
elif path == "/api/v1/users/me" and method == "GET":
self._send_json(200, self._get_actor(request_id), request_id)
elif path == "/api/v1/chats" and method == "POST":
self._handle_create_chat(request_id)
elif path == "/api/v1/chats" and method == "GET":
self._handle_list_chats(query, request_id)
elif path.startswith("/api/v1/chats/") and path.endswith("/messages") and method in ("GET", "POST"):
chat_id = path.rsplit("/", 2)[1]
if method == "POST":
self._handle_create_message(chat_id, request_id)
else:
self._handle_list_messages(chat_id, query, request_id)
elif path.startswith("/api/v1/chats/") and method == "GET":
self._handle_get_chat(path.rsplit("/", 1)[1], request_id)
else:
raise ApiError(404, "not_found", "Object not found")
except ApiError as exc:
self._send_error(exc.status, exc.code, exc.message, request_id, exc.headers)
except CacheUnavailableError:
self._send_error(503, "unavailable", "Session cache is unavailable", request_id)
except (PoolTimeout, psycopg.OperationalError, psycopg.InterfaceError):
self._send_error(503, "unavailable", "Database is unavailable", request_id)
except psycopg.errors.UniqueViolation:
self._send_error(409, "conflict", "Username already exists", request_id)
except psycopg.errors.ForeignKeyViolation:
self._send_error(400, "invalid_request", "Unknown member", request_id)
except Exception as exc:
LOGGER.error("request %s failed with %s", request_id, type(exc).__name__)
self._send_error(500, "internal_error", "Internal server error", request_id)
finally:
if acquired:
REQUEST_GUARD.release()
def _read_json(self, request_id: str) -> Any:
if self.headers.get_content_type() != "application/json":
raise ApiError(415, "unsupported_media_type", "Content-Type must be application/json")
length = self.headers.get("Content-Length")
if length is None:
raise ApiError(400, "invalid_request", "Missing Content-Length")
try:
size = int(length)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size < 0:
raise ApiError(400, "invalid_request", "Invalid Content-Length")
if size > MAX_REQUEST_BODY_BYTES:
raise ApiError(413, "payload_too_large", "Request body too large")
raw = self.rfile.read(size)
if not raw:
raise ApiError(400, "invalid_request", "Empty body")
try:
return json.loads(raw.decode("utf-8"))
except (UnicodeDecodeError, json.JSONDecodeError):
raise ApiError(400, "invalid_request", "Malformed JSON")
def _get_actor(self, request_id: str) -> Dict[str, Any]:
header = self.headers.get("Authorization", "")
scheme, separator, token = header.partition(" ")
if scheme.lower() != "bearer" or not separator or not token or " " in token:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
user = self.store.session_user(token)
if user is None:
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
return user
def _handle_register(self, request_id: str):
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"username", "display_name", "password"}:
raise ApiError(400, "invalid_request", "Expected username, display_name and password")
username = body["username"]
display_name = body["display_name"]
password = body["password"]
if not isinstance(username, str) or not USER_RE.fullmatch(username):
raise ApiError(400, "invalid_request", "Invalid username")
if not isinstance(display_name, str) or not display_name.strip() or len(display_name) > 100:
raise ApiError(400, "invalid_request", "Invalid display_name")
if not isinstance(password, str) or not (12 <= len(password) <= 128):
raise ApiError(400, "invalid_request", "Invalid password")
user = self.store.create_user(username, display_name, hash_password(password))
self._send_json(201, user, request_id)
def _basic_credentials(self, request_id: str) -> Tuple[str, str]:
value = self.headers.get("Authorization", "")
scheme, separator, encoded = value.partition(" ")
challenge = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
if scheme.lower() != "basic" or not separator or not encoded:
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
try:
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
username, password = decoded.split(":", 1)
except (ValueError, UnicodeError):
raise ApiError(401, "unauthorized", "Invalid credentials", challenge)
return username, password
def _handle_create_session(self, request_id: str):
username, password = self._basic_credentials(request_id)
user = self.store.authenticate(username, password)
if user is None:
raise ApiError(
401,
"unauthorized",
"Invalid credentials",
{"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'},
)
token, expires_at = self.store.create_session(user["id"])
self._send_json(
201,
{"token": token, "token_type": "Bearer", "expires_at": isoformat(expires_at), "user": user},
request_id,
)
def _handle_revoke_session(self, request_id: str):
self._get_actor(request_id)
token = self.headers["Authorization"].partition(" ")[2]
if not self.store.revoke_session(token):
raise ApiError(
401,
"unauthorized",
"Authentication required",
{"WWW-Authenticate": 'Bearer realm="messenger"'},
)
self.send_response(204)
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
self.end_headers()
def _handle_create_chat(self, request_id: str):
actor = self._get_actor(request_id)
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"title", "member_ids"}:
raise ApiError(400, "invalid_request", "Expected title and member_ids")
title, member_ids = body["title"], body["member_ids"]
if not isinstance(title, str) or not title.strip() or len(title) > 100:
raise ApiError(400, "invalid_request", "Invalid title")
if (
not isinstance(member_ids, list)
or not 1 <= len(member_ids) <= 100
or any(not isinstance(item, str) or not valid_uuid(item) for item in member_ids)
or len(member_ids) != len(set(member_ids))
or actor["id"] not in member_ids
):
raise ApiError(400, "invalid_request", "Invalid member_ids")
chat = self.store.create_chat(title, member_ids, actor["id"])
self._send_json(201, chat, request_id)
def _handle_list_chats(self, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
chats, next_cursor = self.store.list_chats(actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
self._send_json(200, {"items": chats, "next_cursor": next_cursor}, request_id)
def _handle_get_chat(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
chat = self.store.get_chat(chat_id, actor["id"])
if chat is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(200, chat, request_id)
def _handle_create_message(self, chat_id: str, request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
body = self._read_json(request_id)
if not isinstance(body, dict) or set(body) != {"text"}:
raise ApiError(400, "invalid_request", "Expected text")
text = body["text"]
if not isinstance(text, str) or not text.strip() or len(text) > 4000:
raise ApiError(400, "invalid_request", "Invalid message text")
message = self.store.create_message(chat_id, actor["id"], text)
if message is None:
raise ApiError(404, "not_found", "Chat not found")
self._send_json(201, message, request_id)
def _handle_list_messages(self, chat_id: str, query: Dict[str, List[str]], request_id: str):
actor = self._get_actor(request_id)
if not valid_uuid(chat_id):
raise ApiError(400, "invalid_request", "Invalid chat id")
try:
limit = int(query.get("limit", ["50"])[0])
except ValueError:
raise ApiError(400, "invalid_request", "Invalid limit")
if not 1 <= limit <= 100:
raise ApiError(400, "invalid_request", "Invalid limit")
cursor = query.get("cursor", [None])[0]
try:
result = self.store.list_messages(chat_id, actor["id"], limit, cursor)
except ValueError:
raise ApiError(400, "invalid_request", "Invalid cursor")
if result is None:
raise ApiError(404, "not_found", "Chat not found")
items, next_cursor = result
self._send_json(200, {"items": items, "next_cursor": next_cursor}, request_id)
def _send_json(
self,
status: int,
payload: Dict[str, Any],
request_id: str,
headers: Optional[Dict[str, str]] = None,
):
body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
self.send_response(status)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.send_header("X-Request-Id", request_id)
self.send_header("X-Instance-Id", INSTANCE_ID)
if headers:
for key, value in headers.items():
self.send_header(key, value)
self.end_headers()
self.wfile.write(body)
def _send_error(
self,
status: int,
code: str,
message: str,
request_id: str,
extra_headers: Optional[Dict[str, str]] = None,
):
self._send_json(
status,
{"error": {"code": code, "message": message, "request_id": request_id}},
request_id,
extra_headers,
)
+24
View File
@@ -0,0 +1,24 @@
#!/usr/bin/env python3
import logging
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.store import Store
else:
from .store import Store
def main():
logging.basicConfig(level="INFO")
store = Store()
store.open()
try:
store.migrate()
finally:
store.close()
if __name__ == "__main__":
main()
+53
View File
@@ -0,0 +1,53 @@
import base64
import hashlib
import hmac
import secrets
import uuid
from datetime import datetime, timezone
def isoformat(value: datetime) -> str:
return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def valid_uuid(value: str) -> bool:
try:
uuid.UUID(value)
return True
except (ValueError, TypeError, AttributeError):
return False
def hash_password(password: str) -> str:
salt = secrets.token_bytes(16)
derived = hashlib.scrypt(
password.encode("utf-8"), salt=salt, n=32768, r=8, p=1, dklen=32, maxmem=64 * 1024 * 1024
)
salt_text = base64.urlsafe_b64encode(salt).decode("ascii")
hash_text = base64.urlsafe_b64encode(derived).decode("ascii")
return f"scrypt$32768$8$1${salt_text}${hash_text}"
def verify_password(password: str, stored: str) -> bool:
try:
scheme, n, r, p, salt_text, hash_text = stored.split("$")
if scheme != "scrypt":
return False
salt = base64.urlsafe_b64decode(salt_text.encode("ascii"))
expected = base64.urlsafe_b64decode(hash_text.encode("ascii"))
actual = hashlib.scrypt(
password.encode("utf-8"),
salt=salt,
n=int(n),
r=int(r),
p=int(p),
dklen=len(expected),
maxmem=64 * 1024 * 1024,
)
return hmac.compare_digest(actual, expected)
except (ValueError, TypeError, UnicodeError):
return False
+13
View File
@@ -0,0 +1,13 @@
#!/usr/bin/env python3
import sys
from pathlib import Path
if __package__ in (None, ""):
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.app import main
else:
from .app import main
if __name__ == "__main__":
main()
+348
View File
@@ -0,0 +1,348 @@
import base64
import hashlib
import json
import logging
import math
import os
import secrets
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple
import redis
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from .config import (
DB_LOCK_TIMEOUT_MS,
DB_POOL_MAX_SIZE,
DB_POOL_TIMEOUT_SECONDS,
DB_STATEMENT_TIMEOUT_MS,
SESSION_TTL_SECONDS,
)
from .errors import CacheUnavailableError
from .security import isoformat, utc_now, valid_uuid, verify_password
LOGGER = logging.getLogger("messenger.store")
def database_url() -> str:
value = os.getenv("DATABASE_URL")
if value:
return value
file_path = os.getenv("DATABASE_URL_FILE")
if file_path:
return Path(file_path).read_text(encoding="utf-8").strip()
raise RuntimeError("Set DATABASE_URL or DATABASE_URL_FILE")
class Store:
"""PostgreSQL source of truth and optional Redis session lookup cache."""
def __init__(self):
self.pool = ConnectionPool(
conninfo=database_url(),
min_size=1,
max_size=DB_POOL_MAX_SIZE,
timeout=DB_POOL_TIMEOUT_SECONDS,
kwargs={
"row_factory": dict_row,
"connect_timeout": 5,
"options": (
f"-c statement_timeout={DB_STATEMENT_TIMEOUT_MS} "
f"-c lock_timeout={DB_LOCK_TIMEOUT_MS} "
"-c idle_in_transaction_session_timeout=10000"
),
},
open=False,
)
cache_url = os.getenv("CACHE_URL")
self.cache = (
redis.Redis.from_url(
cache_url,
decode_responses=True,
socket_connect_timeout=0.25,
socket_timeout=0.25,
retry_on_timeout=False,
max_connections=16,
)
if cache_url
else None
)
def open(self):
self.pool.open(wait=True, timeout=10)
def close(self):
if self.cache is not None:
self.cache.close()
self.pool.close()
def ready(self) -> bool:
with self.pool.connection(timeout=2) as conn:
conn.execute("SELECT 1")
return True
def migrate(self):
migrations = Path(__file__).resolve().parents[1] / "migrations"
with self.pool.connection() as conn:
with conn.transaction():
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations "
"(version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())"
)
conn.execute("SELECT pg_advisory_xact_lock(2741662026)")
applied = {row["version"] for row in conn.execute("SELECT version FROM schema_migrations")}
for migration in sorted(migrations.glob("*.sql")):
if migration.name in applied:
continue
conn.execute(migration.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (migration.name,))
@staticmethod
def user(row: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"username": row["username"],
"display_name": row["display_name"],
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def chat(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"title": row["title"],
"member_ids": member_ids,
"created_at": isoformat(row["created_at"]),
}
@staticmethod
def message(row: Dict[str, Any]) -> Dict[str, Any]:
return {
"id": str(row["id"]),
"chat_id": str(row["chat_id"]),
"sender_id": str(row["sender_id"]),
"text": row["text"],
"created_at": isoformat(row["created_at"]),
}
def create_user(self, username: str, display_name: str, password_hash: str) -> Dict[str, Any]:
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO users (username, display_name, password_hash) "
"VALUES (%s, %s, %s) RETURNING id, username, display_name, created_at",
(username, display_name, password_hash),
).fetchone()
return self.user(row)
def get_user(self, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s",
(user_id,),
).fetchone()
return self.user(row) if row else None
def authenticate(self, username: str, password: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT id, username, display_name, created_at, password_hash "
"FROM users WHERE username = %s",
(username,),
).fetchone()
if not row or not verify_password(password, row["password_hash"]):
return None
return self.user(row)
def create_session(self, user_id: str) -> Tuple[str, datetime]:
token = secrets.token_urlsafe(32)
expires_at = utc_now() + timedelta(seconds=SESSION_TTL_SECONDS)
token_hash = hashlib.sha256(token.encode("ascii")).digest()
with self.pool.connection() as conn:
conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash, user_id, expires_at),
)
return token, expires_at
@staticmethod
def _session_lock_key(token_hash: bytes) -> int:
return int.from_bytes(token_hash[:8], byteorder="big", signed=True)
def session_user(self, token: str) -> Optional[Dict[str, Any]]:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
cache_key = "messenger:session:" + token_hash.hex()
if self.cache is not None:
try:
cached = self.cache.get(cache_key)
if cached:
return json.loads(cached)
except (redis.RedisError, ValueError, TypeError):
LOGGER.warning("session cache read failed; falling back to PostgreSQL")
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
row = conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at "
"FROM sessions s JOIN users u ON u.id = s.user_id "
"WHERE s.token_hash = %s AND s.expires_at > now()",
(token_hash,),
).fetchone()
if not row:
return None
user = self.user(row)
if self.cache is not None:
ttl_ms = math.floor((row["expires_at"] - utc_now()).total_seconds() * 1000)
if ttl_ms > 0:
try:
self.cache.set(cache_key, json.dumps(user, separators=(",", ":")), px=ttl_ms)
except redis.RedisError:
LOGGER.warning("session cache write failed; PostgreSQL remains authoritative")
return user
def revoke_session(self, token: str) -> bool:
token_hash = hashlib.sha256(token.encode("utf-8")).digest()
with self.pool.connection() as conn:
with conn.transaction():
conn.execute("SELECT pg_advisory_xact_lock(%s)", (self._session_lock_key(token_hash),))
if self.cache is not None:
try:
self.cache.delete("messenger:session:" + token_hash.hex())
except redis.RedisError as exc:
raise CacheUnavailableError from exc
result = conn.execute("DELETE FROM sessions WHERE token_hash = %s", (token_hash,))
return result.rowcount == 1
def create_chat(self, title: str, member_ids: List[str], actor_id: str) -> Dict[str, Any]:
if actor_id not in member_ids or len(member_ids) != len(set(member_ids)):
raise ValueError("invalid members")
with self.pool.connection() as conn:
with conn.transaction():
row = conn.execute(
"INSERT INTO chats (title) VALUES (%s) RETURNING id, title, created_at",
(title,),
).fetchone()
for member_id in member_ids:
conn.execute(
"INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)",
(row["id"], member_id),
)
return self.chat(row, member_ids)
def get_chat(self, chat_id: str, user_id: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
row = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id "
"WHERE c.id = %s AND m.user_id = %s",
(chat_id, user_id),
).fetchone()
if not row:
return None
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(chat_id,),
).fetchall()
return self.chat(row, [str(item["user_id"]) for item in members])
def list_chats(
self, user_id: str, limit: int, cursor: Optional[str]
) -> Tuple[List[Dict[str, Any]], Optional[str]]:
where = "own.user_id = %s"
params: List[Any] = [user_id]
if cursor is not None:
try:
raw = base64.b64decode(
cursor.encode("ascii") + b"=" * (-len(cursor) % 4),
altchars=b"-_",
validate=True,
)
cursor_user, timestamp, chat_id = raw.decode("utf-8").split("|", 2)
if cursor_user != user_id or not valid_uuid(chat_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
if created_at.tzinfo is None or created_at.utcoffset() is None:
raise ValueError("cursor timestamp must include a timezone")
where += " AND (c.created_at, c.id) > (%s, %s)"
params.extend((created_at, chat_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
with self.pool.connection() as conn:
rows = conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members own ON own.chat_id = c.id "
f"WHERE {where} ORDER BY c.created_at, c.id LIMIT %s",
params,
).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
result = []
for row in page:
members = conn.execute(
"SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY joined_at, user_id",
(row["id"],),
).fetchall()
result.append(self.chat(row, [str(item["user_id"]) for item in members]))
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{user_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return result, next_cursor
def create_message(self, chat_id: str, sender_id: str, text: str) -> Optional[Dict[str, Any]]:
with self.pool.connection() as conn:
with conn.transaction():
member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, sender_id),
).fetchone()
if not member:
return None
row = conn.execute(
"INSERT INTO messages (chat_id, sender_id, text) "
"VALUES (%s, %s, %s) RETURNING id, chat_id, sender_id, text, created_at",
(chat_id, sender_id, text),
).fetchone()
return self.message(row)
def list_messages(
self, chat_id: str, user_id: str, limit: int, cursor: Optional[str]
) -> Optional[Tuple[List[Dict[str, Any]], Optional[str]]]:
with self.pool.connection() as conn:
is_member = conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id),
).fetchone()
if not is_member:
return None
where = "chat_id = %s"
params: List[Any] = [chat_id]
if cursor:
try:
raw = base64.urlsafe_b64decode(cursor.encode("ascii") + b"=" * (-len(cursor) % 4))
decoded_chat, timestamp, message_id = raw.decode("utf-8").split("|", 2)
if decoded_chat != chat_id or not valid_uuid(message_id):
raise ValueError("cursor scope")
created_at = datetime.fromisoformat(timestamp.replace("Z", "+00:00"))
where += " AND (created_at, id) > (%s, %s)"
params.extend((created_at, message_id))
except (ValueError, UnicodeError, TypeError) as exc:
raise ValueError("invalid cursor") from exc
params.append(limit + 1)
rows = conn.execute(
"SELECT id, chat_id, sender_id, text, created_at FROM messages "
f"WHERE {where} ORDER BY created_at, id LIMIT %s",
params,
).fetchall()
has_more = len(rows) > limit
page = rows[:limit]
next_cursor = None
if has_more and page:
last = page[-1]
cursor_data = f"{chat_id}|{isoformat(last['created_at'])}|{last['id']}".encode("utf-8")
next_cursor = base64.urlsafe_b64encode(cursor_data).decode("ascii").rstrip("=")
return [self.message(row) for row in page], next_cursor
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""Integration checks for the lab2 level-4 rubric; requires the local Compose stack."""
import base64
from concurrent.futures import ThreadPoolExecutor
import hashlib
import os
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure, check # noqa: E402
def compose(*args):
result = subprocess.run(
["docker", "compose", "--env-file", ".env", *args],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=False,
)
if result.returncode:
raise Failure("Docker Compose operation failed: " + args[0])
return result.stdout.strip()
def psql_scalar(sql):
result = subprocess.run(
[
"docker",
"compose",
"--env-file",
".env",
"exec",
"-T",
"postgres",
"sh",
"-c",
'psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "$1"',
"level4-test",
sql,
],
cwd=ROOT,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
check=False,
)
if result.returncode:
raise Failure("PostgreSQL verification query failed")
return result.stdout.strip()
def assert_ready(client, expected=200, deadline_seconds=45):
deadline = time.monotonic() + deadline_seconds
while time.monotonic() < deadline:
try:
client.call("GET", "/health/ready", expected, schema="Health" if expected == 200 else None)
return
except Failure:
if expected != 200:
raise
time.sleep(0.5)
raise Failure("API did not become ready")
def expire_only(args):
client = Client(args.base_url)
actor = client.user("level4expiry")
time.sleep(args.ttl_seconds + 1)
client.call("GET", "/api/v1/users/me", 401, actor=actor)
print("PASS: expired session rejected")
def main():
import argparse
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://127.0.0.1:8080")
parser.add_argument("--expiry-only", action="store_true")
parser.add_argument("--ttl-seconds", type=int, default=2)
args = parser.parse_args()
if args.expiry_only:
check(args.ttl_seconds >= 1, "TTL must be positive")
expire_only(args)
return
client = Client(args.base_url)
user = client.user("level4")
username = user["registration"]["username"]
wrong = base64.b64encode((username + ":not-the-correct-password").encode()).decode()
client.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": "Basic " + wrong})
client.call("GET", "/api/v1/users/me", actor=user, schema="User")
token_hash = hashlib.sha256(user["token"].encode("utf-8")).hexdigest()
redis_key = "messenger:session:" + token_hash
ttl_ms = int(
compose(
"exec",
"-T",
"redis",
"redis-cli",
"--raw",
"PTTL",
redis_key,
)
)
check(ttl_ms > 0, "session cache entry must have a positive Redis TTL")
unique_name = "race_" + uuid.uuid4().hex[:20]
registration = {
"username": unique_name,
"display_name": "Concurrent registration",
"password": "Concurrent-test-password-42",
}
barrier = threading.Barrier(4)
def register_once(_):
barrier.wait(timeout=10)
return Client(args.base_url).call(
"POST", "/api/v1/users", expected=[201, 409], body=registration
)["status"]
with ThreadPoolExecutor(max_workers=4) as executor:
statuses = list(executor.map(register_once, range(4)))
check(statuses.count(201) == 1 and statuses.count(409) == 3, "username race was not serialized")
check(
psql_scalar("SELECT count(*) FROM users WHERE username = '" + unique_name + "'") == "1",
"username race persisted other than one user",
)
chat_title = "rollback_" + uuid.uuid4().hex
missing_user_id = str(uuid.uuid4())
client.call(
"POST",
"/api/v1/chats",
400,
actor=user,
body={"title": chat_title, "member_ids": [user["id"], missing_user_id]},
)
check(
psql_scalar("SELECT count(*) FROM chats WHERE title = '" + chat_title + "'") == "0",
"failed chat creation left a partial chat row",
)
compose("stop", "redis")
try:
client.call("GET", "/api/v1/users/me", actor=user, schema="User")
finally:
compose("start", "redis")
deadline = time.monotonic() + 30
while True:
try:
if compose("exec", "-T", "redis", "redis-cli", "ping") == "PONG":
break
except Failure:
if time.monotonic() >= deadline:
raise Failure("Redis did not recover")
time.sleep(0.5)
client.call("GET", "/api/v1/users/me", actor=user, schema="User")
compose("stop", "postgres")
try:
client.call("GET", "/health/ready", 503)
client.call("GET", "/api/v1/chats?limit=10", 503, actor=user)
finally:
compose("start", "postgres")
assert_ready(client)
client.call("GET", "/api/v1/users/me", actor=user, schema="User")
print(
"PASS: level-4 checks (password failure, username race, atomic rollback, "
"Redis TTL/fallback, bounded PostgreSQL outage)"
)
if __name__ == "__main__":
try:
main()
except (Failure, OSError, subprocess.SubprocessError) as exc:
print("FAIL:", str(exc))
raise SystemExit(1)
+107
View File
@@ -0,0 +1,107 @@
#!/usr/bin/env python3
"""Deterministically verify that a cache miss cannot repopulate a logged-out session."""
import hashlib
import secrets
import threading
from src.store import Store
class BlockingCache:
def __init__(self, cache):
self.cache = cache
self.set_started = threading.Event()
self.allow_set = threading.Event()
def __getattr__(self, name):
return getattr(self.cache, name)
def set(self, key, value, **kwargs):
self.set_started.set()
if not self.allow_set.wait(timeout=10):
raise RuntimeError("timed out waiting to release the paused cache write")
return self.cache.set(key, value, **kwargs)
def main():
store = Store()
store.open()
underlying_cache = store.cache
if underlying_cache is None:
store.close()
raise RuntimeError("Redis cache is required for this test")
username = "sessionrace_" + secrets.token_hex(8)
user = store.create_user(username, "Session race test", "unused-test-hash")
token, _ = store.create_session(user["id"])
token_hash = hashlib.sha256(token.encode("ascii")).digest()
cache_key = "messenger:session:" + token_hash.hex()
blocking_cache = BlockingCache(underlying_cache)
store.cache = blocking_cache
outcomes = {}
failures = []
revoker_done = threading.Event()
def read_session():
try:
outcomes["reader"] = store.session_user(token)
except Exception as exc:
failures.append(exc)
def revoke_session():
try:
outcomes["revoker"] = store.revoke_session(token)
except Exception as exc:
failures.append(exc)
finally:
revoker_done.set()
reader = threading.Thread(target=read_session)
revoker = None
try:
underlying_cache.delete(cache_key)
reader.start()
if not blocking_cache.set_started.wait(timeout=10):
raise RuntimeError("session read did not reach the cache write")
revoker_started = threading.Event()
def start_revocation():
revoker_started.set()
revoke_session()
revoker = threading.Thread(target=start_revocation)
revoker.start()
if not revoker_started.wait(timeout=5):
raise RuntimeError("logout thread did not start")
if revoker_done.wait(timeout=0.2):
raise RuntimeError("logout passed the read's session lock before cache write completed")
blocking_cache.allow_set.set()
reader.join(timeout=10)
revoker.join(timeout=10)
if reader.is_alive() or revoker.is_alive():
raise RuntimeError("session race test threads did not finish")
if failures:
raise RuntimeError("session race worker failed") from failures[0]
if outcomes.get("reader") != user or outcomes.get("revoker") is not True:
raise RuntimeError("session read/logout returned unexpected results")
if underlying_cache.get(cache_key) is not None:
raise RuntimeError("logged-out session remains in Redis")
if store.session_user(token) is not None:
raise RuntimeError("logged-out session is still accepted")
print("PASS: concurrent session cache fill and logout leave no cached or valid session")
finally:
blocking_cache.allow_set.set()
reader.join(timeout=10)
if revoker is not None:
revoker.join(timeout=10)
store.cache = underlying_cache
underlying_cache.delete(cache_key)
with store.pool.connection() as conn:
conn.execute("DELETE FROM users WHERE id = %s", (user["id"],))
store.close()
if __name__ == "__main__":
main()
+344
View File
@@ -0,0 +1,344 @@
#!/usr/bin/env python3
"""Reproducible 100k-message plan/index/write-cost and backup-restore experiment."""
import json
import os
import platform
import secrets
import statistics
import subprocess
import tempfile
import time
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
EVIDENCE = ROOT / "evidence" / "track5_query_restore.json"
MARKDOWN = ROOT / "evidence" / "track5_query_restore.md"
COMPOSE = ["docker", "compose", "--env-file", ".env"]
def run_container(*args, input_text=None, stdin=None, stdout=None, check=True):
result = subprocess.run(
[*COMPOSE, "exec", "-T", "postgres", *args],
cwd=ROOT,
input=input_text,
stdin=stdin,
stdout=stdout if stdout is not None else subprocess.PIPE,
stderr=subprocess.PIPE,
text=input_text is not None,
check=False,
)
if check and result.returncode:
detail = result.stderr.decode("utf-8", "replace") if isinstance(result.stderr, bytes) else result.stderr
raise RuntimeError("PostgreSQL benchmark command failed: " + detail[-1000:].strip())
return result
def psql(database, sql):
result = run_container(
"sh",
"-c",
'psql -X -qAt -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$1"',
"track5",
database,
input_text=sql,
)
return result.stdout.strip()
def stdout_text(result):
if isinstance(result.stdout, bytes):
return result.stdout.decode("utf-8", "replace").strip()
return result.stdout.strip()
def admin(command, database):
return run_container(
"sh",
"-c",
f'{command} -U "$POSTGRES_USER" "$1"',
"track5",
database,
check=command != "dropdb",
)
def plan(database, table, query):
raw = psql(database, f"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) {query};")
document = json.loads(raw)[0]
root = document["Plan"]
return {
"execution_ms": document["Execution Time"],
"planning_ms": document["Planning Time"],
"actual_rows": root.get("Actual Rows"),
"shared_hit_blocks": root.get("Shared Hit Blocks", 0),
"shared_read_blocks": root.get("Shared Read Blocks", 0),
"plan": root,
}
def container_metadata():
container_id = subprocess.run(
[*COMPOSE, "ps", "-q", "postgres"],
cwd=ROOT,
check=True,
stdout=subprocess.PIPE,
text=True,
).stdout.strip()
inspected = subprocess.run(
["docker", "inspect", "--format", "{{.HostConfig.Memory}} {{.HostConfig.NanoCpus}}", container_id],
check=True,
stdout=subprocess.PIPE,
text=True,
).stdout.strip()
memory, nano_cpus = (int(value) for value in inspected.split())
try:
host_memory = int(
subprocess.run(
["sysctl", "-n", "hw.memsize"],
check=True,
stdout=subprocess.PIPE,
text=True,
).stdout.strip()
)
except (OSError, subprocess.CalledProcessError, ValueError):
host_memory = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
pg_version = stdout_text(run_container(
"sh",
"-c",
'psql -X -qAt -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT current_setting(\'server_version\')"',
"track5",
))
return {
"host_platform": platform.platform(),
"host_architecture": platform.machine(),
"host_logical_cpus": os.cpu_count(),
"host_memory_bytes": host_memory,
"postgres_image": "postgres:16.4-alpine",
"postgres_version": pg_version,
"postgres_container_memory_limit_bytes": memory,
"postgres_container_cpu_limit_nanos": nano_cpus,
"warmup": "initial data load followed by ANALYZE; plan query is executed 5 times",
}
def dataset_sql():
return """
CREATE TABLE messages_chat_only (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
chat_id integer NOT NULL,
sender_id integer NOT NULL,
text varchar(200) NOT NULL,
created_at timestamptz NOT NULL
);
CREATE TABLE messages_composite (LIKE messages_chat_only INCLUDING ALL);
INSERT INTO messages_chat_only (chat_id, sender_id, text, created_at)
SELECT CASE WHEN n <= 80000 THEN 1 ELSE 2 + ((n - 80001) % 100) END,
1 + (n % 3),
'synthetic-message-' || lpad(n::text, 6, '0'),
timestamptz '2025-01-01 00:00:00+00' + n * interval '1 second'
FROM generate_series(1, 100000) AS n;
INSERT INTO messages_composite (chat_id, sender_id, text, created_at)
SELECT chat_id, sender_id, text, created_at FROM messages_chat_only ORDER BY id;
CREATE INDEX messages_chat_only_idx ON messages_chat_only (chat_id);
CREATE INDEX messages_chat_history_idx ON messages_composite (chat_id, created_at, id);
ANALYZE messages_chat_only;
ANALYZE messages_composite;
"""
def history_query(table):
return (
f"SELECT id, created_at, text FROM {table} "
"WHERE chat_id = 2 "
"AND (created_at, id) > (timestamptz '2025-01-01 23:36:40+00', 0) "
"ORDER BY created_at, id LIMIT 25"
)
def write_samples(database, table):
sql = (
"BEGIN; "
"EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) "
f"INSERT INTO {table} (chat_id, sender_id, text, created_at) "
"SELECT 2 + (n % 100), 1, 'synthetic-write-' || n::text, "
"timestamptz '2025-01-01 00:00:00+00' + n * interval '1 second' "
"FROM generate_series(100001, 102000) AS n; "
"ROLLBACK;"
)
samples = []
for _ in range(5):
raw = psql(database, sql)
document = json.loads(raw)[0]
samples.append(
{
"execution_ms": document["Execution Time"],
"planning_ms": document["Planning Time"],
"shared_write_blocks": document["Plan"].get("Shared Written Blocks", 0),
}
)
return {
"samples": samples,
"median_execution_ms": statistics.median(item["execution_ms"] for item in samples),
"median_shared_write_blocks": statistics.median(item["shared_write_blocks"] for item in samples),
"rows_per_sample": 2000,
"rolled_back_after_each_measurement": True,
}
def render_markdown(result):
small = result["query_plans"]
writes = result["write_cost"]
restore = result["backup_restore"]
return f"""# Lab2 level-5 track 2 — query plans and recovery
- Run UTC: {result['run_utc']}
- Host: {result['environment']['host_platform']} ({result['environment']['host_architecture']}), logical CPUs: {result['environment']['host_logical_cpus']}
- Host physical memory: {result['environment']['host_memory_bytes']} bytes.
- PostgreSQL: {result['environment']['postgres_version']} in `{result['environment']['postgres_image']}`; container memory limit: {result['environment']['postgres_container_memory_limit_bytes']} bytes; CPU limit: {result['environment']['postgres_container_cpu_limit_nanos']} nanocores (0 means unlimited).
- Dataset: 100,000 synthetic rows in each comparison table; one chat has 80,000 rows and 100 small chats share the remaining 20,000. Only generated text is used.
- Query: keyset page from chat 2, identical SELECT and cursor for both indexes; each plan was executed five times after ANALYZE.
| Index | Median execution (ms) | Shared hits | Shared reads | Rows |
|---|---:|---:|---:|---:|
| `(chat_id)` | {statistics.median(item['execution_ms'] for item in small['chat_id_only']['samples']):.3f} | {small['chat_id_only']['samples'][0]['shared_hit_blocks']} | {small['chat_id_only']['samples'][0]['shared_read_blocks']} | {small['chat_id_only']['samples'][0]['actual_rows']} |
| `(chat_id, created_at, id)` | {statistics.median(item['execution_ms'] for item in small['composite_history']['samples']):.3f} | {small['composite_history']['samples'][0]['shared_hit_blocks']} | {small['composite_history']['samples'][0]['shared_read_blocks']} | {small['composite_history']['samples'][0]['actual_rows']} |
| Index | Median time for 2,000 inserted rows (ms) | Median shared write blocks |
|---|---:|---:|
| `(chat_id)` | {writes['chat_id_only']['median_execution_ms']:.3f} | {writes['chat_id_only']['median_shared_write_blocks']} |
| `(chat_id, created_at, id)` | {writes['composite_history']['median_execution_ms']:.3f} | {writes['composite_history']['median_shared_write_blocks']} |
Each write-cost query was measured five times inside a transaction and rolled back. This isolates execution and index-maintenance work but does not measure durable commit/fsync latency.
Backup/restore: custom-format dump {restore['dump_bytes']} bytes; dump {restore['dump_seconds']:.3f}s; restore into a separately created empty database {restore['restore_seconds']:.3f}s. Restored row counts and deterministic checksums for the selected messages were verified: **PASS**.
Raw JSON plans, samples, environment and restore checksums are in `track5_query_restore.json`. This single-host synthetic run is a reproducible comparison, not a production capacity or HA claim.
"""
def main():
tag = secrets.token_hex(5)
source = "lab2_track5_source_" + tag
target = "lab2_track5_restore_" + tag
created = []
try:
for database in (source, target):
admin("createdb", database)
created.append(database)
psql(source, dataset_sql())
select_chat = "SELECT id, chat_id, sender_id, text, created_at FROM {table} WHERE chat_id IN (1, 2) ORDER BY chat_id, created_at, id"
plans = {
"chat_id_only": [
plan(source, "messages_chat_only", history_query("messages_chat_only"))
for _ in range(5)
],
"composite_history": [
plan(source, "messages_composite", history_query("messages_composite"))
for _ in range(5)
],
}
writes = {
"chat_id_only": write_samples(source, "messages_chat_only"),
"composite_history": write_samples(source, "messages_composite"),
}
selected_hash = {}
for table in ("messages_chat_only", "messages_composite"):
selected_hash[table] = psql(
source,
"SELECT md5(string_agg(md5(row_to_json(r)::text), '' ORDER BY r.chat_id, r.created_at, r.id)) "
f"FROM ({select_chat.format(table=table)}) AS r;",
)
with tempfile.NamedTemporaryFile(prefix="lab2-track5-", suffix=".dump") as dump_file:
started = time.monotonic()
dump = run_container(
"sh",
"-c",
'pg_dump -U "$POSTGRES_USER" -Fc "$1"',
"track5",
source,
stdout=dump_file,
)
if dump.returncode:
raise RuntimeError("PostgreSQL benchmark backup failed")
dump_seconds = time.monotonic() - started
dump_file.flush()
dump_size = os.fstat(dump_file.fileno()).st_size
started = time.monotonic()
dump_file.seek(0)
restore = run_container(
"sh",
"-c",
'pg_restore -U "$POSTGRES_USER" --exit-on-error -d "$1"',
"track5",
target,
stdin=dump_file,
stdout=subprocess.DEVNULL,
)
if restore.returncode:
raise RuntimeError("PostgreSQL benchmark restore failed")
restore_seconds = time.monotonic() - started
restored_counts = {
table: int(psql(target, f"SELECT count(*) FROM {table};"))
for table in ("messages_chat_only", "messages_composite")
}
restored_hashes = {
table: psql(
target,
"SELECT md5(string_agg(md5(row_to_json(r)::text), '' ORDER BY r.chat_id, r.created_at, r.id)) "
f"FROM ({select_chat.format(table=table)}) AS r;",
)
for table in ("messages_chat_only", "messages_composite")
}
if any(count != 100000 for count in restored_counts.values()):
raise RuntimeError("Restored message counts do not match the 100,000-row source tables")
if restored_hashes != selected_hash:
raise RuntimeError("Restored selected-message checksums differ from the source")
env = container_metadata()
env["postgres_memory_setting"] = psql(source, "SHOW shared_buffers;")
env["postgres_work_mem"] = psql(source, "SHOW work_mem;")
result = {
"run_utc": datetime.now(timezone.utc).isoformat(),
"environment": env,
"method": {
"rows_per_index_variant": 100000,
"distribution": "80,000 messages in one chat; 20,000 distributed evenly across 100 chats",
"warmup": "ANALYZE followed by five executed EXPLAIN ANALYZE samples per query",
"query": history_query("messages"),
},
"query_plans": {
"chat_id_only": {"index": "(chat_id)", "samples": plans["chat_id_only"]},
"composite_history": {
"index": "(chat_id, created_at, id)",
"samples": plans["composite_history"],
},
},
"write_cost": writes,
"backup_restore": {
"format": "pg_dump custom format",
"dump_bytes": dump_size,
"dump_seconds": dump_seconds,
"restore_seconds": restore_seconds,
"restored_counts": restored_counts,
"source_selected_checksums": selected_hash,
"restored_selected_checksums": restored_hashes,
"verification": "PASS",
},
}
EVIDENCE.write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
MARKDOWN.write_text(render_markdown(result), encoding="utf-8")
print("PASS: level-5 track 2; evidence written to evidence/track5_query_restore.md")
finally:
for database in reversed(created):
admin("dropdb", database)
if __name__ == "__main__":
try:
main()
except (OSError, subprocess.SubprocessError, ValueError, KeyError, RuntimeError) as exc:
print("FAIL:", str(exc))
raise SystemExit(1)
+1 -1
View File
@@ -9,7 +9,7 @@ make test BASE_URL=http://localhost:8080
python3 tests/smoke.py --base-url http://localhost:8080
```
`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → чат → сообщения → пагинация → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite.
`smoke.py` выполняет **один сквозной сценарий**, автоматически выбирая профиль из `lab.json`: health → три пользователя → вход с lab2 → несколько чатов и их keyset-пагинация → сообщения и пагинация истории → ошибки доступа → идемпотентность с lab3 → файл с lab4 → thumbnail с lab5 → logout. Он проверяет JSON по используемому подмножеству схемы, коды/headers, Unicode, отсутствие повторов/пропусков при пагинации, SHA-256 оригинала и фактические JPEG-размеры thumbnail. Это публичный минимум, не полный fuzz/security/load suite.
Каждый запуск создаёт уникальных синтетических пользователей, чат и сообщения. Данные автоматически не удаляются: в API курса нет delete. Для чистого повтора используйте отдельный тестовый стенд/volume и осознанный сброс своего окружения. Session tokens хранятся только в памяти процесса и не печатаются/не сохраняются.
+48 -1
View File
@@ -20,6 +20,54 @@ def main():
check(any(x['id']==chat['id'] for x in listed['items']),'member cannot list chat')
hidden = c.call('GET','/api/v1/chats?limit=100',actor=eve,schema='ChatPage')['json']
check(all(x['id']!=chat['id'] for x in hidden['items']),'chat leaks to outsider')
page_chats = [
c.call(
'POST',
'/api/v1/chats',
201,
actor=alice,
body={'title':'Pagination test %d' % index,'member_ids':[alice['id'],bob['id']]},
schema='Chat',
)['json']
for index in range(4)
]
from urllib.parse import quote
chat_pages, seen_cursors = [], set()
cursor = None
while True:
query = '/api/v1/chats?limit=2'
if cursor is not None:
query += '&cursor=' + quote(cursor, safe='')
page = c.call('GET',query,actor=alice,schema='ChatPage')['json']
check(1 <= len(page['items']) <= 2,'chat page must contain at most the requested limit')
chat_pages.extend(page['items'])
cursor = page['next_cursor']
if cursor is None:
break
check(cursor not in seen_cursors,'chat pagination cursor repeated')
seen_cursors.add(cursor)
check(len(seen_cursors) <= 3,'chat pagination did not finish within expected pages')
chat_ids = [item['id'] for item in chat_pages]
check(len(chat_ids)==len(set(chat_ids)),'chat pagination duplicated an item')
check({item['id'] for item in page_chats + [chat]} <= set(chat_ids),'chat pagination lost a chat')
ordered_chats = sorted(
chat_pages,
key=lambda item:(timestamp(item['created_at']),uuid.UUID(item['id']).int),
)
check(chat_pages==ordered_chats,'chat pagination is not ordered by (created_at, id)')
check(
chat_pages == c.call('GET','/api/v1/chats?limit=100',actor=alice,schema='ChatPage')['json']['items'],
'chat pages differ from the complete chat list',
)
c.call('GET','/api/v1/chats?limit=2&cursor=not-a-valid-cursor',400,actor=alice)
if page['next_cursor'] is None and seen_cursors:
foreign_cursor = next(iter(seen_cursors))
c.call(
'GET',
'/api/v1/chats?limit=2&cursor=' + quote(foreign_cursor,safe=''),
400,
actor=bob,
)
key = uuid.uuid4().hex
first = c.message(alice,chat,'Привет, мир 👋',key=key)['json']
second = c.message(bob,chat,'Сообщение Боба')['json']
@@ -28,7 +76,6 @@ def main():
check(first['text']=='Привет, мир 👋' and second['text']=='Сообщение Боба','message text changed')
page1 = c.call('GET',path+'/messages?limit=1',actor=bob,schema='MessagePage')['json']
check(len(page1['items'])==1 and page1['next_cursor'],'first page must have one item and cursor')
from urllib.parse import quote
page2 = c.call('GET',path+'/messages?limit=1&cursor='+quote(page1['next_cursor'],safe=''),actor=alice,schema='MessagePage')['json']
check(len(page2['items'])==1 and page2['next_cursor'] is None,'second page must finish history')
ordered = sorted([first,second],key=lambda x:(timestamp(x['created_at']),uuid.UUID(x['id']).int))