ЛР 2 — БВТ2302 — Боданюк — на 5 #42

Closed
wfa wants to merge 7 commits from wfa/mtusi2026lab2:solution/bvt2302-bodanyuk into master
77 changed files with 5494 additions and 19 deletions
+12
View File
@@ -18,3 +18,15 @@ __pycache__
node_modules
evidence
.DS_Store
# Not needed in the image (only requirements.txt, src/, migrations/ and the contract are copied)
.pytest_cache
student-tests
bench
tests
contracts/*
!contracts/openapi.json
examples
scripts
infra
docs
*.md
+13 -1
View File
@@ -5,7 +5,19 @@ APP_PORT=8080
APP_ENV=lab
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
# The files are created by `python scripts/init_secrets.py` and mounted by compose as secrets.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
# valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`.
SESSION_STORE=valkey
CACHE_URL_FILE=/run/secrets/cache_url
# Additional settings of this implementation (defaults shown; all optional):
LOG_LEVEL=info
DB_POOL_MIN=1
DB_POOL_MAX=10
DB_TIMEOUT_SECONDS=3
CACHE_TIMEOUT_SECONDS=0.5
PASSWORD_HASH_CONCURRENCY=2
DOCS_ENABLED=true
# Host port of the API (compose publishes it on 127.0.0.1 only)
APP_PUBLISH_PORT=8080
+5
View File
@@ -0,0 +1,5 @@
# LF everywhere: identical files on Windows/Linux and inside `git archive | docker build -`.
* text=auto eol=lf
*.png binary
# cmd.exe expects CRLF in batch files.
*.cmd text eol=crlf
+32
View File
@@ -0,0 +1,32 @@
name: Application (lab2)
on: [push, pull_request]
permissions:
contents: read
jobs:
app:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- name: Synthetic secrets and env (CI only, never committed)
run: python scripts/init_secrets.py && cp .env.example .env
- name: Build, migrate, start and wait for healthy services
run: docker compose --env-file .env up -d --build --wait
- name: Contract and public smoke
run: make check test PYTHON=python
- name: Persistence across restart of API and stores
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python
- name: Own tests (unit + stand, failures restored in finally)
run: |
python -m pip install --require-hashes --no-deps -r requirements.txt
python -m pip install -r requirements-test.txt
make student-test PYTHON=python
- name: Logs
if: always()
run: docker compose logs --tail 100
- name: Remove CI stand and its synthetic data
if: always()
run: docker compose down -v
+1
View File
@@ -30,3 +30,4 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.pytest_cache/
+53
View File
@@ -0,0 +1,53 @@
# syntax=docker/dockerfile:1
# Messenger API, lab 2 (PostgreSQL + Valkey). Built by compose.yaml (services api, migrate).
#
# Python is interpreted, so there is no separate compile stage; the image
# contains only the interpreter, locked dependencies and the source code.
# Base image is pinned by version AND digest (python 3.13.16, Debian 12 bookworm).
FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1 \
PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /app
# 1. Dependencies first (layer cache survives source edits). Every package,
# including transitive ones, is pinned with sha256 hashes in requirements.txt;
# --no-deps guarantees nothing outside the lock file is installed.
COPY requirements.txt ./
RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt
# 2. Unprivileged runtime user. Code stays owned by root, so the process
# cannot modify its own code.
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
COPY src/ ./src/
# Versioned SQL migrations, applied by `python -m messenger.migrate` (compose service migrate).
COPY migrations/ ./migrations/
# Course contract, served read-only at /openapi.json for Swagger UI (/docs).
COPY contracts/openapi.json ./contracts/openapi.json
# Non-secret defaults; compose.yaml passes the rest. Passwords come only as files
# (DATABASE_URL_FILE / CACHE_URL_FILE -> Docker secrets), never as ENV or COPY.
ENV PYTHONPATH=/app/src \
APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
LOG_LEVEL=info \
SHUTDOWN_TIMEOUT_SECONDS=8 \
DOCS_ENABLED=true \
SESSION_TTL_SECONDS=3600 \
DATABASE_URL_FILE=/run/secrets/database_url
USER app:app
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "-m", "messenger.healthcheck"]
# Exec form: the Python server is PID 1 and receives SIGTERM directly (no shell in between).
CMD ["python", "-m", "messenger"]
+35
View File
@@ -24,3 +24,38 @@ load:
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# ---------------------------------------------------------------------------
# Student solution (lab 2, Python). On Windows without make run the commands from
# REPORT.md directly (public tests there need PYTHONUTF8=1; restart hook: scripts/restart.cmd).
.PHONY: secrets up down logs ps migrate student-test unit-test explain
secrets:
$(PYTHON) scripts/init_secrets.py
# Build, apply migrations (service migrate) and wait until every service is healthy.
up: secrets
docker compose up -d --build --wait
# Safe stop: containers stop, named volumes (data) stay.
down:
docker compose down
logs:
docker compose logs --tail 50 api
ps:
docker compose ps
migrate:
docker compose run --rm migrate
# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv).
student-test:
$(PYTHON) -m pytest student-tests -v
unit-test:
$(PYTHON) -m pytest student-tests -m "not stack" -q
explain:
$(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt
+240 -18
View File
@@ -1,37 +1,259 @@
# Отчёт — лабораторная 2
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Боданюк Алексей, БВТ2302
- Целевая оценка: **5**, трек 2 «План запроса и восстановление»
- Ветка: `solution/bvt2302-bodanyuk`. Форк: `https://gitea.privcloud.tech/wfa/mtusi2026lab2`. PR: «ЛР 2 — БВТ2302 — Боданюк — на 5» в mtusi2026/mtusi2026lab2
- Стек: Python 3.13.16, Starlette 1.7.0, uvicorn 0.54.0, psycopg 3.3.6 + psycopg-pool 3.3.3, redis-py 8.1.0 (протокол Valkey), argon2-cffi 25.1.0; PostgreSQL 18.6, Valkey 8.1.3. Все образы закреплены по тегу и digest, все Python-пакеты — по хешам ([requirements.txt](requirements.txt))
- Источники: README, COURSE, контракт ([openapi.json](contracts/openapi.json), [семантика](contracts/README.md)), документация PostgreSQL (transactions, EXPLAIN, pg_dump/pg_restore, pgbench), Valkey persistence, RFC 7617, OWASP Password Storage Cheat Sheet (параметры Argon2id). Учтены замечания преподавателя к PR других студентов этого репозитория (см. раздел «Проверки, на которых ловили другие работы»)
- Помощь ИИ: код, тесты, скрипты эксперимента и черновик отчёта написаны с помощью ИИ-ассистента Claude Code (Anthropic). Все команды и замеры реально выполнены на стенде ниже, их вывод лежит в [evidence/](evidence/). Пояснения для защиты — [docs/DEFENSE.md](docs/DEFENSE.md), запуск на Windows и сценарий защиты по шагам — [docs/WINDOWS.md](docs/WINDOWS.md)
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Нужны Git, Docker с Compose v2 (проверено: Docker 29.8.0, Compose 5.5.1) и Python 3.9+ (secrets, публичные тесты). Для своих тестов нужен venv. Учётные данные не хранятся в Git: их создаёт скрипт.
```sh
git clone https://gitea.privcloud.tech/wfa/mtusi2026lab2 mtusi2026lab2 && cd mtusi2026lab2 && git checkout solution/bvt2302-bodanyuk
python3 scripts/init_secrets.py # случайные локальные пароли -> ./secrets (git-ignored), идемпотентно
cp .env.example .env # необязательно: значения по умолчанию заданы в compose.yaml
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait # postgres+valkey healthy -> migrate -> api healthy
curl -i http://127.0.0.1:8080/health/ready # 200 {"status":"ok"}; Swagger: http://127.0.0.1:8080/docs
make check && make test
make test-persistence ACTION_SCRIPT=scripts/restart.sh
```
- **Миграции:** выполняет одноразовый сервис `migrate` (`python -m messenger.migrate`) перед стартом `api`. Повторный запуск: `docker compose run --rm migrate` → `0001_init: already applied, skipped`.
- **Свои тесты:** `python3 -m venv .venv`, затем `pip install --require-hashes --no-deps -r requirements.txt` и `pip install -r requirements-test.txt`, затем `make student-test`. Все тесты — около 3,5 мин. Те, что не требуют стенда: `make unit-test`.
- **Windows без make:** цели Makefile — однострочные команды. Перед публичными тестами нужно `$env:PYTHONUTF8=1`. Hook сохранности на Windows — `scripts/restart.cmd`: он запускает тот же `restart.sh` через Git Bash, потому что Windows не исполняет `.sh` напрямую.
- **Остановка:** безопасно — `docker compose down` (volumes сохраняются). Намеренное удаление учебных данных — `docker compose down -v` и `rm -rf secrets`. После этого стенд поднимается с нуля той же последовательностью.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
клиент ──HTTP──> api (uvicorn+Starlette, 1 процесс, пользователь app, PID 1)
├─ PostgreSQL 18.6 — источник истины: users, chats, chat_members, messages (volume pgdata)
└─ Valkey 8.1.3 — единственное хранилище сессий, AOF appendfsync always (volume valkeydata)
migrate (одноразовый) ──> PostgreSQL: версионированные SQL-миграции, advisory lock, checksum
```
| Роль | Service | Образ | Health / readiness | Volume | Сеть / порт | Credentials |
| --- | --- | --- | --- | --- | --- | --- |
| API | `api` | `mtusi-messenger:lab2` (собирается из исходников) | `HEALTHCHECK` → `/health/ready` | — | `127.0.0.1:8080→8080` | secrets `database_url`, `cache_url` |
| БД | `postgres` | `postgres:18.6-bookworm@sha256:afc7e2d4…` | `pg_isready` | `pgdata` | только внутренняя | secret `postgres_password` |
| Сессии | `valkey` | `valkey/valkey:8.1.3-bookworm@sha256:fea8b3e6…` | `valkey-cli ping` | `valkeydata` | только внутренняя | secret `valkey_password` → include-файл конфига (не в аргументах процесса) |
| Миграции | `migrate` | тот же образ API | завершение с кодом 0 | — | внутренняя | secret `database_url` |
- **Модель данных и транзакции** — [docs/DATA_MODEL.md](docs/DATA_MODEL.md), схема — [migrations/0001_init.sql](migrations/0001_init.sql).
- PK, FK (в том числе `messages (chat_id, sender_id) → chat_members`: отправитель обязан быть участником), `UNIQUE (username)`, CHECK.
- Свободный текст хранится в `bytea`, потому что контракт разрешает U+0000, а `text` в PostgreSQL его не хранит.
- **Создание чата** — одна транзакция: `INSERT chats` → `INSERT chat_members` из `unnest(member_ids)`. Неизвестный участник ломает второй INSERT по FK, и откатывается всё, включая уже вставленный чат.
- **Порядок сообщений.** `UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + 1 мкс) … RETURNING` блокирует строку чата до COMMIT. Поэтому писатели одного чата идут по очереди, а время строго растёт в порядке коммитов. Это перенос инварианта lab1 («время назначается под замком») в БД.
- **Пагинация** — тот же keyset-cursor, что в lab1, но в SQL: `(created_at, id) > ($2,$3)` по индексу `(chat_id, created_at, id)`. Запросов два — первая страница и после cursor, без `IS NULL OR`. Cursor проверяет диапазон времени (до 9999 года): дата вне диапазона даёт 400, а не 500.
- **Аутентификация** ([authhdr.py](src/messenger/authhdr.py), [sessions.py](src/messenger/sessions.py), [passwords.py](src/messenger/passwords.py)):
- `Basic` принимается только на `POST /auth/sessions`. Любой битый заголовок (не base64, не UTF-8, без `:`) даёт 401, пароль может содержать `:`;
- пароль: 12–128 code points, не из одних пробелов; Argon2id (m=19 MiB, t=2, p=1) считается в отдельном потоке с ограничением параллельности 2;
- неизвестный пользователь проверяется против фиктивного хеша: время и ответ такие же, как при неверном пароле;
- токен — `token_urlsafe(32)` (256 бит), хранится только SHA-256 токена;
- `X-User-Id` и `Basic` на бизнес-endpoint дают 401 с `WWW-Authenticate: Bearer realm="messenger"`.
- **Сессии — роль Valkey (уровень 4).** Источник истины для сессий — Valkey, и это **не** кэш над PostgreSQL. Значение ключа — `{user_id, expires_at}`, срок — абсолютный `PXAT`.
- **Expiry:** `expires_at` проверяется при каждом чтении; рестарт срок не продлевает.
- **Инвалидация:** logout = `DEL`.
- **Persistence:** `appendonly yes`, `appendfsync always`, `maxmemory-policy noeviction` — подтверждённые вход и logout переживают рестарт и падение, а сессии не вытесняются молча.
- **Отказ Valkey:** проверить сессию нельзя → 503 на защищённые запросы и на logout; анонимный доступ никогда не открывается.
Копия одна, поэтому нет сценария «logout потерялся в кэше и токен ожил». Переменная `SESSION_STORE=postgres` переключает сессии на таблицу `sessions` (проверено тестами и hook сохранности).
- **Ограничения ресурсов** ([db.py](src/messenger/db.py)):
- пул psycopg: min 1, max 10, ожидание соединения ≤ 3 с;
- сервер: `statement_timeout` = 2,5 с, `lock_timeout` = 1,25 с, `idle_in_transaction_session_timeout` = 10 с;
- клиентский дедлайн 3 с. Он работает в отдельной задаче: при замороженном PostgreSQL (`docker pause`) psycopg после отмены ещё до 10 с ждёт ответа на cancel. Без этого ответ приходил через 13 с, с этим — через 3,0 с;
- Valkey: таймауты сокета 0,5 с, без скрытых повторов;
- все отказы зависимостей → 503 `unavailable`.
- **Readiness** — 200, только если отвечают PostgreSQL и Valkey (обе зависимости критичны для обязательного пути). Liveness — только жизнь процесса.
- **Из lab1 перенесено** без изменений: формат ошибок и `X-Request-Id`, строгая валидация JSON (лишние поля, типы, NaN, суррогаты, склеенные объекты), JSON-логи без тел и текстов, graceful SIGTERM (≤ 8 с, readiness 503), Swagger. **Изменено:** хранилище, идентификация, cursor — с проверкой диапазона. Перенос сделан отдельным коммитом `893beaa`.
Соответствие `.env.example` реальной конфигурации ([config.py](src/messenger/config.py), [compose.yaml](compose.yaml)):
| Переменная | Значение | Где используется |
| --- | --- | --- |
| `APP_HOST`, `APP_PORT`, `APP_ENV` | `0.0.0.0`, `8080`, `lab` | bind uvicorn, стартовый лог |
| `SESSION_TTL_SECONDS` | `3600` | срок сессии; compose передаёт его в контейнер (проверено `printenv` в тесте TTL=2) |
| `DATABASE_URL_FILE` | `/run/secrets/database_url` | строка подключения из secret-файла |
| `SESSION_STORE` | `valkey` (или `postgres`; `redis` — синоним `valkey`) | выбор хранилища сессий |
| `CACHE_URL_FILE` | `/run/secrets/cache_url` | адрес и пароль Valkey из secret-файла |
| `DB_POOL_MIN/MAX`, `DB_TIMEOUT_SECONDS`, `CACHE_TIMEOUT_SECONDS`, `PASSWORD_HASH_CONCURRENCY` | 1/10, 3, 0.5, 2 | пул и таймауты, параллельность Argon2id |
| `LOG_LEVEL`, `DOCS_ENABLED`, `APP_PUBLISH_PORT` | `info`, `true`, `8080` | логи, Swagger, порт на хосте (только `127.0.0.1`) |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Стенд: Windows 10 22H2, AMD Ryzen 5 3400G (4 ядра / 8 потоков), 16 ГБ RAM, Docker Desktop 29.8.0 (WSL2), дата 2026-10-12. Все проверки уровней 3 и 4 ниже выполнены на стенде, поднятом **из чистого clone** с пустыми volumes (отдельный compose-проект на порту 8081).
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий README | Команда / сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Чистый clone, secrets, миграции с нуля | `init_secrets.py`, `compose up --build --wait`, `compose logs migrate`, повтор `run migrate` | все сервисы healthy; `0001_init: applied`, повтор — `already applied, skipped`; uid 10001, PID 1; `secrets/` и `.env` игнорируются Git | [clean-clone.txt](evidence/clean-clone.txt) |
| Контракт | `make check`, `make validate` | PASS, PASS | [make-check-validate.txt](evidence/make-check-validate.txt) |
| **3:** публичный smoke | `make test` | `PASS: lab2 public API scenario` | [make-test.txt](evidence/make-test.txt) |
| **3:** данные и сессии переживают рестарт API и хранилищ | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; `StartedAt` api, postgres, valkey изменились, volumes те же | [test-persistence-valkey.txt](evidence/test-persistence-valkey.txt), [restart-hook-output.txt](evidence/restart-hook-output.txt) |
| то же для `SESSION_STORE=postgres` | тот же тест после `SESSION_STORE=postgres compose up -d api` | PASS, сессии в таблице `sessions` | [test-persistence-postgres.txt](evidence/test-persistence-postgres.txt) |
| **3:** PK/FK, unique username, миграции с нуля | [0001_init.sql](migrations/0001_init.sql), `test_migrations_rerun_is_a_noop` | схема применяется к пустому volume; повтор ничего не меняет | [student-tests.txt](evidence/student-tests.txt) |
| **3:** атомарное создание чата | `test_create_chat_rolls_back_after_the_chat_row_was_inserted` | `n_tup_ins` таблицы `chats` вырос (INSERT был), строки нет, членство не изменилось | там же |
| **3:** Basic только на входе, Bearer, expiry, отзыв, `X-User-Id` не даёт доступ, KDF | `test_api.py` (18 проверок аутентификации), `test_short_ttl_expires`, `test_secrets_are_not_stored_in_plain_text` | 401 с нужным challenge; logout → 401; TTL=2 → 401; хеш `$argon2id$v=19$m=19456,t=2,p=1$`; в Valkey только `session:<sha256>` | там же |
| **3:** secrets и volumes не в Git | `git check-ignore secrets/database_url .env` | оба игнорируются; в отчёте и evidence нет паролей и токенов | [clean-clone.txt](evidence/clean-clone.txt), [logs-check.txt](evidence/logs-check.txt) |
| **4:** Valkey реально хранит сессии с TTL и persistence | `test_logout_during_valkey_outage_is_never_lost`, `test_restart_does_not_extend_ttl`, persistence-тест | сессии переживают рестарт Valkey; TTL не продлевается; отозванный токен остаётся 401 после рестарта Valkey и API (10 проверок GET и POST) | [student-tests.txt](evidence/student-tests.txt) |
| **4:** параметризованные запросы, ограниченный пул, таймауты | [repo.py](src/messenger/repo.py) (`%s`), [db.py](src/messenger/db.py) | max 10 соединений; пауза PostgreSQL → 503 за 3,0 с; Valkey → 0,5 с | `test_postgres_failure[pause]`, `test_valkey_failure[pause]` |
| **4:** гонка регистрации | `test_registration_race_exactly_one_201_and_seven_409` | 8 одновременных → ровно `[201] + [409]×7` (сравниваются HTTP-коды), в БД 1 строка | [student-tests.txt](evidence/student-tests.txt) |
| **4:** rollback составной операции | см. «атомарное создание чата» | откат **после** первого INSERT | там же |
| **4:** индекс истории и EXPLAIN на ≥ 10 000 | `make explain`, `test_late_page_uses_index_range_not_filter` | 50 189 сообщений, 20 000 в чате; поздняя страница при generic plan: `Index Cond: … ROW(created_at, id) > ROW($2, $3)`, 5 буферов, 0,11 мс; вариант с `IS NULL OR` — `Rows Removed by Filter: 19950`, 526 буферов, 3,3 мс; OFFSET — 526 буферов, 4,4 мс | [explain-history.txt](evidence/explain-history.txt) |
| **4:** expiry, logout, неверный пароль | `test_short_ttl_expires[valkey/postgres]`, `test_logout_revokes_only_this_token`, `test_wrong_password_and_unknown_user_give_the_same_401` | 401 после `expires_at`; logout отзывает только этот токен; неверный пароль и неизвестный пользователь — одинаковый 401 | [student-tests.txt](evidence/student-tests.txt) |
| **4:** недоступность Redis/БД, аноним не доверенный | `test_failures.py`: stop/pause Valkey и PostgreSQL с **действующим** токеном, `LOCK TABLE`, восстановление в `finally` | 503 в пределах дедлайна; аноним — 401; readiness 503; после восстановления тот же токен и данные на месте | там же |
| Логи без секретов и текстов | grep по логу api после трафика | 0 текстов сообщений, паролей, токенов, паролей БД; события `login_failed`, `session_store_unavailable`, access-строки | [logs-check.txt](evidence/logs-check.txt) |
| SIGTERM сохранён из lab1 | `docker compose stop api` × 3 | 1,1–1,2 с, exit 0, без OOM | [sigterm.txt](evidence/sigterm.txt) |
| **5:** трек 2 | `bench/track2/*.py` | см. ниже | [evidence/track2/](evidence/track2/) |
Итог своих тестов на стенде из чистого clone: **108 passed** (45 unit + 63 против стенда), 3 мин 28 с ([student-tests.txt](evidence/student-tests.txt)).
### Проверки, на которых ловили другие работы, и наш результат
| Замечание к чужим PR | У нас | Где проверено |
| --- | --- | --- |
| logout при упавшем Redis → после восстановления токен снова 200 | logout при упавшем Valkey = 503 (ничего не отозвано, клиент это видит). После восстановления logout → 401, и 401 сохраняется после рестарта Valkey и API | `test_logout_during_valkey_outage_is_never_lost` |
| истёкшая сессия принимается из кэша | expiry проверяется при каждом чтении; ключ живёт до абсолютного `PXAT` | `test_short_ttl_expires` |
| пауза PostgreSQL → запрос висит 8+ с; ограничено ожидание пула, но не SQL | клиентский дедлайн в отдельной задаче + `statement_timeout` / `lock_timeout` | `test_postgres_failure[pause]`, `test_locked_table_gives_bounded_503` |
| отказ БД маскируется под 401 | 503 `unavailable` для действующего токена, в обоих режимах сессий | `test_postgres_failure`, `test_postgres_session_store` |
| cursor в `Filter` при generic plan | два отдельных запроса | `test_explain.py` |
| `{"text":"hello\u0000world"}` → 500 | 201, текст сохраняется и читается побайтно | `test_unicode_limits_are_code_points_and_nul_is_stored` |
| пароль из 12 пробелов принимается; `Basic é` → 500 | 400; 401 | `test_bad_password_on_registration_is_400`, `test_malformed_basic_is_401_not_500` |
| bcrypt и пароли длиной 73–128 символов → 500 | Argon2id без ограничения длины; 128 кириллических символов работают | `test_edge_passwords_register_and_login` |
| дата в cursor вне диапазона → 500 | 400 | `test_broken_cursors_are_400_not_500` |
| `SESSION_TTL_SECONDS` не доходит до контейнера | compose передаёт его, тест читает `printenv` в контейнере | `test_short_ttl_expires` |
| в инструкции нет создания secrets | `scripts/init_secrets.py` в первой команде | [clean-clone.txt](evidence/clean-clone.txt) |
| нет executable bit у `restart.sh` | `100755` в Git | `git ls-files -s scripts/restart.sh` |
| тесты отказов засчитывают любую ошибку | сравниваются конкретные коды и время, восстановление в `finally` | `test_failures.py` |
## Трек 2. План запроса и восстановление
### Гипотезы
1. С индексом `(chat_id, created_at, id)` и keyset-запросом стоимость страницы почти не зависит от размера чата и позиции страницы. С индексом только по `(chat_id)` keyset-запросу приходится сортировать весь чат. OFFSET читает весь префикс до позиции. Без индекса читается вся таблица.
2. Каждый вторичный индекс удорожает запись: больше WAL и места, ниже пропускная способность.
3. `pg_dump` даёт согласованный снимок. Восстановление 100 000+ сообщений в новое хранилище занимает секунды, сверка сходится полностью, а граница потери данных — момент снимка.
### Методика
- **Изолированная БД** ([plans.py](bench/track2/plans.py)): отдельный контейнер `postgres:18.6` (тот же digest), `--cpus 2 --memory 2g`, настройки по умолчанию, без опубликованных портов. Схема — наши миграции.
- **Данные** — детерминированный seed: **121 120 сообщений в 200 чатах** неравномерно — 1 × 60 000, 2 × 15 000, 8 × 2 000, 189 × 80. Текст 10–380 байт.
- **Генератор:** pgbench (`-M prepared`, как у драйвера) в отдельном контейнере со своими 2 CPU.
- **Чтение:** одна страница `LIMIT 51` после **случайной** позиции чата (200 позиций по всей длине). Запрос keyset — это SQL приложения (`HISTORY_AFTER`), OFFSET — для сравнения. 4 клиента, прогрев 3 с, замер 10 с, 3 повтора. Индексы: `keyset_index` (наш), `chat_id_index`, `no_index` (только PK).
- **Запись:** транзакция сообщения приложения (`UPDATE chats … RETURNING` + `INSERT`) в случайный из 200 чатов.
- TPS и задержки: 8 клиентов, 5 повторов по 10 с.
- WAL: детерминированно — 5 000 транзакций в 1 клиент, autovacuum таблицы выключен, `CHECKPOINT` перед прогоном, 3 повтора. Первые замеры WAL под 8 клиентами давали разброс в 2 раза из-за checkpoint и autovacuum посреди прогона, поэтому методика заменена.
- Индексы: без вторичных, наш, наш + `(sender_id, created_at)`.
- **Размер индексов** — отдельно на одинаковых 121 120 строках ([index_sizes.py](bench/track2/index_sizes.py)).
- **Backup/restore** ([backup_restore.py](bench/track2/backup_restore.py)) на **живой БД стенда**: к данным стенда добавлены те же 121 120 сообщений, всего 171 410–171 898. Во время дампа писатель отправляет сообщения через API (~40/с).
- `pg_dump -Fc -Z 6` → новый контейнер PostgreSQL с **новым volume** → `pg_restore -j 2` → `ANALYZE` → сверка.
- Сверка: количество строк по таблицам, MD5 всех сообщений (`id | created_at | text`) **каждого** чата, первые и последние 50 сообщений самого большого чата, `schema_migrations`, список индексов. Сообщения писателя в копии должны быть **точным префиксом** подтверждённых.
- Затем наш образ API запускается на восстановленной БД: вход по старому паролю и чтение чата писателя через API.
- 3 раунда.
```sh
python bench/track2/plans.py # чтение и запись (~25 мин)
python bench/track2/plans.py --writes-only # только запись (~6 мин)
python bench/track2/index_sizes.py # размеры индексов
python bench/track2/backup_restore.py --repeats 3
pip install -r requirements-bench.txt && python bench/track2/summarize.py # summary.md + plans.png
```
### Результаты: чтение одной страницы
Медиана 3 повторов; в скобках — min–max TPS между повторами. Полная таблица — [summary.md](evidence/track2/summary.md), сырые данные — [reads.csv](evidence/track2/reads.csv).
| Индекс | Запрос | Чат | TPS | p50, мс | p95, мс | p99, мс |
| --- | --- | ---: | ---: | ---: | ---: | ---: |
| **(chat_id, created_at, id)** | **keyset** | 60 000 | **2 655** (2 402–3 486) | **0,81** | **2,28** | 32,8 |
| (chat_id, created_at, id) | keyset | 15 000 | 2 991 | 0,76 | 1,68 | 32,8 |
| (chat_id, created_at, id) | keyset | 80 | 4 018 | 0,58 | 1,69 | 11,9 |
| (chat_id, created_at, id) | OFFSET | 60 000 | 180 | 11,6 | 69,2 | 81,0 |
| (chat_id, created_at, id) | OFFSET | 15 000 | 791 | 2,53 | 43,9 | 52,6 |
| (chat_id) | keyset | 60 000 | 89,5 | 27,0 | 80,9 | 99,0 |
| (chat_id) | keyset | 15 000 | 362 | 5,64 | 55,7 | 59,4 |
| (chat_id) | OFFSET | 60 000 | 28,3 | 120 | 278 | 389 |
| только PK | keyset | 60 000 | 29,0 | 112 | 197 | 293 |
| только PK | keyset | 80 | 36,3 | 102 | 182 | 193 |
![Трек 2](evidence/track2/plans.png)
- **Наш индекс + keyset:** p50 и p95 почти не зависят от размера чата (0,6–0,8 мс и 1,7–2,3 мс от 80 до 60 000 сообщений). Узел плана `Index Scan … Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))` читает ровно 51 строку ([explain/keyset_index-big-late-page.txt](evidence/track2/explain/keyset_index-big-late-page.txt)).
- **OFFSET с тем же индексом:** стоимость растёт с позицией. На чате в 60 000 TPS в 15 раз ниже, p95 в 30 раз выше: индекс читается с начала чата до смещения.
- **Индекс только по `chat_id`:** keyset-запрос вынужден взять все строки чата и сделать top-N sort. На 60 000 TPS в 30 раз ниже (89,5 против 2 655). На маленьких чатах (80) разницы нет — сортировать почти нечего.
- **Без индекса:** около 30 TPS на любом чате, потому что каждый запрос сканирует всю таблицу. Размер чата почти не влияет, влияет размер таблицы.
- **Почему p99 нашего варианта ~33 мс при p50 < 1 мс.** Это хвост стенда, а не плана: Docker Desktop в VM на Windows, CFS-квота `--cpus 2` и общий CPU с генератором. Тот же хвост виден на маленьком чате, где работы почти нет. В плане такого хвоста нет (`Execution Time` 0,1 мс).
### Результаты: стоимость записи
| Вторичные индексы messages | TPS (8 клиентов, min–max) | p50, мс | p95, мс | WAL на вставку, байт (min–max) | Размер вторичных индексов на 121 120 строк |
| --- | ---: | ---: | ---: | ---: | ---: |
| нет | 1 954 (1 540–2 055) | 3,55 | 7,14 | 1 930 (1 884–1 940) | 0 |
| (chat_id, created_at, id) — наш | 1 577 (1 002–1 630) | 4,73 | 8,50 | 2 910 (2 800–2 926) | 6,8 МиБ (59 байт/строку) |
| наш + (sender_id, created_at) | 1 223 (896–1 607) | 6,25 | 9,30 | 3 318 (3 310–3 414) | 9,8 МиБ (85 байт/строку) |
- **WAL** — самая надёжная метрика (разброс 1–3%). Наш индекс добавляет **+51%** WAL на вставку, второй индекс — ещё +14%. WAL — это и запись на диск, и трафик репликации, и объём архива для PITR.
- **TPS** падает примерно на 19% и 37% (медианы), но разброс повторов большой (до ±30%), поэтому это тенденция, а не точная цифра. Транзакция также упирается в блокировку строки чата и fsync COMMIT, общие для всех вариантов.
- Индекс только по `chat_id` занимает 0,9 МиБ против 6,8 МиБ у нашего: B-tree PostgreSQL ≥ 13 дедуплицирует одинаковые ключи. Ради поздних страниц мы сознательно платим местом и записью.
- **Вывод:** наш единственный вторичный индекс окупается — чтение страницы большого чата в 30 раз дешевле альтернатив, а запись дорожает умеренно. Второй индекс без запроса, который им пользуется, — чистая потеря.
### Результаты: backup → восстановление в новое хранилище → сверка
| Раунд | Сообщений | Дамп, с | Размер дампа | Восстановление, с | ANALYZE, с | Сверка | Писатель: подтверждено / в копии / после снимка | API на восстановленной БД |
| ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | --- |
| 1 | 171 410 | 1,47 | 5,1 МиБ | 2,21 | 0,57 | всё совпало (257 чатов) | 237 / 101 / 136 | вход ✓, история = копия ✓ |
| 2 | 171 655 | 1,70 | 5,1 МиБ | 1,90 | 0,49 | всё совпало (258 чатов) | 242 / 109 / 133 | ✓ |
| 3 | 171 898 | 1,40 | 5,1 МиБ | 1,76 | 0,49 | всё совпало (259 чатов) | 250 / 110 / 140 | ✓ |
Данные — [backup.json](evidence/track2/backup.json). «Всё совпало» означает, что одновременно выполнено:
- MD5 сообщений каждого чата;
- первые и последние 50 сообщений чата в 60 000;
- число пользователей, чатов и участников;
- `schema_migrations` и список индексов;
- сообщения писателя в копии — **точный префикс** подтверждённых.
### Время восстановления и границы потери данных
- **Согласованность.** `pg_dump` работает в одной транзакции REPEATABLE READ, то есть это MVCC-снимок на момент старта дампа. В копии нет «рваного» состояния: сообщения писателя — непрерывный префикс без пропусков. Например, в раунде 1 последнее сообщение копии — 17:06:57.831, первое потерянное — 17:06:57.857, а записи шли всё время дампа.
- **RPO.** Теряется всё, что подтверждено **после старта снимка**: в эксперименте 133–140 сообщений за ~3,5 с (дамп плюс 2 с после него). В эксплуатации RPO равен интервалу между дампами: при ночном дампе — до 24 часов подтверждённых сообщений. Сессии в Valkey в этот бэкап не входят: после восстановления пользователи входят заново, что безопасно.
- **RTO на этих данных:** около 2,3–2,8 с = `pg_restore` (1,8–2,2 с, включая перестройку индексов) + `ANALYZE` (0,5 с), плюс переключение `DATABASE_URL` и старт API (~2 с). Время восстановления растёт линейно с объёмом данных и индексов. Дамп 5,1 МиБ на 172 000 сообщений — около 31 байта на сообщение после сжатия.
- **Как уменьшить RPO/RTO:** непрерывный архив WAL и PITR (pgBackRest или WAL-G) сокращают RPO до секунд и позволяют восстановить состояние на любой момент. Потоковая реплика сокращает RTO до времени переключения. Синхронная реплика — RPO≈0 ценой задержки COMMIT. Проверять восстановление нужно регулярно, как этот скрипт: копия, которую не восстанавливали, — ещё не копия.
### Ограничения эксперимента
- Данные синтетические: равномерное время сообщений, один отправитель на чат в seed. На реальном распределении селективность индексов другая.
- Настройки PostgreSQL по умолчанию; данные помещаются в `shared_buffers` и кэш ОС, поэтому замерено «тёплое» чтение. На холодном диске разница между вариантами будет больше.
- pgbench в режиме closed loop; один хост с Docker Desktop (VM WSL2) — хвосты p99 и TPS записи шумные (разброс указан). Повторов 3 и 5, без статистических тестов.
- Backup — логический (`pg_dump`) на небольшом объёме. Для больших баз нужны физические бэкапы с WAL. Valkey не входит в эксперимент восстановления: сессии одноразовые.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `restart.sh`: stop api → restart postgres, valkey → start api | данные и тот же токен работают | PASS (оба режима сессий), все три `StartedAt` новые | ~11 с до готовности; потерь нет |
| `docker compose stop postgres` с действующим токеном | 503, аноним 401, readiness 503 | 503 < 3,6 с на `/users/me`, `/chats`, историю | после старта тот же токен и сообщение на месте |
| `docker pause postgres` | ограниченное время ответа | 503 через 3,0 с (до исправления — 13 с) | после unpause всё работает |
| `LOCK TABLE messages` на 6 с | быстрый отказ, а не ожидание | 503 < 3,6 с (`lock_timeout`) | после снятия блокировки 200 |
| stop или pause Valkey | 503, аноним 401 | 503 за 0,5 с | после старта прежние сессии действуют (AOF) |
| logout во время отказа Valkey | отзыв не теряется | 503 (не отозван) → после восстановления logout 204 → 401 и после рестарта Valkey и API | токен не «оживает» |
| `SESSION_TTL_SECONDS=2` | 401 после `expires_at` | 401 (Valkey и PostgreSQL) | — |
| TTL 12 с + рестарт Valkey и API | срок не продлевается | 200 до `expires_at`, 401 после | — |
| 8 одновременных регистраций одного имени | 1 × 201, 7 × 409 | ровно так, одна строка в БД | — |
| 10 клиентов × 10 сообщений | 100 уникальных, строгий порядок | 100, 15 страниц по 7, уникальное время | — |
| backup во время записи → restore в новое хранилище | согласованный снимок | префикс без пропусков, сверка 100% | RPO = момент снимка; RTO ≈ 2,5 с на 172 000 сообщений |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **Выдерживается:**
- рестарт API и обоих хранилищ без потери подтверждённых данных и сессий;
- отказ или зависание PostgreSQL или Valkey без зависаний и без анонимного доступа;
- гонки регистрации и записи в один чат.
- **Не выдерживается:**
- потеря хоста или диска (один узел, нет реплик и архива WAL);
- отказ Valkey означает отказ всех защищённых запросов: это сознательная плата за отсутствие второй копии сессий. Повысить доступность можно репликой Valkey или `SESSION_STORE=postgres`.
- **Учебные упрощения:** HTTP без TLS (только локальный стенд с синтетическими данными); secret-файлы читаемы на хосте (0644, чтобы их могли прочитать пользователи контейнеров); один экземпляр API (реплики и балансировщик — lab3).
- **Список чатов при одновременном создании.** Он сортируется по `created_at` без глобального замка. Чат, закоммиченный «задним числом», может не попасть в уже идущий обход, но виден при новом. Для сообщений эта проблема закрыта блокировкой строки чата.
- **Подтверждено замерами:** выигрыш индекса и keyset, стоимость WAL, восстановление и сверка. **Гипотезы** (не проверялись): поведение на холодном диске и на данных в десятки ГБ; выигрыш PITR.
+281
View File
@@ -0,0 +1,281 @@
#!/usr/bin/env python3
"""Track 2, part 2: backup -> restore into NEW storage -> reconciliation, with RPO/RTO.
python bench/track2/backup_restore.py --repeats 3 # stand must be running
1. The live stand database gets (once) >= 100 000 synthetic messages with uneven chat sizes.
2. A writer sends messages through the real API (POST /messages) the whole time.
3. `pg_dump -Fc` of the live database (one consistent MVCC snapshot) to ./backups (git-ignored).
4. Restore into NEW storage: a fresh PostgreSQL container with a NEW volume (`pg_restore -j 2`).
5. Reconciliation source vs restored: row counts, per-chat MD5 of every message
(id, created_at, text), the first/last 50 messages of the biggest chat, schema_migrations.
Writer messages: the restored ones must be an exact PREFIX of the confirmed ones
(consistent snapshot, no torn state); the rest is the data-loss window (RPO).
6. The app itself is started against the restored database and reads the writer's chat
with a fresh login.
Results: evidence/track2/backup.json. No passwords or tokens are written.
"""
from __future__ import annotations
import argparse
import base64
import datetime as dt
import json
import secrets
import subprocess
import sys
import threading
import time
import uuid
from pathlib import Path
import httpx2 as httpx
ROOT = Path(__file__).resolve().parents[2]
IMAGE = "postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c"
APP_IMAGE = "mtusi-messenger:lab2"
RESTORE_PG, RESTORE_API, RESTORE_NET, RESTORE_VOL = "mtusi-restore-pg", "mtusi-restore-api", "mtusi-restore-net", "mtusi-restore-pgdata"
BASE = "http://127.0.0.1:8080"
SEED_USER = "00000000-0000-4000-8000-0000000c0001"
SEED_LIVE = f"""
\\set ON_ERROR_STOP on
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{SEED_USER}', 'track2_live', convert_to('Track 2 live', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
CREATE TEMP TABLE sizes AS
SELECT n, CASE WHEN n = 1 THEN 60000 WHEN n <= 3 THEN 15000 WHEN n <= 11 THEN 2000 ELSE 80 END AS size,
md5('track2-live-chat-' || n)::uuid AS chat_id
FROM generate_series(1::int, 200::int) AS n;
INSERT INTO chats (id, title, created_by, created_at)
SELECT chat_id, convert_to('track2 live ' || n, 'UTF8'), '{SEED_USER}', timestamptz '2026-02-01 00:00:00+00'
FROM sizes ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT chat_id, '{SEED_USER}', 0, timestamptz '2026-02-01 00:00:00+00' FROM sizes ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), s.chat_id, '{SEED_USER}',
convert_to(left(repeat('сообщение ', 40), 5 + (random() * 195)::int), 'UTF8'),
timestamptz '2026-02-01 00:00:00+00' + g * interval '1 second'
FROM sizes s CROSS JOIN LATERAL generate_series(1::int, s.size) AS g
WHERE NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = s.chat_id);
UPDATE chats c SET last_message_at = m.mx FROM (SELECT chat_id, max(created_at) mx FROM messages GROUP BY chat_id) m
WHERE m.chat_id = c.id AND c.created_by = '{SEED_USER}';
COMMIT;
"""
RECONCILE_SQL = """
\\pset footer off
SELECT 'count:' || t || ':' || n FROM (
SELECT 'users' t, count(*) n FROM users UNION ALL SELECT 'chats', count(*) FROM chats
UNION ALL SELECT 'chat_members', count(*) FROM chat_members UNION ALL SELECT 'messages', count(*) FROM messages
UNION ALL SELECT 'schema_migrations', count(*) FROM schema_migrations) x ORDER BY t;
SELECT 'chat:' || chat_id || ':' || count(*) || ':' ||
md5(string_agg(id::text || '|' || created_at::text || '|' || encode(text, 'hex'), ',' ORDER BY created_at, id))
FROM messages WHERE chat_id <> '{writer_chat}' GROUP BY chat_id ORDER BY chat_id;
SELECT 'big:' || md5(string_agg(x, ',')) FROM (
(SELECT id::text || '|' || encode(text, 'hex') AS x FROM messages WHERE chat_id = md5('track2-live-chat-1')::uuid
ORDER BY created_at, id LIMIT 50)
UNION ALL
(SELECT * FROM (SELECT id::text || '|' || encode(text, 'hex') FROM messages WHERE chat_id = md5('track2-live-chat-1')::uuid
ORDER BY created_at DESC, id DESC LIMIT 50) y)) z;
SELECT 'mig:' || string_agg(version || '=' || checksum, ',' ORDER BY version) FROM schema_migrations;
SELECT 'idx:' || string_agg(indexname, ',' ORDER BY indexname) FROM pg_indexes WHERE schemaname = 'public';
SELECT 'writer:' || string_agg(id::text, ',' ORDER BY created_at, id) FROM messages WHERE chat_id = '{writer_chat}';
"""
def sh(args, *, input_bytes: bytes | None = None, check: bool = True, timeout: float = 600) -> subprocess.CompletedProcess:
result = subprocess.run(args, input=input_bytes, capture_output=True, timeout=timeout, cwd=ROOT)
if check and result.returncode != 0:
raise RuntimeError(f"{' '.join(map(str, args[:5]))}: {result.stderr.decode(errors='replace')[-1500:]}")
return result
def live_psql(sql: str) -> str:
return sh(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-X", "-q", "-At", "-f", "-"], input_bytes=sql.encode()).stdout.decode().strip()
def restored_psql(sql: str) -> str:
return sh(["docker", "exec", "-i", RESTORE_PG, "psql", "-U", "messenger", "-d", "messenger",
"-X", "-q", "-At", "-f", "-"], input_bytes=sql.encode()).stdout.decode().strip()
def parse(text: str) -> dict:
out: dict = {"count": {}, "chat": {}}
for line in text.splitlines():
kind, _, rest = line.partition(":")
if kind == "count":
name, _, value = rest.partition(":")
out["count"][name] = int(value)
elif kind == "chat":
chat, _, digest = rest.partition(":")
out["chat"][chat] = digest
else:
out[kind] = rest
return out
class Writer(threading.Thread):
"""Real API traffic during the backup: one message every ~10 ms, confirmations recorded."""
def __init__(self):
super().__init__(daemon=True)
self.client = httpx.Client(base_url=BASE, timeout=10)
self.username = "writer_" + uuid.uuid4().hex[:10]
self.password = secrets.token_urlsafe(16)
user = self.client.post("/api/v1/users", json={"username": self.username, "display_name": "Writer",
"password": self.password}).json()
auth = base64.b64encode(f"{self.username}:{self.password}".encode()).decode()
token = self.client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + auth}).json()["token"]
self.headers = {"Authorization": "Bearer " + token}
self.chat = self.client.post("/api/v1/chats", headers=self.headers,
json={"title": "writer", "member_ids": [user["id"]]}).json()["id"]
self.confirmed: list[tuple[str, str]] = []
self.errors = 0
self.stop = threading.Event()
def run(self) -> None:
n = 0
while not self.stop.is_set():
r = self.client.post(f"/api/v1/chats/{self.chat}/messages", headers=self.headers, json={"text": f"w{n}"})
if r.status_code == 201:
self.confirmed.append((r.json()["id"], r.json()["created_at"]))
else:
self.errors += 1
n += 1
time.sleep(0.01)
def new_storage() -> str:
for cmd in (["docker", "rm", "-f", "-v", RESTORE_API, RESTORE_PG], ["docker", "volume", "rm", "-f", RESTORE_VOL],
["docker", "network", "rm", RESTORE_NET]):
sh(cmd, check=False)
sh(["docker", "network", "create", RESTORE_NET])
password = secrets.token_urlsafe(18)
sh(["docker", "run", "-d", "--name", RESTORE_PG, "--network", RESTORE_NET, "--cpus", "2", "--memory", "2g",
"-e", f"POSTGRES_PASSWORD={password}", "-e", "POSTGRES_USER=messenger", "-e", "POSTGRES_DB=messenger",
"-v", f"{RESTORE_VOL}:/var/lib/postgresql", IMAGE])
for _ in range(120):
if sh(["docker", "exec", RESTORE_PG, "pg_isready", "-U", "messenger", "-d", "messenger", "-q"], check=False).returncode == 0:
time.sleep(1)
return password
time.sleep(0.5)
raise RuntimeError("restore PostgreSQL did not start")
def app_on_restored(password: str, writer: Writer, expected_ids: list[str]) -> dict:
"""Run our API image against the restored DB (sessions in PostgreSQL) and read the writer chat."""
url = f"postgresql://messenger:{password}@{RESTORE_PG}:5432/messenger"
sh(["docker", "run", "-d", "--name", RESTORE_API, "--network", RESTORE_NET, "-p", "127.0.0.1:18090:8080",
"-e", "SESSION_STORE=postgres", "-e", f"DATABASE_URL={url}", "-e", "DATABASE_URL_FILE=", APP_IMAGE])
client = httpx.Client(base_url="http://127.0.0.1:18090", timeout=10)
for _ in range(60):
try:
if client.get("/health/ready").status_code == 200:
break
except httpx.HTTPError:
pass
time.sleep(0.5)
auth = base64.b64encode(f"{writer.username}:{writer.password}".encode()).decode()
token = client.post("/api/v1/auth/sessions", headers={"Authorization": "Basic " + auth}).json()["token"]
seen, cursor = [], None
while True:
q = "?limit=100" + (f"&cursor={cursor}" if cursor else "")
page = client.get(f"/api/v1/chats/{writer.chat}/messages{q}", headers={"Authorization": "Bearer " + token}).json()
seen += [m["id"] for m in page["items"]]
cursor = page["next_cursor"]
if not cursor:
break
sh(["docker", "rm", "-f", RESTORE_API], check=False)
return {"login_ok": True, "history_via_api_equals_restored": seen == expected_ids, "messages_read": len(seen)}
def one_round(round_no: int, backups: Path) -> dict:
writer = Writer()
writer.start()
time.sleep(2) # writer is in steady state before the backup starts
dump_file = backups / f"messenger-round{round_no}.dump"
started_wall = dt.datetime.now(dt.timezone.utc)
started = time.monotonic()
with dump_file.open("wb") as f:
proc = subprocess.run(["docker", "compose", "exec", "-T", "postgres", "pg_dump", "-U", "messenger", "-d", "messenger",
"-Fc", "-Z", "6"], stdout=f, stderr=subprocess.PIPE, cwd=ROOT, timeout=600)
if proc.returncode != 0:
raise RuntimeError(proc.stderr.decode()[-1000:])
dump_s = time.monotonic() - started
time.sleep(2) # keep writing after the dump: these confirmed messages are not in the backup
writer.stop.set()
writer.join()
source = parse(live_psql(RECONCILE_SQL.format(writer_chat=writer.chat)))
password = new_storage()
started = time.monotonic()
sh(["docker", "exec", "-i", RESTORE_PG, "sh", "-c", "cat > /tmp/backup.dump"], input_bytes=dump_file.read_bytes())
sh(["docker", "exec", RESTORE_PG, "pg_restore", "-U", "messenger", "-d", "messenger", "--no-owner", "-j", "2",
"--exit-on-error", "/tmp/backup.dump"])
restore_s = time.monotonic() - started
started = time.monotonic()
restored_psql("ANALYZE;")
analyze_s = time.monotonic() - started
restored = parse(restored_psql(RECONCILE_SQL.format(writer_chat=writer.chat)))
confirmed_ids = [m[0] for m in writer.confirmed]
restored_writer = restored.get("writer", "").split(",") if restored.get("writer") else []
prefix_ok = restored_writer == confirmed_ids[:len(restored_writer)]
lost = confirmed_ids[len(restored_writer):]
last_in_backup = writer.confirmed[len(restored_writer) - 1][1] if restored_writer else None
first_lost = writer.confirmed[len(restored_writer)][1] if lost else None
other_counts_equal = {k: source["count"][k] == restored["count"][k] for k in source["count"] if k != "messages"}
api_check = app_on_restored(password, writer, restored_writer)
result = {
"round": round_no, "backup_started_utc": started_wall.isoformat(timespec="seconds"),
"dump_seconds": round(dump_s, 2), "dump_bytes": dump_file.stat().st_size,
"restore_seconds": round(restore_s, 2), "analyze_seconds": round(analyze_s, 2),
"rto_seconds_restore_plus_analyze": round(restore_s + analyze_s, 2),
"source_counts_after_writer_stopped": source["count"], "restored_counts": restored["count"],
"messages_in_restored": restored["count"]["messages"],
"per_chat_md5_equal": source["chat"] == restored["chat"], "chats_compared": len(source["chat"]),
"big_chat_first_last_50_equal": source["big"] == restored["big"],
"schema_migrations_equal": source["mig"] == restored["mig"], "indexes_equal": source["idx"] == restored["idx"],
"other_table_counts_equal": other_counts_equal,
"writer": {"confirmed": len(confirmed_ids), "errors": writer.errors, "in_backup": len(restored_writer),
"restored_is_exact_prefix": prefix_ok, "lost_confirmed_after_snapshot": len(lost),
"last_message_in_backup_at": last_in_backup, "first_lost_message_at": first_lost},
"app_on_restored_db": api_check,
}
result["reconciled"] = bool(result["per_chat_md5_equal"] and result["big_chat_first_last_50_equal"]
and result["schema_migrations_equal"] and result["indexes_equal"]
and all(other_counts_equal.values()) and prefix_ok
and api_check["history_via_api_equals_restored"])
return result
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--out", default=str(ROOT / "evidence/track2/backup.json"))
args = parser.parse_args()
backups = ROOT / "backups"
backups.mkdir(exist_ok=True)
live_psql(SEED_LIVE)
rounds = []
try:
for i in range(1, args.repeats + 1):
r = one_round(i, backups)
rounds.append(r)
print(json.dumps({k: r[k] for k in ("round", "dump_seconds", "dump_bytes", "restore_seconds", "messages_in_restored",
"reconciled")} | {"writer": r["writer"]}, ensure_ascii=False), flush=True)
finally:
for cmd in (["docker", "rm", "-f", "-v", RESTORE_API, RESTORE_PG], ["docker", "volume", "rm", "-f", RESTORE_VOL],
["docker", "network", "rm", RESTORE_NET]):
sh(cmd, check=False)
Path(args.out).parent.mkdir(parents=True, exist_ok=True)
Path(args.out).write_text(json.dumps({"image": IMAGE, "rounds": rounds}, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8", newline="\n")
print("done:", args.out)
return 0 if all(r["reconciled"] for r in rounds) else 1
if __name__ == "__main__":
sys.exit(main())
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/env python3
"""Index sizes of every variant on the SAME 121 120-row dataset (no writes in between).
python bench/track2/index_sizes.py # ~2 min, writes evidence/track2/index-sizes.csv
"""
from __future__ import annotations
import csv
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).parent))
import plans # noqa: E402 (same image, limits, schema and deterministic seed)
def main() -> int:
plans.start_container("2", "2g")
try:
plans.psql("".join(p.read_text(encoding="utf-8") for p in sorted((plans.ROOT / "migrations").glob("*.sql"))))
plans.psql(plans.SEED)
rows = int(plans.psql("SELECT count(*) FROM messages;"))
out = []
for variant in plans.VARIANTS:
info = plans.use_variant(variant)
secondary = sum(int(b) for name, b in info["index_bytes"].items() if name != "messages_pkey")
out.append({"variant": variant, "rows": rows, "table_bytes": info["table_bytes"],
"pkey_bytes": int(info["index_bytes"]["messages_pkey"]), "secondary_index_bytes": secondary,
"secondary_bytes_per_row": round(secondary / rows, 1), "index_build_seconds": info["index_build_s"]})
print(out[-1], flush=True)
finally:
plans.sh(["docker", "rm", "-f", "-v", plans.CLIENT, plans.NAME], check=False)
path = plans.ROOT / "evidence/track2/index-sizes.csv"
with path.open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=list(out[0]))
writer.writeheader()
writer.writerows(out)
print("done:", path)
return 0
if __name__ == "__main__":
sys.exit(main())
+286
View File
@@ -0,0 +1,286 @@
#!/usr/bin/env python3
"""Track 2, part 1: query plans and write cost on >= 100 000 messages with uneven chat sizes.
python bench/track2/plans.py # ~20 min, results in evidence/track2/
An ISOLATED PostgreSQL container (same pinned image as compose.yaml, --cpus 2 --memory 2g,
no published ports) gets the schema from migrations/*.sql and a deterministic synthetic
dataset: 1 chat x 60 000, 2 x 15 000, 8 x 2 000, 189 x 80 messages = 121 120 messages,
texts of 10..400 bytes. The live stand is not touched: indexes are dropped and created
here freely.
For every index variant the script runs pgbench (prepared statements, like the driver):
* reads — one history page (LIMIT 51) after a RANDOM position of a big/medium/small chat,
keyset query (the app's SQL) vs OFFSET; 4 clients;
* writes — the app's message transaction (UPDATE chats ... RETURNING + INSERT); 8 clients;
and records TPS, p50/p95/p99 latency, WAL bytes per insert, index/table sizes, EXPLAIN.
"""
from __future__ import annotations
import argparse
import csv
import json
import math
import platform
import secrets
import statistics
import subprocess
import sys
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
sys.path.insert(0, str(ROOT / "src"))
from messenger.repo import HISTORY_AFTER # noqa: E402 (the app's real statement)
IMAGE = "postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c"
NAME = "mtusi-track2-pg"
CLIENT = "mtusi-track2-client" # pgbench runs here: it must not eat the server's CPU quota
PASSWORD = secrets.token_urlsafe(16) # throwaway, never printed
USER_ID = "00000000-0000-4000-8000-0000000b0001"
VARIANTS = {
# name: (DDL of secondary indexes on messages, description)
"keyset_index": (["CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id)"],
"(chat_id, created_at, id) — used by the app"),
"chat_id_index": (["CREATE INDEX messages_chat_only ON messages (chat_id)"],
"(chat_id) only"),
"no_index": ([], "primary key only"),
"keyset_plus_sender": (["CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id)",
"CREATE INDEX messages_sender_time ON messages (sender_id, created_at)"],
"app index + a second index (sender_id, created_at)"),
}
READ_VARIANTS = ["keyset_index", "chat_id_index", "no_index"]
WRITE_VARIANTS = ["no_index", "keyset_index", "keyset_plus_sender"]
KINDS = {"big": 60_000, "medium": 15_000, "small": 80}
def sh(args: list[str], input_text: str | None = None, check: bool = True) -> str:
result = subprocess.run(args, input=input_text, capture_output=True, text=True, encoding="utf-8")
if check and result.returncode != 0:
raise RuntimeError(f"{' '.join(args[:4])}: {result.stderr.strip()[-1500:]}")
return result.stdout
def psql(sql: str) -> str:
return sh(["docker", "exec", "-i", NAME, "psql", "-U", "postgres", "-d", "bench", "-X", "-q", "-At",
"-v", "ON_ERROR_STOP=1", "-f", "-"], input_text=sql).strip()
def put(path: str, content: str) -> None:
sh(["docker", "exec", "-i", CLIENT, "sh", "-c", f"cat > {path}"], input_text=content)
def start_container(cpus: str, memory: str) -> None:
sh(["docker", "rm", "-f", "-v", CLIENT, NAME], check=False)
sh(["docker", "run", "-d", "--name", NAME, "--cpus", cpus, "--memory", memory,
"-e", f"POSTGRES_PASSWORD={PASSWORD}", "-e", "POSTGRES_DB=bench", IMAGE])
# Load generator: separate container (own CPU quota) sharing the server's network namespace.
sh(["docker", "run", "-d", "--name", CLIENT, "--network", f"container:{NAME}", "--cpus", "2",
"-e", f"PGPASSWORD={PASSWORD}", "--entrypoint", "sleep", IMAGE, "infinity"])
for _ in range(120):
if subprocess.run(["docker", "exec", NAME, "pg_isready", "-U", "postgres", "-d", "bench", "-q"]).returncode == 0:
time.sleep(1)
return
time.sleep(0.5)
raise RuntimeError("experiment PostgreSQL did not start")
SEED = f"""
SELECT setseed(0.42);
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{USER_ID}', 'track2_bench', convert_to('Track 2', 'UTF8'), 'not-a-login');
CREATE TABLE bench_sizes AS
SELECT n, CASE WHEN n = 1 THEN 60000 WHEN n <= 3 THEN 15000 WHEN n <= 11 THEN 2000 ELSE 80 END AS size,
md5('track2-chat-' || n)::uuid AS chat_id
FROM generate_series(1::int, 200::int) AS n;
INSERT INTO chats (id, title, created_by, created_at)
SELECT chat_id, convert_to('chat ' || n, 'UTF8'), '{USER_ID}', timestamptz '2026-01-01 00:00:00+00' FROM bench_sizes;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT chat_id, '{USER_ID}', 0, timestamptz '2026-01-01 00:00:00+00' FROM bench_sizes;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), s.chat_id, '{USER_ID}',
convert_to(left(repeat('сообщение ', 40), 5 + (random() * 195)::int), 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM bench_sizes s CROSS JOIN LATERAL generate_series(1::int, s.size) AS g;
UPDATE chats c SET last_message_at = m.mx FROM (SELECT chat_id, max(created_at) mx FROM messages GROUP BY chat_id) m
WHERE m.chat_id = c.id;
CREATE TABLE bench_chats AS SELECT n, chat_id AS id FROM bench_sizes;
CREATE TABLE bench_cursors AS
SELECT kind, row_number() OVER (PARTITION BY kind ORDER BY rn)::int AS n, created_at AS ts, id AS mid, rn::int AS off
FROM (SELECT CASE s.n WHEN 1 THEN 'big' WHEN 2 THEN 'medium' ELSE 'small' END AS kind, m.created_at, m.id,
row_number() OVER (PARTITION BY m.chat_id ORDER BY m.created_at, m.id) AS rn, s.size
FROM messages m JOIN bench_sizes s ON s.chat_id = m.chat_id WHERE s.n IN (1, 2, 12)) x
WHERE rn % greatest(1, size / 200) = 0 AND rn < size;
CREATE INDEX ON bench_cursors (kind, n);
VACUUM ANALYZE;
"""
def pg_param(sql: str) -> str:
"""psycopg %s placeholders -> pgbench :variables (passed as parameters in prepared mode)."""
names = iter([":chat", ":cts", ":cid", "51"])
out = sql
while "%s" in out:
out = out.replace("%s", next(names), 1)
return out
READ_SCRIPTS = {
"keyset": "\\set k random(1, :ncur)\n"
"SELECT ts AS cts, mid AS cid FROM bench_cursors WHERE kind = :kind AND n = :k \\gset\n"
+ pg_param(HISTORY_AFTER) + ";\n",
"offset": "\\set k random(1, :ncur)\n"
"SELECT off FROM bench_cursors WHERE kind = :kind AND n = :k \\gset\n"
"SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = :chat "
"ORDER BY created_at, id OFFSET :off LIMIT 51;\n",
}
WRITE_SCRIPT = (
"\\set c random(1, 200)\n"
"SELECT id AS chat FROM bench_chats WHERE n = :c \\gset\n"
"BEGIN;\n"
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + interval '1 microsecond') "
"WHERE id = :chat RETURNING last_message_at AS ts \\gset\n"
f"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES "
f"(gen_random_uuid(), :chat, '{USER_ID}', convert_to(repeat('w', 120), 'UTF8'), :ts);\n"
"END;\n"
)
def pgbench(script_path: str, clients: int, seconds: int, defines: dict[str, str]) -> dict:
"""Run pgbench with per-transaction logging; return TPS and latency percentiles (ms)."""
sh(["docker", "exec", CLIENT, "sh", "-c", "rm -f /tmp/pgblog*"])
args = ["docker", "exec", CLIENT, "sh", "-c",
"cd /tmp && pgbench -h 127.0.0.1 -U postgres -n -M prepared -c {c} -j {c} -T {t} -l --log-prefix=pgblog {d} -f {f} bench".format(
c=clients, t=seconds, f=script_path, d=" ".join(f"-D {k}={v}" for k, v in defines.items()))]
out = sh(args)
tps = float(next(l for l in out.splitlines() if l.startswith("tps = ")).split()[2])
failed = next((l for l in out.splitlines() if "number of failed transactions" in l), "0").split(":")[-1].split()[0]
lat = sorted(int(line.split()[2]) / 1000 for line in sh(["docker", "exec", CLIENT, "sh", "-c", "cat /tmp/pgblog*"]).splitlines())
def pct(p: float) -> float:
return round(lat[max(0, math.ceil(p * len(lat)) - 1)], 3)
return {"tps": round(tps, 1), "transactions": len(lat), "failed": int(failed),
"p50_ms": pct(.50), "p95_ms": pct(.95), "p99_ms": pct(.99), "mean_ms": round(statistics.mean(lat), 3)}
def use_variant(name: str) -> dict:
psql("DROP INDEX IF EXISTS messages_chat_keyset; DROP INDEX IF EXISTS messages_chat_only; "
"DROP INDEX IF EXISTS messages_sender_time;")
started = time.monotonic()
for ddl in VARIANTS[name][0]:
psql(ddl + ";")
build_s = round(time.monotonic() - started, 2)
psql("VACUUM ANALYZE messages;")
sizes = psql("SELECT indexrelname || '=' || pg_relation_size(indexrelid) FROM pg_stat_user_indexes "
"WHERE relname = 'messages' ORDER BY 1;").split()
return {"index_build_s": build_s, "index_bytes": dict(s.split("=") for s in sizes),
"table_bytes": int(psql("SELECT pg_relation_size('messages');"))}
def explain_late(kind: str) -> str:
chat = psql(f"SELECT chat_id FROM bench_sizes WHERE n = {dict(big=1, medium=2, small=12)[kind]};")
return psql(f"""
SET plan_cache_mode = force_generic_plan;
PREPARE p(uuid, timestamptz, uuid, int) AS {HISTORY_AFTER.replace('%s', '$1', 1).replace('%s', '$2', 1).replace('%s', '$3', 1).replace('%s', '$4', 1)};
SELECT ts, mid FROM bench_cursors WHERE kind = '{kind}' ORDER BY n DESC LIMIT 1 \\gset
EXPLAIN (ANALYZE, BUFFERS) EXECUTE p('{chat}', :'ts', :'mid', 51);
""")
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--seconds", type=int, default=10)
parser.add_argument("--warmup", type=int, default=3)
parser.add_argument("--repeats", type=int, default=3)
parser.add_argument("--out", default=str(ROOT / "evidence/track2"))
parser.add_argument("--writes-only", action="store_true", help="re-measure writes, keep reads.csv")
parser.add_argument("--write-repeats", type=int, default=5)
args = parser.parse_args()
out = Path(args.out)
(out / "explain").mkdir(parents=True, exist_ok=True)
start_container("2", "2g")
schema = "".join(p.read_text(encoding="utf-8") for p in sorted((ROOT / "migrations").glob("*.sql")))
psql(schema)
seed_started = time.monotonic()
psql(SEED)
seed_s = round(time.monotonic() - seed_started, 1)
stats = psql("SELECT count(*), count(DISTINCT chat_id), min(octet_length(text)), max(octet_length(text)) FROM messages;")
sizes = psql("SELECT size || 'x' || count(*) FROM bench_sizes GROUP BY size ORDER BY size DESC;").split()
for name, script in READ_SCRIPTS.items():
put(f"/tmp/read_{name}.sql", script)
put("/tmp/write.sql", WRITE_SCRIPT)
chats = {k: psql(f"SELECT chat_id FROM bench_sizes WHERE n = {dict(big=1, medium=2, small=12)[k]};") for k in KINDS}
ncur = {k: int(psql(f"SELECT count(*) FROM bench_cursors WHERE kind = '{k}';")) for k in KINDS}
meta = {"image": IMAGE, "limits": {"server_cpus": 2, "server_memory": "2g", "pgbench_client_cpus": 2}, "pg_settings": "image defaults",
"dataset": {"messages_chats_minbytes_maxbytes": stats, "chat_sizes": sizes, "seed_seconds": seed_s},
"pgbench": {"mode": "prepared", "seconds": args.seconds, "warmup": args.warmup, "repeats": args.repeats,
"read_clients": 4, "write_clients": 8, "page": "LIMIT 51 after a random position"},
"host": {"platform": platform.platform(), "docker": sh(["docker", "version", "-f", "{{.Server.Version}}"]).strip()},
"variants": {k: v[1] for k, v in VARIANTS.items()}, "sizes": {}}
print("dataset:", stats, sizes, flush=True)
reads, writes = [], []
if args.writes_only:
old = json.loads((out / "plans-meta.json").read_text(encoding="utf-8"))
meta["sizes"] = old.get("sizes", {})
for variant in ([] if args.writes_only else READ_VARIANTS):
meta["sizes"][variant] = use_variant(variant)
for kind in KINDS:
(out / "explain" / f"{variant}-{kind}-late-page.txt").write_text(explain_late(kind), encoding="utf-8", newline="\n")
for rep in range(1, args.repeats + 1):
for query in READ_SCRIPTS:
for kind in KINDS:
defines = {"chat": chats[kind], "kind": kind, "ncur": str(ncur[kind])}
pgbench(f"/tmp/read_{query}.sql", 4, args.warmup, defines) # warm-up, discarded
row = {"variant": variant, "query": query, "chat": kind, "chat_size": KINDS[kind], "repeat": rep,
**pgbench(f"/tmp/read_{query}.sql", 4, args.seconds, defines)}
reads.append(row)
print("read ", row, flush=True)
wal_rows = []
for variant in WRITE_VARIANTS:
meta["sizes"][variant] = use_variant(variant)
# (a) throughput/latency under concurrency: 8 clients, time-based, autovacuum as in production
for rep in range(1, args.write_repeats + 1):
pgbench("/tmp/write.sql", 8, args.warmup, {})
row = {"variant": variant, "repeat": rep, **pgbench("/tmp/write.sql", 8, args.seconds, {})}
writes.append(row)
print("write", row, flush=True)
# (b) WAL volume per insert, deterministic: the same 5 000 app transactions (1 client),
# autovacuum off for the table and a CHECKPOINT right before, so every run starts from
# the same state and only the index set differs.
psql("ALTER TABLE messages SET (autovacuum_enabled = off);")
for rep in range(1, 4):
psql("CHECKPOINT;")
lsn0 = psql("SELECT pg_current_wal_lsn();")
sh(["docker", "exec", CLIENT, "pgbench", "-h", "127.0.0.1", "-U", "postgres", "-n", "-M", "prepared",
"-c", "1", "-t", "5000", "-f", "/tmp/write.sql", "bench"])
wal = int(psql(f"SELECT pg_current_wal_lsn() - '{lsn0}'::pg_lsn;"))
wal_row = {"variant": variant, "repeat": rep, "inserts": 5000, "wal_bytes": wal,
"wal_bytes_per_insert": round(wal / 5000)}
wal_rows.append(wal_row)
print("wal ", wal_row, flush=True)
psql("ALTER TABLE messages RESET (autovacuum_enabled);")
sh(["docker", "rm", "-f", "-v", CLIENT, NAME], check=False)
with (out / "wal.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=list(wal_rows[0]))
writer.writeheader()
writer.writerows(wal_rows)
meta["pgbench"]["write_repeats"] = args.write_repeats
meta["pgbench"]["wal_method"] = "5000 app transactions, 1 client, autovacuum off, CHECKPOINT before; 3 repeats"
for name, rows in (("reads", reads), ("writes", writes)):
if not rows:
continue
with (out / f"{name}.csv").open("w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=list(rows[0]))
writer.writeheader()
writer.writerows(rows)
(out / "plans-meta.json").write_text(json.dumps(meta, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print("done:", out)
return 0
if __name__ == "__main__":
sys.exit(main())
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Aggregate track 2 raw data into tables (summary.md) and charts (plans.png).
python bench/track2/summarize.py # matplotlib from requirements-bench.txt for the chart
"""
from __future__ import annotations
import csv
import json
import statistics
import sys
from collections import defaultdict
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
OUT = ROOT / "evidence/track2"
SIZE_ORDER = {"big": 0, "medium": 1, "small": 2}
def load(name: str) -> list[dict]:
with (OUT / name).open(encoding="utf-8") as f:
return list(csv.DictReader(f))
def agg(rows: list[dict], keys: tuple[str, ...], fields: tuple[str, ...]) -> dict:
groups: dict = defaultdict(list)
for r in rows:
groups[tuple(r[k] for k in keys)].append(r)
out = {}
for k, g in groups.items():
out[k] = {f: statistics.median(float(r[f]) for r in g) for f in fields}
out[k]["tps_min"] = min(float(r["tps"]) for r in g)
out[k]["tps_max"] = max(float(r["tps"]) for r in g)
out[k]["n"] = len(g)
return out
def fmt(x: float) -> str:
return f"{x:,.0f}".replace(",", " ") if x >= 100 else (f"{x:.1f}" if x >= 10 else f"{x:.2f}")
def main() -> int:
sys.stdout.reconfigure(encoding="utf-8")
reads, writes, wal = load("reads.csv"), load("writes.csv"), load("wal.csv")
sizes_rows = {r["variant"]: r for r in load("index-sizes.csv")}
meta = json.loads((OUT / "plans-meta.json").read_text(encoding="utf-8"))
r = agg(reads, ("variant", "query", "chat"), ("tps", "p50_ms", "p95_ms", "p99_ms"))
w = agg(writes, ("variant",), ("tps", "p50_ms", "p95_ms", "p99_ms"))
wal_by = {v: [int(r["wal_bytes_per_insert"]) for r in wal if r["variant"] == v] for v in {r["variant"] for r in wal}}
for v, values in wal_by.items():
w[(v,)]["wal_bytes_per_insert"] = statistics.median(values)
w[(v,)]["wal_min"], w[(v,)]["wal_max"] = min(values), max(values)
sizes = {"big": 60000, "medium": 15000, "small": 80}
md = ["## Чтение: одна страница (LIMIT 51) со случайной позиции чата", "",
"Медиана 3 повторов по 10 с, 4 клиента pgbench, prepared statements. TPS min–max — разброс повторов.", "",
"| Индекс | Запрос | Чат (сообщений) | TPS (min–max) | p50, мс | p95, мс | p99, мс |",
"| --- | --- | ---: | ---: | ---: | ---: | ---: |"]
for (variant, query, chat), v in sorted(r.items(), key=lambda kv: (kv[0][0] != "keyset_index", kv[0][0], kv[0][1], SIZE_ORDER[kv[0][2]])):
md.append(f"| {variant} | {query} | {chat} ({sizes[chat]:,}) | {fmt(v['tps'])} ({fmt(v['tps_min'])}–{fmt(v['tps_max'])}) "
f"| {fmt(v['p50_ms'])} | {fmt(v['p95_ms'])} | {fmt(v['p99_ms'])} |".replace(",", " ", 1))
md += ["", "## Запись: транзакция сообщения приложения (UPDATE chats … RETURNING + INSERT)", "",
"TPS и задержка — медиана 5 повторов по 10 с, 8 клиентов. WAL — медиана 3 детерминированных прогонов: "
"5 000 транзакций, 1 клиент, autovacuum выключен, CHECKPOINT перед прогоном.", "",
"| Индексы messages | TPS (min–max) | p50, мс | p95, мс | p99, мс | WAL, байт/вставку (min–max) | Доп. индексы на 121 120 строк, МиБ |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |"]
for variant in ("no_index", "keyset_index", "keyset_plus_sender"):
v = w[(variant,)]
idx_mib = int(sizes_rows[variant]["secondary_index_bytes"]) / 2**20 # same dataset for every variant
md.append(f"| {variant} | {fmt(v['tps'])} ({fmt(v['tps_min'])}–{fmt(v['tps_max'])}) | {fmt(v['p50_ms'])} | {fmt(v['p95_ms'])} "
f"| {fmt(v['p99_ms'])} | {v['wal_bytes_per_insert']:.0f} ({v['wal_min']}–{v['wal_max']}) | {idx_mib:.1f} |")
md += ["", "Размеры индексов измерены отдельно на одинаковых 121 120 строках (bench/track2/index_sizes.py); "
"первичный ключ во всех вариантах — " + f"{int(sizes_rows['no_index']['pkey_bytes']) / 2**20:.1f} МиБ, таблица — "
f"{int(sizes_rows['no_index']['table_bytes']) / 2**20:.1f} МиБ."]
(OUT / "summary.md").write_text("\n".join(md) + "\n", encoding="utf-8", newline="\n")
print("\n".join(md))
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
except ImportError:
print("matplotlib is not installed: skipping plans.png")
return 0
fig, axes = plt.subplots(1, 2, figsize=(15, 5))
chats = ["small", "medium", "big"]
x = [sizes[c] for c in chats]
styles = {("keyset_index", "keyset"): ("#12a150", "-o"), ("keyset_index", "offset"): ("#12a150", "--s"),
("chat_id_index", "keyset"): ("#e07b00", "-o"), ("chat_id_index", "offset"): ("#e07b00", "--s"),
("no_index", "keyset"): ("#b03060", "-o"), ("no_index", "offset"): ("#b03060", "--s")}
for (variant, query), (color, style) in styles.items():
axes[0].plot(x, [r[(variant, query, c)]["p95_ms"] for c in chats], style, color=color, label=f"{variant}, {query}")
axes[0].set_xscale("log")
axes[0].set_yscale("log")
axes[0].set_xlabel("messages in the chat")
axes[0].set_ylabel("p95 latency of one page, ms (log)")
axes[0].set_title("Read: random page of a chat")
axes[0].grid(True, which="both", alpha=0.3)
axes[0].legend(fontsize=8)
names = ["no_index", "keyset_index", "keyset_plus_sender"]
axes[1].bar([n.replace("_", "\n") for n in names], [w[(n,)]["wal_bytes_per_insert"] for n in names], color="#2a6fdb")
for i, n in enumerate(names):
axes[1].text(i, w[(n,)]["wal_bytes_per_insert"], f"{w[(n,)]['tps']:.0f} TPS", ha="center", va="bottom")
axes[1].set_ylabel("WAL bytes per inserted message")
axes[1].set_title("Write cost of indexes (WAL: 5000 inserts; TPS: 8 clients)")
fig.suptitle("Track 2: 121 120 messages, chats of 60 000 / 15 000 / 2 000 / 80; PostgreSQL 18.6, 2 CPU / 2 GiB")
fig.tight_layout()
fig.savefig(OUT / "plans.png", dpi=120)
print("chart:", OUT / "plans.png")
return 0
if __name__ == "__main__":
sys.exit(main())
+108
View File
@@ -0,0 +1,108 @@
# Lab 2 stand: API + PostgreSQL + Valkey. Start from a clean clone:
# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored)
# cp .env.example .env # optional overrides
# docker compose up -d --build --wait
# Images are pinned by version AND digest. Passwords are passed only as secret files.
name: mtusi-lab2
x-api-env: &api-env
APP_ENV: ${APP_ENV:-lab}
LOG_LEVEL: ${LOG_LEVEL:-info}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: ${SESSION_STORE:-valkey}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
DB_POOL_MIN: ${DB_POOL_MIN:-1}
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3}
CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5}
PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2}
DOCS_ENABLED: ${DOCS_ENABLED:-true}
services:
postgres:
image: postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes:
- pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -q"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery
restart: unless-stopped
valkey:
image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571
# The password goes into a config include readable only by the valkey user,
# so it never appears in the process arguments. docker-entrypoint.sh drops root.
command:
- sh
- -c
- >-
umask 077 &&
printf 'requirepass %s\n' "$$(cat /run/secrets/valkey_password)" > /tmp/requirepass.conf &&
chown valkey /tmp/requirepass.conf &&
exec docker-entrypoint.sh valkey-server /etc/valkey/valkey.conf
secrets: [valkey_password]
volumes:
- ./infra/valkey/valkey.conf:/etc/valkey/valkey.conf:ro
- valkeydata:/data
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/valkey_password)\" valkey-cli ping | grep -q PONG"]
interval: 2s
timeout: 3s
retries: 30
start_period: 5s
restart: unless-stopped
migrate:
build: .
image: mtusi-messenger:lab2
command: ["python", "-m", "messenger.migrate"]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
depends_on:
postgres:
condition: service_healthy
healthcheck:
disable: true
restart: "no"
api:
build: .
image: mtusi-messenger:lab2
environment: *api-env
secrets: [database_url, cache_url]
ports:
- "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" # only on localhost of the host
depends_on:
postgres:
condition: service_healthy
valkey:
condition: service_healthy
migrate:
condition: service_completed_successfully
stop_grace_period: 10s
restart: unless-stopped
volumes:
pgdata:
valkeydata:
secrets:
postgres_password:
file: ./secrets/postgres_password
valkey_password:
file: ./secrets/valkey_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
+41
View File
@@ -0,0 +1,41 @@
# Модель данных lab2
Схема: [migrations/0001_init.sql](../migrations/0001_init.sql). Источник истины для пользователей, чатов, участников и сообщений — PostgreSQL 18. Для сессий источник истины — хранилище, выбранное через `SESSION_STORE`. По умолчанию это Valkey с AOF, его можно переключить на таблицу `sessions` в PostgreSQL.
## Таблицы и ограничения
| Таблица | Ключ | Ограничения | Зачем |
| --- | --- | --- | --- |
| `users` | `id uuid` | `UNIQUE (username)`, `CHECK` формата username, длина `display_name` | уникальность имени держит БД, а не код: гонку регистрации разрешает уникальный индекс |
| `chats` | `id uuid` | FK `created_by → users`, длина `title` | `last_message_at` — «часы» чата для упорядочивания сообщений |
| `chat_members` | `(chat_id, user_id)` | FK на `chats` и `users`, `UNIQUE (chat_id, position)` | `position` хранит порядок `member_ids`; `chat_created_at` — копия времени чата для keyset-списка чатов |
| `messages` | `id uuid` | FK `(chat_id, sender_id) → chat_members`, длина `text` | отправитель обязан быть участником — это гарантирует сама БД |
| `sessions` | `token_hash bytea` | FK `user_id → users`, длина хеша 32 байта | хранится SHA-256 токена; сам токен не хранится нигде |
| `schema_migrations` | `version` | checksum файла | какие миграции применены; изменённую задним числом миграцию мигратор не применит |
**Почему `bytea` для текста.** Контракт разрешает символ U+0000, а тип `text` в PostgreSQL не может содержать байт NUL (`DataError`). Поэтому `display_name`, `title` и `text` хранятся как байты UTF-8. Длину в code points проверяет API, а `CHECK` в байтах — страховка (1 code point ≤ 4 байта).
## Индексы и запросы
- `messages_chat_keyset (chat_id, created_at, id)`. Первая страница — `WHERE chat_id=$1 ORDER BY created_at, id LIMIT n`, следующая — `WHERE chat_id=$1 AND (created_at, id) > ($2, $3) ORDER BY ... LIMIT n`. Сравнение строк `(created_at, id) > (...)` попадает в `Index Cond`: поиск начинается прямо с позиции cursor и читает только `n+1` строк. Запросов два, а не один с `($2 IS NULL OR ...)`: иначе при generic plan условие cursor уходит в `Filter`, и поздняя страница сканирует всю историю чата до неё.
- `chat_members_user_keyset (user_id, chat_created_at, chat_id)` — список чатов пользователя тем же keyset-способом.
- `users_username_key` — вход по username и уникальность.
- `sessions_expires_at` — очистка истёкших сессий.
## Транзакционные границы
| Операция | Транзакция | Что гарантирует |
| --- | --- | --- |
| Регистрация | один `INSERT` | `UniqueViolation` → 409; ровно одна строка при гонке |
| Создание чата | `INSERT chats` → `INSERT chat_members` (все участники одним запросом) → `COMMIT` | неизвестный участник вызывает FK-ошибку **после** вставки чата; откат удаляет и чат, и частичное членство |
| Сообщение | проверка членства → `UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + 1 мкс) RETURNING` → `INSERT messages` → `COMMIT` | строка чата заблокирована до конца транзакции, поэтому писатели одного чата идут по очереди. Время сообщений строго растёт **в порядке коммитов**: страница по cursor не пропустит сообщение, закоммиченное «задним числом» |
| Вход | `SELECT` пользователя, проверка Argon2id, запись сессии | неверный пароль ничего не пишет |
| Logout | `DELETE` сессии (или `DEL` в Valkey) | повтор с тем же токеном — 401 |
Чтение истории не требует snapshot между HTTP-запросами (так в контракте): новое сообщение появится на следующей странице. Список чатов сортируется по `chats.created_at = clock_timestamp()` без блокировки. Два чата, создаваемые одновременно, могут закоммититься не в порядке времени. Тогда у обхода, который уже начался, такой чат может оказаться «до» cursor; новый обход его увидит. Чаты создаются редко, поэтому мы не сериализуем их глобальным замком; это ограничение описано в REPORT.md.
## Сессии
- Токен: `secrets.token_urlsafe(32)` — 256 бит случайности, в ответе 43 символа. В хранилище лежит только SHA-256 токена. Для случайного 256-битного значения быстрого хеша достаточно, медленный KDF нужен только для низкоэнтропийных паролей.
- `SESSION_STORE=valkey` (по умолчанию): ключ `session:<sha256-hex>`, значение `{user_id, expires_at}`. Срок жизни задаётся абсолютным `PXAT expires_at`, поэтому рестарт его не продлевает. Включён `appendonly yes` + `appendfsync always`: подтверждённые вход и logout сохраняются до ответа клиенту. Valkey — **единственное** хранилище сессий, это не кэш над PostgreSQL. Поэтому нет второй копии, которая могла бы «оживить» отозванный токен после отказа. Если Valkey недоступен, проверить сессию нельзя: защищённые запросы и logout получают 503, анонимный доступ не открывается.
- `SESSION_STORE=postgres`: таблица `sessions`, проверка `expires_at > now()` в запросе.
+83
View File
@@ -0,0 +1,83 @@
# Шпаргалка к защите ЛР 2
Отчёт с доказательствами — [REPORT.md](../REPORT.md), модель данных — [DATA_MODEL.md](DATA_MODEL.md).
## Что изменилось по сравнению с lab1
| Было в lab1 | Стало в lab2 | Где |
| --- | --- | --- |
| словари в памяти (`store.py`) | PostgreSQL 18, данные переживают рестарт | [repo.py](../src/messenger/repo.py), [migrations/0001_init.sql](../migrations/0001_init.sql) |
| `X-User-Id` | вход по `Basic` → opaque `Bearer` с expiry, logout | [authhdr.py](../src/messenger/authhdr.py), [sessions.py](../src/messenger/sessions.py), [app.py](../src/messenger/app.py) |
| нет паролей | Argon2id (m=19 MiB, t=2, p=1) в отдельном потоке | [passwords.py](../src/messenger/passwords.py) |
| `threading.Lock` + свой счётчик времени | блокировка строки чата `UPDATE chats … RETURNING` в транзакции | `add_message` в repo.py |
| `bisect` по списку | индекс `(chat_id, created_at, id)` и row-comparison в `Index Cond` | `HISTORY_AFTER` в repo.py |
| один контейнер | `compose.yaml`: postgres, valkey, migrate, api; secrets-файлы; named volumes | [compose.yaml](../compose.yaml) |
Что осталось из lab1: формат ошибок и `X-Request-Id`, строгая валидация, cursor (теперь с проверкой диапазона времени), JSON-логи, graceful SIGTERM, healthcheck, Swagger на `/docs`.
## Путь запроса `GET /chats/{id}/messages?cursor=…` в lab2
1. `parse_bearer`: в заголовке ровно `Bearer <43 символа>`, иначе 401 с `WWW-Authenticate: Bearer`.
2. `sessions.user_id_for`: в Valkey ключ `session:<sha256(token)>`, проверка `expires_at`. Valkey недоступен — 503, нет ключа — 401.
3. `path_uuid` → 400; `ensure_member` (`SELECT 1 FROM chat_members …`) → 404 для чужого чата.
4. `cursor.decode` → `(время в мкс, id)`, привязка к чату → иначе 400.
5. `HISTORY_AFTER`: `WHERE chat_id=$1 AND (created_at, id) > ($2,$3) ORDER BY … LIMIT 51` — индекс начинает чтение прямо с cursor.
6. Каждая работа с БД — `db.run(...)`: отдельная задача, транзакция, дедлайн 3 с. Отказ или зависание БД → 503 ровно через дедлайн.
## Вопросы из README
**Что подтверждает COMMIT и какие гарантии зависят от настройки durability?**
COMMIT возвращается клиенту, когда запись транзакции в WAL сброшена на диск (`fsync=on`, `synchronous_commit=on`, это значения по умолчанию). Транзакция переживёт падение процесса PostgreSQL или ОС: при старте PostgreSQL проиграет WAL. От настроек зависит:
- `synchronous_commit=off` — ответ приходит раньше сброса, и последние ~0,6 с подтверждённых транзакций могут пропасть (база при этом останется целой);
- `fsync=off` — после сбоя возможна порча данных;
- кэш записи диска, который «врёт» о сбросе, ломает и то, и другое.
COMMIT на одном узле не защищает от потери самого диска или хоста: для этого нужны реплики (`synchronous_standby_names`) или резервные копии с архивом WAL (трек 2).
У Valkey свой аналог — `appendfsync`:
- у нас `always`: fsync до ответа, ничего не теряется;
- `everysec`: можно потерять до 1 с записей;
- `no`: когда сбросить на диск, решает ОС.
**Где хранятся сессии и что будет после flush/restart Redis (Valkey)?**
По умолчанию (`SESSION_STORE=valkey`) сессия существует **только** в Valkey: ключ `session:<sha256>`, значение `{user_id, expires_at}` и абсолютный срок `PXAT`.
- **Restart:** Valkey перечитывает AOF с volume, сессии на месте (тест `make test-persistence`). TTL не продлевается.
- **FLUSHALL** или потеря volume: все сессии исчезают, все пользователи получают 401 и входят заново. Отказ безопасный: доступ не выдаётся никому.
- **Valkey недоступен:** 503 на защищённые запросы и на logout. Анонимный доступ не открывается.
Почему не кэш над PostgreSQL: у одной копии нечему расходиться. У студентов с кэшем отозванный токен «оживал» после восстановления Redis, у нас такой ситуации быть не может (тест `test_logout_during_valkey_outage_is_never_lost`). Альтернатива `SESSION_STORE=postgres` хранит сессии в таблице `sessions`.
**Чем аутентификация отличается от доступа к конкретному чату?**
- **Аутентификация** — кто ты: действительная Bearer-сессия → `user_id`. Нет или недействительна → 401.
- **Авторизация доступа** — можно ли этому пользователю этот чат: есть ли строка в `chat_members`. Нет → 404, а не 403, чтобы не раскрывать существование чужого чата.
Сначала всегда аутентификация, потом доступ. Отправитель сообщения берётся из сессии, а БД сама проверяет `FOREIGN KEY (chat_id, sender_id) → chat_members`.
**Почему hash пароля и быстрый checksum имеют разные задачи?**
- **Checksum или быстрый хеш (SHA-256)** — контроль целостности и быстрый поиск. Он должен быть быстрым.
- **Хеш пароля** — защита низкоэнтропийного секрета на случай утечки базы. Он должен быть медленным, с солью и memory-hard (Argon2id), чтобы перебор словаря стоил дорого и не ускорялся на GPU.
Токен сессии — 256 бит случайности, перебрать его невозможно. Поэтому для токена хватает SHA-256: храним хеш, чтобы утечка базы не дала готовых токенов. Для пароля SHA-256 нельзя: миллиарды попыток в секунду на GPU.
## Частые ловушки (на чём преподаватель ловил других) и наш ответ
| Проверка преподавателя | У нас | Тест |
| --- | --- | --- |
| logout при упавшем Redis → после восстановления токен снова 200 | logout при упавшем Valkey = 503 (ничего не отозвано); после восстановления logout → 401, и после рестарта Valkey и API тоже 401 | `test_logout_during_valkey_outage_is_never_lost` |
| истёкшая сессия живёт в кэше | expiry проверяется при каждом чтении, ключ живёт до абсолютного `PXAT` | `test_short_ttl_expires[valkey/postgres]` |
| `docker pause postgres` → запрос висит 8+ с | клиентский дедлайн в отдельной задаче: 503 через 3,0 с | `test_postgres_failure[pause]` |
| БД упала → 401 вместо 503 | 503 `unavailable`, анонимный — 401 | `test_postgres_failure`, `test_postgres_session_store` |
| cursor в `Filter` при generic plan | два запроса без `IS NULL OR`, cursor в `Index Cond` | `test_explain.py`, `evidence/explain-history.txt` |
| `\u0000` → 500 | текст в `bytea`, NUL сохраняется | `test_unicode_limits_are_code_points_and_nul_is_stored` |
| пароль из пробелов, `Basic é` → 500 | 400 и 401 | `test_bad_password_on_registration_is_400`, `test_malformed_basic_is_401_not_500` |
| гонка регистрации считает любые ошибки за 409 | проверяются коды: ровно `[201] + [409]*7`, в БД 1 строка | `test_registration_race_exactly_one_201_and_seven_409` |
| rollback «до первого INSERT» | чат вставляется первым, FK участника падает после; `n_tup_ins` вырос, строк нет | `test_create_chat_rolls_back_after_the_chat_row_was_inserted` |
| `SESSION_TTL_SECONDS` не доходит до контейнера | compose передаёт его из `.env`, тест читает `printenv` внутри контейнера | `test_short_ttl_expires` |
| нет команды создания secrets | `python scripts/init_secrets.py` (идемпотентно) | раздел «Запуск из чистого clone» |
## Трек 2 — коротко
- **Гипотеза.** С индексом `(chat_id, created_at, id)` и keyset-запросом стоимость страницы не зависит от размера чата и позиции. С индексом только по `chat_id` нужна сортировка всего чата, с OFFSET — чтение всего префикса. Каждый индекс удорожает запись.
- **Методика.** Отдельный PostgreSQL (2 CPU / 2 GiB), 121 120 сообщений, чаты 60 000 / 15 000 / 2 000 / 80. pgbench с prepared statements; генератор нагрузки — в своём контейнере. 3 повтора. Плюс EXPLAIN для поздней страницы.
- **Backup.** `pg_dump -Fc` живой БД во время записи через API → новый контейнер и новый volume → `pg_restore` → сверка MD5 по каждому чату и по первым/последним сообщениям → наше же API на восстановленной БД читает чат после нового входа.
- **RPO и RTO.** `pg_dump` — снимок на момент старта. Всё подтверждённое после старта дампа в копию не попадает; это и есть граница потери данных (на практике — время с последнего дампа). RTO — время `pg_restore` с перестройкой индексов плюс переключение приложения. Чтобы уменьшить RPO до секунд, нужен архив WAL и PITR или реплика.
+134
View File
@@ -0,0 +1,134 @@
# Запуск на ноутбуке с Windows и сценарий защиты
Все команды — в **PowerShell**, из папки проекта. Внутри одного окна PowerShell переменные `$env:...` действуют до его закрытия.
## 1. Что установить один раз
| Программа | Зачем | Важно |
| --- | --- | --- |
| [Git for Windows](https://git-scm.com/download/win) | clone и Git Bash | ставить в путь по умолчанию `C:\Program Files\Git`: оттуда `scripts\restart.cmd` запускает `restart.sh` |
| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | PostgreSQL, Valkey, API | backend WSL 2; в BIOS включена виртуализация; после установки перезагрузка |
| [Python 3.12+](https://www.python.org/downloads/windows/) | secrets, публичные тесты, свои тесты | галочка **Add python.exe to PATH**. Если `python` открывает Microsoft Store — отключить «Псевдонимы выполнения приложений» для python в настройках Windows |
Проверка:
```powershell
git --version; docker version --format '{{.Server.Version}}'; docker compose version; python --version
```
Docker Desktop должен быть **запущен** (кит в трее не мигает).
## 2. Развернуть проект (дома, заранее — нужен интернет)
```powershell
git clone https://gitea.privcloud.tech/wfa/mtusi2026lab2 mtusi2026lab2
cd mtusi2026lab2
git checkout solution/bvt2302-bodanyuk
python scripts\init_secrets.py
Copy-Item .env.example .env
docker compose --env-file .env up -d --build --wait
```
Первый запуск скачивает образы (~700 МБ) и собирает API — 3–10 минут. Сделайте это **дома**: в аудитории сеть может быть медленной. Последняя команда завершается, когда все сервисы `healthy`.
Проверка в браузере: http://127.0.0.1:8080/health/ready → `{"status":"ok"}`, Swagger — http://127.0.0.1:8080/docs.
Окружение для своих тестов (один раз):
```powershell
python -m venv .venv
.venv\Scripts\python -m pip install --require-hashes --no-deps -r requirements.txt
.venv\Scripts\python -m pip install -r requirements-test.txt -r requirements-dev.txt
```
Активировать venv не нужно: везде вызывается `.venv\Scripts\python` напрямую, поэтому политика выполнения PowerShell не мешает.
## 3. Перед защитой (в аудитории)
```powershell
cd mtusi2026lab2
docker compose up -d --wait # если стенд был остановлен
$env:PYTHONUTF8 = "1" # публичные тесты читают UTF-8 JSON
python tests\smoke.py # PASS: lab2 public API scenario
```
## 4. Сценарий защиты из README — команды по шагам
**Шаг 1. Миграции к пустому volume, пользователи, вход, сообщение.**
```powershell
docker compose down -v # стереть учебные данные (volumes)
docker compose up -d --wait # postgres и valkey пустые -> migrate применяет схему
docker compose logs migrate # "0001_init: applied"
docker compose run --rm migrate # повтор: "already applied, skipped"
```
Дальше в Swagger (http://127.0.0.1:8080/docs):
1. `POST /api/v1/users` — создать двух пользователей (пароль 12+ символов).
2. **Authorize → basicAuth**: логин и пароль первого; `POST /api/v1/auth/sessions` — скопировать `token`.
3. **Authorize → bearerAuth**: вставить токен.
4. `POST /api/v1/chats` (в `member_ids` — оба id) → `POST /api/v1/chats/{chat_id}/messages` → `GET` истории.
**Шаги 2–3. Перезапуск API и хранилищ с сохранением volumes, тот же токен; logout → 401; короткий TTL.**
```powershell
python tests\resilience.py --mode persistence --action-script scripts\restart.cmd # PASS
docker compose ps # у api, postgres, valkey свежее время запуска
```
В Swagger тот же токен по-прежнему работает. `DELETE /api/v1/auth/sessions/current` → 204, затем любой запрос → 401.
Короткий TTL:
```powershell
$env:SESSION_TTL_SECONDS = "2"; docker compose up -d --wait api
docker compose exec api printenv SESSION_TTL_SECONDS # 2
# в Swagger: войти, сразу GET /users/me -> 200, через 3 секунды -> 401
Remove-Item Env:SESSION_TTL_SECONDS; docker compose up -d --wait api # вернуть 3600
```
**Шаг 4. Роль Valkey, план SQL, ошибка зависимости.**
```powershell
# Сессии лежат только в Valkey: ключи session:<sha256 токена>, самих токенов там нет
# (без двойных кавычек внутри: PowerShell 5.1 ломает их при передаче в docker)
docker compose exec valkey sh -c 'REDISCLI_AUTH=$(cat /run/secrets/valkey_password) valkey-cli --scan --pattern ''session:*'''
# План: поздняя страница по индексу (Index Cond) против IS NULL OR и OFFSET, 20 000 сообщений
.venv\Scripts\python scripts\explain_history.py
# Ошибка зависимости: с действующим токеном -> 503, без токена -> 401, после старта всё работает
docker compose stop valkey # или: docker compose pause postgres
# в Swagger: GET /users/me -> 503 unavailable; GET /health/ready -> 503
docker compose start valkey # или: docker compose unpause postgres
```
Всё то же автоматически, с восстановлением в `finally`, около 1 минуты:
```powershell
.venv\Scripts\python -m pytest student-tests\test_failures.py -v
```
**Трек 2 вживую.**
```powershell
.venv\Scripts\python bench\track2\backup_restore.py --repeats 1 # ~1 мин: dump во время записи -> restore в новый контейнер -> сверка
```
Скрипт печатает время дампа и восстановления, `reconciled: true` и сколько сообщений записано после снимка (RPO). Полный замер планов идёт ~25 минут, поэтому на защите показывайте готовые результаты: `evidence\track2\summary.md` и `evidence\track2\plans.png`. При желании можно запустить `.venv\Scripts\python bench\track2\index_sizes.py` (~2 мин).
**Все свои тесты** (~3,5 мин): `.venv\Scripts\python -m pytest student-tests -v`.
## 5. Если что-то пошло не так
| Симптом | Что сделать |
| --- | --- |
| `error during connect … dockerDesktopLinuxEngine` | Docker Desktop не запущен — запустить и подождать минуту |
| `port is already allocated` на 8080 | в `.env` поставить `APP_PUBLISH_PORT=8081`; дальше тесты с `--base-url http://127.0.0.1:8081` (smoke и resilience), для pytest — `$env:BASE_URL="http://127.0.0.1:8081"` |
| `secrets/... not found` при `up` | не выполнен `python scripts\init_secrets.py` |
| `UnicodeDecodeError` в `tests\*.py` | не задан `$env:PYTHONUTF8 = "1"` |
| `restart.cmd`: «не удаётся найти путь» | Git установлен не в `C:\Program Files\Git` — поправить путь в `scripts\restart.cmd` |
| api `unhealthy` после сна ноутбука | `docker compose restart` и подождать `docker compose ps` |
| нужно начать с чистого листа | `docker compose down -v` (удаляет учебные данные) → `docker compose up -d --wait` |
Остановить стенд после защиты: `docker compose down` (данные сохранятся) или `docker compose down -v` (данные удаляются).
+40
View File
@@ -0,0 +1,40 @@
# Clean clone -> build -> fresh volumes -> migrations from zero, 2026-10-12T17:08:32Z
$ git clone --branch solution/bvt2302-bodanyuk <repo> mtusi2026lab2 && cd mtusi2026lab2
eb92e8e bench: трек 2 — планы запросов, стоимость записи, backup/restore со сверкой
$ python scripts/init_secrets.py
created secrets/postgres_password
created secrets/valkey_password
created secrets/database_url
created secrets/cache_url
$ cp .env.example .env # + COMPOSE_PROJECT_NAME/APP_PUBLISH_PORT only to run beside the main stand
$ docker compose --env-file .env config --quiet
config OK
$ docker compose --env-file .env up -d --build --wait
Container mtusi-lab2-clean-api-1 Healthy
Container mtusi-lab2-clean-migrate-1 Exited
Container mtusi-lab2-clean-postgres-1 Healthy
Container mtusi-lab2-clean-valkey-1 Healthy
$ docker compose ps
SERVICE IMAGE STATUS
api mtusi-messenger:lab2 Up 6 seconds (healthy)
postgres postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c Up 13 seconds (healthy)
valkey valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 Up 13 seconds (healthy)
$ docker compose logs migrate
migrate: 0001_init: applied
migrate: done, 1 migration file(s) checked
$ docker compose volume ls
mtusi-lab2-clean_pgdata
mtusi-lab2-clean_valkeydata
$ curl -i http://127.0.0.1:8081/health/ready
HTTP/1.1 200 OK
x-request-id: 38be4c9665124450b66c5d9218c596d9
{"status":"ok"}
$ docker compose exec api id; PID 1
uid=10001(app) gid=10001(app) groups=10001(app)
PID 1: python -m messenger
$ docker compose run --rm migrate # second run: idempotent
migrate: 0001_init: already applied, skipped
migrate: done, 1 migration file(s) checked
$ git status --short # secrets/.env must not be tracked
secrets/database_url
.env
+63
View File
@@ -0,0 +1,63 @@
$ python scripts/explain_history.py --output evidence/explain-history.txt
messages_total | messages_in_target_chat
----------------+-------------------------
50146 | 20000
(1 row)
=== 1. first page, app query HISTORY_FIRST, generic plan
QUERY PLAN
------------------------------------------------------------------------------------------------------------------------------------------------
Limit (cost=0.41..122.63 rows=119 width=55) (actual time=0.019..0.030 rows=51.00 loops=1)
Buffers: shared hit=4
-> Index Scan using messages_chat_keyset on messages (cost=0.41..1226.64 rows=1194 width=55) (actual time=0.017..0.025 rows=51.00 loops=1)
Index Cond: (chat_id = $1)
Index Searches: 1
Buffers: shared hit=4
Planning:
Buffers: shared hit=10
Planning Time: 0.157 ms
Execution Time: 0.067 ms
(10 rows)
=== 2. late page (cursor at 19950), app query HISTORY_AFTER, generic plan
QUERY PLAN
----------------------------------------------------------------------------------------------------------------------------------------------
Limit (cost=0.41..59.95 rows=40 width=55) (actual time=0.022..0.036 rows=50.00 loops=1)
Buffers: shared hit=5
-> Index Scan using messages_chat_keyset on messages (cost=0.41..592.75 rows=398 width=55) (actual time=0.020..0.030 rows=50.00 loops=1)
Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))
Index Searches: 1
Buffers: shared hit=5
Planning Time: 0.163 ms
Execution Time: 0.112 ms
(8 rows)
=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan
QUERY PLAN
-----------------------------------------------------------------------------------------------------------------------------------------------
Limit (cost=0.41..123.02 rows=40 width=55) (actual time=3.242..3.257 rows=50.00 loops=1)
Buffers: shared hit=526
-> Index Scan using messages_chat_keyset on messages (cost=0.41..1232.61 rows=402 width=55) (actual time=3.239..3.251 rows=50.00 loops=1)
Index Cond: (chat_id = $1)
Filter: (($2 IS NULL) OR (ROW(created_at, id) > ROW($2, $3)))
Rows Removed by Filter: 19950
Index Searches: 1
Buffers: shared hit=526
Planning Time: 0.125 ms
Execution Time: 3.282 ms
(10 rows)
=== 4. REJECTED: OFFSET pagination, same late page
QUERY PLAN
---------------------------------------------------------------------------------------------------------------------------------------------------
Limit (cost=122.63..244.84 rows=119 width=55) (actual time=4.404..4.418 rows=50.00 loops=1)
Buffers: shared hit=526
-> Index Scan using messages_chat_keyset on messages (cost=0.41..1226.64 rows=1194 width=55) (actual time=0.015..3.790 rows=20000.00 loops=1)
Index Cond: (chat_id = $1)
Index Searches: 1
Buffers: shared hit=526
Planning Time: 0.091 ms
Execution Time: 4.438 ms
(8 rows)
+22
View File
@@ -0,0 +1,22 @@
# api log checks after smoke + valkey pause + unicode/NUL + wrong password tests, 2026-10-12T17:15:19Z
lines in api log: 93
message texts sent by tests ('Привет, мир', 'Сообщение Боба', 'до отказа', 'hello', 'чужое') found: 0
test passwords ('Test-password', 'T3st-') found: 0
Authorization values (Bearer/Basic + credential) found: 0
postgres/valkey passwords found: 0
event types:
62 "event": "request"
8 "event": "user_created"
7 "event": "session_created"
3 "event": "session_store_unavailable"
3 "event": "login_failed"
3 "event": "chat_created"
1 "event": "starting"
1 "event": "session_revoked"
1 "event": "dependencies_configured"
sample lines:
{"ts": "2026-10-12T17:15:10.934Z", "level": "info", "logger": "messenger", "event": "login_failed", "request_id": "350860a9d9ef4ad4b7fd6a9be0964981"}
{"ts": "2026-10-12T17:15:10.963Z", "level": "info", "logger": "messenger", "event": "session_revoked", "request_id": "5e8711e330724ed2a09f734947793877"}
{"ts": "2026-10-12T17:15:13.849Z", "level": "warning", "logger": "messenger.sessions", "event": "session_store_unavailable", "error": "TimeoutError"}
{"ts": "2026-10-12T17:15:13.850Z", "level": "info", "logger": "messenger.access", "event": "request", "request_id": "14fd3d975da3447e855d860eec8b1832", "method": "GET", "path": "/api/v1/users/me", "status": 503, "duration_ms": 501.281}
{"ts": "2026-10-12T17:15:14.354Z", "level": "info", "logger": "messenger.access", "event": "request", "request_id": "e2ddf53f97a1403fbb635d94d52ea9ad", "method": "GET", "path": "/api/v1/chats/3d96c3ee-a9c8-46eb-b8fe-7f12ba8ec2f0/messages", "status": 503, "duration_ms": 503.088}
+4
View File
@@ -0,0 +1,4 @@
$ make check # python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ make validate # (venv with requirements-dev.txt)
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+2
View File
@@ -0,0 +1,2 @@
$ make test BASE_URL=http://127.0.0.1:8081
PASS: lab2 public API scenario; infrastructure and grading evidence remain separate.
+20
View File
@@ -0,0 +1,20 @@
$ sh scripts/restart.sh # what the hook does (run by hand to see its output)
Container mtusi-lab2-clean-api-1 Stopping
Container mtusi-lab2-clean-api-1 Stopped
Container mtusi-lab2-clean-postgres-1 Restarting
Container mtusi-lab2-clean-valkey-1 Restarting
Container mtusi-lab2-clean-valkey-1 Started
Container mtusi-lab2-clean-postgres-1 Started
Container mtusi-lab2-clean-postgres-1 Waiting
Container mtusi-lab2-clean-postgres-1 Healthy
Container mtusi-lab2-clean-migrate-1 Starting
Container mtusi-lab2-clean-migrate-1 Started
Container mtusi-lab2-clean-postgres-1 Waiting
Container mtusi-lab2-clean-valkey-1 Waiting
Container mtusi-lab2-clean-migrate-1 Waiting
Container mtusi-lab2-clean-valkey-1 Healthy
Container mtusi-lab2-clean-postgres-1 Healthy
Container mtusi-lab2-clean-migrate-1 Exited
Container mtusi-lab2-clean-api-1 Starting
Container mtusi-lab2-clean-api-1 Started
restart.sh: api, postgres and valkey restarted with volumes preserved
+8
View File
@@ -0,0 +1,8 @@
# SIGTERM (docker compose stop = SIGTERM, then SIGKILL after stop_grace_period 10s)
run 1: docker compose stop api -> 1195 ms, exit code 0, OOMKilled=false
run 2: docker compose stop api -> 1089 ms, exit code 0, OOMKilled=false
run 3: docker compose stop api -> 1098 ms, exit code 0, OOMKilled=false
last shutdown log lines:
{"ts": "2026-10-12T17:16:17.419Z", "level": "info", "logger": "messenger", "event": "shutdown_requested", "signal": 15, "timeout_s": 8.0}
{"ts": "2026-10-12T17:16:17.541Z", "level": "info", "logger": "uvicorn.error", "event": "Finished server process [1]"}
{"ts": "2026-10-12T17:16:17.542Z", "level": "info", "logger": "messenger", "event": "stopped"}
+117
View File
@@ -0,0 +1,117 @@
$ BASE_URL=http://127.0.0.1:8081 python -m pytest student-tests -v # stand from the clean clone
============================= test session starts =============================
platform win32 -- Python 3.14.6, pytest-9.1.1, pluggy-1.6.0 -- <repo>/.venv/Scripts/python.exe
rootdir: <clean clone of the repo>
plugins: anyio-4.15.1
collecting ... collected 108 items
student-tests/test_api.py::test_register_never_returns_password_and_login_returns_bearer PASSED
student-tests/test_api.py::test_duplicate_username_is_409 PASSED
student-tests/test_api.py::test_bad_password_on_registration_is_400[short-11ch] PASSED
student-tests/test_api.py::test_bad_password_on_registration_is_400[ ] PASSED
student-tests/test_api.py::test_bad_password_on_registration_is_400[ \t \n \t \n \t \n ] PASSED
student-tests/test_api.py::test_bad_password_on_registration_is_400[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED
student-tests/test_api.py::test_edge_passwords_register_and_login[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED
student-tests/test_api.py::test_edge_passwords_register_and_login[with:colons:inside] PASSED
student-tests/test_api.py::test_edge_passwords_register_and_login[Twelve chars] PASSED
student-tests/test_api.py::test_wrong_password_and_unknown_user_give_the_same_401 PASSED
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic \xe9] PASSED
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic !!!] PASSED
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic bm9fY29sb24=] PASSED
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Basic //46eA==] PASSED
student-tests/test_api.py::test_malformed_basic_is_401_not_500[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED
student-tests/test_api.py::test_malformed_basic_is_401_not_500[] PASSED
student-tests/test_api.py::test_logout_revokes_only_this_token PASSED
student-tests/test_api.py::test_x_user_id_basic_and_garbage_do_not_authenticate PASSED
student-tests/test_api.py::test_malformed_json_is_400[{] PASSED
student-tests/test_api.py::test_malformed_json_is_400[] PASSED
student-tests/test_api.py::test_malformed_json_is_400[[]] PASSED
student-tests/test_api.py::test_malformed_json_is_400[null] PASSED
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}] PASSED
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}] PASSED
student-tests/test_api.py::test_malformed_json_is_400[{"username": "abc_def", "display_name": "\ud800", "password": "Twelve-chars"}] PASSED
student-tests/test_api.py::test_invalid_registration_fields_are_400[body0] PASSED
student-tests/test_api.py::test_invalid_registration_fields_are_400[body1] PASSED
student-tests/test_api.py::test_invalid_registration_fields_are_400[body2] PASSED
student-tests/test_api.py::test_invalid_registration_fields_are_400[body3] PASSED
student-tests/test_api.py::test_invalid_registration_fields_are_400[body4] PASSED
student-tests/test_api.py::test_invalid_registration_fields_are_400[body5] PASSED
student-tests/test_api.py::test_unicode_limits_are_code_points_and_nul_is_stored PASSED
student-tests/test_api.py::test_oversized_body_and_wrong_content_type PASSED
student-tests/test_api.py::test_chat_validation PASSED
student-tests/test_api.py::test_foreign_chat_is_indistinguishable_from_missing PASSED
student-tests/test_api.py::test_chat_list_order_pagination_and_isolation PASSED
student-tests/test_api.py::test_history_pages_without_loss_or_duplicates PASSED
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=0] PASSED
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=101] PASSED
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=abc] PASSED
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[limit=1&limit=2] PASSED
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=] PASSED
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=garbage] PASSED
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDF8Mg] PASSED
student-tests/test_api.py::test_bad_limit_or_cursor_is_400[cursor=djF8bXx4fDk5OTk5OTk5OTk5OTk5OTk5OTk5fHk=] PASSED
student-tests/test_api.py::test_cursor_bound_to_chat PASSED
student-tests/test_api.py::test_read_after_write PASSED
student-tests/test_db.py::test_registration_race_exactly_one_201_and_seven_409 PASSED
student-tests/test_db.py::test_create_chat_rolls_back_after_the_chat_row_was_inserted PASSED
student-tests/test_db.py::test_migrations_rerun_is_a_noop PASSED
student-tests/test_db.py::test_100_messages_from_10_clients_keep_order_and_cursor_walk PASSED
student-tests/test_db.py::test_secrets_are_not_stored_in_plain_text PASSED
student-tests/test_explain.py::test_late_page_uses_index_range_not_filter PASSED
student-tests/test_failures.py::test_valkey_failure[stop] PASSED
student-tests/test_failures.py::test_valkey_failure[pause] PASSED
student-tests/test_failures.py::test_postgres_failure[stop] PASSED
student-tests/test_failures.py::test_postgres_failure[pause] PASSED
student-tests/test_failures.py::test_locked_table_gives_bounded_503 PASSED
student-tests/test_failures.py::test_logout_during_valkey_outage_is_never_lost PASSED
student-tests/test_sessions.py::test_short_ttl_expires[valkey] PASSED
student-tests/test_sessions.py::test_short_ttl_expires[postgres] PASSED
student-tests/test_sessions.py::test_restart_does_not_extend_ttl PASSED
student-tests/test_sessions.py::test_postgres_session_store PASSED
student-tests/unit/test_unit.py::test_valid_passwords[Twelve-chars] PASSED
student-tests/unit/test_unit.py::test_valid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED
student-tests/unit/test_unit.py::test_valid_passwords[\u043f\u0430\u0440\u043e\u043b\u044c:\u0441:\u0434\u0432\u043e\u0435\u0442\u043e\u0447\u0438\u044f\u043c\u0438] PASSED
student-tests/unit/test_unit.py::test_valid_passwords[\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600\U0001f600] PASSED
student-tests/unit/test_unit.py::test_valid_passwords[a b c d e f g] PASSED
student-tests/unit/test_unit.py::test_invalid_passwords[short-11ch] PASSED
student-tests/unit/test_unit.py::test_invalid_passwords[\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f\u044f] PASSED
student-tests/unit/test_unit.py::test_invalid_passwords[ ] PASSED
student-tests/unit/test_unit.py::test_invalid_passwords[\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000\t\n\u3000] PASSED
student-tests/unit/test_unit.py::test_invalid_passwords[\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800\ud800] PASSED
student-tests/unit/test_unit.py::test_invalid_passwords[None] PASSED
student-tests/unit/test_unit.py::test_invalid_passwords[123456789012] PASSED
student-tests/unit/test_unit.py::test_text_may_contain_nul_but_not_lone_surrogates PASSED
student-tests/unit/test_unit.py::test_cursor_roundtrip_and_binding PASSED
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|253402300800000000|11111111-1111-4111-8111-111111111111] PASSED
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|9999999999999999999|11111111-1111-4111-8111-111111111111] PASSED
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|-1|11111111-1111-4111-8111-111111111111] PASSED
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|not-a-uuid] PASSED
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v1|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-11111111111G] PASSED
student-tests/unit/test_unit.py::test_broken_cursors_are_400_not_500[v2|m|3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11|1|11111111-1111-4111-8111-111111111111] PASSED
student-tests/unit/test_unit.py::test_time_helpers_are_exact PASSED
student-tests/unit/test_unit.py::test_basic_is_parsed_and_password_may_contain_colons PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[None] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic \xe9] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic !!!!] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic bm8tY29sb24=] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic //46cHc=] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic OnBhc3N3b3Jk] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic YWxpY2U6] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic 0JDQu9C40YHQsDpwdw==] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED
student-tests/unit/test_unit.py::test_bad_basic_is_401_with_basic_challenge[Basic AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA] PASSED
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[None] PASSED
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[] PASSED
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer] PASSED
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer short] PASSED
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!] PASSED
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Basic YWxpY2U6cGFzc3dvcmQtMTIz] PASSED
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Token aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa] PASSED
student-tests/unit/test_unit.py::test_bad_bearer_is_401_with_bearer_challenge[Bearer \xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9\xe9] PASSED
student-tests/unit/test_unit.py::test_bearer_ok PASSED
student-tests/unit/test_unit.py::test_argon2id_hash_and_verify PASSED
student-tests/unit/test_unit.py::test_settings_read_secret_files_and_contract_names PASSED
======================= 108 passed in 208.53s (0:03:28) =======================
+24
View File
@@ -0,0 +1,24 @@
# Same persistence test with SESSION_STORE=postgres (sessions in table sessions)
$ SESSION_STORE=postgres docker compose up -d --wait api
Container mtusi-lab2-clean-api-1 Healthy
Container mtusi-lab2-clean-api-1 Recreated
Container mtusi-lab2-clean-postgres-1 Healthy
Container mtusi-lab2-clean-valkey-1 Healthy
api env SESSION_STORE=postgres
containers BEFORE:
api StartedAt=2026-10-12T17:10:11.785377847Z
postgres StartedAt=2026-10-12T17:09:31.655693328Z
valkey StartedAt=2026-10-12T17:09:31.612351204Z
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
containers AFTER:
api StartedAt=2026-10-12T17:10:30.772464788Z
postgres StartedAt=2026-10-12T17:10:23.581636896Z
valkey StartedAt=2026-10-12T17:10:23.504892649Z
sessions rows in PostgreSQL: 3
$ docker compose up -d --wait api # back to the default SESSION_STORE=valkey
Container mtusi-lab2-clean-api-1 Healthy
Container mtusi-lab2-clean-postgres-1 Healthy
Container mtusi-lab2-clean-valkey-1 Healthy
api env SESSION_STORE=valkey
+14
View File
@@ -0,0 +1,14 @@
# make test-persistence ACTION_SCRIPT=scripts/restart.sh (Windows: scripts/restart.cmd runs the same restart.sh via Git Bash)
containers BEFORE:
api StartedAt=2026-10-12T17:09:00.366421789Z
postgres StartedAt=2026-10-12T17:08:53.162313311Z
valkey StartedAt=2026-10-12T17:08:53.503522499Z
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8081
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
containers AFTER (all three restarted, volumes kept):
api StartedAt=2026-10-12T17:09:38.831312693Z
postgres StartedAt=2026-10-12T17:09:31.655693328Z
valkey StartedAt=2026-10-12T17:09:31.612351204Z
mtusi-lab2-clean_pgdata
mtusi-lab2-clean_valkeydata
+155
View File
@@ -0,0 +1,155 @@
{
"image": "postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c",
"rounds": [
{
"round": 1,
"backup_started_utc": "2026-10-12T17:06:57+00:00",
"dump_seconds": 1.47,
"dump_bytes": 5349383,
"restore_seconds": 2.21,
"analyze_seconds": 0.57,
"rto_seconds_restore_plus_analyze": 2.79,
"source_counts_after_writer_stopped": {
"chat_members": 330,
"chats": 290,
"messages": 171546,
"schema_migrations": 1,
"users": 117
},
"restored_counts": {
"chat_members": 330,
"chats": 290,
"messages": 171410,
"schema_migrations": 1,
"users": 117
},
"messages_in_restored": 171410,
"per_chat_md5_equal": true,
"chats_compared": 257,
"big_chat_first_last_50_equal": true,
"schema_migrations_equal": true,
"indexes_equal": true,
"other_table_counts_equal": {
"chat_members": true,
"chats": true,
"schema_migrations": true,
"users": true
},
"writer": {
"confirmed": 237,
"errors": 0,
"in_backup": 101,
"restored_is_exact_prefix": true,
"lost_confirmed_after_snapshot": 136,
"last_message_in_backup_at": "2026-10-12T17:06:57.831240Z",
"first_lost_message_at": "2026-10-12T17:06:57.856678Z"
},
"app_on_restored_db": {
"login_ok": true,
"history_via_api_equals_restored": true,
"messages_read": 101
},
"reconciled": true
},
{
"round": 2,
"backup_started_utc": "2026-10-12T17:07:13+00:00",
"dump_seconds": 1.7,
"dump_bytes": 5362829,
"restore_seconds": 1.9,
"analyze_seconds": 0.49,
"rto_seconds_restore_plus_analyze": 2.39,
"source_counts_after_writer_stopped": {
"chat_members": 331,
"chats": 291,
"messages": 171788,
"schema_migrations": 1,
"users": 118
},
"restored_counts": {
"chat_members": 331,
"chats": 291,
"messages": 171655,
"schema_migrations": 1,
"users": 118
},
"messages_in_restored": 171655,
"per_chat_md5_equal": true,
"chats_compared": 258,
"big_chat_first_last_50_equal": true,
"schema_migrations_equal": true,
"indexes_equal": true,
"other_table_counts_equal": {
"chat_members": true,
"chats": true,
"schema_migrations": true,
"users": true
},
"writer": {
"confirmed": 242,
"errors": 0,
"in_backup": 109,
"restored_is_exact_prefix": true,
"lost_confirmed_after_snapshot": 133,
"last_message_in_backup_at": "2026-10-12T17:07:14.212768Z",
"first_lost_message_at": "2026-10-12T17:07:14.237207Z"
},
"app_on_restored_db": {
"login_ok": true,
"history_via_api_equals_restored": true,
"messages_read": 109
},
"reconciled": true
},
{
"round": 3,
"backup_started_utc": "2026-10-12T17:07:30+00:00",
"dump_seconds": 1.4,
"dump_bytes": 5371057,
"restore_seconds": 1.76,
"analyze_seconds": 0.49,
"rto_seconds_restore_plus_analyze": 2.25,
"source_counts_after_writer_stopped": {
"chat_members": 332,
"chats": 292,
"messages": 172038,
"schema_migrations": 1,
"users": 119
},
"restored_counts": {
"chat_members": 332,
"chats": 292,
"messages": 171898,
"schema_migrations": 1,
"users": 119
},
"messages_in_restored": 171898,
"per_chat_md5_equal": true,
"chats_compared": 259,
"big_chat_first_last_50_equal": true,
"schema_migrations_equal": true,
"indexes_equal": true,
"other_table_counts_equal": {
"chat_members": true,
"chats": true,
"schema_migrations": true,
"users": true
},
"writer": {
"confirmed": 250,
"errors": 0,
"in_backup": 110,
"restored_is_exact_prefix": true,
"lost_confirmed_after_snapshot": 140,
"last_message_in_backup_at": "2026-10-12T17:07:31.075846Z",
"first_lost_message_at": "2026-10-12T17:07:31.095982Z"
},
"app_on_restored_db": {
"login_ok": true,
"history_via_api_equals_restored": true,
"messages_read": 110
},
"reconciled": true
}
]
}
@@ -0,0 +1,20 @@
Limit (cost=1673.53..1673.58 rows=20 width=238) (actual time=14.400..14.407 rows=51.00 loops=1)
Buffers: shared hit=2124 read=53
-> Sort (cost=1673.53..1674.03 rows=202 width=238) (actual time=14.398..14.402 rows=51.00 loops=1)
Sort Key: created_at, id
Sort Method: top-N heapsort Memory: 44kB
Buffers: shared hit=2124 read=53
-> Bitmap Heap Scan on messages (cost=8.89..1665.79 rows=202 width=238) (actual time=13.229..14.305 rows=300.00 loops=1)
Recheck Cond: (chat_id = $1)
Filter: (ROW(created_at, id) > ROW($2, $3))
Rows Removed by Filter: 59700
Heap Blocks: exact=2118
Buffers: shared hit=2118 read=53
-> Bitmap Index Scan on messages_chat_only (cost=0.00..8.84 rows=606 width=0) (actual time=1.829..1.829 rows=60000.00 loops=1)
Index Cond: (chat_id = $1)
Index Searches: 1
Buffers: shared read=53
Planning:
Buffers: shared hit=97
Planning Time: 0.456 ms
Execution Time: 14.511 ms
@@ -0,0 +1,20 @@
Limit (cost=1673.53..1673.58 rows=20 width=238) (actual time=4.411..4.418 rows=51.00 loops=1)
Buffers: shared hit=571 read=14
-> Sort (cost=1673.53..1674.03 rows=202 width=238) (actual time=4.410..4.413 rows=51.00 loops=1)
Sort Key: created_at, id
Sort Method: quicksort Memory: 52kB
Buffers: shared hit=571 read=14
-> Bitmap Heap Scan on messages (cost=8.89..1665.79 rows=202 width=238) (actual time=4.306..4.361 rows=75.00 loops=1)
Recheck Cond: (chat_id = $1)
Filter: (ROW(created_at, id) > ROW($2, $3))
Rows Removed by Filter: 14925
Heap Blocks: exact=564
Buffers: shared hit=565 read=14
-> Bitmap Index Scan on messages_chat_only (cost=0.00..8.84 rows=606 width=0) (actual time=0.764..0.765 rows=15000.00 loops=1)
Index Cond: (chat_id = $1)
Index Searches: 1
Buffers: shared hit=1 read=14
Planning:
Buffers: shared hit=97
Planning Time: 0.339 ms
Execution Time: 4.475 ms
@@ -0,0 +1,20 @@
Limit (cost=1673.53..1673.58 rows=20 width=238) (actual time=0.129..0.130 rows=1.00 loops=1)
Buffers: shared hit=11 read=1
-> Sort (cost=1673.53..1674.03 rows=202 width=238) (actual time=0.127..0.128 rows=1.00 loops=1)
Sort Key: created_at, id
Sort Method: quicksort Memory: 25kB
Buffers: shared hit=11 read=1
-> Bitmap Heap Scan on messages (cost=8.89..1665.79 rows=202 width=238) (actual time=0.104..0.104 rows=1.00 loops=1)
Recheck Cond: (chat_id = $1)
Filter: (ROW(created_at, id) > ROW($2, $3))
Rows Removed by Filter: 79
Heap Blocks: exact=4
Buffers: shared hit=5 read=1
-> Bitmap Index Scan on messages_chat_only (cost=0.00..8.84 rows=606 width=0) (actual time=0.064..0.064 rows=80.00 loops=1)
Index Cond: (chat_id = $1)
Index Searches: 1
Buffers: shared hit=1 read=1
Planning:
Buffers: shared hit=97
Planning Time: 0.368 ms
Execution Time: 0.207 ms
@@ -0,0 +1,10 @@
Limit (cost=0.42..77.92 rows=20 width=237) (actual time=0.456..0.514 rows=51.00 loops=1)
Buffers: shared hit=6 read=4
-> Index Scan using messages_chat_keyset on messages (cost=0.42..783.20 rows=202 width=237) (actual time=0.451..0.504 rows=51.00 loops=1)
Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))
Index Searches: 1
Buffers: shared hit=6 read=4
Planning:
Buffers: shared hit=120
Planning Time: 0.921 ms
Execution Time: 0.636 ms
@@ -0,0 +1,10 @@
Limit (cost=0.42..77.92 rows=20 width=237) (actual time=0.087..0.114 rows=51.00 loops=1)
Buffers: shared hit=5 read=2
-> Index Scan using messages_chat_keyset on messages (cost=0.42..783.20 rows=202 width=237) (actual time=0.084..0.108 rows=51.00 loops=1)
Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))
Index Searches: 1
Buffers: shared hit=5 read=2
Planning:
Buffers: shared hit=110
Planning Time: 0.551 ms
Execution Time: 0.156 ms
@@ -0,0 +1,10 @@
Limit (cost=0.42..77.92 rows=20 width=237) (actual time=0.071..0.072 rows=1.00 loops=1)
Buffers: shared hit=2 read=2
-> Index Scan using messages_chat_keyset on messages (cost=0.42..783.20 rows=202 width=237) (actual time=0.069..0.070 rows=1.00 loops=1)
Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))
Index Searches: 1
Buffers: shared hit=2 read=2
Planning:
Buffers: shared hit=110
Planning Time: 0.373 ms
Execution Time: 0.112 ms
@@ -0,0 +1,20 @@
Limit (cost=6162.88..6165.20 rows=20 width=238) (actual time=12.084..15.940 rows=51.00 loops=1)
Buffers: shared hit=4307
-> Gather Merge (cost=6162.88..6186.40 rows=202 width=238) (actual time=12.082..15.934 rows=51.00 loops=1)
Workers Planned: 2
Workers Launched: 2
Buffers: shared hit=4307
-> Sort (cost=5162.85..5163.06 rows=84 width=238) (actual time=8.686..8.690 rows=44.33 loops=3)
Sort Key: created_at, id
Sort Method: quicksort Memory: 57kB
Buffers: shared hit=4307
Worker 0: Sort Method: top-N heapsort Memory: 41kB
Worker 1: Sort Method: quicksort Memory: 58kB
-> Parallel Seq Scan on messages (cost=0.00..5160.17 rows=84 width=238) (actual time=1.775..8.612 rows=100.00 loops=3)
Filter: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))
Rows Removed by Filter: 40273
Buffers: shared hit=4277
Planning:
Buffers: shared hit=89
Planning Time: 0.335 ms
Execution Time: 16.001 ms
@@ -0,0 +1,20 @@
Limit (cost=6162.88..6165.20 rows=20 width=238) (actual time=19.403..27.573 rows=51.00 loops=1)
Buffers: shared hit=4307
-> Gather Merge (cost=6162.88..6186.40 rows=202 width=238) (actual time=19.400..27.552 rows=51.00 loops=1)
Workers Planned: 2
Workers Launched: 2
Buffers: shared hit=4307
-> Sort (cost=5162.85..5163.06 rows=84 width=238) (actual time=12.274..12.277 rows=21.00 loops=3)
Sort Key: created_at, id
Sort Method: quicksort Memory: 44kB
Buffers: shared hit=4307
Worker 0: Sort Method: quicksort Memory: 27kB
Worker 1: Sort Method: quicksort Memory: 30kB
-> Parallel Seq Scan on messages (cost=0.00..5160.17 rows=84 width=238) (actual time=5.204..12.212 rows=25.00 loops=3)
Filter: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))
Rows Removed by Filter: 40348
Buffers: shared hit=4277
Planning:
Buffers: shared hit=85
Planning Time: 0.565 ms
Execution Time: 27.653 ms
@@ -0,0 +1,20 @@
Limit (cost=6162.88..6165.20 rows=20 width=238) (actual time=12.515..16.342 rows=1.00 loops=1)
Buffers: shared hit=4307
-> Gather Merge (cost=6162.88..6186.40 rows=202 width=238) (actual time=12.513..16.339 rows=1.00 loops=1)
Workers Planned: 2
Workers Launched: 2
Buffers: shared hit=4307
-> Sort (cost=5162.85..5163.06 rows=84 width=238) (actual time=8.455..8.456 rows=0.33 loops=3)
Sort Key: created_at, id
Sort Method: quicksort Memory: 25kB
Buffers: shared hit=4307
Worker 0: Sort Method: quicksort Memory: 25kB
Worker 1: Sort Method: quicksort Memory: 25kB
-> Parallel Seq Scan on messages (cost=0.00..5160.17 rows=84 width=238) (actual time=7.451..8.415 rows=0.33 loops=3)
Filter: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))
Rows Removed by Filter: 40373
Buffers: shared hit=4277
Planning:
Buffers: shared hit=85
Planning Time: 0.289 ms
Execution Time: 16.395 ms
+5
View File
@@ -0,0 +1,5 @@
variant,rows,table_bytes,pkey_bytes,secondary_index_bytes,secondary_bytes_per_row,index_build_seconds
keyset_index,121120,35037184,4808704,7159808,59.1,0.3
chat_id_index,121120,35037184,4808704,876544,7.2,0.3
no_index,121120,35037184,4808704,0,0.0,0.0
keyset_plus_sender,121120,35037184,4808704,10240000,84.5,0.59
1 variant rows table_bytes pkey_bytes secondary_index_bytes secondary_bytes_per_row index_build_seconds
2 keyset_index 121120 35037184 4808704 7159808 59.1 0.3
3 chat_id_index 121120 35037184 4808704 876544 7.2 0.3
4 no_index 121120 35037184 4808704 0 0.0 0.0
5 keyset_plus_sender 121120 35037184 4808704 10240000 84.5 0.59
+74
View File
@@ -0,0 +1,74 @@
{
"image": "postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c",
"limits": {
"server_cpus": 2,
"server_memory": "2g",
"pgbench_client_cpus": 2
},
"pg_settings": "image defaults",
"dataset": {
"messages_chats_minbytes_maxbytes": "121120|200|10|380",
"chat_sizes": [
"60000x1",
"15000x2",
"2000x8",
"80x189"
],
"seed_seconds": 3.3
},
"pgbench": {
"mode": "prepared",
"seconds": 10,
"warmup": 3,
"repeats": 3,
"read_clients": 4,
"write_clients": 8,
"page": "LIMIT 51 after a random position",
"write_repeats": 5,
"wal_method": "5000 app transactions, 1 client, autovacuum off, CHECKPOINT before; 3 repeats"
},
"host": {
"platform": "Windows-10-10.0.19045-SP0",
"docker": "29.8.0"
},
"variants": {
"keyset_index": "(chat_id, created_at, id) — used by the app",
"chat_id_index": "(chat_id) only",
"no_index": "primary key only",
"keyset_plus_sender": "app index + a second index (sender_id, created_at)"
},
"sizes": {
"keyset_index": {
"index_build_s": 0.64,
"index_bytes": {
"messages_chat_keyset": "15097856",
"messages_pkey": "10051584"
},
"table_bytes": 64290816
},
"chat_id_index": {
"index_build_s": 0.25,
"index_bytes": {
"messages_chat_only": "876544",
"messages_pkey": "4775936"
},
"table_bytes": 35037184
},
"no_index": {
"index_build_s": 0.0,
"index_bytes": {
"messages_pkey": "4849664"
},
"table_bytes": 35037184
},
"keyset_plus_sender": {
"index_build_s": 1.67,
"index_bytes": {
"messages_chat_keyset": "21102592",
"messages_pkey": "15286272",
"messages_sender_time": "12468224"
},
"table_bytes": 86335488
}
}
}
Binary file not shown.

After

Width:  |  Height:  |  Size: 115 KiB

+55
View File
@@ -0,0 +1,55 @@
variant,query,chat,chat_size,repeat,tps,transactions,failed,p50_ms,p95_ms,p99_ms,mean_ms
keyset_index,keyset,big,60000,1,2401.7,23998,0,0.873,2.88,32.812,1.651
keyset_index,keyset,medium,15000,1,2991.2,29887,0,0.76,1.681,34.535,1.327
keyset_index,keyset,small,80,1,4017.5,40162,0,0.576,1.691,11.886,0.986
keyset_index,offset,big,60000,1,169.5,1696,0,13.118,70.429,81.03,23.566
keyset_index,offset,medium,15000,1,665.7,6672,0,2.909,47.698,53.811,5.995
keyset_index,offset,small,80,1,3402.6,34012,0,0.658,1.68,29.655,1.165
keyset_index,keyset,big,60000,2,2655.3,26611,0,0.815,2.278,33.597,1.495
keyset_index,keyset,medium,15000,2,1887.8,18889,0,0.905,6.01,32.787,2.101
keyset_index,keyset,small,80,2,2439.0,24420,0,0.659,5.636,25.202,1.622
keyset_index,offset,big,60000,2,180.4,1814,0,11.618,69.186,84.59,22.064
keyset_index,offset,medium,15000,2,791.1,7909,0,2.528,43.867,52.614,5.044
keyset_index,offset,small,80,2,4059.2,40598,0,0.604,1.05,9.089,0.977
keyset_index,keyset,big,60000,3,3486.0,34836,0,0.676,1.191,31.965,1.139
keyset_index,keyset,medium,15000,3,3499.1,34950,0,0.675,1.229,31.369,1.134
keyset_index,keyset,small,80,3,5575.1,55692,0,0.489,0.791,1.744,0.711
keyset_index,offset,big,60000,3,234.7,2357,0,9.301,63.742,68.861,17.016
keyset_index,offset,medium,15000,3,852.0,8550,0,2.398,6.87,52.109,4.683
keyset_index,offset,small,80,3,4251.9,42587,0,0.584,0.973,3.529,0.933
chat_id_index,keyset,big,60000,1,89.8,902,0,27.035,80.852,100.066,44.495
chat_id_index,keyset,medium,15000,1,364.0,3653,0,5.595,55.042,58.907,10.973
chat_id_index,keyset,small,80,1,4115.2,41258,0,0.6,1.015,4.808,0.964
chat_id_index,offset,big,60000,1,28.3,287,0,120.756,277.986,388.776,140.219
chat_id_index,offset,medium,15000,1,135.4,1360,0,16.801,71.112,77.884,29.509
chat_id_index,offset,small,80,1,4214.0,42116,0,0.587,0.978,4.666,0.941
chat_id_index,keyset,big,60000,2,89.4,899,0,28.031,80.795,96.807,44.655
chat_id_index,keyset,medium,15000,2,362.5,3623,0,5.642,55.669,59.361,11.022
chat_id_index,keyset,small,80,2,4068.0,40695,0,0.603,1.026,5.892,0.975
chat_id_index,offset,big,60000,2,28.0,283,0,120.306,292.288,396.718,142.499
chat_id_index,offset,medium,15000,2,140.0,1401,0,15.77,71.238,74.833,28.531
chat_id_index,offset,small,80,2,4222.1,42290,0,0.579,1.019,3.828,0.939
chat_id_index,keyset,big,60000,3,89.5,901,0,27.0,81.494,98.989,44.63
chat_id_index,keyset,medium,15000,3,345.7,3473,0,5.885,55.874,59.633,11.54
chat_id_index,keyset,small,80,3,4140.1,41444,0,0.59,1.026,4.533,0.958
chat_id_index,offset,big,60000,3,29.3,296,0,119.134,208.472,381.353,135.64
chat_id_index,offset,medium,15000,3,134.8,1354,0,16.795,71.384,76.114,29.634
chat_id_index,offset,small,80,3,4266.0,42637,0,0.581,0.967,3.792,0.93
no_index,keyset,big,60000,1,29.0,291,0,111.727,197.369,292.929,137.59
no_index,keyset,medium,15000,1,34.3,345,0,104.45,184.828,192.39,116.54
no_index,keyset,small,80,1,36.3,364,0,102.442,182.084,192.872,110.02
no_index,offset,big,60000,1,25.7,259,0,132.386,293.23,376.222,155.228
no_index,offset,medium,15000,1,63.3,635,0,75.84,94.867,109.291,63.124
no_index,offset,small,80,1,105.1,1055,0,19.797,72.802,78.183,38.004
no_index,keyset,big,60000,2,30.7,309,0,110.372,191.978,200.211,130.118
no_index,keyset,medium,15000,2,33.9,340,0,105.192,185.337,190.981,117.813
no_index,keyset,small,80,2,37.4,377,0,100.838,180.405,188.141,106.88
no_index,offset,big,60000,2,26.4,265,0,134.06,222.582,384.567,151.443
no_index,offset,medium,15000,2,49.7,498,0,85.725,118.239,170.86,80.332
no_index,offset,small,80,2,84.1,843,0,30.964,82.362,94.9,47.488
no_index,keyset,big,60000,3,21.6,219,0,192.061,288.029,303.799,184.125
no_index,keyset,medium,15000,3,26.0,263,0,169.475,207.861,283.698,153.341
no_index,keyset,small,80,3,25.1,255,0,178.656,220.368,289.13,158.613
no_index,offset,big,60000,3,18.0,182,0,205.371,381.47,517.852,221.459
no_index,offset,medium,15000,3,44.9,450,0,90.847,150.378,187.415,88.956
no_index,offset,small,80,3,77.6,781,0,68.094,86.015,98.542,51.458
1 variant query chat chat_size repeat tps transactions failed p50_ms p95_ms p99_ms mean_ms
2 keyset_index keyset big 60000 1 2401.7 23998 0 0.873 2.88 32.812 1.651
3 keyset_index keyset medium 15000 1 2991.2 29887 0 0.76 1.681 34.535 1.327
4 keyset_index keyset small 80 1 4017.5 40162 0 0.576 1.691 11.886 0.986
5 keyset_index offset big 60000 1 169.5 1696 0 13.118 70.429 81.03 23.566
6 keyset_index offset medium 15000 1 665.7 6672 0 2.909 47.698 53.811 5.995
7 keyset_index offset small 80 1 3402.6 34012 0 0.658 1.68 29.655 1.165
8 keyset_index keyset big 60000 2 2655.3 26611 0 0.815 2.278 33.597 1.495
9 keyset_index keyset medium 15000 2 1887.8 18889 0 0.905 6.01 32.787 2.101
10 keyset_index keyset small 80 2 2439.0 24420 0 0.659 5.636 25.202 1.622
11 keyset_index offset big 60000 2 180.4 1814 0 11.618 69.186 84.59 22.064
12 keyset_index offset medium 15000 2 791.1 7909 0 2.528 43.867 52.614 5.044
13 keyset_index offset small 80 2 4059.2 40598 0 0.604 1.05 9.089 0.977
14 keyset_index keyset big 60000 3 3486.0 34836 0 0.676 1.191 31.965 1.139
15 keyset_index keyset medium 15000 3 3499.1 34950 0 0.675 1.229 31.369 1.134
16 keyset_index keyset small 80 3 5575.1 55692 0 0.489 0.791 1.744 0.711
17 keyset_index offset big 60000 3 234.7 2357 0 9.301 63.742 68.861 17.016
18 keyset_index offset medium 15000 3 852.0 8550 0 2.398 6.87 52.109 4.683
19 keyset_index offset small 80 3 4251.9 42587 0 0.584 0.973 3.529 0.933
20 chat_id_index keyset big 60000 1 89.8 902 0 27.035 80.852 100.066 44.495
21 chat_id_index keyset medium 15000 1 364.0 3653 0 5.595 55.042 58.907 10.973
22 chat_id_index keyset small 80 1 4115.2 41258 0 0.6 1.015 4.808 0.964
23 chat_id_index offset big 60000 1 28.3 287 0 120.756 277.986 388.776 140.219
24 chat_id_index offset medium 15000 1 135.4 1360 0 16.801 71.112 77.884 29.509
25 chat_id_index offset small 80 1 4214.0 42116 0 0.587 0.978 4.666 0.941
26 chat_id_index keyset big 60000 2 89.4 899 0 28.031 80.795 96.807 44.655
27 chat_id_index keyset medium 15000 2 362.5 3623 0 5.642 55.669 59.361 11.022
28 chat_id_index keyset small 80 2 4068.0 40695 0 0.603 1.026 5.892 0.975
29 chat_id_index offset big 60000 2 28.0 283 0 120.306 292.288 396.718 142.499
30 chat_id_index offset medium 15000 2 140.0 1401 0 15.77 71.238 74.833 28.531
31 chat_id_index offset small 80 2 4222.1 42290 0 0.579 1.019 3.828 0.939
32 chat_id_index keyset big 60000 3 89.5 901 0 27.0 81.494 98.989 44.63
33 chat_id_index keyset medium 15000 3 345.7 3473 0 5.885 55.874 59.633 11.54
34 chat_id_index keyset small 80 3 4140.1 41444 0 0.59 1.026 4.533 0.958
35 chat_id_index offset big 60000 3 29.3 296 0 119.134 208.472 381.353 135.64
36 chat_id_index offset medium 15000 3 134.8 1354 0 16.795 71.384 76.114 29.634
37 chat_id_index offset small 80 3 4266.0 42637 0 0.581 0.967 3.792 0.93
38 no_index keyset big 60000 1 29.0 291 0 111.727 197.369 292.929 137.59
39 no_index keyset medium 15000 1 34.3 345 0 104.45 184.828 192.39 116.54
40 no_index keyset small 80 1 36.3 364 0 102.442 182.084 192.872 110.02
41 no_index offset big 60000 1 25.7 259 0 132.386 293.23 376.222 155.228
42 no_index offset medium 15000 1 63.3 635 0 75.84 94.867 109.291 63.124
43 no_index offset small 80 1 105.1 1055 0 19.797 72.802 78.183 38.004
44 no_index keyset big 60000 2 30.7 309 0 110.372 191.978 200.211 130.118
45 no_index keyset medium 15000 2 33.9 340 0 105.192 185.337 190.981 117.813
46 no_index keyset small 80 2 37.4 377 0 100.838 180.405 188.141 106.88
47 no_index offset big 60000 2 26.4 265 0 134.06 222.582 384.567 151.443
48 no_index offset medium 15000 2 49.7 498 0 85.725 118.239 170.86 80.332
49 no_index offset small 80 2 84.1 843 0 30.964 82.362 94.9 47.488
50 no_index keyset big 60000 3 21.6 219 0 192.061 288.029 303.799 184.125
51 no_index keyset medium 15000 3 26.0 263 0 169.475 207.861 283.698 153.341
52 no_index keyset small 80 3 25.1 255 0 178.656 220.368 289.13 158.613
53 no_index offset big 60000 3 18.0 182 0 205.371 381.47 517.852 221.459
54 no_index offset medium 15000 3 44.9 450 0 90.847 150.378 187.415 88.956
55 no_index offset small 80 3 77.6 781 0 68.094 86.015 98.542 51.458
+36
View File
@@ -0,0 +1,36 @@
## Чтение: одна страница (LIMIT 51) со случайной позиции чата
Медиана 3 повторов по 10 с, 4 клиента pgbench, prepared statements. TPS min–max — разброс повторов.
| Индекс | Запрос | Чат (сообщений) | TPS (min–max) | p50, мс | p95, мс | p99, мс |
| --- | --- | ---: | ---: | ---: | ---: | ---: |
| keyset_index | keyset | big (60 000) | 2 655 (2 402–3 486) | 0.81 | 2.28 | 32.8 |
| keyset_index | keyset | medium (15 000) | 2 991 (1 888–3 499) | 0.76 | 1.68 | 32.8 |
| keyset_index | keyset | small (80) | 4 018 (2 439–5 575) | 0.58 | 1.69 | 11.9 |
| keyset_index | offset | big (60 000) | 180 (170–235) | 11.6 | 69.2 | 81.0 |
| keyset_index | offset | medium (15 000) | 791 (666–852) | 2.53 | 43.9 | 52.6 |
| keyset_index | offset | small (80) | 4 059 (3 403–4 252) | 0.60 | 1.05 | 9.09 |
| chat_id_index | keyset | big (60 000) | 89.5 (89.4–89.8) | 27.0 | 80.9 | 99.0 |
| chat_id_index | keyset | medium (15 000) | 362 (346–364) | 5.64 | 55.7 | 59.4 |
| chat_id_index | keyset | small (80) | 4 115 (4 068–4 140) | 0.60 | 1.03 | 4.81 |
| chat_id_index | offset | big (60 000) | 28.3 (28.0–29.3) | 120 | 278 | 389 |
| chat_id_index | offset | medium (15 000) | 135 (135–140) | 16.8 | 71.2 | 76.1 |
| chat_id_index | offset | small (80) | 4 222 (4 214–4 266) | 0.58 | 0.98 | 3.83 |
| no_index | keyset | big (60 000) | 29.0 (21.6–30.7) | 112 | 197 | 293 |
| no_index | keyset | medium (15 000) | 33.9 (26.0–34.3) | 105 | 185 | 192 |
| no_index | keyset | small (80) | 36.3 (25.1–37.4) | 102 | 182 | 193 |
| no_index | offset | big (60 000) | 25.7 (18.0–26.4) | 134 | 293 | 385 |
| no_index | offset | medium (15 000) | 49.7 (44.9–63.3) | 85.7 | 118 | 171 |
| no_index | offset | small (80) | 84.1 (77.6–105) | 31.0 | 82.4 | 94.9 |
## Запись: транзакция сообщения приложения (UPDATE chats … RETURNING + INSERT)
TPS и задержка — медиана 5 повторов по 10 с, 8 клиентов. WAL — медиана 3 детерминированных прогонов: 5 000 транзакций, 1 клиент, autovacuum выключен, CHECKPOINT перед прогоном.
| Индексы messages | TPS (min–max) | p50, мс | p95, мс | p99, мс | WAL, байт/вставку (min–max) | Доп. индексы на 121 120 строк, МиБ |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| no_index | 1 954 (1 540–2 055) | 3.55 | 7.14 | 12.2 | 1930 (1884–1940) | 0.0 |
| keyset_index | 1 577 (1 002–1 630) | 4.73 | 8.50 | 13.0 | 2910 (2800–2926) | 6.8 |
| keyset_plus_sender | 1 223 (896–1 607) | 6.25 | 9.30 | 13.9 | 3318 (3310–3414) | 9.8 |
Размеры индексов измерены отдельно на одинаковых 121 120 строках (bench/track2/index_sizes.py); первичный ключ во всех вариантах — 4.6 МиБ, таблица — 33.4 МиБ.
+10
View File
@@ -0,0 +1,10 @@
variant,repeat,inserts,wal_bytes,wal_bytes_per_insert
no_index,1,5000,9419936,1884
no_index,2,5000,9700304,1940
no_index,3,5000,9652184,1930
keyset_index,1,5000,14000160,2800
keyset_index,2,5000,14548936,2910
keyset_index,3,5000,14630336,2926
keyset_plus_sender,1,5000,17071344,3414
keyset_plus_sender,2,5000,16548592,3310
keyset_plus_sender,3,5000,16589232,3318
1 variant repeat inserts wal_bytes wal_bytes_per_insert
2 no_index 1 5000 9419936 1884
3 no_index 2 5000 9700304 1940
4 no_index 3 5000 9652184 1930
5 keyset_index 1 5000 14000160 2800
6 keyset_index 2 5000 14548936 2910
7 keyset_index 3 5000 14630336 2926
8 keyset_plus_sender 1 5000 17071344 3414
9 keyset_plus_sender 2 5000 16548592 3310
10 keyset_plus_sender 3 5000 16589232 3318
+16
View File
@@ -0,0 +1,16 @@
variant,repeat,tps,transactions,failed,p50_ms,p95_ms,p99_ms,mean_ms
no_index,1,2055.1,20532,0,3.43,6.677,10.312,3.882
no_index,2,1953.5,19529,0,3.547,7.139,12.24,4.082
no_index,3,2001.8,20012,0,3.487,6.848,10.324,3.983
no_index,4,1572.9,15729,0,4.303,9.726,16.331,5.067
no_index,5,1539.6,15382,0,4.367,9.093,16.95,5.178
keyset_index,1,1002.2,10036,0,5.796,19.448,26.224,7.955
keyset_index,2,1222.9,12222,0,6.34,10.159,14.613,6.529
keyset_index,3,1629.0,16281,0,4.33,8.497,12.283,4.896
keyset_index,4,1629.7,16291,0,4.393,8.13,12.979,4.896
keyset_index,5,1577.4,15763,0,4.732,8.336,11.991,5.058
keyset_plus_sender,1,937.7,9387,0,6.629,17.527,30.246,8.507
keyset_plus_sender,2,895.6,8954,0,6.249,22.727,33.156,8.916
keyset_plus_sender,3,1606.9,16067,0,4.58,8.243,11.774,4.965
keyset_plus_sender,4,1441.4,14415,0,5.61,8.542,11.815,5.535
keyset_plus_sender,5,1223.3,12264,0,6.29,9.296,13.881,6.526
1 variant repeat tps transactions failed p50_ms p95_ms p99_ms mean_ms
2 no_index 1 2055.1 20532 0 3.43 6.677 10.312 3.882
3 no_index 2 1953.5 19529 0 3.547 7.139 12.24 4.082
4 no_index 3 2001.8 20012 0 3.487 6.848 10.324 3.983
5 no_index 4 1572.9 15729 0 4.303 9.726 16.331 5.067
6 no_index 5 1539.6 15382 0 4.367 9.093 16.95 5.178
7 keyset_index 1 1002.2 10036 0 5.796 19.448 26.224 7.955
8 keyset_index 2 1222.9 12222 0 6.34 10.159 14.613 6.529
9 keyset_index 3 1629.0 16281 0 4.33 8.497 12.283 4.896
10 keyset_index 4 1629.7 16291 0 4.393 8.13 12.979 4.896
11 keyset_index 5 1577.4 15763 0 4.732 8.336 11.991 5.058
12 keyset_plus_sender 1 937.7 9387 0 6.629 17.527 30.246 8.507
13 keyset_plus_sender 2 895.6 8954 0 6.249 22.727 33.156 8.916
14 keyset_plus_sender 3 1606.9 16067 0 4.58 8.243 11.774 4.965
15 keyset_plus_sender 4 1441.4 14415 0 5.61 8.542 11.815 5.535
16 keyset_plus_sender 5 1223.3 12264 0 6.29 9.296 13.881 6.526
+17
View File
@@ -0,0 +1,17 @@
# Valkey is the single store of sessions (SESSION_STORE=valkey), so it must be durable.
# AOF with fsync on every write: a confirmed login or logout survives a crash/restart,
# a logout can never be lost and "resurrect" a token. Sessions are small and rare
# writes, so the fsync cost is acceptable here (see REPORT.md).
appendonly yes
appendfsync always
appenddirname "appendonlydir"
# RDB snapshots are not needed on top of AOF.
save ""
dir /data
port 6379
protected-mode yes
# Sessions are never evicted silently: if memory is full, writes fail and login returns 503.
maxmemory 128mb
maxmemory-policy noeviction
# Password, generated by scripts/init_secrets.py and written here at container start.
include /tmp/requirepass.conf
+65
View File
@@ -0,0 +1,65 @@
-- 0001: initial schema of the messenger (lab 2). Design rationale: docs/DATA_MODEL.md
-- Applied once by `python -m messenger.migrate`; the version is recorded in schema_migrations.
-- User-supplied free text (display_name, title, message text) is stored as UTF-8 in BYTEA:
-- the contract allows U+0000, and PostgreSQL TEXT cannot contain NUL bytes.
-- Lengths are limited in code points by the API; the CHECKs below are byte-level safety nets
-- (one code point is at most 4 bytes in UTF-8).
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL,
display_name bytea NOT NULL,
password_hash text NOT NULL, -- Argon2id PHC string, never the password
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_key UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (octet_length(display_name) BETWEEN 1 AND 400)
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title bytea NOT NULL,
created_by uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
-- Per-chat clock: created_at of the newest message. Updating this row serialises writers
-- of one chat, so message timestamps strictly increase in commit order (see DATA_MODEL.md).
last_message_at timestamptz,
CONSTRAINT chats_title_len CHECK (octet_length(title) BETWEEN 1 AND 400)
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id),
user_id uuid NOT NULL REFERENCES users (id),
position smallint NOT NULL, -- order of member_ids sent by the creator
chat_created_at timestamptz NOT NULL, -- copy of chats.created_at for keyset listing
PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position)
);
-- GET /chats of one user: (user_id, created_at, id) range scan, no sort.
CREATE INDEX chat_members_user_keyset ON chat_members (user_id, chat_created_at, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text bytea NOT NULL,
created_at timestamptz NOT NULL,
-- The sender must be a member of this chat: enforced by the database, not only by the API.
CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id),
CONSTRAINT messages_text_len CHECK (octet_length(text) BETWEEN 1 AND 16000)
);
-- History page: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n
-- becomes an index range scan that starts exactly at the cursor.
CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id);
-- Sessions for SESSION_STORE=postgres (the default store is Valkey; see docs/DATA_MODEL.md).
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY, -- SHA-256 of the token; the token is not stored
user_id uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32)
);
CREATE INDEX sessions_expires_at ON sessions (expires_at);
+2
View File
@@ -0,0 +1,2 @@
# Only for bench/track2/summarize.py charts (optional).
matplotlib==3.11.2
+4
View File
@@ -0,0 +1,4 @@
# Student tests (in-process, via Starlette TestClient). Runtime deps come from requirements.txt.
pytest==9.1.1
httpx2==2.13.1
jsonschema==4.26.0
+16
View File
@@ -0,0 +1,16 @@
# Direct runtime dependencies. The pinned lock with hashes is requirements.txt:
# docker run --rm -v "$PWD:/w" -w /w python:3.13.16-slim-bookworm \
# sh -c "pip install -q pip-tools==7.6.1 && pip-compile --generate-hashes --strip-extras --no-emit-index-url -o requirements.txt requirements.in"
starlette==1.7.0
uvicorn==0.54.0
# Fast C parser/event loop used by uvicorn in the container (Linux only).
httptools==0.9.0
uvloop==0.23.0 ; sys_platform != "win32"
# PostgreSQL driver (libpq bundled in the binary wheel) and its async connection pool.
psycopg==3.3.6
psycopg-binary==3.3.6
psycopg-pool==3.3.3
# Valkey client (Valkey speaks the Redis protocol).
redis==8.1.0
# Argon2id password KDF.
argon2-cffi==25.1.0
+452
View File
@@ -0,0 +1,452 @@
#
# This file is autogenerated by pip-compile with Python 3.13
# by the following command:
#
# pip-compile --generate-hashes --no-emit-index-url --no-index --output-file=requirements.txt --strip-extras requirements.in
#
anyio==4.15.1 \
--hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \
--hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94
# via starlette
argon2-cffi==25.1.0 \
--hash=sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1 \
--hash=sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741
# via -r requirements.in
argon2-cffi-bindings==26.1.0 \
--hash=sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2 \
--hash=sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e \
--hash=sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605 \
--hash=sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a \
--hash=sha256:19b562b1de4b9052ef1214a2821c44b6e6f22945daa102c32ae4eff929d8b6d8 \
--hash=sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4 \
--hash=sha256:1af817e84578ef8b7295ad17de0f9896e4c8520dbf2233c7aa5aa3d487256fc4 \
--hash=sha256:1b0bcac4d490a237e18cf91f57352920c29f77f2fa39efd0813fb81298bf17ba \
--hash=sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb \
--hash=sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2 \
--hash=sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81 \
--hash=sha256:242bb0cda2ae3650764fc194593d9ea45fc9e72729acd89778c7cfe184cec2a5 \
--hash=sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29 \
--hash=sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31 \
--hash=sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8 \
--hash=sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e \
--hash=sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728 \
--hash=sha256:49d525938467d52c923a890153c99087c9d5a937d1f6b585dbdba34ec82e397a \
--hash=sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35 \
--hash=sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a \
--hash=sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d \
--hash=sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca \
--hash=sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98 \
--hash=sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1 \
--hash=sha256:7014ab7e6f5d8511af92544667a0346ea6dfc314ea9a7cad1dba9fdb5c9a6e33 \
--hash=sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36 \
--hash=sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69 \
--hash=sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1 \
--hash=sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb \
--hash=sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f \
--hash=sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083 \
--hash=sha256:b70225b5fd1e0d2ef4f7fd30d24658454535f0924dff0caca5dc08efbbbadfbb \
--hash=sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08 \
--hash=sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6 \
--hash=sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440 \
--hash=sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d \
--hash=sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e \
--hash=sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210 \
--hash=sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990 \
--hash=sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638 \
--hash=sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4
# via argon2-cffi
cffi==2.1.1 \
--hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \
--hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \
--hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \
--hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \
--hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \
--hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \
--hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \
--hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \
--hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \
--hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \
--hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \
--hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \
--hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \
--hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \
--hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \
--hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \
--hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \
--hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \
--hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \
--hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \
--hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \
--hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \
--hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \
--hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \
--hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \
--hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \
--hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \
--hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \
--hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \
--hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \
--hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \
--hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \
--hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \
--hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \
--hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \
--hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \
--hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \
--hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \
--hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \
--hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \
--hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \
--hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \
--hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \
--hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \
--hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \
--hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \
--hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \
--hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \
--hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \
--hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \
--hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \
--hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \
--hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \
--hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \
--hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \
--hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \
--hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \
--hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \
--hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \
--hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \
--hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \
--hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \
--hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \
--hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \
--hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \
--hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \
--hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \
--hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \
--hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \
--hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \
--hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \
--hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \
--hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \
--hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \
--hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \
--hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \
--hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \
--hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \
--hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \
--hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \
--hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \
--hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \
--hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \
--hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \
--hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \
--hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \
--hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \
--hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \
--hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \
--hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \
--hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \
--hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \
--hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \
--hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \
--hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \
--hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \
--hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \
--hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \
--hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \
--hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264
# via argon2-cffi-bindings
click==8.5.0 \
--hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \
--hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34
# via uvicorn
h11==0.16.0 \
--hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \
--hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86
# via uvicorn
httptools==0.9.0 \
--hash=sha256:02bc5b3dcb6394b9d825fd62a7bfa0b2943063a3c89abc4492ad45e334a20eb5 \
--hash=sha256:050f7ab098121873c8f13e35857f97ab60a76185c8302bde9a384939bb7c3b96 \
--hash=sha256:050f84b7ec46a6efe0e5f521cf8729e3397c1cef4384f62ed8d5d68ca0045776 \
--hash=sha256:06bfe7fad972a417269d8a5fc53b87e4eca970354abf5e9e24336fd06d64292e \
--hash=sha256:088de1738e1af624466a01c35d652dbe6fb825be887c76d68aa850621d81db88 \
--hash=sha256:0adc974916efe1fbf89d0363a86dcb2c746727643e362ff398de1a4b50b6bc77 \
--hash=sha256:0cc339a807c156d840b54f8bf050ba0fc265eb81692c24bca8535b52fbd797c6 \
--hash=sha256:0fd73d0bbf700a30dd87e4412adf41cfa71542a533d6b390c7244bbb8a1152bb \
--hash=sha256:130635fea6e611a6b2026120037965ddb88b3dafd11bb64e264b101a70a76630 \
--hash=sha256:13873eb8aef5972fcfee614f63d47064312ad4efbfe65ade15b8a3b77f8c8659 \
--hash=sha256:18d800aaa2d6bff7d889df810d1b19a5fde72b1f6c0ca96e8d9f28a692fe5460 \
--hash=sha256:1a4050a651e1f2faf05eb028ce9f2168abbcee9e24b209f5c1f2eb96d8c569e4 \
--hash=sha256:1a7f1df31829c258158be01bb04eb668c4fba7df1ddf2262131a972962e651b6 \
--hash=sha256:1b01c0fcd6725a8d79a164ecdc4116866282479d68bb3d6d74a909bf994656c4 \
--hash=sha256:1b95775f6292d72cb452c33e5c0f8b8551807c29a10e3c1671fef7f61361370a \
--hash=sha256:1f6da814aeecbc6cb8872d6d3e85ed16e8ab1653f9557cea8658725ce212348a \
--hash=sha256:2095207b75a83c9e947346da9c127fb7e4fb29f41589df2643764f06b750989c \
--hash=sha256:22ab1b10b06d357f01092e60f5e6856a0d479ed79b0ec2166a339ea26c699be2 \
--hash=sha256:2319858018eedd0c0b2f950a620413c0a9d1352607be4267eb28209eca8b1e3f \
--hash=sha256:268d18601feb5367885c6ebf6f402c18fc25a324cee215784adafe0a1eef925f \
--hash=sha256:26e1d9629f3bf70d23f0d22238152aec51c837a7c9e384cb74f356fdccad7eb3 \
--hash=sha256:272db0c51e8b71e953c1f2ecbe63402b819680e4564be2ef285cfd4584ee8355 \
--hash=sha256:289f213d2a3dde2e8312c415ffecec5a01698589ec6249ec4e8fb3b47c0444ba \
--hash=sha256:29b0d823e3c1e7cd1093a5dc889245db693ef13ada624cd66e2262421ef38867 \
--hash=sha256:310266a2db1377ffae3bdf6556ab4973f4f94508a8ce37b2f6bb096a89bcefa1 \
--hash=sha256:3238e198429cb8909ec42951b82d6a33fe0fdfcf86371732f8f09311c5b8ac32 \
--hash=sha256:34266cec8c1d4e3e91fcca7efe38971d6bdda64a7944f2a46ab576da15173680 \
--hash=sha256:36fac804b8cfd6b935ae64f71349f833d2b6298404626d017a2c57bb942bc643 \
--hash=sha256:3af4e45ff455fce5511fdf2653c1ce428ef09c56fe37a83eb4d924c2d474f31e \
--hash=sha256:3e3201fe4d46e0d15d7ff9fafc94a605da9eb82d2c5b9837f0368acb325481f1 \
--hash=sha256:45b3002392948dcf578029c89f6318e1289a993a1a5ec38a4161560fab60f811 \
--hash=sha256:465bc1526debf53a3be92022a16ca0c38f891ea3b5c1587af4f52e44020f8a07 \
--hash=sha256:48c705bd0b1afb6253ed71eca9f9ba7ac7d47838e5fed1ef7891d67f21ecd4de \
--hash=sha256:4a4d8c2c7e73ba5967be74d7c3a5ff81fde815ee1b48d9c5c0f14de8463a847b \
--hash=sha256:4a85401b0c3f893cf5695c1199e8679fbf673f7f78c2f6c11d6b1850f8c7e358 \
--hash=sha256:4c58dc91aefb31adad500aa68054334f429b840b36dd29e34e834101044cb2ef \
--hash=sha256:4efbee349138a3fee7a4cc3a95abd2d499fae70dd5bff9fed9138d6f570f4283 \
--hash=sha256:4fb995082fe41ec410b33c48b54fb1d44abb8a6ee762c31e8c42519e8c3a30a9 \
--hash=sha256:5042aa1c7e2b1a24c17dab31d8770b63a5101c9abc25f832c6aef6b201e1ca4f \
--hash=sha256:52fe0176682a25b15370f23f5b0f1366a84771df89144fb0cd979cb72a94b5ca \
--hash=sha256:5332a020a60bbe32ede4bda1a62b3d56c4831d309cdf0932842c0fca8ad6aaa3 \
--hash=sha256:563e4568217dc907a91843f38c737be865222c0400a38cdcd0d26ce92b3db271 \
--hash=sha256:581b27663c6e9f4df68068f32fe6d1cd7647b31fac90237221a66f8821c342eb \
--hash=sha256:58a1b0ec4cbb930e69669f9771715b2c7898d3cdf064d9811f7a66afef96b544 \
--hash=sha256:5cc5d3a29f9ec86ce406e5ec09c241dd8dc4d30e838f74f68d728b89131a3acf \
--hash=sha256:63d38e9a9a10a20fb57593742e63c6b1e78dd7f6ef5472de8e0b1e4cf4f3db26 \
--hash=sha256:6b1ac7f1bc6c0dbf90684b77571a51a21b2463909fd916ce0ac9bfc4d566dc75 \
--hash=sha256:6b900073e7b8481ef1aaf4f6c1789d210a1db01a9da8789821578cfeb4c2d540 \
--hash=sha256:6c12d0393a903b58bc5f5a7406d6c5290acfb8284290d68547ce620c06f7d133 \
--hash=sha256:6e2780e33a58a93f27cc3bb74a55bae6f9a8278a1dbabdff392940d30d381671 \
--hash=sha256:6ebd39ee26db460cfe5ab8b71a15d1149b289139a0d3981522757d6af620887e \
--hash=sha256:6f8b41299b203ce8f627db670cfea82067d9638853dbeaf86dccd93878879b85 \
--hash=sha256:6f9549ca354a1d6d6167c458a1f1b12147726b968f02dd64b6a5801dba91ae0f \
--hash=sha256:6ff0145b34610e57c9fae20df4e133c8d54266447387de6fcc0bdabfe4db4569 \
--hash=sha256:6ff5f0ed70783dcb9562dbd20edca51c3d4d277f128223709e3da6b75986d1d4 \
--hash=sha256:714bf348f468532d86bed670837e7d5ddff3834dd7f5d3c08066da400c86f088 \
--hash=sha256:757e3f79cb865a7db94e0db5f4d0ed3284a69e39d53568f433982ea13c60cac1 \
--hash=sha256:7e32b83bd8c2f8b6fa726ef34e63e21c4d7eddc277d40d4ef7245ea3ed28e5b6 \
--hash=sha256:805b0f2618e5d4c3e28f45b731eb1a0539691ae4a2f97b4ce014de0bf96a1ff5 \
--hash=sha256:80eae881cfb69383303e9a4d7961a478025b89c24f38f2e69b30c516fa0d57f2 \
--hash=sha256:813a32f94991b9627795528053c73a57d2ce3eb98ede89f0e1c7a31095938e81 \
--hash=sha256:8463b34ebde3f000627e9dbd8a545f995ad49fbf7ff9dd5abc0cd507da98a603 \
--hash=sha256:8a59c749a73fbdbc8e63b895a3079825fa085d752e75bc0a500042cb8a801e48 \
--hash=sha256:8d90d10e9b6594c28f27896a68fab97fd784c43804e9fe419dab8e8dcfcf4b02 \
--hash=sha256:8e1e037bb57dbc549c6fe20370b763ea74bdb09413cdcf857e4f14d9e4e2fb13 \
--hash=sha256:931f45f84e15daafec5f82cc92e6710569e1f50933f3253d206eab4132bec678 \
--hash=sha256:995b52f7c260ac7023640221f27472303968753cb6fc6fce1ddfb0e9db59a398 \
--hash=sha256:9b4da5789d7cf576c7e81f0088c632f6ee3786d87d17f08e90e703c22ce15633 \
--hash=sha256:a3ed60ea9a7c352c590182c67404599e6b5a0c901e75ae4cceee9a9fd6bfa455 \
--hash=sha256:a4d1ecad62e83cc65b411ea0125972cf3af98821e8117129947fd1e3a113f8d2 \
--hash=sha256:ae9bb62a7902e2ab65782447cd3eeb753510feace4e3ea03937a85489b01b16b \
--hash=sha256:b2ab3aad55d75d0b8df8d8a1b5920baaec9b161112cd5e95984848b4d2cd3dfe \
--hash=sha256:b2cc6991f16f6d666d48e4b57318104e7b29109e32e2f6b86e9d44c4e6a27f4e \
--hash=sha256:b5a3f5f70967a1aa2bc47fec42a1e19d2fb38c61700e3ee62b63a4af4f4fd001 \
--hash=sha256:b68fb053b37c258a473ab67f4965c3b439500dc160fe364667035a6833eaf50a \
--hash=sha256:b6ee42112d785a913dd63ec0335435a3dddbea5040c151252db815b0095cf066 \
--hash=sha256:b928ab0ecaa664e8caecc529dcb8bc881b6b35bb2b74bf9a39ae25f982ee8812 \
--hash=sha256:b9430f65db521db7962ad951571d446171213686f96c998a54dc18ed574821e2 \
--hash=sha256:b9cd15cb7cf0d5cc41f649fd789aae12c56c3b83eff593f8e095c1d4555ad5c3 \
--hash=sha256:bb1533541c729ad422f870a780d8b4af924f9817d45b5f580390418cda72eaa2 \
--hash=sha256:bbf7377fbd41b7c87d47820e25b9876724963681c2a1d6f6ff2adb4db46ac174 \
--hash=sha256:bca180cbe84e4fba7807eb408a8655295f697928512324517e30a091ede522a8 \
--hash=sha256:beb2c8a34cc90fb4d862b7284eafdb322030d6a8b2ee5eb6a744f84205beedc3 \
--hash=sha256:bfdabac0c6d3d6a5be8c2a100a001c92c14a39bbafd5999545a675c493626e64 \
--hash=sha256:c0e45def4d9ce7073e2226535572442d9d6efb4047c7a5fd8960807e877ce70a \
--hash=sha256:c0f537e5e8152e8d9cae82804024790cb973061abd3b7ef8f66f46e2b5c7bb51 \
--hash=sha256:c195a69df0ab2541252ab5b1d76e3c182e5688ac2a9b708e5e6f66aaeda91e9a \
--hash=sha256:c271bfb832be5c5c020b4e2fcbc1e70a0b990adba6de874b0bba1184b89cdea3 \
--hash=sha256:c42424213c28804f8d0e20f5692106cfb57bf72e1dbc4092b8481fb2f9e4c707 \
--hash=sha256:c4fa57d3c31889722f64bfa785545a5e603a893b6f29ac1a41bfa830abeaefd5 \
--hash=sha256:cb2bb3ac0af7fdab2311b895c9eb95442b45deb14cc949b9e65545e74aa0be69 \
--hash=sha256:cb3e7a4fd0168e362673a980380bf4fd6ae3b1555150e60c5390b4b10d9c50c4 \
--hash=sha256:cbbfcd5d15056fbd1edd5e725cf3feeb47c7cbccbe205927ebab422cc229f417 \
--hash=sha256:cd3e55223a77d6e08d5730ebacb4930ecca5d2ce7c57e7ba10833be7e52903f1 \
--hash=sha256:ce8e723b4637034b76f5382a30a6b725518c332273e8d62a6c7d46e90837c947 \
--hash=sha256:d1e329a1866981efe0201d05a374617f6c6cf14434a501d78ab22793d1ab1fa6 \
--hash=sha256:d20ba5c84cf0592afb2713336f07e2b6ced082e4ae803ceada153a85613efc9f \
--hash=sha256:d2b095129b9a98eb46a271ee9631089529c4e40354576b4aa74e24de9d2bf2f7 \
--hash=sha256:d3906b5c549ff2ad2473cb711e1fc65d76715c2726a402108fbf55eab6c6b49d \
--hash=sha256:d484ebb7e3a3f3597b0f645fbd1b85633674ca808c1f5ba11c2caf7c66f5c8b6 \
--hash=sha256:db735a23ecb0f0450d2b24e0a05fb00a8a35c9db172919c4d3e023e7c7ee4c9b \
--hash=sha256:dbc9fd1521e573045d71b6afab7398439c5cc259e8cb9d416fe62d485c4899c6 \
--hash=sha256:df3867518b205be3648e2fbd522bf380c851b5c2500588047505afdd786b6669 \
--hash=sha256:e0acbd474d0af4afacc6e66c4273f8a19e25f8af4379fc816388095ea6b01371 \
--hash=sha256:eacf0f45ca3ff84c01481c60c15da9ee56711f7292f66663df0f57af61e011c2 \
--hash=sha256:ead1a40543a033a6732a9e1e515944979a19db3737ce77363fc0660e38554344 \
--hash=sha256:eae4e9c7a0785a1a715de0a74fb822ab40084c060f444f18f075d05e322aa7ef \
--hash=sha256:ecf7037e491c220cd73987838c1ac3958d787bb098c3be0bfaf7f04204a6162c \
--hash=sha256:ecfeee649184ffd800955068be9a6b579a0f33fc3c98535d685d5779cb59347f \
--hash=sha256:edd5aa045fa3cc57143db018dd32ce7962bd5b525d05230709015d7e570100aa \
--hash=sha256:f0ef48ce353f6b6a52232ba23d0983d4c2c84c84a778899404e34b4718509bf2 \
--hash=sha256:f1734bd6f588975ffc246211e8b96c11933344087ca280d2cbcbf35cf835d7a9 \
--hash=sha256:f67db0ba2bedafec15b8e5330d40da1e1c7921559fa715af021252bfef81a6f8 \
--hash=sha256:f6ac1414556b910a879c108d79736f77e797871f9919ed0d2c3cf8cf3ecca986 \
--hash=sha256:f78f7ae1c2e5aabf29583fc0d302d8081a663776f84578025662eb6f5d63a921 \
--hash=sha256:f9489c1d87160c126f73b004742fe8654fa1ce37ed89e9e01330a1c10aaecde4 \
--hash=sha256:f9ccc9884241efceb4547a92955d128574c864681f11b7ea3ecbde295fafbe8b \
--hash=sha256:fc1a4f9d18d32a6e0a0a0a382986a60a2126f5144dd08715be7adb8df18e8a46
# via -r requirements.in
idna==3.20 \
--hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \
--hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c
# via anyio
psycopg==3.3.6 \
--hash=sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631 \
--hash=sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2
# via -r requirements.in
psycopg-binary==3.3.6 \
--hash=sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781 \
--hash=sha256:0a52991594ac4db888c7d39bccef331797e30cb31a95cae02cf2607f83a42dc2 \
--hash=sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475 \
--hash=sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372 \
--hash=sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de \
--hash=sha256:198a48e68cc99ccac03ba95ac857e73aa66f3bf6be77019fafb0832a05f7ad03 \
--hash=sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840 \
--hash=sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79 \
--hash=sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b \
--hash=sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e \
--hash=sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5 \
--hash=sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9 \
--hash=sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f \
--hash=sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe \
--hash=sha256:37e517c146b185f9c0c6e8d0a0ebbdeeeb67896af28466e032bc810d0c7dc7a7 \
--hash=sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138 \
--hash=sha256:3c9e663b2e800e3218994cf948c11bcc2844e6491b34aa80d089baf6531827bf \
--hash=sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d \
--hash=sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a \
--hash=sha256:566dd827f17728efdf7d88a5b066f815170f6fdad13967ae952842d90e6aaa9f \
--hash=sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4 \
--hash=sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6 \
--hash=sha256:5ea8beeb5541780b4b50b462eeacbc4f594ce3b911dc20c81c75f267876f71d2 \
--hash=sha256:5f598f19fa9a91540b5cee17932ffd227b7b53a481605bcc4573c0eafa647300 \
--hash=sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0 \
--hash=sha256:6ff05561e4a067d35507dc5c90f1deb2ec1c9703ac5cccc1bc26e08a197f9c5a \
--hash=sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6 \
--hash=sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7 \
--hash=sha256:7beb3e41c9a1e509f3ed85263386588cbe3e975aa67be21f79f44fd35ffaeefc \
--hash=sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e \
--hash=sha256:889e42acec10450185e0cdfb396f375e2c1a8d7737c114830a7fde4654f59e30 \
--hash=sha256:910ace140e3e7b7596898d083f37a8fe90c5c40684252ad4e682364b2cd3deba \
--hash=sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2 \
--hash=sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22 \
--hash=sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef \
--hash=sha256:9b2f11794e017ce340934e35de46181c46ef71ec75ea3d85dd75cd836761c01e \
--hash=sha256:a2e44a342d2aee40508e28a563d8961c39d9bbd8cae36d8578f0a3c6658aab0f \
--hash=sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c \
--hash=sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c \
--hash=sha256:a9348c5b43a3bb5ef8c2e89d5237c9c87eeafb01d338c84a7aebbc5cd0313299 \
--hash=sha256:aa73160077345ec21b3f51e8e24b3de2e99586217e497629326eb9b2ea88c52e \
--hash=sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638 \
--hash=sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba \
--hash=sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a \
--hash=sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9 \
--hash=sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc \
--hash=sha256:b979a42815410432420275412633960807178b1ce26591a16ce06e78a5bd4bb2 \
--hash=sha256:be4f9b3c9338ac5dd217c5847e21521b396c8117f78dc420d495a5c49bbef874 \
--hash=sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c \
--hash=sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e \
--hash=sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312 \
--hash=sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8 \
--hash=sha256:c7f92daa0d2a1c76f07264abddf8cbabd30152a2f09c3270e50f0c7efdf5dcac \
--hash=sha256:cbd5f73073ed19c378d4c35499db1e3e703a5b1a324e521204065967bfaa7a18 \
--hash=sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269 \
--hash=sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb \
--hash=sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10 \
--hash=sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f \
--hash=sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1 \
--hash=sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784 \
--hash=sha256:f0535693ce476a722b718b002d5d2c27d47e71ca945276ac194409c98e74c492 \
--hash=sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc \
--hash=sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52 \
--hash=sha256:f87dbdc42e78ee0f7ea180c03f8c78e80a949e373066629bd90fefff10552dff \
--hash=sha256:fa34eb47969297471db7b7f193622c7e3ee839ec05abd05f1fe104d5b1b1dcf4 \
--hash=sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8
# via -r requirements.in
psycopg-pool==3.3.3 \
--hash=sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37 \
--hash=sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d
# via -r requirements.in
pycparser==3.11 \
--hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 \
--hash=sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc
# via cffi
redis==8.1.0 \
--hash=sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25 \
--hash=sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb
# via -r requirements.in
starlette==1.7.0 \
--hash=sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e \
--hash=sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d
# via -r requirements.in
typing-extensions==4.16.0 \
--hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \
--hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5
# via
# anyio
# psycopg-pool
uvicorn==0.54.0 \
--hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \
--hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620
# via -r requirements.in
uvloop==0.23.0 ; sys_platform != "win32" \
--hash=sha256:0305871ac712f54b62af73f943dbf21ae3ce80a44bc0f0151424484affa85645 \
--hash=sha256:090865d8ce7a03986755a3ce711b7dd0d4b44eb14ab74368b717f3fad1180208 \
--hash=sha256:098a85e1393ef5202767b7e5fb41a32cd8bd81e6ee4af364c179801c4aa3f6d4 \
--hash=sha256:0efdd55bddbd36bb2fcb842d64c0d5f6407c6958c68088cc25df8c09edc5b5fd \
--hash=sha256:12634f15e6625f78b3f2922f91404c4d7173487eba11746764153f556e9852dc \
--hash=sha256:1748321e3c59a14a75404b1ae8d5a8d81c4e201803ea0e14c1b6fd84421024b5 \
--hash=sha256:19c64108b507cd0bc140e400e3396bacebd9d504956aa7726272bf6de7d9aabb \
--hash=sha256:1e84575f11873c109cf3962ad0bdf679094466184125f4cadcc41a73febff41f \
--hash=sha256:24c58ae4a83e93a04c504bcc678125e36a0bfc44af928ad69444880c60f187a5 \
--hash=sha256:28d160f51ab4da3b187063652e643dea6831072add4adc1e6d62afbe73b6be27 \
--hash=sha256:2dcff2d69be43e6559e5dad2c5a7a2dbfb60e05a77311b6c4b7a4a8123d86c65 \
--hash=sha256:31e0cf90bc8fd88784f6802cdba968a51fb1aec1cc3feec74d862b2d371d1330 \
--hash=sha256:378188efbb1524f2219d05246a3e1e5907217848d2882144dff59585f1b81d55 \
--hash=sha256:42feced24b9b44b856c633eafb5cc5dec354972da55ce77598db6844c054bc7c \
--hash=sha256:4448e9124537620f9c25d004c227bb5104440b58955c19bbd312d910af919a63 \
--hash=sha256:4a08875543bbd4519faf30497506c9cda8a48470467ffdf967c7313c7a5981a8 \
--hash=sha256:4b8e207c67d207a8608fec57e116511030af3495dc0109b8c333cf9cb412b16f \
--hash=sha256:4bb7f5d0b62b5afaaaea2b7b60d508921c24b0fe39c22c1438bec1811ffe10ec \
--hash=sha256:4f1798f56c6f4ba5ac11fa2869e5717926e4470d97a1dd42b4f59219d43b5027 \
--hash=sha256:514698d3683189031dcbfdc31e87115992e5ce9e1b19fe5359941323f2df800c \
--hash=sha256:53c2c5d7e2024e46776c2d90e6c637d01102126b61aaf5faa5edaf05f8b5722a \
--hash=sha256:55d6f4135d914305929fe9e9c44d8b5383a9b3fa1bee3bfcf60ee97e01af07ea \
--hash=sha256:5a2bbad3a63007f7e9524d4903ba04fee252557c2acd86f9a3d4f91786695254 \
--hash=sha256:5a3e0f56ec19bfd9ad1605572878dd6ff7f01b325f4fc154812ae70d615c3aff \
--hash=sha256:5bb9be71d9ee39b4359b832f9569518ec9bc08704194034e79e4958e6bc4d46d \
--hash=sha256:60ec798c40a1810d282ee046f61ecac1c5675cb898763d9f08d97d53a5e00a81 \
--hash=sha256:6b3cbc4f96ddfa1fb88a78a69dd851369825b7816d9702eee8c4461505ba172e \
--hash=sha256:6c7ef4701a96553514b2688e342ef1bf2beae6cfd172d89a76c768292aabf405 \
--hash=sha256:7337b06a9f9ed9ea3049f04b76f65819db9b19bb832ee598e97b388eadf25e5f \
--hash=sha256:76345f51367fb1f23e08605c6efb18374f669be5b223658fbab6b17627950507 \
--hash=sha256:7e35c9bc977760981693e1a7a51493b58ee5a501f9ebb1e547565ee40b6c6208 \
--hash=sha256:80cac5cb90ed7b9b72a217a1d6982b15b829cdbd0ee6bc19b93e3a9e47fb0ac9 \
--hash=sha256:8af88fe5c7dd68fe1fec6dea8155caa1a47155d219a750ff34049541cf536a5e \
--hash=sha256:8fcd721113260ffb5e38bf14a8725b17d431f34209f7d1c7005b667946e630b3 \
--hash=sha256:93087a845cdfb35753e539354ac9551bdd2ff528c202a98df0ae46e852bcf021 \
--hash=sha256:93935ab27b6eaef4c3e5489aebc84284f0644592f7ab516df60ee1b27eaf5eb3 \
--hash=sha256:9bf08e4b6362dd1c08623bbfa2d061e8bac0f1da8fc2007062cfe1dc360a49fa \
--hash=sha256:a6ac96da66c35bf789bdcde78a88dc7d56b7907d8379648c54adc1c61594575d \
--hash=sha256:ab17b3a8aa754be0de0e397f7b95f13b14e56f077a4c6ae295e3d4afd199b325 \
--hash=sha256:b0d106d9314546d69b3df1b5352639aa628530ec3ecef8a98a21942d2a2a64f5 \
--hash=sha256:b90397a50ad6332ed3e459c648ac20d182cce24a557354363ad85fc9ea4a17cd \
--hash=sha256:bbbdb8fcd5e7062e546eec1ac78c28bb21ae7df54c18f8e4b06e15a18d661a49 \
--hash=sha256:bd6f2f81c7b9da99d301c0b16b82044e76fe887086e42e1590ecf520b94dbdac \
--hash=sha256:be53e1d5f83de43dc175c87612ecc128d444b38e5c56cb3f807f5a73d6887476 \
--hash=sha256:c3f23f403a273900d57de6ee5ca0614c650f7f58563065dad1a4744498960e53 \
--hash=sha256:cbe8d03d4efcccdb7fcedecbaa1e1fa02913eaf3a74cb933634a6bc6d2ea9e2a \
--hash=sha256:ce17bc317d089f361b33521654c13e30eacfd3d2034fd34e613ca9c51c969686 \
--hash=sha256:d918d6f304a309222a784bbd140b85ec5594d97e4dc0e79f590549d28970663a \
--hash=sha256:dc61e4f9e37b507069dc7e659ae28bca7adcb04c993c3508214315d12c63f848 \
--hash=sha256:e095f9e105af76593b4c183bb0bcbdae64bd913a59ec595732dc108b48730ab5 \
--hash=sha256:e2cba180d6451822763eda8364f342435a873bcfb3849cbd82fdeca248ca65eb \
--hash=sha256:e49eba8f1e28e7c03648b7a476e1ba05309e087ccdea859fc6dd659564aa8d7e \
--hash=sha256:f1341c6abcee1c31277cfe28d34e46196f2143ec3d755e6efe7452126e1f626d \
--hash=sha256:f3fbfe82829d8e381426a289b87e59e585278728361db9ce975b88b51f64f410 \
--hash=sha256:f50b580fad005a092ed87c5a3a4683459b21d1620497d6a5bccad203bee4c071 \
--hash=sha256:f5576e8ae1723ece60d8f93c6710abf784714e99388bcf023ba9ca800bc587f6 \
--hash=sha256:f673d835bdb1a60229cc3609a113fd2c9ce3f4a3c75ad4eaed111180c00199d2 \
--hash=sha256:f7548ede3ee908cfabc0d068106e303a9a2d811af959cdf6ab85676344cedcda \
--hash=sha256:fa8ed556fcc87a4091cf61587ef172fa104323dc89ecc085a618ba7ff8629a8f \
--hash=sha256:fefea5cf8cdda9053b962ca8a90216fb0b1d40907dcb6819382b42e483e6e9f6 \
--hash=sha256:ff7144d8167e513fe39fbb46bffb4f6f192dfb1f4b0b4e9102e1fd4f212e4747
# via -r requirements.in
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env python3
"""EXPLAIN (ANALYZE, BUFFERS) of the history query on >= 10 000 messages (level 4).
python scripts/explain_history.py --output evidence/explain-history.txt
Creates (once, idempotent) a synthetic chat with 20 000 messages plus 30 000 messages in
30 other chats, then explains the EXACT statements of src/messenger/repo.py:
1. first page (HISTORY_FIRST)
2. late page via cursor (HISTORY_AFTER) — generic plan forced, as a prepared statement
reused by the driver would get it
3. the rejected variant `($2 IS NULL OR (created_at, id) > ...)` — generic plan
4. the rejected OFFSET pagination for the same late page
Runs psql inside the postgres container; no database port is published.
"""
from __future__ import annotations
import argparse
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
from messenger.repo import HISTORY_AFTER, HISTORY_FIRST # noqa: E402 (the app's real SQL)
CHAT = "00000000-0000-4000-8000-00000000e002"
USER = "00000000-0000-4000-8000-00000000e001"
TARGET = 20_000
LATE = 19_950 # cursor position: the 50 newest messages remain
SEED = f"""
\\set ON_ERROR_STOP on
BEGIN;
INSERT INTO users (id, username, display_name, password_hash)
VALUES ('{USER}', 'explain_seed', convert_to('Explain seed', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING;
INSERT INTO chats (id, title, created_by, created_at)
SELECT ('00000000-0000-4000-8000-0000000' || lpad(to_hex(n), 5, '0'))::uuid, convert_to('explain ' || n, 'UTF8'),
'{USER}', timestamptz '2026-01-01 00:00:00+00'
FROM generate_series(57346::int, 57376::int) AS n -- e002 .. e020: target + 30 noise chats
ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position, chat_created_at)
SELECT id, '{USER}', 0, created_at FROM chats WHERE created_by = '{USER}' ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), '{CHAT}', '{USER}', convert_to('synthetic message ' || g, 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM generate_series(1::int, {TARGET}::int) AS g
WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '{CHAT}');
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), c.id, '{USER}', convert_to('noise ' || g, 'UTF8'),
timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second'
FROM chats c CROSS JOIN generate_series(1::int, 1000::int) AS g
WHERE c.created_by = '{USER}' AND c.id <> '{CHAT}'
AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id);
COMMIT;
ANALYZE messages;
"""
def pg(sql_param: str) -> str:
"""Turn psycopg %s placeholders into $1..$n for a PREPARE statement."""
parts = sql_param.split("%s")
return "".join(p + (f"${i + 1}" if i < len(parts) - 1 else "") for i, p in enumerate(parts))
EXPLAIN = f"""
\\set ON_ERROR_STOP on
\\pset pager off
SELECT count(*) AS messages_total, count(*) FILTER (WHERE chat_id = '{CHAT}') AS messages_in_target_chat FROM messages;
SELECT created_at AS cur_ts, id AS cur_id FROM messages WHERE chat_id = '{CHAT}'
ORDER BY created_at, id OFFSET {LATE - 1} LIMIT 1 \\gset
SET plan_cache_mode = force_generic_plan;
PREPARE first_page(uuid, int) AS {pg(HISTORY_FIRST)};
PREPARE after_page(uuid, timestamptz, uuid, int) AS {pg(HISTORY_AFTER)};
PREPARE or_page(uuid, timestamptz, uuid, int) AS
SELECT id, sender_id, text, created_at FROM messages
WHERE chat_id = $1 AND ($2 IS NULL OR (created_at, id) > ($2, $3)) ORDER BY created_at, id LIMIT $4;
PREPARE offset_page(uuid, int, int) AS
SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id OFFSET $2 LIMIT $3;
\\echo '=== 1. first page, app query HISTORY_FIRST, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE first_page('{CHAT}', 51);
\\echo '=== 2. late page (cursor at {LATE}), app query HISTORY_AFTER, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE after_page('{CHAT}', :'cur_ts', :'cur_id', 51);
\\echo '=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE or_page('{CHAT}', :'cur_ts', :'cur_id', 51);
\\echo '=== 4. REJECTED: OFFSET pagination, same late page'
EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE offset_page('{CHAT}', {LATE}, 51);
"""
def psql(script: str) -> str:
result = subprocess.run(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-f", "-"],
cwd=ROOT, input=script, capture_output=True, text=True, encoding="utf-8", check=True)
return result.stdout
def run() -> str:
psql(SEED)
return psql(EXPLAIN)
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--output", default=None)
args = parser.parse_args()
text = run()
print(text)
if args.output:
Path(args.output).write_text(text, encoding="utf-8", newline="\n")
return 0
if __name__ == "__main__":
sys.exit(main())
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env python3
"""Create local synthetic credentials for the stand in ./secrets (ignored by Git).
python scripts/init_secrets.py
Idempotent: existing files are kept, so re-running never changes passwords of a
database that already has data. Prints only file names, never values.
Delete ./secrets together with the volumes (`docker compose down -v`) to start over.
"""
from __future__ import annotations
import os
import secrets
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
SECRETS = ROOT / "secrets"
def write(name: str, value: str) -> None:
path = SECRETS / name
if path.exists():
print(f"keep secrets/{name}")
return
path.write_text(value + "\n", encoding="utf-8", newline="\n")
# Readable by the container users (postgres, valkey, app uid 10001) through the
# compose bind mount. Local synthetic lab credentials only.
os.chmod(path, 0o644)
print(f"created secrets/{name}")
def read(name: str) -> str:
return (SECRETS / name).read_text(encoding="utf-8").strip()
def main() -> int:
SECRETS.mkdir(exist_ok=True)
write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs
write("valkey_password", secrets.token_urlsafe(24))
write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger")
write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0")
return 0
if __name__ == "__main__":
sys.exit(main())
+6
View File
@@ -0,0 +1,6 @@
@echo off
rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows
rem cannot execute a .sh file directly. Runs the same scripts\restart.sh with Git Bash.
rem python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
"%ProgramFiles%\Git\bin\bash.exe" "%~dp0restart.sh"
exit /b %ERRORLEVEL%
+15
View File
@@ -0,0 +1,15 @@
#!/bin/sh
# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
# Restarts the API and ALL state stores (PostgreSQL, Valkey) of compose.yaml.
# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed.
# Output is suppressed by the test; run it by hand to see the steps.
set -eu
cd "$(dirname "$0")/.."
# 1. API first: no request is served while the stores restart underneath it.
docker compose stop api
# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync), then start again.
docker compose restart postgres valkey
# 3. API again; it waits for healthy stores via depends_on and reconnects its pool.
docker compose start api
echo "restart.sh: api, postgres and valkey restarted with volumes preserved"
+3
View File
@@ -0,0 +1,3 @@
"""In-memory messenger for lab 1 (MTUSI, «Высоконагруженные приложения»)."""
__version__ = "1.0.0"
+65
View File
@@ -0,0 +1,65 @@
"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker).
Shutdown sequence on SIGTERM/SIGINT:
1. readiness flips to 503 (`app.state.draining = True`);
2. uvicorn closes the listening socket, so new connections are refused;
3. in-flight requests get up to SHUTDOWN_TIMEOUT_SECONDS (default 8 s < Docker's 10 s) to finish;
4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0.
Data stays in PostgreSQL and Valkey volumes.
"""
from __future__ import annotations
import logging
import sys
from types import FrameType
import uvicorn
from . import __version__, logging_setup
from .app import create_app
from .config import ConfigError, Settings
log = logging.getLogger("messenger")
class GracefulServer(uvicorn.Server):
"""uvicorn.Server that marks the app as draining before the normal graceful stop."""
def handle_exit(self, sig: int, frame: FrameType | None) -> None:
if not self.should_exit:
log.info("shutdown_requested", extra={"fields": {"signal": sig, "timeout_s": self.config.timeout_graceful_shutdown}})
self.config.app.state.draining = True # config.app is our Starlette instance
super().handle_exit(sig, frame)
def main() -> int:
try:
settings = Settings.from_env()
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
logging_setup.configure(settings.log_level)
app = create_app(settings)
config = uvicorn.Config(
app,
host=settings.host,
port=settings.port,
log_config=None,
access_log=False,
server_header=False,
proxy_headers=False,
lifespan="on",
timeout_graceful_shutdown=settings.shutdown_timeout,
)
server = GracefulServer(config)
log.info("starting", extra={"fields": {
"version": __version__, "env": settings.env, "host": settings.host, "port": settings.port,
"python": sys.version.split()[0], "storage": "postgresql",
}})
server.run()
log.info("stopped")
return 0
if __name__ == "__main__":
sys.exit(main())
+332
View File
@@ -0,0 +1,332 @@
"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping."""
from __future__ import annotations
import asyncio
import contextlib
import logging
import re
import time
import uuid
from typing import Any
from starlette.applications import Starlette
from starlette.exceptions import HTTPException
from starlette.requests import Request
from starlette.responses import JSONResponse, Response
from starlette.routing import Route
from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send
from . import authhdr, cursor, docs, passwords, repo, validation
from .config import Settings
from .db import Database
from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, PayloadTooLarge, Unauthorized, Unavailable
from .sessions import SessionStore, build_store
from .timefmt import rfc3339
log = logging.getLogger("messenger")
access_log = logging.getLogger("messenger.access")
REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
# --------------------------------------------------------------------------
# Responses
# --------------------------------------------------------------------------
def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse:
body = {"error": {"code": code, "message": message, "request_id": request_id}}
return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})})
async def api_error_handler(request: Request, exc: Exception) -> JSONResponse:
assert isinstance(exc, ApiError)
return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers)
async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse:
"""Unknown routes and wrong methods also answer in the contract error format."""
assert isinstance(exc, HTTPException)
if exc.status_code == 404:
return error_response(request.state.request_id, 404, "not_found", "Объект не найден")
if exc.status_code == 405:
return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers)
return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос")
# --------------------------------------------------------------------------
# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions)
# --------------------------------------------------------------------------
class RequestContextMiddleware:
"""Assigns X-Request-Id to every response, writes one access-log line, turns crashes into 500."""
def __init__(self, app: ASGIApp):
self.app = app
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
await self.app(scope, receive, send)
return
incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1")
request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex
scope.setdefault("state", {})["request_id"] = request_id
status = 500
started = False
begin = time.perf_counter()
async def send_with_id(message: AsgiMessage) -> None:
nonlocal status, started
if message["type"] == "http.response.start":
started = True
status = message["status"]
headers = [(k, v) for k, v in message.get("headers", []) if k.lower() != b"x-request-id"]
headers.append((b"x-request-id", request_id.encode("latin-1")))
message = {**message, "headers": headers}
await send(message)
try:
await self.app(scope, receive, send_with_id)
except asyncio.CancelledError:
# uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires:
# answer 503 (retryable) instead of a generic 500, then let the cancellation proceed.
if not started:
status = 503
response = error_response(request_id, 503, "unavailable", "Сервер завершает работу")
await response(scope, receive, send)
raise
except Exception:
log.exception("unhandled_error", extra={"fields": {"request_id": request_id}})
if not started:
status = 500
response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера")
await response(scope, receive, send)
finally:
access_log.info("request", extra={"fields": {
"request_id": request_id,
"method": scope["method"],
"path": scope["path"], # ids only; query (cursor) and bodies are never logged
"status": status,
"duration_ms": round((time.perf_counter() - begin) * 1000, 3),
}})
# --------------------------------------------------------------------------
# Request helpers
# --------------------------------------------------------------------------
def db_of(request: Request) -> Database:
return request.app.state.db
def sessions_of(request: Request) -> SessionStore:
return request.app.state.sessions
async def authenticate(request: Request) -> str:
"""User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2.
Store failures are 503 (Unavailable), never 401 and never anonymous access.
"""
token = authhdr.parse_bearer(request.headers.get("authorization"))
user_id = await sessions_of(request).user_id_for(token)
if user_id is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return user_id
def path_uuid(request: Request, name: str) -> str:
return validation.parse_uuid(request.path_params[name], name)
async def read_json(request: Request) -> dict[str, Any]:
content_type = request.headers.get("content-type", "")
media_type, _, params = content_type.partition(";")
if media_type.strip().lower() != "application/json":
raise InvalidRequest("Content-Type должен быть application/json")
charset = params.strip().lower()
if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}:
raise InvalidRequest("Поддерживается только кодировка UTF-8")
limit: int = request.app.state.settings.max_body_bytes
declared = request.headers.get("content-length")
if declared is not None and declared.isdigit() and int(declared) > limit:
raise PayloadTooLarge()
chunks: list[bytes] = []
size = 0
async for chunk in request.stream():
size += len(chunk)
if size > limit:
raise PayloadTooLarge()
chunks.append(chunk)
return validation.parse_json_object(b"".join(chunks))
def page_params(request: Request) -> tuple[int, str | None]:
query = request.query_params.multi_items()
limit = validation.parse_limit(validation.single_query_param(query, "limit"))
raw_cursor = validation.single_query_param(query, "cursor")
if raw_cursor is not None and raw_cursor == "":
raise InvalidRequest("cursor не может быть пустым")
return limit, raw_cursor
def ok(body: Any, status: int = 200) -> JSONResponse:
return JSONResponse(body, status_code=status)
# --------------------------------------------------------------------------
# Handlers
# --------------------------------------------------------------------------
async def live(request: Request) -> JSONResponse:
return ok({"status": "ok"})
async def ready(request: Request) -> JSONResponse:
"""200 only if the mandatory synchronous path can be served: PostgreSQL and,
with SESSION_STORE=valkey, Valkey answer. 503 while draining after SIGTERM."""
if request.app.state.draining:
raise Unavailable("Сервер завершает работу")
db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping())
if not db_ok:
raise Unavailable("PostgreSQL недоступен")
if not store_ok:
raise Unavailable("Хранилище сессий недоступно")
return ok({"status": "ok"})
async def register_user(request: Request) -> JSONResponse:
data = await read_json(request)
validation.check_fields(data, {"username", "display_name", "password"})
username = validation.username_field(data)
display_name = validation.text_field(data, "display_name", 100)
password = validation.password_field(data)
password_hash = await passwords.hash_password(password)
user = await repo.create_user(db_of(request), username, display_name, password_hash)
log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok(user.to_json(), 201)
async def create_session(request: Request) -> JSONResponse:
"""Login: only `Authorization: Basic`; wrong user and wrong password give the same 401."""
username, password = authhdr.parse_basic(request.headers.get("authorization"))
found = await repo.get_credentials(db_of(request), username)
user, stored_hash = found if found else (None, None)
if not await passwords.verify_password(stored_hash, password) or user is None:
log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}})
raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
ttl = request.app.state.settings.session_ttl_seconds
token, expires_at = await sessions_of(request).create(user.id, ttl)
log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}})
return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201)
async def revoke_session(request: Request) -> Response:
token = authhdr.parse_bearer(request.headers.get("authorization"))
if not await sessions_of(request).revoke(token):
raise Unauthorized("Сессия недействительна, истекла или отозвана")
log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}})
return Response(status_code=204)
async def me(request: Request) -> JSONResponse:
user_id = await authenticate(request)
user = await repo.get_user(db_of(request), user_id)
if user is None:
raise Unauthorized("Сессия недействительна, истекла или отозвана")
return ok(user.to_json())
async def create_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
data = await read_json(request)
validation.check_fields(data, {"title", "member_ids"})
title = validation.text_field(data, "title", 100)
member_ids = validation.member_ids_field(data)
chat = await repo.create_chat(db_of(request), user_id, title, member_ids)
log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}})
return ok(chat.to_json(), 201)
async def list_chats(request: Request) -> JSONResponse:
user_id = await authenticate(request)
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None
items, has_more = await repo.list_chats(db_of(request), user_id, after, limit)
next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None
return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor})
async def get_chat(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json())
async def create_message(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before the body is read
data = await read_json(request)
validation.check_fields(data, {"text"})
text = validation.text_field(data, "text", 4000)
# sender_id is derived from the session, never taken from the body.
message = await repo.add_message(db, user_id, chat_id, text)
return ok(message.to_json(), 201)
async def list_messages(request: Request) -> JSONResponse:
user_id = await authenticate(request)
chat_id = path_uuid(request, "chat_id")
db = db_of(request)
await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak
limit, raw_cursor = page_params(request)
after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None
items, has_more = await repo.list_messages(db, chat_id, after, limit)
next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None
return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor})
def create_app(settings: Settings | None = None) -> Starlette:
settings = settings or Settings.from_env()
db = Database(settings)
sessions = build_store(settings, db)
@contextlib.asynccontextmanager
async def lifespan(app: Starlette):
passwords.configure(settings.hash_concurrency)
await db.open()
log.info("dependencies_configured", extra={"fields": {
"session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds,
"db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds}})
try:
yield
finally:
await sessions.close()
await db.close()
routes = [
Route("/health/live", live, methods=["GET"]),
Route("/health/ready", ready, methods=["GET"]),
Route("/api/v1/users", register_user, methods=["POST"]),
Route("/api/v1/users/me", me, methods=["GET"]),
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]),
Route("/api/v1/chats", create_chat, methods=["POST"]),
Route("/api/v1/chats", list_chats, methods=["GET"]),
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
]
if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract
routes += [
Route("/docs", docs.swagger_page, methods=["GET"]),
Route("/openapi.json", docs.openapi_json, methods=["GET"]),
]
app = Starlette(
routes=routes,
exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler},
lifespan=lifespan,
)
app.add_middleware(RequestContextMiddleware)
app.state.settings = settings
app.state.db = db
app.state.sessions = sessions
app.state.draining = False
return app
+48
View File
@@ -0,0 +1,48 @@
"""Parsing of the Authorization header. Any malformed value is 401, never 500.
Basic (RFC 7617) is accepted only by POST /auth/sessions; everything else needs
Bearer. Basic is an encoding, not encryption: plain HTTP is acceptable only for the
local synthetic lab stand (lab 7 adds TLS).
"""
from __future__ import annotations
import base64
import binascii
import re
from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, Unauthorized
from .validation import USERNAME_RE
B64_RE = re.compile(r"[A-Za-z0-9+/]+={0,2}")
TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") # token_urlsafe(32) gives 43 characters
def _split(header: str | None) -> tuple[str, str]:
if not header:
return "", ""
scheme, _, value = header.strip().partition(" ")
return scheme.lower(), value.strip()
def parse_basic(header: str | None) -> tuple[str, str]:
"""(username, password) from `Basic base64(username:password)`; the password may contain ':'."""
invalid = Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE)
scheme, value = _split(header)
if scheme != "basic" or B64_RE.fullmatch(value) is None or len(value) > 1024:
raise invalid
try:
decoded = base64.b64decode(value, validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise invalid from None
username, sep, password = decoded.partition(":") # split at the FIRST colon
if not sep or USERNAME_RE.fullmatch(username) is None or not password:
raise invalid
return username, password
def parse_bearer(header: str | None) -> str:
scheme, value = _split(header)
if scheme != "bearer" or TOKEN_RE.fullmatch(value) is None:
# Missing header, Basic on a business endpoint, garbage token: one answer.
raise Unauthorized("Требуется действительная Bearer-сессия", challenge=BEARER_CHALLENGE)
return value
+127
View File
@@ -0,0 +1,127 @@
"""Runtime configuration read from environment variables (12-factor style).
Non-secret settings have safe defaults. Connection strings with passwords are
read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in
/run/secrets), so they never appear in `docker inspect`, compose files or logs.
Invalid values stop the process at startup with a clear message.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from pathlib import Path
class ConfigError(ValueError):
pass
def _int(name: str, default: int, low: int, high: int) -> int:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = int(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be an integer, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _float(name: str, default: float, low: float, high: float) -> float:
raw = os.environ.get(name)
if raw is None or raw.strip() == "":
return default
try:
value = float(raw.strip())
except ValueError:
raise ConfigError(f"{name} must be a number, got {raw!r}") from None
if not low <= value <= high:
raise ConfigError(f"{name} must be in [{low}, {high}], got {value}")
return value
def _bool(name: str, default: bool) -> bool:
raw = os.environ.get(name, "").strip().lower()
if raw == "":
return default
if raw not in {"true", "false", "1", "0"}:
raise ConfigError(f"{name} must be true/false, got {raw!r}")
return raw in {"true", "1"}
def _secret(name: str, required: bool) -> str:
"""Value of <NAME>_FILE (preferred, Docker secret) or <NAME> (local runs/tests)."""
path = os.environ.get(f"{name}_FILE", "").strip()
if path:
try:
value = Path(path).read_text(encoding="utf-8").strip()
except OSError as exc:
# The message names the file, never its content.
raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None
else:
value = os.environ.get(name, "").strip()
if required and not value:
raise ConfigError(f"{name}_FILE (or {name}) must be set")
return value
@dataclass(frozen=True)
class Settings:
host: str
port: int
env: str
log_level: str
shutdown_timeout: float
max_body_bytes: int
docs_enabled: bool = True
database_url: str = ""
session_store: str = "postgres"
cache_url: str = ""
session_ttl_seconds: int = 3600
db_pool_min: int = 1
db_pool_max: int = 10
db_timeout_seconds: float = 3.0
cache_timeout_seconds: float = 0.5
hash_concurrency: int = 2
@property
def db_statement_timeout_ms(self) -> int:
# The server stops a statement a bit before the client-side deadline fires.
return max(100, int(self.db_timeout_seconds * 1000) - 500)
@classmethod
def from_env(cls) -> "Settings":
log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info"
if log_level not in {"debug", "info", "warning", "error"}:
raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}")
session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres"
if session_store == "redis":
session_store = "valkey" # same protocol; the course allows Redis OR Valkey
if session_store not in {"postgres", "valkey"}:
raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}")
db_pool_min = _int("DB_POOL_MIN", 1, 0, 50)
db_pool_max = _int("DB_POOL_MAX", 10, 1, 100)
if db_pool_min > db_pool_max:
raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX")
return cls(
host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0",
port=_int("APP_PORT", 8080, 1, 65535),
env=os.environ.get("APP_ENV", "lab").strip() or "lab",
log_level=log_level,
# Docker sends SIGKILL 10 s after SIGTERM by default; finish earlier.
shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 9.0),
max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024),
docs_enabled=_bool("DOCS_ENABLED", True),
database_url=_secret("DATABASE_URL", required=True),
session_store=session_store,
cache_url=_secret("CACHE_URL", required=session_store == "valkey"),
# Contract name. Minimum 1 s so the expiry test can use 2 s.
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
db_pool_min=db_pool_min,
db_pool_max=db_pool_max,
db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0),
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0),
hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16),
)
+56
View File
@@ -0,0 +1,56 @@
"""Opaque keyset cursors.
A cursor encodes the sort key `(created_at, id)` of the last item on the page
plus the resource it belongs to, e.g. `v1|m|<chat_id>|<ts_us>|<message_id>`,
packed in URL-safe base64. The client treats it as an opaque string; the server
rejects cursors of another chat/user (400) and any non-canonical encoding.
"""
from __future__ import annotations
import base64
import re
from .errors import InvalidRequest
from .validation import UUID_RE
B64_RE = re.compile(r"[A-Za-z0-9_-]{1,512}")
TS_RE = re.compile(r"[0-9]{1,19}")
# Latest instant both Python datetime and PostgreSQL accept: 9999-12-31T23:59:59.999999Z.
MAX_TS_US = 253402300799999999
MESSAGES = "m"
CHATS = "c"
SortKey = tuple[int, str]
def encode(scope: str, owner_id: str, key: SortKey) -> str:
ts, item_id = key
raw = f"v1|{scope}|{owner_id}|{ts}|{item_id}".encode("ascii")
return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii")
def decode(cursor: str, scope: str, owner_id: str) -> SortKey:
"""Return the sort key stored in the cursor or raise InvalidRequest."""
bad = InvalidRequest("Неверный cursor")
if B64_RE.fullmatch(cursor) is None:
raise bad
try:
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)).decode("ascii")
except (ValueError, UnicodeDecodeError):
raise bad from None
parts = raw.split("|")
if len(parts) != 5 or parts[0] != "v1":
raise bad
_, got_scope, got_owner, ts, item_id = parts
if TS_RE.fullmatch(ts) is None or UUID_RE.fullmatch(item_id) is None or item_id != item_id.lower():
raise bad
key = (int(ts), item_id)
if key[0] > MAX_TS_US:
raise bad # out-of-range time must be 400, not an overflow (500) in datetime/SQL
if encode(got_scope, got_owner, key) != cursor:
raise bad # only the canonical form produced by the server is accepted
if got_scope != scope or got_owner != owner_id:
raise InvalidRequest("cursor относится к другому ресурсу")
return key
+109
View File
@@ -0,0 +1,109 @@
"""PostgreSQL access: a bounded connection pool and a hard deadline for every unit of work.
Three independent limits keep a request from hanging when the database misbehaves:
* pool `timeout` — waiting for a free connection (pool exhausted);
* server `statement_timeout` / `lock_timeout` — PostgreSQL itself cancels a slow
statement or a statement stuck behind a lock;
* client deadline — covers what the server cannot report, e.g. a
paused/frozen PostgreSQL container or a dead network.
The client deadline needs care: when a psycopg query is cancelled, psycopg sends a
cancel request to the server and waits for it (up to 5 s + 5 s) before giving up.
With a frozen server that would stretch a 3 s deadline to 13 s. Therefore every unit
of work runs as its own task: at the deadline the request stops waiting and gets 503
at once, while the task finishes its cleanup (cancel, close, return to the pool) in
the background. Any failure becomes 503 `unavailable` — never 401, never a hang.
"""
from __future__ import annotations
import asyncio
import logging
from collections.abc import Awaitable, Callable
from typing import TypeVar
import psycopg
from psycopg_pool import AsyncConnectionPool, PoolTimeout
from .config import Settings
from .errors import Unavailable
log = logging.getLogger("messenger.db")
T = TypeVar("T")
Work = Callable[[psycopg.AsyncConnection], Awaitable[T]]
# Errors that mean "the database is unavailable right now", not "the request is wrong".
# QueryCanceled (statement_timeout), LockNotAvailable (lock_timeout) and AdminShutdown
# (server restart) are subclasses of OperationalError.
UNAVAILABLE_ERRORS = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TimeoutError)
def _forget(task: asyncio.Task) -> None:
"""Done-callback for abandoned tasks: retrieve the result so nothing is logged as 'never retrieved'."""
if not task.cancelled():
task.exception()
class Database:
def __init__(self, settings: Settings):
self.deadline = settings.db_timeout_seconds
options = (
f"-c statement_timeout={settings.db_statement_timeout_ms} "
f"-c lock_timeout={max(100, settings.db_statement_timeout_ms // 2)} "
"-c idle_in_transaction_session_timeout=10000"
)
self.pool = AsyncConnectionPool(
conninfo=settings.database_url,
min_size=settings.db_pool_min,
max_size=settings.db_pool_max,
timeout=settings.db_timeout_seconds, # max wait for a free connection
open=False,
name="messenger",
check=AsyncConnectionPool.check_connection, # drop connections broken by a DB restart
kwargs={
"options": options,
"connect_timeout": max(1, int(settings.db_timeout_seconds)),
"application_name": "messenger-api",
},
)
async def open(self) -> None:
# wait=False: the API starts (liveness 200) even if PostgreSQL is not up yet;
# readiness stays 503 until the database answers.
await self.pool.open(wait=False)
async def close(self) -> None:
await self.pool.close(timeout=2)
async def run(self, work: Work[T], deadline: float | None = None) -> T:
"""Run `work(conn)` in one transaction (COMMIT on success, ROLLBACK on any exception).
Domain errors raised by `work` (NotFound, Conflict, ...) propagate unchanged;
database unavailability and the deadline become Unavailable (503).
"""
async def unit() -> T:
async with self.pool.connection() as conn:
async with conn.transaction():
return await work(conn)
task = asyncio.ensure_future(unit())
try:
return await asyncio.wait_for(asyncio.shield(task), deadline or self.deadline)
except UNAVAILABLE_ERRORS as exc:
log.warning("database_unavailable", extra={"fields": {"error": type(exc).__name__}})
raise Unavailable("База данных временно недоступна") from None
finally:
if not task.done():
# Deadline or cancelled request: stop waiting now, let psycopg clean up later.
task.cancel()
task.add_done_callback(_forget)
async def ping(self, timeout: float = 1.0) -> bool:
async def select_one(conn: psycopg.AsyncConnection) -> None:
await conn.execute("SELECT 1")
try:
await self.run(select_one, deadline=timeout)
return True
except Unavailable:
return False
+57
View File
@@ -0,0 +1,57 @@
"""Swagger UI for manual experiments: http://127.0.0.1:8080/docs
The page shows the course contract contracts/openapi.json as is (it is not
generated from our code). Only `servers` is replaced with "/" so that the
"Try it out" requests go to the same host the page was opened from.
Swagger UI itself (JS/CSS) is loaded by the browser from the jsDelivr CDN.
Disable with DOCS_ENABLED=false.
"""
from __future__ import annotations
import json
from functools import lru_cache
from pathlib import Path
from starlette.requests import Request
from starlette.responses import HTMLResponse, JSONResponse
SPEC_PATH = Path(__file__).resolve().parents[2] / "contracts" / "openapi.json"
SWAGGER_UI = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5.33.0"
PAGE = f"""<!doctype html>
<html lang="ru">
<head>
<meta charset="utf-8">
<title>Messenger lab1 — Swagger UI</title>
<link rel="stylesheet" href="{SWAGGER_UI}/swagger-ui.css">
</head>
<body>
<div id="swagger-ui"></div>
<script src="{SWAGGER_UI}/swagger-ui-bundle.js"></script>
<script>
window.ui = SwaggerUIBundle({{
url: "/openapi.json",
dom_id: "#swagger-ui",
tryItOutEnabled: true, // fields are editable right away
persistAuthorization: true, // X-User-Id from "Authorize" survives page reload
displayRequestDuration: true,
}});
</script>
</body>
</html>
"""
@lru_cache(maxsize=1)
def _spec() -> dict:
spec = json.loads(SPEC_PATH.read_text(encoding="utf-8"))
spec["servers"] = [{"url": "/", "description": "этот сервер"}]
return spec
async def swagger_page(request: Request) -> HTMLResponse:
return HTMLResponse(PAGE)
async def openapi_json(request: Request) -> JSONResponse:
return JSONResponse(_spec())
+65
View File
@@ -0,0 +1,65 @@
"""Domain errors mapped 1:1 to the contract error codes (contracts/README.md)."""
from __future__ import annotations
BEARER_CHALLENGE = 'Bearer realm="messenger"'
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
class ApiError(Exception):
status = 500
code = "internal_error"
default_message = "Внутренняя ошибка сервера"
def __init__(self, message: str | None = None):
self.message = message or self.default_message
super().__init__(self.message)
@property
def headers(self) -> dict[str, str]:
return {}
class InvalidRequest(ApiError):
status = 400
code = "invalid_request"
default_message = "Неверный запрос"
class Unauthorized(ApiError):
"""401 with the challenge of the scheme the endpoint expects (RFC 7235)."""
status = 401
code = "unauthorized"
default_message = "Требуется действительная сессия"
def __init__(self, message: str | None = None, challenge: str = BEARER_CHALLENGE):
super().__init__(message)
self.challenge = challenge
@property
def headers(self) -> dict[str, str]:
return {"WWW-Authenticate": self.challenge}
class NotFound(ApiError):
status = 404
code = "not_found"
default_message = "Объект не найден"
class Conflict(ApiError):
status = 409
code = "conflict"
default_message = "Конфликт"
class PayloadTooLarge(ApiError):
status = 413
code = "payload_too_large"
default_message = "Тело запроса слишком большое"
class Unavailable(ApiError):
status = 503
code = "unavailable"
default_message = "Сервис временно недоступен"
+19
View File
@@ -0,0 +1,19 @@
"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`."""
from __future__ import annotations
import os
import sys
import urllib.request
def main() -> int:
port = os.environ.get("APP_PORT", "8080").strip() or "8080"
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response:
return 0 if response.status == 200 else 1
except Exception:
return 1
if __name__ == "__main__":
sys.exit(main())
+36
View File
@@ -0,0 +1,36 @@
"""JSON-lines logging to stdout. Logs never contain message texts or request bodies."""
from __future__ import annotations
import datetime as dt
import json
import logging
import sys
class JsonFormatter(logging.Formatter):
def format(self, record: logging.LogRecord) -> str:
entry = {
"ts": dt.datetime.fromtimestamp(record.created, dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"),
"level": record.levelname.lower(),
"logger": record.name,
"event": record.getMessage(),
}
fields = getattr(record, "fields", None)
if fields:
entry.update(fields)
if record.exc_info:
# Stack traces go only to the server log, never to the HTTP client.
entry["exception"] = self.formatException(record.exc_info)
return json.dumps(entry, ensure_ascii=False)
def configure(level: str) -> None:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
root.handlers[:] = [handler]
root.setLevel(level.upper())
# uvicorn's own access log is disabled; our middleware writes one line per request.
for name in ("uvicorn", "uvicorn.error"):
logging.getLogger(name).handlers[:] = []
logging.getLogger(name).propagate = True
+76
View File
@@ -0,0 +1,76 @@
"""Versioned SQL migrations: `python -m messenger.migrate` (compose service `migrate`).
* Files `migrations/NNNN_name.sql` are applied in name order, each in its own
transaction together with its row in `schema_migrations` (DDL is transactional
in PostgreSQL, so a failed migration leaves nothing behind).
* Re-running is safe: applied versions are skipped, so data is never duplicated.
* The SHA-256 of every applied file is stored; if an applied file was edited later,
the run stops instead of silently diverging from the real schema.
* `pg_advisory_lock` serialises concurrent runs (two API replicas starting at once).
"""
from __future__ import annotations
import hashlib
import sys
import time
from pathlib import Path
import psycopg
from .config import ConfigError, _secret
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations"
LOCK_ID = 2026_02 # arbitrary application-wide advisory lock id
def say(text: str) -> None:
print(f"migrate: {text}", flush=True)
def connect(url: str, wait_seconds: float = 60) -> psycopg.Connection:
deadline = time.monotonic() + wait_seconds
while True:
try:
return psycopg.connect(url, autocommit=True, connect_timeout=3, application_name="messenger-migrate")
except psycopg.OperationalError as exc:
if time.monotonic() > deadline:
raise SystemExit(f"migrate: database is not reachable: {type(exc).__name__}") from None
time.sleep(1)
def main() -> int:
try:
url = _secret("DATABASE_URL", required=True)
except ConfigError as exc:
print(f"configuration error: {exc}", file=sys.stderr)
return 2
files = sorted(MIGRATIONS_DIR.glob("*.sql"))
with connect(url) as conn:
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY, checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT now())")
applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall())
for path in files:
version, sql = path.stem, path.read_bytes()
checksum = hashlib.sha256(sql).hexdigest()
if version in applied:
if applied[version] != checksum:
say(f"{version}: file changed after it was applied; create a new migration instead")
return 1
say(f"{version}: already applied, skipped")
continue
with conn.transaction():
conn.execute(sql.decode("utf-8"))
conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (version, checksum))
say(f"{version}: applied")
finally:
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
say(f"done, {len(files)} migration file(s) checked")
return 0
if __name__ == "__main__":
sys.exit(main())
+53
View File
@@ -0,0 +1,53 @@
"""Password hashing with Argon2id (memory-hard password KDF; also mandatory in lab 7).
* Parameters follow the OWASP minimum for Argon2id: m=19 MiB, t=2, p=1. Unlike bcrypt
there is no 72-byte limit, so any 12-128 code point UTF-8 password is hashed whole.
* Hashing takes tens of milliseconds of CPU and 19 MiB of RAM. It runs in a worker
thread (argon2-cffi releases the GIL), so the event loop keeps serving other
requests, and a semaphore caps parallel hashes (memory/CPU DoS protection).
* An unknown username is checked against a dummy hash, so "no such user" and
"wrong password" take the same time and return the same 401.
"""
from __future__ import annotations
import asyncio
from argon2 import PasswordHasher
from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError
_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1, hash_len=32, salt_len=16)
_semaphore: asyncio.Semaphore | None = None
_dummy_hash: str | None = None
def configure(concurrency: int) -> None:
global _semaphore
_semaphore = asyncio.Semaphore(concurrency)
async def _run(func, *args):
if _semaphore is None:
configure(2)
async with _semaphore:
return await asyncio.to_thread(func, *args)
async def hash_password(password: str) -> str:
return await _run(_hasher.hash, password)
def _verify(stored_hash: str, password: str) -> bool:
try:
return _hasher.verify(stored_hash, password)
except (VerifyMismatchError, VerificationError, InvalidHashError):
return False
async def verify_password(stored_hash: str | None, password: str) -> bool:
global _dummy_hash
if stored_hash is None:
if _dummy_hash is None:
_dummy_hash = await hash_password("dummy-password-for-timing")
await _run(_verify, _dummy_hash, password)
return False
return await _run(_verify, stored_hash, password)
+244
View File
@@ -0,0 +1,244 @@
"""SQL for users, chats and messages. All statements are parameterised (%s placeholders):
values travel separately from the SQL text, so input can never change the query.
Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is
encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md.
"""
from __future__ import annotations
import datetime as dt
import uuid
from dataclasses import dataclass
import psycopg
from psycopg import errors as pg_errors
from .cursor import SortKey
from .db import Database
from .errors import Conflict, InvalidRequest, NotFound
from .timefmt import from_us, rfc3339, to_us
def _b(text: str) -> bytes:
return text.encode("utf-8")
def _s(raw: bytes | memoryview) -> str:
return bytes(raw).decode("utf-8")
@dataclass(slots=True, frozen=True)
class User:
id: str
username: str
display_name: str
created_at: dt.datetime
def to_json(self) -> dict:
return {"id": self.id, "username": self.username, "display_name": self.display_name,
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Chat:
id: str
title: str
member_ids: tuple[str, ...]
created_at: dt.datetime
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids),
"created_at": rfc3339(self.created_at)}
@dataclass(slots=True, frozen=True)
class Message:
id: str
chat_id: str
sender_id: str
text: str
created_at: dt.datetime
@property
def key(self) -> SortKey:
return (to_us(self.created_at), self.id)
def to_json(self) -> dict:
return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text,
"created_at": rfc3339(self.created_at)}
def _user(row) -> User:
return User(str(row[0]), row[1], _s(row[2]), row[3])
# ------------------------------------------------------------------ users
async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User:
async def work(conn: psycopg.AsyncConnection) -> User:
cur = await conn.execute(
"INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) "
"RETURNING id, username, display_name, created_at",
(uuid.uuid4(), username, _b(display_name), password_hash))
return _user(await cur.fetchone())
try:
return await db.run(work)
except pg_errors.UniqueViolation:
# Two concurrent registrations: the unique index lets exactly one INSERT commit.
raise Conflict("Имя пользователя уже занято") from None
async def get_user(db: Database, user_id: str) -> User | None:
async def work(conn: psycopg.AsyncConnection) -> User | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,))
row = await cur.fetchone()
return _user(row) if row else None
return await db.run(work)
async def get_credentials(db: Database, username: str) -> tuple[User, str] | None:
async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None:
cur = await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,))
row = await cur.fetchone()
return (_user(row), row[4]) if row else None
return await db.run(work)
# ------------------------------------------------------------------ chats
async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat:
"""Atomic: the chat row and all memberships commit together or not at all.
The chat is inserted FIRST; an unknown member then fails the membership INSERT
with a foreign-key violation and the whole transaction (chat row included) is
rolled back. student-tests check that nothing is left behind.
"""
if creator_id not in member_ids:
raise InvalidRequest("member_ids должен включать текущего пользователя")
chat_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
cur = await conn.execute(
"INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at",
(chat_id, _b(title), creator_id))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) "
"SELECT %s, m.user_id, m.ord - 1, %s "
"FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
(chat_id, created_at, member_ids))
return created_at
try:
created_at = await db.run(work)
except pg_errors.ForeignKeyViolation:
raise InvalidRequest("member_ids содержит несуществующих пользователей") from None
return Chat(str(chat_id), title, tuple(member_ids), created_at)
async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]:
if not rows:
return []
ids = [r[0] for r in rows]
cur = await conn.execute(
"SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,))
members: dict[uuid.UUID, list[str]] = {i: [] for i in ids}
for chat_id, user_id in await cur.fetchall():
members[chat_id].append(str(user_id))
return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows]
async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat:
"""The chat if the user is a member; a missing chat and a foreign chat are both 404."""
async def work(conn: psycopg.AsyncConnection) -> Chat:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chats c "
"JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
(user_id, chat_id))
row = await cur.fetchone()
if row is None:
raise NotFound("Чат не найден")
return (await _load_chats(conn, [row]))[0]
return await db.run(work)
async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None:
cur = await conn.execute(
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id))
if await cur.fetchone() is None:
raise NotFound("Чат не найден")
async def ensure_member(db: Database, user_id: str, chat_id: str) -> None:
async def work(conn: psycopg.AsyncConnection) -> None:
await _check_member(conn, user_id, chat_id)
await db.run(work)
async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]:
async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]:
if after is None:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, limit + 1))
else:
cur = await conn.execute(
"SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id "
"WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) "
"ORDER BY m.chat_created_at, m.chat_id LIMIT %s",
(user_id, from_us(after[0]), after[1], limit + 1))
rows = await cur.fetchall()
return await _load_chats(conn, rows[:limit]), len(rows) > limit
return await db.run(work)
# ------------------------------------------------------------------ messages
async def add_message(db: Database, user_id: str, chat_id: str, text: str) -> Message:
"""Insert with a per-chat strictly increasing timestamp.
`UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one
chat are serialised: a message with a smaller created_at can never commit after
a message with a bigger one, and a reader paging by cursor cannot skip it.
"""
message_id = uuid.uuid4()
async def work(conn: psycopg.AsyncConnection) -> dt.datetime:
await _check_member(conn, user_id, chat_id)
cur = await conn.execute(
"UPDATE chats SET last_message_at = greatest(clock_timestamp(), "
"last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at",
(chat_id,))
(created_at,) = await cur.fetchone()
await conn.execute(
"INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)",
(message_id, chat_id, user_id, _b(text), created_at))
return created_at
created_at = await db.run(work)
return Message(str(message_id), chat_id, user_id, text, created_at)
# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR
# would push the cursor into a Filter and late pages would scan the whole prefix.
HISTORY_FIRST = (
"SELECT id, sender_id, text, created_at FROM messages "
"WHERE chat_id = %s ORDER BY created_at, id LIMIT %s")
HISTORY_AFTER = (
"SELECT id, sender_id, text, created_at FROM messages "
"WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s")
async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]:
async def work(conn: psycopg.AsyncConnection) -> list:
if after is None:
cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1))
else:
cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1))
return await cur.fetchall()
rows = await db.run(work)
items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3]) for r in rows[:limit]]
return items, len(rows) > limit
+160
View File
@@ -0,0 +1,160 @@
"""Server-side sessions behind an opaque Bearer token.
* Token: `secrets.token_urlsafe(32)` = 256 random bits. Only SHA-256(token) is
stored, so a leaked database/AOF does not contain usable tokens. A fast hash is
enough for a 256-bit random value; passwords need a slow KDF (passwords.py).
* Expiry is absolute: `expires_at = login time + SESSION_TTL_SECONDS`. Every lookup
compares it with the current time, and a restart never extends it.
Two interchangeable stores (SESSION_STORE):
* `valkey` (compose default): the ONLY copy of a session is a Valkey key with
`PXAT expires_at`, persisted by AOF with `appendfsync always`. There is no second
copy (no cache over PostgreSQL), so a logout cannot be "forgotten" and a revoked
token cannot come back after Valkey fails and recovers. If Valkey is unreachable
a session cannot be checked: protected requests and logout get 503, never access.
* `postgres`: table `sessions` (migrations/0001_init.sql).
"""
from __future__ import annotations
import asyncio
import datetime as dt
import hashlib
import json
import logging
import secrets
from typing import Protocol
from redis import exceptions as redis_errors
from redis.asyncio import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
from .config import Settings
from .db import Database
from .errors import Unavailable
from .timefmt import UTC, from_us, to_us
log = logging.getLogger("messenger.sessions")
def new_token() -> str:
return secrets.token_urlsafe(32)
def token_hash(token: str) -> bytes:
return hashlib.sha256(token.encode("ascii")).digest()
def now() -> dt.datetime:
return dt.datetime.now(UTC)
class SessionStore(Protocol):
name: str
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: ...
async def user_id_for(self, token: str) -> str | None: ...
async def revoke(self, token: str) -> bool: ...
async def ping(self) -> bool: ...
async def close(self) -> None: ...
class PostgresSessionStore:
name = "postgres"
def __init__(self, db: Database):
self.db = db
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
token = new_token()
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
async def work(conn) -> None:
await conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)",
(token_hash(token), user_id, expires_at))
# Housekeeping: drop this user's expired sessions (bounded by the user's own rows).
await conn.execute("DELETE FROM sessions WHERE user_id = %s AND expires_at <= %s", (user_id, now()))
await self.db.run(work)
return token, expires_at
async def user_id_for(self, token: str) -> str | None:
async def work(conn):
cur = await conn.execute(
"SELECT user_id FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
return await cur.fetchone()
row = await self.db.run(work)
return str(row[0]) if row else None
async def revoke(self, token: str) -> bool:
async def work(conn) -> bool:
cur = await conn.execute(
"DELETE FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now()))
return cur.rowcount == 1
return await self.db.run(work)
async def ping(self) -> bool:
return True # covered by the database readiness check
async def close(self) -> None:
return None
class ValkeySessionStore:
name = "valkey"
PREFIX = "session:"
def __init__(self, settings: Settings):
self.timeout = settings.cache_timeout_seconds
self.client = Redis.from_url(
settings.cache_url,
socket_timeout=self.timeout,
socket_connect_timeout=self.timeout,
retry=Retry(NoBackoff(), 0), # no hidden retries: a failure is reported at once
decode_responses=False,
)
def _key(self, token: str) -> str:
return self.PREFIX + token_hash(token).hex()
async def _call(self, coro):
try:
async with asyncio.timeout(self.timeout * 2):
return await coro
except (redis_errors.ConnectionError, redis_errors.TimeoutError, TimeoutError, OSError) as exc:
log.warning("session_store_unavailable", extra={"fields": {"error": type(exc).__name__}})
raise Unavailable("Хранилище сессий временно недоступно") from None
async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]:
token = new_token()
expires_at = now() + dt.timedelta(seconds=ttl_seconds)
value = json.dumps({"user_id": user_id, "expires_at_us": to_us(expires_at)}).encode()
# PXAT = absolute expiry time: survives restarts (AOF) without being extended.
await self._call(self.client.set(self._key(token), value, pxat=to_us(expires_at) // 1000))
return token, expires_at
async def user_id_for(self, token: str) -> str | None:
raw = await self._call(self.client.get(self._key(token)))
if raw is None:
return None
data = json.loads(raw)
if from_us(data["expires_at_us"]) <= now():
return None # do not rely only on Valkey's lazy expiry
return data["user_id"]
async def revoke(self, token: str) -> bool:
return await self._call(self.client.delete(self._key(token))) == 1
async def ping(self) -> bool:
try:
return bool(await self._call(self.client.ping()))
except Unavailable:
return False
async def close(self) -> None:
await self.client.aclose()
def build_store(settings: Settings, db: Database) -> SessionStore:
return ValkeySessionStore(settings) if settings.session_store == "valkey" else PostgresSessionStore(db)
+22
View File
@@ -0,0 +1,22 @@
"""Conversions between PostgreSQL timestamptz values, cursor integers and RFC 3339 strings."""
from __future__ import annotations
import datetime as dt
UTC = dt.timezone.utc
_EPOCH = dt.datetime(1970, 1, 1, tzinfo=UTC)
def to_us(value: dt.datetime) -> int:
"""Microseconds since the Unix epoch (exact: timestamptz has microsecond precision)."""
delta = value - _EPOCH
return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds
def from_us(us: int) -> dt.datetime:
return _EPOCH + dt.timedelta(microseconds=us)
def rfc3339(value: dt.datetime) -> str:
"""RFC 3339 UTC with fixed microsecond precision, e.g. 2026-10-12T12:00:00.000001Z."""
return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+117
View File
@@ -0,0 +1,117 @@
"""Strict request validation according to contracts/openapi.json + contracts/README.md.
Written by hand instead of a schema library so that every rule is visible:
extra fields, wrong JSON types, whitespace-only strings and lengths counted in
Unicode code points all produce 400 invalid_request.
"""
from __future__ import annotations
import json
import re
from typing import Any
from .errors import InvalidRequest
USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}")
# Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case.
UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
LIMIT_RE = re.compile(r"[0-9]{1,3}")
SURROGATES_RE = re.compile("[\ud800-\udfff]")
DEFAULT_LIMIT = 50
MAX_LIMIT = 100
def _reject_constant(name: str) -> Any:
# Python's json accepts NaN/Infinity, which are not valid JSON.
raise ValueError(f"invalid JSON constant {name}")
def parse_json_object(body: bytes) -> dict[str, Any]:
"""Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400)."""
try:
text = body.decode("utf-8")
value = json.loads(text, parse_constant=_reject_constant)
except (UnicodeDecodeError, ValueError):
raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None
if not isinstance(value, dict):
raise InvalidRequest("Тело запроса должно быть JSON-объектом")
return value
def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[str] = frozenset()) -> None:
extra = set(data) - required - optional
if extra:
raise InvalidRequest("Неизвестные поля: " + ", ".join(sorted(extra)))
missing = required - set(data)
if missing:
raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing)))
def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1) -> str:
value = data[name]
if not isinstance(value, str):
raise InvalidRequest(f"Поле {name} должно быть строкой")
if SURROGATES_RE.search(value):
# "\ud800" escapes decode to lone surrogates that cannot be encoded as UTF-8.
raise InvalidRequest(f"Поле {name} содержит недопустимые символы")
# len() of a Python str is the number of Unicode code points, as the contract requires.
if not min_len <= len(value) <= max_len:
raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов")
if value.strip() == "":
raise InvalidRequest(f"Поле {name} не может состоять только из пробелов")
return value
def username_field(data: dict[str, Any]) -> str:
value = data["username"]
if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None:
raise InvalidRequest("username должен соответствовать [a-z0-9_]{3,32}")
return value
def password_field(data: dict[str, Any]) -> str:
"""12-128 Unicode code points, not whitespace-only; any other character (':' too) is allowed."""
value = data["password"]
if not isinstance(value, str):
raise InvalidRequest("Поле password должно быть строкой")
if SURROGATES_RE.search(value):
raise InvalidRequest("Поле password содержит недопустимые символы")
if not 12 <= len(value) <= 128:
raise InvalidRequest("Длина пароля должна быть от 12 до 128 символов")
if value.strip() == "":
raise InvalidRequest("Пароль не может состоять только из пробелов")
return value
def parse_uuid(value: Any, what: str = "UUID") -> str:
if not isinstance(value, str) or UUID_RE.fullmatch(value) is None:
raise InvalidRequest(f"Неверный {what}")
return value.lower()
def member_ids_field(data: dict[str, Any]) -> list[str]:
value = data["member_ids"]
if not isinstance(value, list):
raise InvalidRequest("member_ids должен быть массивом")
if not 1 <= len(value) <= 100:
raise InvalidRequest("member_ids должен содержать от 1 до 100 элементов")
ids = [parse_uuid(item, "UUID участника") for item in value]
if len(set(ids)) != len(ids):
raise InvalidRequest("member_ids содержит дубликаты")
return ids
def single_query_param(query: list[tuple[str, str]], name: str) -> str | None:
values = [v for k, v in query if k == name]
if len(values) > 1:
raise InvalidRequest(f"Параметр {name} указан несколько раз")
return values[0] if values else None
def parse_limit(raw: str | None) -> int:
if raw is None:
return DEFAULT_LIMIT
if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT:
raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}")
return int(raw)
+226
View File
@@ -0,0 +1,226 @@
"""Shared helpers for the student tests.
* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services.
* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose
stand (BASE_URL, default http://127.0.0.1:8080) plus `docker compose` for database
checks (psql inside the postgres container) and failure injection. Every failure
test restores the stopped/paused service in `finally`.
python -m pytest student-tests # everything (stand must be up)
python -m pytest student-tests -m "not stack" # unit only
"""
from __future__ import annotations
import base64
import contextlib
import json
import os
import subprocess
import sys
import time
import uuid
from pathlib import Path
import httpx2 as httpx
import pytest
from jsonschema import Draft202012Validator
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "src"))
SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8"))
BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/")
PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose
def pytest_configure(config):
config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)")
def schema_errors(value, name: str) -> list[str]:
"""Validate against components/schemas/<name> of the course contract (JSON Schema 2020-12)."""
schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]}
validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER)
return [e.message for e in validator.iter_errors(value)]
def basic(username: str, password: str) -> str:
return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode()
# ------------------------------------------------------------------ docker / psql
def compose(*args: str, check: bool = True, timeout: float = 120) -> subprocess.CompletedProcess:
return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True,
encoding="utf-8", check=check, timeout=timeout)
def psql(sql: str) -> str:
"""Run SQL inside the postgres container (no DB port is published) and return unaligned output."""
result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
"-At", "-v", "ON_ERROR_STOP=1", "-c", sql)
return result.stdout.strip()
def wait_ready(timeout: float = 90) -> None:
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
try:
if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200:
return
except httpx.HTTPError:
pass
time.sleep(0.3)
raise AssertionError("stand did not become ready")
def wait_healthy(*services: str, timeout: float = 120) -> None:
"""Poll Docker health until every service is healthy again. `compose up --wait` is not used:
it gives up at once on a container that was marked unhealthy while paused."""
deadline = time.monotonic() + timeout
pending = set(services)
while pending and time.monotonic() < deadline:
for service in list(pending):
ids = compose("ps", "-q", service, check=False).stdout.split()
if ids:
state = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]],
capture_output=True, text=True).stdout.strip()
if state == "healthy":
pending.discard(service)
time.sleep(0.5)
assert not pending, f"not healthy: {pending}"
def restart_services(*services: str) -> None:
compose("restart", *services)
wait_healthy(*services)
wait_ready()
@contextlib.contextmanager
def service_down(service: str, how: str = "stop"):
"""Stop or pause a compose service; ALWAYS bring it back and wait for readiness."""
assert how in {"stop", "pause"}
compose(how, service)
try:
yield
finally:
compose("unpause" if how == "pause" else "start", service)
wait_healthy(service)
wait_ready()
@contextlib.contextmanager
def api_with_env(**overrides: str):
"""Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2),
then ALWAYS recreate it with the normal configuration."""
env = {**os.environ, **overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env,
capture_output=True, text=True, check=True, timeout=180)
wait_ready()
try:
yield
finally:
clean = {k: v for k, v in os.environ.items() if k not in overrides}
subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean,
capture_output=True, text=True, check=True, timeout=180)
wait_ready()
def api_env(name: str) -> str:
"""Current value of an environment variable inside the running api container."""
return compose("exec", "-T", "api", "printenv", name).stdout.strip()
# ------------------------------------------------------------------ HTTP
class Api:
"""HTTP helper that checks the invariants every response must satisfy."""
def __init__(self, base_url: str = BASE_URL):
self.client = httpx.Client(base_url=base_url, timeout=15)
def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None,
json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None):
h = dict(headers or {})
if token is not None:
h["Authorization"] = "Bearer " + token
if json_body is not None:
raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8")
h.setdefault("Content-Type", "application/json")
response = self.client.request(method, path, content=raw, headers=h)
allowed = (expected,) if isinstance(expected, int) else expected
assert response.status_code in allowed, (method, path, response.status_code, response.text[:300])
request_id = response.headers.get("x-request-id")
assert request_id, "every response must carry X-Request-Id"
if response.status_code >= 400:
body = response.json()
assert schema_errors(body, "Error") == [], response.text
assert body["error"]["request_id"] == request_id
assert "Traceback" not in response.text and "psycopg" not in response.text
if response.status_code == 401:
challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer"
assert response.headers.get("www-authenticate", "").lower().startswith(challenge)
elif schema:
assert schema_errors(response.json(), schema) == [], response.text
return response
def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict:
username = f"{label}_{uuid.uuid4().hex[:12]}"[:32]
body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password}
user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json()
return {"id": user["id"], "user": user, "username": username, "password": password}
def login(self, actor: dict) -> str:
session = self.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], actor["password"])},
schema="Session").json()
assert session["user"] == actor["user"] and session["token_type"] == "Bearer"
return session["token"]
def user(self, label: str = "u", **kwargs) -> dict:
actor = self.register(label, **kwargs)
actor["token"] = self.login(actor)
return actor
def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict:
ids = [owner["id"], *(m["id"] for m in members)]
return self.call("POST", "/api/v1/chats", 201, token=owner["token"],
json_body={"title": title, "member_ids": ids}, schema="Chat").json()
def send(self, actor: dict, chat: dict, text: str) -> dict:
return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=actor["token"],
json_body={"text": text}, schema="Message").json()
def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]:
"""Walk all pages; returns (items, number_of_pages)."""
items, pages, cursor = [], 0, None
while True:
query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "")
page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200,
token=actor["token"], schema="MessagePage").json()
pages += 1
items.extend(page["items"])
cursor = page["next_cursor"]
if cursor is None:
return items, pages
assert len(page["items"]) == limit, "a non-final page must be full"
assert len(items) < 100_000, "cursor loop"
def _stand_is_up() -> bool:
try:
return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200
except httpx.HTTPError:
return False
def pytest_collection_modifyitems(config, items):
if any("stack" in item.keywords for item in items) and not _stand_is_up():
skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait")
for item in items:
if "stack" in item.keywords:
item.add_marker(skip)
@pytest.fixture(scope="session")
def api() -> Api:
return Api()
+201
View File
@@ -0,0 +1,201 @@
"""Contract behaviour over real HTTP (stand must be running). Lab 1 checks moved to Bearer + new lab 2 auth."""
from __future__ import annotations
import base64
import uuid
import pytest
from conftest import PASSWORD, basic
pytestmark = pytest.mark.stack
def key(item: dict) -> tuple[str, int]:
return item["created_at"], uuid.UUID(item["id"]).int
# ================================================================== registration & login
def test_register_never_returns_password_and_login_returns_bearer(api):
actor = api.register("alice", display_name="Алиса 👩‍💻")
assert set(actor["user"]) == {"id", "username", "display_name", "created_at"}
session = api.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], PASSWORD)}, schema="Session").json()
assert session["token_type"] == "Bearer" and len(session["token"]) >= 43
assert actor["username"] not in session["token"] and actor["id"] not in session["token"]
assert api.call("GET", "/api/v1/users/me", 200, token=session["token"], schema="User").json() == actor["user"]
def test_duplicate_username_is_409(api):
actor = api.register("dup")
body = {"username": actor["username"], "display_name": "Другой", "password": PASSWORD}
assert api.call("POST", "/api/v1/users", 409, json_body=body).json()["error"]["code"] == "conflict"
@pytest.mark.parametrize("password", ["short-11ch", " " * 12, " \t \n " * 3, "я" * 129])
def test_bad_password_on_registration_is_400(api, password):
body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": password}
api.call("POST", "/api/v1/users", 400, json_body=body)
@pytest.mark.parametrize("password", ["я" * 128, "with:colons:inside", "Twelve chars"])
def test_edge_passwords_register_and_login(api, password):
actor = api.user("edge", password=password)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
def test_wrong_password_and_unknown_user_give_the_same_401(api):
actor = api.register("victim")
wrong = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(actor["username"], "Wrong-password-1")})
unknown = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("nobody_" + uuid.uuid4().hex[:8], PASSWORD)})
assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"]
assert wrong.headers["www-authenticate"] == 'Basic realm="messenger", charset="UTF-8"'
@pytest.mark.parametrize("header", ["Basic é", "Basic !!!", "Basic " + base64.b64encode(b"no_colon").decode(),
"Basic " + base64.b64encode(b"\xff\xfe:x").decode(), "Bearer " + "a" * 43, ""])
def test_malformed_basic_is_401_not_500(api, header):
# Raw UTF-8 bytes, as curl would send them (httpx refuses non-ASCII str headers).
api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": header.encode("utf-8")} if header else {})
def test_logout_revokes_only_this_token(api):
actor = api.user("logout")
second = api.login(actor)
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=actor["token"])
api.call("GET", "/api/v1/users/me", 200, token=second) # other session of the same user survives
def test_x_user_id_basic_and_garbage_do_not_authenticate(api):
actor = api.user("hdr")
for headers in ({"X-User-Id": actor["id"]},
{"Authorization": basic(actor["username"], PASSWORD)},
{"Authorization": "Bearer " + "A" * 43},
{"Authorization": "Bearer"}):
api.call("GET", "/api/v1/users/me", 401, headers=headers)
me = api.call("GET", "/api/v1/users/me", 200, token=actor["token"], headers={"X-User-Id": str(uuid.uuid4())})
assert me.json()["id"] == actor["id"]
# ================================================================== validation / errors (from lab 1)
@pytest.mark.parametrize("raw", [b"{", b"", b"[]", b"null", b'{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}',
b'{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}',
b'{"username": "abc_def", "display_name": "\\ud800", "password": "Twelve-chars"}'])
def test_malformed_json_is_400(api, raw):
api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"})
@pytest.mark.parametrize("body", [
{"username": "abc_def", "display_name": "x"}, # no password
{"username": "abc_def", "display_name": "x", "password": "Twelve-chars", "extra": 1},
{"username": "Abc_def", "display_name": "x", "password": "Twelve-chars"},
{"username": "abc_def", "display_name": " ", "password": "Twelve-chars"},
{"username": "abc_def", "display_name": "x" * 101, "password": "Twelve-chars"},
{"username": 5, "display_name": "x", "password": "Twelve-chars"},
])
def test_invalid_registration_fields_are_400(api, body):
api.call("POST", "/api/v1/users", 400, json_body=body)
def test_unicode_limits_are_code_points_and_nul_is_stored(api):
actor = api.user("uni", display_name="😀" * 100)
assert actor["user"]["display_name"] == "😀" * 100
nul = api.user("nul", display_name="A\u0000B")
assert api.call("GET", "/api/v1/users/me", 200, token=nul["token"]).json()["display_name"] == "A\u0000B"
chat = api.chat(actor, title="T\u0000itle")
assert chat["title"] == "T\u0000itle"
assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world"
assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=actor["token"], json_body={"text": "я" * 4001})
texts = [m["text"] for m in api.history(actor, chat, 100)[0]]
assert texts == ["hello\u0000world", "🙂" * 4000]
def test_oversized_body_and_wrong_content_type(api):
api.call("POST", "/api/v1/users", 413, json_body={"username": "big_body", "display_name": "x" * 70_000, "password": PASSWORD})
api.call("POST", "/api/v1/users", 400, raw=b'{"username":"abc_def"}', headers={"Content-Type": "text/plain"})
# ================================================================== chats & membership
def test_chat_validation(api):
alice, bob = api.user("alice"), api.user("bob")
for body in ({"title": "x", "member_ids": [alice["id"], bob["id"], bob["id"]]},
{"title": "x", "member_ids": [alice["id"], str(uuid.uuid4())]},
{"title": "x", "member_ids": [bob["id"]]},
{"title": " ", "member_ids": [alice["id"]]},
{"title": "x", "member_ids": [alice["id"], "bob"]}):
api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body=body)
def test_foreign_chat_is_indistinguishable_from_missing(api):
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
chat = api.chat(alice, bob)
for chat_id in (chat["id"], str(uuid.uuid4())):
base = f"/api/v1/chats/{chat_id}"
errors = [api.call("GET", base, 404, token=eve["token"]).json(),
api.call("GET", base + "/messages", 404, token=eve["token"]).json(),
api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}).json(),
api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()]
assert len({e["error"]["message"] for e in errors}) == 1
api.call("GET", "/api/v1/chats/not-a-uuid", 401)
api.call("GET", "/api/v1/chats/not-a-uuid", 400, token=alice["token"])
api.call("GET", "/api/v1/chats/NOT-A-UUID/messages", 400, token=alice["token"])
def test_chat_list_order_pagination_and_isolation(api):
alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve")
created = [api.chat(alice, bob, title=f"Чат {i}") for i in range(5)]
api.chat(eve, title="Чат Евы")
walked, cursor = [], None
while True:
query = "?limit=2" + (f"&cursor={cursor}" if cursor else "")
page = api.call("GET", "/api/v1/chats" + query, 200, token=bob["token"], schema="ChatPage").json()
walked += page["items"]
cursor = page["next_cursor"]
if not cursor:
break
assert walked == created
first = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats?limit=1&cursor={first}", 400, token=bob["token"]) # cursor of another user
api.call("GET", "/api/v1/chats?limit=0", 400, token=bob["token"])
# ================================================================== messages & keyset pagination
def test_history_pages_without_loss_or_duplicates(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
sent = [api.send(alice if i % 2 else bob, chat, f"сообщение {i}") for i in range(25)]
items, pages = api.history(bob, chat, 7)
assert pages == 4 and items == sent and items == sorted(items, key=key)
assert sent[0]["sender_id"] == bob["id"] and sent[1]["sender_id"] == alice["id"]
@pytest.mark.parametrize("query", ["limit=0", "limit=101", "limit=abc", "limit=1&limit=2", "cursor=", "cursor=garbage",
"cursor=djF8bXx4fDF8Mg", "cursor=" + base64.urlsafe_b64encode(b"v1|m|x|99999999999999999999|y").decode()])
def test_bad_limit_or_cursor_is_400(api, query):
alice = api.user("alice")
chat = api.chat(alice)
api.call("GET", f"/api/v1/chats/{chat['id']}/messages?{query}", 400, token=alice["token"])
def test_cursor_bound_to_chat(api):
alice = api.user("alice")
a, b = api.chat(alice, title="A"), api.chat(alice, title="B")
for chat in (a, b):
api.send(alice, chat, "1")
api.send(alice, chat, "2")
cursor = api.call("GET", f"/api/v1/chats/{a['id']}/messages?limit=1", 200, token=alice["token"]).json()["next_cursor"]
api.call("GET", f"/api/v1/chats/{b['id']}/messages?cursor={cursor}", 400, token=alice["token"])
empty = api.chat(alice, title="Empty")
api.call("GET", f"/api/v1/chats/{empty['id']}/messages?cursor=garbage", 400, token=alice["token"])
def test_read_after_write(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
for i in range(5):
message = api.send(alice, chat, f"m{i}")
last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1]
assert last == message
+114
View File
@@ -0,0 +1,114 @@
"""Database-level guarantees, checked over HTTP and verified directly in PostgreSQL / Valkey."""
from __future__ import annotations
import threading
import time
import uuid
import pytest
from conftest import PASSWORD, Api, compose, psql
pytestmark = pytest.mark.stack
def run_concurrently(workers: int, target) -> list:
barrier = threading.Barrier(workers)
results: list = [None] * workers
def body(index: int) -> None:
local = Api() # one HTTP client (connection) per worker
barrier.wait() # all workers fire at the same moment
try:
results[index] = target(local, index)
except BaseException as exc: # noqa: BLE001 - reported by the test
results[index] = exc
threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)]
for t in threads:
t.start()
for t in threads:
t.join()
return results
def test_registration_race_exactly_one_201_and_seven_409():
username = "race_" + uuid.uuid4().hex[:10]
def register(client: Api, _index: int) -> int:
body = {"username": username, "display_name": "Гонка", "password": PASSWORD}
response = client.client.post("/api/v1/users", json=body)
return response.status_code
statuses = run_concurrently(8, register)
# Real HTTP codes are compared: a timeout, 500 or exception fails the test instead of counting as 409.
assert sorted(statuses, key=str) == [201] + [409] * 7, statuses
assert psql(f"SELECT count(*) FROM users WHERE username = '{username}'") == "1"
def _ins_counter() -> int:
return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'"))
def test_create_chat_rolls_back_after_the_chat_row_was_inserted(api):
"""The chat INSERT runs first; the unknown member then breaks the membership INSERT (FK).
Proof that the first INSERT really happened and was rolled back: PostgreSQL counts
attempted inserts in pg_stat_user_tables.n_tup_ins even for aborted transactions,
while the table contains no such chat and no new membership rows.
"""
alice = api.user("rollback")
title = "rollback-" + uuid.uuid4().hex
members_before = int(psql("SELECT count(*) FROM chat_members"))
attempts_before = _ins_counter()
error = api.call("POST", "/api/v1/chats", 400, token=alice["token"],
json_body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}).json()
assert "несуществующих" in error["error"]["message"]
deadline = time.monotonic() + 20 # statistics are flushed by the API backend asynchronously
while _ins_counter() == attempts_before and time.monotonic() < deadline:
time.sleep(0.5)
assert _ins_counter() > attempts_before, "the chat INSERT was never attempted"
assert psql(f"SELECT count(*) FROM chats WHERE title = convert_to('{title}', 'UTF8')") == "0"
assert int(psql("SELECT count(*) FROM chat_members")) == members_before
def test_migrations_rerun_is_a_noop():
users_before = psql("SELECT count(*) FROM users")
result = compose("run", "--rm", "--no-deps", "migrate")
assert "0001_init: already applied, skipped" in result.stdout
assert psql("SELECT count(*) FROM schema_migrations") == "1"
assert psql("SELECT count(*) FROM users") == users_before
def test_100_messages_from_10_clients_keep_order_and_cursor_walk(api):
clients = [api.user(f"w{i}") for i in range(10)]
chat = api.chat(clients[0], *clients[1:])
def writer(client: Api, index: int) -> list[dict]:
return [client.send(clients[index], chat, f"c{index}-m{j}") for j in range(10)]
results = run_concurrently(10, writer)
assert all(isinstance(r, list) for r in results), results
confirmed = [m for batch in results for m in batch]
history, pages = api.history(clients[5], chat, 7)
ids = [m["id"] for m in history]
assert len(confirmed) == 100 and len(history) == 100 and len(set(ids)) == 100 and pages == 15
assert set(ids) == {m["id"] for m in confirmed}
keys = [(m["created_at"], uuid.UUID(m["id"]).int) for m in history]
assert all(a < b for a, b in zip(keys, keys[1:])), "history is not strictly ordered"
stamps = [m["created_at"] for m in history]
assert len(set(stamps)) == 100, "per-chat clock must give unique timestamps"
for i, client in enumerate(clients):
own = [m["text"] for m in history if m["sender_id"] == client["id"]]
assert own == [f"c{i}-m{j}" for j in range(10)]
def test_secrets_are_not_stored_in_plain_text(api):
actor = api.user("store")
stored = psql(f"SELECT password_hash FROM users WHERE id = '{actor['id']}'")
assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and PASSWORD not in stored
token_hex = actor["token"].encode().hex()
keys = compose("exec", "-T", "valkey", "sh", "-c",
'REDISCLI_AUTH="$(cat /run/secrets/valkey_password)" valkey-cli --scan --pattern "session:*"').stdout.split()
assert keys and all(len(k) == len("session:") + 64 for k in keys)
assert not any(actor["token"] in k or token_hex in k for k in keys)
+35
View File
@@ -0,0 +1,35 @@
"""The history index really bounds the work of a late page (>= 10 000 messages, generic plan)."""
from __future__ import annotations
import re
import sys
import pytest
from conftest import ROOT
pytestmark = pytest.mark.stack
sys.path.insert(0, str(ROOT / "scripts"))
import explain_history # noqa: E402
def section(text: str, number: int) -> str:
parts = re.split(r"^=== ", text, flags=re.M)
return next(p for p in parts if p.startswith(f"{number}."))
def buffers(plan: str) -> int:
return int(re.search(r"Buffers: shared hit=(\d+)", plan).group(1))
def test_late_page_uses_index_range_not_filter():
text = explain_history.run()
assert "20000" in text # the target chat holds 20 000 messages
app_late = section(text, 2)
assert "Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))" in app_late
assert "Rows Removed by Filter" not in app_late and "Sort" not in app_late
rejected = section(text, 3)
assert "Rows Removed by Filter: 19950" in rejected
# The app query touches a handful of pages; the rejected variants touch the whole prefix.
assert buffers(app_late) <= 10 < buffers(rejected) and buffers(app_late) <= 10 < buffers(section(text, 4))
+99
View File
@@ -0,0 +1,99 @@
"""Dependency failures with a VALID session: bounded 503, never anonymous access, recovery afterwards.
Every test stops/pauses a real container and restores it in `finally` (service_down).
Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5.
"""
from __future__ import annotations
import subprocess
import time
import pytest
from conftest import ROOT, restart_services, service_down
pytestmark = pytest.mark.stack
DB_BOUND = 3.6 # DB_TIMEOUT_SECONDS + network/HTTP overhead
CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead
def timed(api, *args, **kwargs):
started = time.monotonic()
response = api.call(*args, **kwargs)
return response, time.monotonic() - started
@pytest.fixture
def prepared(api):
alice, bob = api.user("alice"), api.user("bob")
chat = api.chat(alice, bob)
message = api.send(alice, chat, "до отказа")
return alice, bob, chat, message
def check_intact(api, alice, bob, chat, message):
"""After recovery the SAME tokens work and confirmed data is unchanged."""
assert api.call("GET", "/api/v1/users/me", 200, token=alice["token"]).json() == alice["user"]
items = api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json()["items"]
assert items == [message]
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_valkey_failure(api, prepared, how):
alice, bob, chat, message = prepared
with service_down("valkey", how):
response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took
api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected
api.call("GET", "/health/ready", 503)
api.call("GET", "/health/live", 200)
check_intact(api, alice, bob, chat, message)
@pytest.mark.parametrize("how", ["stop", "pause"])
def test_postgres_failure(api, prepared, how):
alice, bob, chat, message = prepared
with service_down("postgres", how):
for path in ("/api/v1/users/me", f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"):
response, took = timed(api, "GET", path, 503, token=alice["token"])
assert response.json()["error"]["code"] == "unavailable", path
assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause)
api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected
api.call("GET", "/health/ready", 503)
api.call("GET", "/health/live", 200)
check_intact(api, alice, bob, chat, message)
def test_locked_table_gives_bounded_503(api, prepared):
"""A statement stuck behind a lock is stopped by lock_timeout, not after minutes."""
alice, bob, chat, message = prepared
locker = subprocess.Popen(
["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c",
"BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(6); COMMIT;"],
cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
try:
time.sleep(1.5) # let the lock be taken
response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"])
assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took
finally:
locker.wait(timeout=30)
check_intact(api, alice, bob, chat, message)
def test_logout_during_valkey_outage_is_never_lost(api, prepared):
"""Valkey is the single source of truth for sessions, so a logout cannot be 'forgotten':
during the outage logout answers 503 (nothing revoked, the client knows it failed);
after recovery logout works, and the revocation survives a Valkey and API restart (AOF)."""
alice, bob, chat, message = prepared
with service_down("valkey", "stop"):
api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"])
api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"])
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
restart_services("valkey", "api")
for _ in range(10): # repeated checks: the revoked token must never come back
api.call("GET", "/api/v1/users/me", 401, token=alice["token"])
api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], json_body={"text": "x"})
api.call("GET", "/api/v1/users/me", 200, token=bob["token"]) # other sessions survived the restart
+64
View File
@@ -0,0 +1,64 @@
"""Session lifetime: expiry with a short SESSION_TTL_SECONDS, no extension by restarts,
and the alternative SESSION_STORE=postgres. The api container is recreated with the
override and ALWAYS recreated back with the normal configuration (api_with_env)."""
from __future__ import annotations
import datetime as dt
import time
import pytest
from conftest import api_env, api_with_env, restart_services, service_down
pytestmark = pytest.mark.stack
def seconds_left(expires_at: str) -> float:
moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00"))
return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds()
def login_session(api, actor) -> dict:
from conftest import basic
return api.call("POST", "/api/v1/auth/sessions", 201,
headers={"Authorization": basic(actor["username"], actor["password"])}, schema="Session").json()
@pytest.mark.parametrize("store", ["valkey", "postgres"])
def test_short_ttl_expires(api, store):
with api_with_env(SESSION_TTL_SECONDS="2", SESSION_STORE=store):
assert api_env("SESSION_TTL_SECONDS") == "2" # the contract variable reaches the container
actor = api.register("ttl")
session = login_session(api, actor)
assert 1.0 < seconds_left(session["expires_at"]) <= 2.1
api.call("GET", "/api/v1/users/me", 200, token=session["token"])
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3)
api.call("GET", "/api/v1/users/me", 401, token=session["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=session["token"])
def test_restart_does_not_extend_ttl(api):
with api_with_env(SESSION_TTL_SECONDS="12"):
actor = api.register("ttlrestart")
session = login_session(api, actor)
restart_services("valkey", "api")
assert seconds_left(session["expires_at"]) > 0, "restart took longer than the TTL; increase it"
api.call("GET", "/api/v1/users/me", 200, token=session["token"])
time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3)
api.call("GET", "/api/v1/users/me", 401, token=session["token"])
def test_postgres_session_store(api):
"""SESSION_STORE=postgres: sessions survive an API restart, logout revokes, and a DB outage
is 503 for a valid token (not 401 and not anonymous access)."""
with api_with_env(SESSION_STORE="postgres"):
assert api_env("SESSION_STORE") == "postgres"
actor = api.user("pgstore")
restart_services("api")
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
with service_down("postgres", "stop"):
assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"]).json()["error"]["code"] == "unavailable"
api.call("GET", "/api/v1/users/me", 401)
api.call("GET", "/api/v1/users/me", 200, token=actor["token"])
api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"])
api.call("GET", "/api/v1/users/me", 401, token=actor["token"])
+134
View File
@@ -0,0 +1,134 @@
"""Pure functions, no services: validation, cursor, Authorization parsing, Argon2id, config."""
from __future__ import annotations
import asyncio
import base64
import datetime as dt
import pytest
from messenger import authhdr, cursor, passwords, validation
from messenger.config import ConfigError, Settings
from messenger.errors import InvalidRequest, Unauthorized
from messenger.timefmt import UTC, from_us, rfc3339, to_us
def b64(raw: bytes) -> str:
return base64.b64encode(raw).decode()
# ------------------------------------------------------------------ password rules
@pytest.mark.parametrize("password", ["Twelve-chars", "я" * 128, "пароль:с:двоеточиями", "😀" * 12, "a b c d e f g"])
def test_valid_passwords(password):
assert validation.password_field({"password": password}) == password
@pytest.mark.parametrize("password", ["short-11ch", "я" * 129, " " * 12, "\t\n " * 5, "\ud800" * 12, None, 123456789012])
def test_invalid_passwords(password):
with pytest.raises(InvalidRequest):
validation.password_field({"password": password})
def test_text_may_contain_nul_but_not_lone_surrogates():
assert validation.text_field({"t": "a\u0000b"}, "t", 10) == "a\u0000b"
with pytest.raises(InvalidRequest):
validation.text_field({"t": "a\ud800"}, "t", 10)
# ------------------------------------------------------------------ cursor
CHAT = "3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11"
def test_cursor_roundtrip_and_binding():
key = (to_us(dt.datetime(2026, 10, 12, 12, 0, 0, 123456, tzinfo=UTC)), "11111111-1111-4111-8111-111111111111")
token = cursor.encode(cursor.MESSAGES, CHAT, key)
assert cursor.decode(token, cursor.MESSAGES, CHAT) == key
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.MESSAGES, "00000000-0000-4000-8000-000000000000") # other chat
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.CHATS, CHAT) # other resource
@pytest.mark.parametrize("payload", [
b"v1|m|" + CHAT.encode() + b"|253402300800000000|11111111-1111-4111-8111-111111111111", # 1 us after 9999-12-31
b"v1|m|" + CHAT.encode() + b"|9999999999999999999|11111111-1111-4111-8111-111111111111", # int64 edge
b"v1|m|" + CHAT.encode() + b"|-1|11111111-1111-4111-8111-111111111111",
b"v1|m|" + CHAT.encode() + b"|1|not-a-uuid",
b"v1|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-11111111111G",
b"v2|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-111111111111",
])
def test_broken_cursors_are_400_not_500(payload):
token = base64.urlsafe_b64encode(payload).rstrip(b"=").decode()
with pytest.raises(InvalidRequest):
cursor.decode(token, cursor.MESSAGES, CHAT)
def test_time_helpers_are_exact():
moment = dt.datetime(2026, 2, 28, 23, 59, 59, 999999, tzinfo=UTC)
assert from_us(to_us(moment)) == moment
assert rfc3339(moment) == "2026-02-28T23:59:59.999999Z"
assert from_us(cursor.MAX_TS_US).year == 9999
# ------------------------------------------------------------------ Authorization header
def test_basic_is_parsed_and_password_may_contain_colons():
assert authhdr.parse_basic("Basic " + b64("alice:pa:ss:wörd".encode())) == ("alice", "pa:ss:wörd")
assert authhdr.parse_basic("basic " + b64(b"bob_1:x" * 1)) == ("bob_1", "x")
@pytest.mark.parametrize("header", [
None, "", "Basic", "Basic é", "Basic !!!!", "Basic " + b64(b"no-colon"), "Basic " + b64(b"\xff\xfe:pw"),
"Basic " + b64(b":password"), "Basic " + b64(b"alice:"), "Basic " + b64("Алиса:pw".encode()),
"Bearer " + "a" * 43, "Basic " + "A" * 2000,
])
def test_bad_basic_is_401_with_basic_challenge(header):
with pytest.raises(Unauthorized) as info:
authhdr.parse_basic(header)
assert info.value.headers["WWW-Authenticate"].startswith("Basic realm=\"messenger\"")
@pytest.mark.parametrize("header", [None, "", "Bearer", "Bearer short", "Bearer " + "a" * 43 + "!",
"Basic " + b64(b"alice:password-123"), "Token " + "a" * 43, "Bearer " + "é" * 43])
def test_bad_bearer_is_401_with_bearer_challenge(header):
with pytest.raises(Unauthorized) as info:
authhdr.parse_bearer(header)
assert info.value.headers["WWW-Authenticate"] == 'Bearer realm="messenger"'
def test_bearer_ok():
token = "A_b-" + "x" * 39
assert authhdr.parse_bearer("Bearer " + token) == token
# ------------------------------------------------------------------ Argon2id
def test_argon2id_hash_and_verify():
async def scenario():
stored = await passwords.hash_password("я" * 128)
assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$")
assert await passwords.verify_password(stored, "я" * 128)
assert not await passwords.verify_password(stored, "я" * 127)
assert not await passwords.verify_password(None, "anything") # unknown user, dummy hash
assert not await passwords.verify_password("not-a-hash", "anything")
assert await passwords.hash_password("same password!") != await passwords.hash_password("same password!") # salt
asyncio.run(scenario())
# ------------------------------------------------------------------ config
def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch):
url_file = tmp_path / "database_url"
url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8")
monkeypatch.setenv("DATABASE_URL_FILE", str(url_file))
monkeypatch.setenv("SESSION_STORE", "redis")
monkeypatch.setenv("CACHE_URL", "redis://:p@valkey:6379/0")
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
settings = Settings.from_env()
assert settings.database_url == "postgresql://u:p@db/x"
assert settings.session_store == "valkey" and settings.session_ttl_seconds == 2
monkeypatch.setenv("SESSION_TTL_SECONDS", "0")
with pytest.raises(ConfigError):
Settings.from_env()
monkeypatch.setenv("SESSION_TTL_SECONDS", "2")
monkeypatch.setenv("DATABASE_URL_FILE", str(tmp_path / "missing"))
with pytest.raises(ConfigError) as info:
Settings.from_env()
assert "u:p" not in str(info.value)