From 4f6cfda37afba95bd0db9d10b7a1a003f224ed87 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:02:13 +0300 Subject: [PATCH 01/18] =?UTF-8?q?chore:=20=D0=BF=D0=B5=D1=80=D0=B5=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=20=D1=80=D0=B5=D1=88=D0=B5=D0=BD=D0=B8=D1=8F=20lab?= =?UTF-8?q?2=20(src,=20=D0=BC=D0=B8=D0=B3=D1=80=D0=B0=D1=86=D0=B8=D0=B8,?= =?UTF-8?q?=20student-tests,=20Dockerfile,=20compose,=20lock)=20=D0=B1?= =?UTF-8?q?=D0=B5=D0=B7=20=D0=B8=D0=B7=D0=BC=D0=B5=D0=BD=D0=B5=D0=BD=D0=B8?= =?UTF-8?q?=D0=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Исходная точка lab3: код из solution/bvt2302-bodanyuk лабы 2, коммит ef44e48 (PR mtusi2026/mtusi2026lab2#42 принят). README, COURSE, contracts/ и tests/ преподавателя не тронуты; к Makefile дописан студенческий раздел lab2, .env.example — версия lab2. Исследование трека 2 lab2 (bench/track2) не переносилось. Co-Authored-By: Claude Opus 5.5 --- .env.example | 16 +- .gitattributes | 5 + .github/workflows/app.yml | 32 +++ Dockerfile | 53 ++++ Makefile | 35 +++ compose.yaml | 108 ++++++++ docs/DATA_MODEL.md | 41 +++ docs/DEFENSE.md | 83 ++++++ docs/WINDOWS.md | 134 ++++++++++ infra/valkey/valkey.conf | 17 ++ migrations/0001_init.sql | 65 +++++ requirements-test.txt | 4 + requirements.in | 16 ++ requirements.txt | 452 ++++++++++++++++++++++++++++++++ scripts/explain_history.py | 114 ++++++++ scripts/init_secrets.py | 47 ++++ scripts/restart.cmd | 6 + scripts/restart.sh | 15 ++ src/messenger/__init__.py | 3 + src/messenger/__main__.py | 65 +++++ src/messenger/app.py | 332 +++++++++++++++++++++++ src/messenger/authhdr.py | 48 ++++ src/messenger/config.py | 127 +++++++++ src/messenger/cursor.py | 56 ++++ src/messenger/db.py | 109 ++++++++ src/messenger/docs.py | 57 ++++ src/messenger/errors.py | 65 +++++ src/messenger/healthcheck.py | 19 ++ src/messenger/logging_setup.py | 36 +++ src/messenger/migrate.py | 76 ++++++ src/messenger/passwords.py | 53 ++++ src/messenger/repo.py | 244 +++++++++++++++++ src/messenger/sessions.py | 160 +++++++++++ src/messenger/timefmt.py | 22 ++ src/messenger/validation.py | 117 +++++++++ student-tests/conftest.py | 226 ++++++++++++++++ student-tests/test_api.py | 201 ++++++++++++++ student-tests/test_db.py | 114 ++++++++ student-tests/test_explain.py | 35 +++ student-tests/test_failures.py | 99 +++++++ student-tests/test_sessions.py | 64 +++++ student-tests/unit/test_unit.py | 134 ++++++++++ 42 files changed, 3702 insertions(+), 3 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/app.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 docs/DATA_MODEL.md create mode 100644 docs/DEFENSE.md create mode 100644 docs/WINDOWS.md create mode 100644 infra/valkey/valkey.conf create mode 100644 migrations/0001_init.sql create mode 100644 requirements-test.txt create mode 100644 requirements.in create mode 100644 requirements.txt create mode 100644 scripts/explain_history.py create mode 100644 scripts/init_secrets.py create mode 100644 scripts/restart.cmd create mode 100755 scripts/restart.sh create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/__main__.py create mode 100644 src/messenger/app.py create mode 100644 src/messenger/authhdr.py create mode 100644 src/messenger/config.py create mode 100644 src/messenger/cursor.py create mode 100644 src/messenger/db.py create mode 100644 src/messenger/docs.py create mode 100644 src/messenger/errors.py create mode 100644 src/messenger/healthcheck.py create mode 100644 src/messenger/logging_setup.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/passwords.py create mode 100644 src/messenger/repo.py create mode 100644 src/messenger/sessions.py create mode 100644 src/messenger/timefmt.py create mode 100644 src/messenger/validation.py create mode 100644 student-tests/conftest.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_db.py create mode 100644 student-tests/test_explain.py create mode 100644 student-tests/test_failures.py create mode 100644 student-tests/test_sessions.py create mode 100644 student-tests/unit/test_unit.py diff --git a/.env.example b/.env.example index d32b9be0..c84d3545 100644 --- a/.env.example +++ b/.env.example @@ -5,9 +5,19 @@ APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. +# The files are created by `python scripts/init_secrets.py` and mounted by compose as secrets. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`. +SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 +# Additional settings of this implementation (defaults shown; all optional): +LOG_LEVEL=info +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_TIMEOUT_SECONDS=3 +CACHE_TIMEOUT_SECONDS=0.5 +PASSWORD_HASH_CONCURRENCY=2 +DOCS_ENABLED=true +# Host port of the API (compose publishes it on 127.0.0.1 only) +APP_PUBLISH_PORT=8080 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..3c10fe54 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +# LF everywhere: identical files on Windows/Linux and inside `git archive | docker build -`. +* text=auto eol=lf +*.png binary +# cmd.exe expects CRLF in batch files. +*.cmd text eol=crlf diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..a9ffaa18 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,32 @@ +name: Application (lab2) +on: [push, pull_request] +permissions: + contents: read +jobs: + app: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Synthetic secrets and env (CI only, never committed) + run: python scripts/init_secrets.py && cp .env.example .env + - name: Build, migrate, start and wait for healthy services + run: docker compose --env-file .env up -d --build --wait + - name: Contract and public smoke + run: make check test PYTHON=python + - name: Persistence across restart of API and stores + run: make test-persistence ACTION_SCRIPT=scripts/restart.sh PYTHON=python + - name: Own tests (unit + stand, failures restored in finally) + run: | + python -m pip install --require-hashes --no-deps -r requirements.txt + python -m pip install -r requirements-test.txt + make student-test PYTHON=python + - name: Logs + if: always() + run: docker compose logs --tail 100 + - name: Remove CI stand and its synthetic data + if: always() + run: docker compose down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..470f37a6 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +# syntax=docker/dockerfile:1 +# Messenger API, lab 2 (PostgreSQL + Valkey). Built by compose.yaml (services api, migrate). +# +# Python is interpreted, so there is no separate compile stage; the image +# contains only the interpreter, locked dependencies and the source code. +# Base image is pinned by version AND digest (python 3.13.16, Debian 12 bookworm). +FROM python:3.13.16-slim-bookworm@sha256:a1165e272e578941b84abc79e4ab38a0305cd12803a5c4247979ac7655f4d641 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /app + +# 1. Dependencies first (layer cache survives source edits). Every package, +# including transitive ones, is pinned with sha256 hashes in requirements.txt; +# --no-deps guarantees nothing outside the lock file is installed. +COPY requirements.txt ./ +RUN pip install --root-user-action=ignore --require-hashes --no-deps -r requirements.txt + +# 2. Unprivileged runtime user. Code stays owned by root, so the process +# cannot modify its own code. +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app + +COPY src/ ./src/ +# Versioned SQL migrations, applied by `python -m messenger.migrate` (compose service migrate). +COPY migrations/ ./migrations/ +# Course contract, served read-only at /openapi.json for Swagger UI (/docs). +COPY contracts/openapi.json ./contracts/openapi.json + +# Non-secret defaults; compose.yaml passes the rest. Passwords come only as files +# (DATABASE_URL_FILE / CACHE_URL_FILE -> Docker secrets), never as ENV or COPY. +ENV PYTHONPATH=/app/src \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + LOG_LEVEL=info \ + SHUTDOWN_TIMEOUT_SECONDS=8 \ + DOCS_ENABLED=true \ + SESSION_TTL_SECONDS=3600 \ + DATABASE_URL_FILE=/run/secrets/database_url + +USER app:app +EXPOSE 8080 +STOPSIGNAL SIGTERM + +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-m", "messenger.healthcheck"] + +# Exec form: the Python server is PID 1 and receives SIGTERM directly (no shell in between). +CMD ["python", "-m", "messenger"] diff --git a/Makefile b/Makefile index 5d86d171..6233ac48 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,38 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --------------------------------------------------------------------------- +# Student solution (lab 2, Python). On Windows without make run the commands from +# REPORT.md directly (public tests there need PYTHONUTF8=1; restart hook: scripts/restart.cmd). +.PHONY: secrets up down logs ps migrate student-test unit-test explain + +secrets: + $(PYTHON) scripts/init_secrets.py + +# Build, apply migrations (service migrate) and wait until every service is healthy. +up: secrets + docker compose up -d --build --wait + +# Safe stop: containers stop, named volumes (data) stay. +down: + docker compose down + +logs: + docker compose logs --tail 50 api + +ps: + docker compose ps + +migrate: + docker compose run --rm migrate + +# Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv). +student-test: + $(PYTHON) -m pytest student-tests -v + +unit-test: + $(PYTHON) -m pytest student-tests -m "not stack" -q + +explain: + $(PYTHON) scripts/explain_history.py --output evidence/explain-history.txt diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..1f76ebe6 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,108 @@ +# Lab 2 stand: API + PostgreSQL + Valkey. Start from a clean clone: +# python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored) +# cp .env.example .env # optional overrides +# docker compose up -d --build --wait +# Images are pinned by version AND digest. Passwords are passed only as secret files. +name: mtusi-lab2 + +x-api-env: &api-env + APP_ENV: ${APP_ENV:-lab} + LOG_LEVEL: ${LOG_LEVEL:-info} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: ${SESSION_STORE:-valkey} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + DB_POOL_MIN: ${DB_POOL_MIN:-1} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3} + CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5} + PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2} + DOCS_ENABLED: ${DOCS_ENABLED:-true} + +services: + postgres: + image: postgres:18.6-bookworm@sha256:afc7e2d441324c0388fa80c3d24f733b4194a4eb7f47dd8ee2b08eb1a24a647c + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -q"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s + stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 + # The password goes into a config include readable only by the valkey user, + # so it never appears in the process arguments. docker-entrypoint.sh drops root. + command: + - sh + - -c + - >- + umask 077 && + printf 'requirepass %s\n' "$$(cat /run/secrets/valkey_password)" > /tmp/requirepass.conf && + chown valkey /tmp/requirepass.conf && + exec docker-entrypoint.sh valkey-server /etc/valkey/valkey.conf + secrets: [valkey_password] + volumes: + - ./infra/valkey/valkey.conf:/etc/valkey/valkey.conf:ro + - valkeydata:/data + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/valkey_password)\" valkey-cli ping | grep -q PONG"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 5s + restart: unless-stopped + + migrate: + build: . + image: mtusi-messenger:lab2 + command: ["python", "-m", "messenger.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: + condition: service_healthy + healthcheck: + disable: true + restart: "no" + + api: + build: . + image: mtusi-messenger:lab2 + environment: *api-env + secrets: [database_url, cache_url] + ports: + - "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" # only on localhost of the host + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + migrate: + condition: service_completed_successfully + stop_grace_period: 10s + restart: unless-stopped + +volumes: + pgdata: + valkeydata: + +secrets: + postgres_password: + file: ./secrets/postgres_password + valkey_password: + file: ./secrets/valkey_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md new file mode 100644 index 00000000..d6d7f3bc --- /dev/null +++ b/docs/DATA_MODEL.md @@ -0,0 +1,41 @@ +# Модель данных lab2 + +Схема: [migrations/0001_init.sql](../migrations/0001_init.sql). Источник истины для пользователей, чатов, участников и сообщений — PostgreSQL 18. Для сессий источник истины — хранилище, выбранное через `SESSION_STORE`. По умолчанию это Valkey с AOF, его можно переключить на таблицу `sessions` в PostgreSQL. + +## Таблицы и ограничения + +| Таблица | Ключ | Ограничения | Зачем | +| --- | --- | --- | --- | +| `users` | `id uuid` | `UNIQUE (username)`, `CHECK` формата username, длина `display_name` | уникальность имени держит БД, а не код: гонку регистрации разрешает уникальный индекс | +| `chats` | `id uuid` | FK `created_by → users`, длина `title` | `last_message_at` — «часы» чата для упорядочивания сообщений | +| `chat_members` | `(chat_id, user_id)` | FK на `chats` и `users`, `UNIQUE (chat_id, position)` | `position` хранит порядок `member_ids`; `chat_created_at` — копия времени чата для keyset-списка чатов | +| `messages` | `id uuid` | FK `(chat_id, sender_id) → chat_members`, длина `text` | отправитель обязан быть участником — это гарантирует сама БД | +| `sessions` | `token_hash bytea` | FK `user_id → users`, длина хеша 32 байта | хранится SHA-256 токена; сам токен не хранится нигде | +| `schema_migrations` | `version` | checksum файла | какие миграции применены; изменённую задним числом миграцию мигратор не применит | + +**Почему `bytea` для текста.** Контракт разрешает символ U+0000, а тип `text` в PostgreSQL не может содержать байт NUL (`DataError`). Поэтому `display_name`, `title` и `text` хранятся как байты UTF-8. Длину в code points проверяет API, а `CHECK` в байтах — страховка (1 code point ≤ 4 байта). + +## Индексы и запросы + +- `messages_chat_keyset (chat_id, created_at, id)`. Первая страница — `WHERE chat_id=$1 ORDER BY created_at, id LIMIT n`, следующая — `WHERE chat_id=$1 AND (created_at, id) > ($2, $3) ORDER BY ... LIMIT n`. Сравнение строк `(created_at, id) > (...)` попадает в `Index Cond`: поиск начинается прямо с позиции cursor и читает только `n+1` строк. Запросов два, а не один с `($2 IS NULL OR ...)`: иначе при generic plan условие cursor уходит в `Filter`, и поздняя страница сканирует всю историю чата до неё. +- `chat_members_user_keyset (user_id, chat_created_at, chat_id)` — список чатов пользователя тем же keyset-способом. +- `users_username_key` — вход по username и уникальность. +- `sessions_expires_at` — очистка истёкших сессий. + +## Транзакционные границы + +| Операция | Транзакция | Что гарантирует | +| --- | --- | --- | +| Регистрация | один `INSERT` | `UniqueViolation` → 409; ровно одна строка при гонке | +| Создание чата | `INSERT chats` → `INSERT chat_members` (все участники одним запросом) → `COMMIT` | неизвестный участник вызывает FK-ошибку **после** вставки чата; откат удаляет и чат, и частичное членство | +| Сообщение | проверка членства → `UPDATE chats SET last_message_at = greatest(clock_timestamp(), last_message_at + 1 мкс) RETURNING` → `INSERT messages` → `COMMIT` | строка чата заблокирована до конца транзакции, поэтому писатели одного чата идут по очереди. Время сообщений строго растёт **в порядке коммитов**: страница по cursor не пропустит сообщение, закоммиченное «задним числом» | +| Вход | `SELECT` пользователя, проверка Argon2id, запись сессии | неверный пароль ничего не пишет | +| Logout | `DELETE` сессии (или `DEL` в Valkey) | повтор с тем же токеном — 401 | + +Чтение истории не требует snapshot между HTTP-запросами (так в контракте): новое сообщение появится на следующей странице. Список чатов сортируется по `chats.created_at = clock_timestamp()` без блокировки. Два чата, создаваемые одновременно, могут закоммититься не в порядке времени. Тогда у обхода, который уже начался, такой чат может оказаться «до» cursor; новый обход его увидит. Чаты создаются редко, поэтому мы не сериализуем их глобальным замком; это ограничение описано в REPORT.md. + +## Сессии + +- Токен: `secrets.token_urlsafe(32)` — 256 бит случайности, в ответе 43 символа. В хранилище лежит только SHA-256 токена. Для случайного 256-битного значения быстрого хеша достаточно, медленный KDF нужен только для низкоэнтропийных паролей. +- `SESSION_STORE=valkey` (по умолчанию): ключ `session:`, значение `{user_id, expires_at}`. Срок жизни задаётся абсолютным `PXAT expires_at`, поэтому рестарт его не продлевает. Включён `appendonly yes` + `appendfsync always`: подтверждённые вход и logout сохраняются до ответа клиенту. Valkey — **единственное** хранилище сессий, это не кэш над PostgreSQL. Поэтому нет второй копии, которая могла бы «оживить» отозванный токен после отказа. Если Valkey недоступен, проверить сессию нельзя: защищённые запросы и logout получают 503, анонимный доступ не открывается. +- `SESSION_STORE=postgres`: таблица `sessions`, проверка `expires_at > now()` в запросе. diff --git a/docs/DEFENSE.md b/docs/DEFENSE.md new file mode 100644 index 00000000..ed6091b5 --- /dev/null +++ b/docs/DEFENSE.md @@ -0,0 +1,83 @@ +# Шпаргалка к защите ЛР 2 + +Отчёт с доказательствами — [REPORT.md](../REPORT.md), модель данных — [DATA_MODEL.md](DATA_MODEL.md). + +## Что изменилось по сравнению с lab1 + +| Было в lab1 | Стало в lab2 | Где | +| --- | --- | --- | +| словари в памяти (`store.py`) | PostgreSQL 18, данные переживают рестарт | [repo.py](../src/messenger/repo.py), [migrations/0001_init.sql](../migrations/0001_init.sql) | +| `X-User-Id` | вход по `Basic` → opaque `Bearer` с expiry, logout | [authhdr.py](../src/messenger/authhdr.py), [sessions.py](../src/messenger/sessions.py), [app.py](../src/messenger/app.py) | +| нет паролей | Argon2id (m=19 MiB, t=2, p=1) в отдельном потоке | [passwords.py](../src/messenger/passwords.py) | +| `threading.Lock` + свой счётчик времени | блокировка строки чата `UPDATE chats … RETURNING` в транзакции | `add_message` в repo.py | +| `bisect` по списку | индекс `(chat_id, created_at, id)` и row-comparison в `Index Cond` | `HISTORY_AFTER` в repo.py | +| один контейнер | `compose.yaml`: postgres, valkey, migrate, api; secrets-файлы; named volumes | [compose.yaml](../compose.yaml) | + +Что осталось из lab1: формат ошибок и `X-Request-Id`, строгая валидация, cursor (теперь с проверкой диапазона времени), JSON-логи, graceful SIGTERM, healthcheck, Swagger на `/docs`. + +## Путь запроса `GET /chats/{id}/messages?cursor=…` в lab2 + +1. `parse_bearer`: в заголовке ровно `Bearer <43 символа>`, иначе 401 с `WWW-Authenticate: Bearer`. +2. `sessions.user_id_for`: в Valkey ключ `session:`, проверка `expires_at`. Valkey недоступен — 503, нет ключа — 401. +3. `path_uuid` → 400; `ensure_member` (`SELECT 1 FROM chat_members …`) → 404 для чужого чата. +4. `cursor.decode` → `(время в мкс, id)`, привязка к чату → иначе 400. +5. `HISTORY_AFTER`: `WHERE chat_id=$1 AND (created_at, id) > ($2,$3) ORDER BY … LIMIT 51` — индекс начинает чтение прямо с cursor. +6. Каждая работа с БД — `db.run(...)`: отдельная задача, транзакция, дедлайн 3 с. Отказ или зависание БД → 503 ровно через дедлайн. + +## Вопросы из README + +**Что подтверждает COMMIT и какие гарантии зависят от настройки durability?** +COMMIT возвращается клиенту, когда запись транзакции в WAL сброшена на диск (`fsync=on`, `synchronous_commit=on`, это значения по умолчанию). Транзакция переживёт падение процесса PostgreSQL или ОС: при старте PostgreSQL проиграет WAL. От настроек зависит: +- `synchronous_commit=off` — ответ приходит раньше сброса, и последние ~0,6 с подтверждённых транзакций могут пропасть (база при этом останется целой); +- `fsync=off` — после сбоя возможна порча данных; +- кэш записи диска, который «врёт» о сбросе, ломает и то, и другое. + +COMMIT на одном узле не защищает от потери самого диска или хоста: для этого нужны реплики (`synchronous_standby_names`) или резервные копии с архивом WAL (трек 2). + +У Valkey свой аналог — `appendfsync`: +- у нас `always`: fsync до ответа, ничего не теряется; +- `everysec`: можно потерять до 1 с записей; +- `no`: когда сбросить на диск, решает ОС. + +**Где хранятся сессии и что будет после flush/restart Redis (Valkey)?** +По умолчанию (`SESSION_STORE=valkey`) сессия существует **только** в Valkey: ключ `session:`, значение `{user_id, expires_at}` и абсолютный срок `PXAT`. +- **Restart:** Valkey перечитывает AOF с volume, сессии на месте (тест `make test-persistence`). TTL не продлевается. +- **FLUSHALL** или потеря volume: все сессии исчезают, все пользователи получают 401 и входят заново. Отказ безопасный: доступ не выдаётся никому. +- **Valkey недоступен:** 503 на защищённые запросы и на logout. Анонимный доступ не открывается. + +Почему не кэш над PostgreSQL: у одной копии нечему расходиться. У студентов с кэшем отозванный токен «оживал» после восстановления Redis, у нас такой ситуации быть не может (тест `test_logout_during_valkey_outage_is_never_lost`). Альтернатива `SESSION_STORE=postgres` хранит сессии в таблице `sessions`. + +**Чем аутентификация отличается от доступа к конкретному чату?** +- **Аутентификация** — кто ты: действительная Bearer-сессия → `user_id`. Нет или недействительна → 401. +- **Авторизация доступа** — можно ли этому пользователю этот чат: есть ли строка в `chat_members`. Нет → 404, а не 403, чтобы не раскрывать существование чужого чата. + +Сначала всегда аутентификация, потом доступ. Отправитель сообщения берётся из сессии, а БД сама проверяет `FOREIGN KEY (chat_id, sender_id) → chat_members`. + +**Почему hash пароля и быстрый checksum имеют разные задачи?** +- **Checksum или быстрый хеш (SHA-256)** — контроль целостности и быстрый поиск. Он должен быть быстрым. +- **Хеш пароля** — защита низкоэнтропийного секрета на случай утечки базы. Он должен быть медленным, с солью и memory-hard (Argon2id), чтобы перебор словаря стоил дорого и не ускорялся на GPU. + +Токен сессии — 256 бит случайности, перебрать его невозможно. Поэтому для токена хватает SHA-256: храним хеш, чтобы утечка базы не дала готовых токенов. Для пароля SHA-256 нельзя: миллиарды попыток в секунду на GPU. + +## Частые ловушки (на чём преподаватель ловил других) и наш ответ + +| Проверка преподавателя | У нас | Тест | +| --- | --- | --- | +| logout при упавшем Redis → после восстановления токен снова 200 | logout при упавшем Valkey = 503 (ничего не отозвано); после восстановления logout → 401, и после рестарта Valkey и API тоже 401 | `test_logout_during_valkey_outage_is_never_lost` | +| истёкшая сессия живёт в кэше | expiry проверяется при каждом чтении, ключ живёт до абсолютного `PXAT` | `test_short_ttl_expires[valkey/postgres]` | +| `docker pause postgres` → запрос висит 8+ с | клиентский дедлайн в отдельной задаче: 503 через 3,0 с | `test_postgres_failure[pause]` | +| БД упала → 401 вместо 503 | 503 `unavailable`, анонимный — 401 | `test_postgres_failure`, `test_postgres_session_store` | +| cursor в `Filter` при generic plan | два запроса без `IS NULL OR`, cursor в `Index Cond` | `test_explain.py`, `evidence/explain-history.txt` | +| `\u0000` → 500 | текст в `bytea`, NUL сохраняется | `test_unicode_limits_are_code_points_and_nul_is_stored` | +| пароль из пробелов, `Basic é` → 500 | 400 и 401 | `test_bad_password_on_registration_is_400`, `test_malformed_basic_is_401_not_500` | +| гонка регистрации считает любые ошибки за 409 | проверяются коды: ровно `[201] + [409]*7`, в БД 1 строка | `test_registration_race_exactly_one_201_and_seven_409` | +| rollback «до первого INSERT» | чат вставляется первым, FK участника падает после; `n_tup_ins` вырос, строк нет | `test_create_chat_rolls_back_after_the_chat_row_was_inserted` | +| `SESSION_TTL_SECONDS` не доходит до контейнера | compose передаёт его из `.env`, тест читает `printenv` внутри контейнера | `test_short_ttl_expires` | +| нет команды создания secrets | `python scripts/init_secrets.py` (идемпотентно) | раздел «Запуск из чистого clone» | + +## Трек 2 — коротко + +- **Гипотеза.** С индексом `(chat_id, created_at, id)` и keyset-запросом стоимость страницы не зависит от размера чата и позиции. С индексом только по `chat_id` нужна сортировка всего чата, с OFFSET — чтение всего префикса. Каждый индекс удорожает запись. +- **Методика.** Отдельный PostgreSQL (2 CPU / 2 GiB), 121 120 сообщений, чаты 60 000 / 15 000 / 2 000 / 80. pgbench с prepared statements; генератор нагрузки — в своём контейнере. 3 повтора. Плюс EXPLAIN для поздней страницы. +- **Backup.** `pg_dump -Fc` живой БД во время записи через API → новый контейнер и новый volume → `pg_restore` → сверка MD5 по каждому чату и по первым/последним сообщениям → наше же API на восстановленной БД читает чат после нового входа. +- **RPO и RTO.** `pg_dump` — снимок на момент старта. Всё подтверждённое после старта дампа в копию не попадает; это и есть граница потери данных (на практике — время с последнего дампа). RTO — время `pg_restore` с перестройкой индексов плюс переключение приложения. Чтобы уменьшить RPO до секунд, нужен архив WAL и PITR или реплика. diff --git a/docs/WINDOWS.md b/docs/WINDOWS.md new file mode 100644 index 00000000..649152a4 --- /dev/null +++ b/docs/WINDOWS.md @@ -0,0 +1,134 @@ +# Запуск на ноутбуке с Windows и сценарий защиты + +Все команды — в **PowerShell**, из папки проекта. Внутри одного окна PowerShell переменные `$env:...` действуют до его закрытия. + +## 1. Что установить один раз + +| Программа | Зачем | Важно | +| --- | --- | --- | +| [Git for Windows](https://git-scm.com/download/win) | clone и Git Bash | ставить в путь по умолчанию `C:\Program Files\Git`: оттуда `scripts\restart.cmd` запускает `restart.sh` | +| [Docker Desktop](https://www.docker.com/products/docker-desktop/) | PostgreSQL, Valkey, API | backend WSL 2; в BIOS включена виртуализация; после установки перезагрузка | +| [Python 3.12+](https://www.python.org/downloads/windows/) | secrets, публичные тесты, свои тесты | галочка **Add python.exe to PATH**. Если `python` открывает Microsoft Store — отключить «Псевдонимы выполнения приложений» для python в настройках Windows | + +Проверка: + +```powershell +git --version; docker version --format '{{.Server.Version}}'; docker compose version; python --version +``` + +Docker Desktop должен быть **запущен** (кит в трее не мигает). + +## 2. Развернуть проект (дома, заранее — нужен интернет) + +```powershell +git clone https://gitea.privcloud.tech/wfa/mtusi2026lab2 mtusi2026lab2 +cd mtusi2026lab2 +git checkout solution/bvt2302-bodanyuk +python scripts\init_secrets.py +Copy-Item .env.example .env +docker compose --env-file .env up -d --build --wait +``` + +Первый запуск скачивает образы (~700 МБ) и собирает API — 3–10 минут. Сделайте это **дома**: в аудитории сеть может быть медленной. Последняя команда завершается, когда все сервисы `healthy`. + +Проверка в браузере: http://127.0.0.1:8080/health/ready → `{"status":"ok"}`, Swagger — http://127.0.0.1:8080/docs. + +Окружение для своих тестов (один раз): + +```powershell +python -m venv .venv +.venv\Scripts\python -m pip install --require-hashes --no-deps -r requirements.txt +.venv\Scripts\python -m pip install -r requirements-test.txt -r requirements-dev.txt +``` + +Активировать venv не нужно: везде вызывается `.venv\Scripts\python` напрямую, поэтому политика выполнения PowerShell не мешает. + +## 3. Перед защитой (в аудитории) + +```powershell +cd mtusi2026lab2 +docker compose up -d --wait # если стенд был остановлен +$env:PYTHONUTF8 = "1" # публичные тесты читают UTF-8 JSON +python tests\smoke.py # PASS: lab2 public API scenario +``` + +## 4. Сценарий защиты из README — команды по шагам + +**Шаг 1. Миграции к пустому volume, пользователи, вход, сообщение.** + +```powershell +docker compose down -v # стереть учебные данные (volumes) +docker compose up -d --wait # postgres и valkey пустые -> migrate применяет схему +docker compose logs migrate # "0001_init: applied" +docker compose run --rm migrate # повтор: "already applied, skipped" +``` + +Дальше в Swagger (http://127.0.0.1:8080/docs): +1. `POST /api/v1/users` — создать двух пользователей (пароль 12+ символов). +2. **Authorize → basicAuth**: логин и пароль первого; `POST /api/v1/auth/sessions` — скопировать `token`. +3. **Authorize → bearerAuth**: вставить токен. +4. `POST /api/v1/chats` (в `member_ids` — оба id) → `POST /api/v1/chats/{chat_id}/messages` → `GET` истории. + +**Шаги 2–3. Перезапуск API и хранилищ с сохранением volumes, тот же токен; logout → 401; короткий TTL.** + +```powershell +python tests\resilience.py --mode persistence --action-script scripts\restart.cmd # PASS +docker compose ps # у api, postgres, valkey свежее время запуска +``` + +В Swagger тот же токен по-прежнему работает. `DELETE /api/v1/auth/sessions/current` → 204, затем любой запрос → 401. + +Короткий TTL: + +```powershell +$env:SESSION_TTL_SECONDS = "2"; docker compose up -d --wait api +docker compose exec api printenv SESSION_TTL_SECONDS # 2 +# в Swagger: войти, сразу GET /users/me -> 200, через 3 секунды -> 401 +Remove-Item Env:SESSION_TTL_SECONDS; docker compose up -d --wait api # вернуть 3600 +``` + +**Шаг 4. Роль Valkey, план SQL, ошибка зависимости.** + +```powershell +# Сессии лежат только в Valkey: ключи session:, самих токенов там нет +# (без двойных кавычек внутри: PowerShell 5.1 ломает их при передаче в docker) +docker compose exec valkey sh -c 'REDISCLI_AUTH=$(cat /run/secrets/valkey_password) valkey-cli --scan --pattern ''session:*''' + +# План: поздняя страница по индексу (Index Cond) против IS NULL OR и OFFSET, 20 000 сообщений +.venv\Scripts\python scripts\explain_history.py + +# Ошибка зависимости: с действующим токеном -> 503, без токена -> 401, после старта всё работает +docker compose stop valkey # или: docker compose pause postgres +# в Swagger: GET /users/me -> 503 unavailable; GET /health/ready -> 503 +docker compose start valkey # или: docker compose unpause postgres +``` + +Всё то же автоматически, с восстановлением в `finally`, около 1 минуты: + +```powershell +.venv\Scripts\python -m pytest student-tests\test_failures.py -v +``` + +**Трек 2 вживую.** + +```powershell +.venv\Scripts\python bench\track2\backup_restore.py --repeats 1 # ~1 мин: dump во время записи -> restore в новый контейнер -> сверка +``` + +Скрипт печатает время дампа и восстановления, `reconciled: true` и сколько сообщений записано после снимка (RPO). Полный замер планов идёт ~25 минут, поэтому на защите показывайте готовые результаты: `evidence\track2\summary.md` и `evidence\track2\plans.png`. При желании можно запустить `.venv\Scripts\python bench\track2\index_sizes.py` (~2 мин). + +**Все свои тесты** (~3,5 мин): `.venv\Scripts\python -m pytest student-tests -v`. + +## 5. Если что-то пошло не так + +| Симптом | Что сделать | +| --- | --- | +| `error during connect … dockerDesktopLinuxEngine` | Docker Desktop не запущен — запустить и подождать минуту | +| `port is already allocated` на 8080 | в `.env` поставить `APP_PUBLISH_PORT=8081`; дальше тесты с `--base-url http://127.0.0.1:8081` (smoke и resilience), для pytest — `$env:BASE_URL="http://127.0.0.1:8081"` | +| `secrets/... not found` при `up` | не выполнен `python scripts\init_secrets.py` | +| `UnicodeDecodeError` в `tests\*.py` | не задан `$env:PYTHONUTF8 = "1"` | +| `restart.cmd`: «не удаётся найти путь» | Git установлен не в `C:\Program Files\Git` — поправить путь в `scripts\restart.cmd` | +| api `unhealthy` после сна ноутбука | `docker compose restart` и подождать `docker compose ps` | +| нужно начать с чистого листа | `docker compose down -v` (удаляет учебные данные) → `docker compose up -d --wait` | + +Остановить стенд после защиты: `docker compose down` (данные сохранятся) или `docker compose down -v` (данные удаляются). diff --git a/infra/valkey/valkey.conf b/infra/valkey/valkey.conf new file mode 100644 index 00000000..510785b2 --- /dev/null +++ b/infra/valkey/valkey.conf @@ -0,0 +1,17 @@ +# Valkey is the single store of sessions (SESSION_STORE=valkey), so it must be durable. +# AOF with fsync on every write: a confirmed login or logout survives a crash/restart, +# a logout can never be lost and "resurrect" a token. Sessions are small and rare +# writes, so the fsync cost is acceptable here (see REPORT.md). +appendonly yes +appendfsync always +appenddirname "appendonlydir" +# RDB snapshots are not needed on top of AOF. +save "" +dir /data +port 6379 +protected-mode yes +# Sessions are never evicted silently: if memory is full, writes fail and login returns 503. +maxmemory 128mb +maxmemory-policy noeviction +# Password, generated by scripts/init_secrets.py and written here at container start. +include /tmp/requirepass.conf diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..aef95c4f --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,65 @@ +-- 0001: initial schema of the messenger (lab 2). Design rationale: docs/DATA_MODEL.md +-- Applied once by `python -m messenger.migrate`; the version is recorded in schema_migrations. + +-- User-supplied free text (display_name, title, message text) is stored as UTF-8 in BYTEA: +-- the contract allows U+0000, and PostgreSQL TEXT cannot contain NUL bytes. +-- Lengths are limited in code points by the API; the CHECKs below are byte-level safety nets +-- (one code point is at most 4 bytes in UTF-8). + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name bytea NOT NULL, + password_hash text NOT NULL, -- Argon2id PHC string, never the password + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (octet_length(display_name) BETWEEN 1 AND 400) +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title bytea NOT NULL, + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + -- Per-chat clock: created_at of the newest message. Updating this row serialises writers + -- of one chat, so message timestamps strictly increase in commit order (see DATA_MODEL.md). + last_message_at timestamptz, + CONSTRAINT chats_title_len CHECK (octet_length(title) BETWEEN 1 AND 400) +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, -- order of member_ids sent by the creator + chat_created_at timestamptz NOT NULL, -- copy of chats.created_at for keyset listing + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_key UNIQUE (chat_id, position) +); +-- GET /chats of one user: (user_id, created_at, id) range scan, no sort. +CREATE INDEX chat_members_user_keyset ON chat_members (user_id, chat_created_at, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text bytea NOT NULL, + created_at timestamptz NOT NULL, + -- The sender must be a member of this chat: enforced by the database, not only by the API. + CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id), + CONSTRAINT messages_text_len CHECK (octet_length(text) BETWEEN 1 AND 16000) +); +-- History page: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n +-- becomes an index range scan that starts exactly at the cursor. +CREATE INDEX messages_chat_keyset ON messages (chat_id, created_at, id); + +-- Sessions for SESSION_STORE=postgres (the default store is Valkey; see docs/DATA_MODEL.md). +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, -- SHA-256 of the token; the token is not stored + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32) +); +CREATE INDEX sessions_expires_at ON sessions (expires_at); diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 00000000..c5ab68ff --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,4 @@ +# Student tests (in-process, via Starlette TestClient). Runtime deps come from requirements.txt. +pytest==9.1.1 +httpx2==2.13.1 +jsonschema==4.26.0 diff --git a/requirements.in b/requirements.in new file mode 100644 index 00000000..bdaae420 --- /dev/null +++ b/requirements.in @@ -0,0 +1,16 @@ +# Direct runtime dependencies. The pinned lock with hashes is requirements.txt: +# docker run --rm -v "$PWD:/w" -w /w python:3.13.16-slim-bookworm \ +# sh -c "pip install -q pip-tools==7.6.1 && pip-compile --generate-hashes --strip-extras --no-emit-index-url -o requirements.txt requirements.in" +starlette==1.7.0 +uvicorn==0.54.0 +# Fast C parser/event loop used by uvicorn in the container (Linux only). +httptools==0.9.0 +uvloop==0.23.0 ; sys_platform != "win32" +# PostgreSQL driver (libpq bundled in the binary wheel) and its async connection pool. +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +# Valkey client (Valkey speaks the Redis protocol). +redis==8.1.0 +# Argon2id password KDF. +argon2-cffi==25.1.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..78a3b4a7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,452 @@ +# +# This file is autogenerated by pip-compile with Python 3.13 +# by the following command: +# +# pip-compile --generate-hashes --no-emit-index-url --no-index --output-file=requirements.txt --strip-extras requirements.in +# +anyio==4.15.1 \ + --hash=sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101 \ + --hash=sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94 + # via starlette +argon2-cffi==25.1.0 \ + --hash=sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1 \ + --hash=sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741 + # via -r requirements.in +argon2-cffi-bindings==26.1.0 \ + --hash=sha256:061a6919145bbf282ebf1f9c59d3135d4833c25313c8595c0d68cf7712ddfce2 \ + --hash=sha256:0cc40f7b4050bb93eb67de95d2d759322fc7ce4930b9d645581ecf4913ec651e \ + --hash=sha256:151dfaad9de753f4af2a7854e707e4784f2acc434340ade64239c5b104b2d605 \ + --hash=sha256:19423e5d7ac1cc354baab59eaabf18db2ec04ef6593b5abe5a34f323c4a8f87a \ + --hash=sha256:19b562b1de4b9052ef1214a2821c44b6e6f22945daa102c32ae4eff929d8b6d8 \ + --hash=sha256:1a0a29ed86960e44eaace7e081bdfab4f08b012fd96ec8edba71e2ad020939e4 \ + --hash=sha256:1af817e84578ef8b7295ad17de0f9896e4c8520dbf2233c7aa5aa3d487256fc4 \ + --hash=sha256:1b0bcac4d490a237e18cf91f57352920c29f77f2fa39efd0813fb81298bf17ba \ + --hash=sha256:1d98e33bd8bd67d7206c124e200bf2229c4cfa8c9c19f7b44a897f0fc71837eb \ + --hash=sha256:21ca0396fe5ec995dd54431c32698189666f9224810acfa752e50d2bd94d9df2 \ + --hash=sha256:224865cbbcb7a2bd1356741dff12b0134df726b6d44bb7b500df8e303cbd9e81 \ + --hash=sha256:242bb0cda2ae3650764fc194593d9ea45fc9e72729acd89778c7cfe184cec2a5 \ + --hash=sha256:27f1821903e2ceadcb88ec2b45ef190897b7682449c772f4d9b53e42c520cf29 \ + --hash=sha256:28524438cd3e723f25412f63d4fd516ff5bae9ae5aa56acbe2a1404398a0cf31 \ + --hash=sha256:2b741888c93147444fdfc851abd81cc207f37f7f7da42062a00deb3888e57da8 \ + --hash=sha256:2c36ff87b5dfaa477d0bd51e9d7f6abdae7c8955d2983c97419085d842154b3e \ + --hash=sha256:34b7d9c24a4165a2c61cc8ae11d44d48c9ce2830fb536cb7914e11fdd9962728 \ + --hash=sha256:49d525938467d52c923a890153c99087c9d5a937d1f6b585dbdba34ec82e397a \ + --hash=sha256:4f84cdd868978d7b7350a566c254042d44216d9e37f241f3a6d3b1dfebeede35 \ + --hash=sha256:62ff20cd130c956c7c9144d5fe35228f98b51c579b2439e988b27ef93e16c02a \ + --hash=sha256:63505c71542a44b68b1e38060450fb006404170da375feb31af153e7f9c6205d \ + --hash=sha256:6376d4b3aca039375ca8bf92f770da0ec424a1ce3a37077a8d3c557411aa56ca \ + --hash=sha256:6a4e68eed961a8de6928d1c17ff3dc2a547e0e923c17f8f1cd79fb7bc9502f98 \ + --hash=sha256:6ab674f668d5962a3a4136ae0812519b0f1586874263723a32181d60d64137e1 \ + --hash=sha256:7014ab7e6f5d8511af92544667a0346ea6dfc314ea9a7cad1dba9fdb5c9a6e33 \ + --hash=sha256:76ae29acace5d33355344612844d588e19deaaba4639d8bb01601e4b1418ef36 \ + --hash=sha256:78de2d65e0b9ea7ce9d1b1c3e87297b2d7305a02c266ee2a2d6910daddd7ee69 \ + --hash=sha256:9bacedc04b0402837586a17f0919e3dfdd95291f441f1f56bd80ec274c2840a1 \ + --hash=sha256:a86c069c91a747a2c4e5c51473590aeb48172fff9b2130d23729a42d98665ecb \ + --hash=sha256:ac82fc756a446b6ccd7139ce70efa9d8bbe541e7ad579a12dcb52764b7175c5f \ + --hash=sha256:af11ac37a7c53dc16cb7950a6190851b0870fe218b6c60c0bb7ac355234e3083 \ + --hash=sha256:b70225b5fd1e0d2ef4f7fd30d24658454535f0924dff0caca5dc08efbbbadfbb \ + --hash=sha256:c49e853a3bef9dd10329f31f702e7fa9b5c58229ff9c2ff6d069efaf09177c08 \ + --hash=sha256:ccaf0a46cbb380f1fd102a874e32aa629fd3cb0c0e94f4943fa1f6d5edc5dac6 \ + --hash=sha256:d157ddfab1e8b21f2f1dedda9c09645d98b5ed0b667b0626be600a345d426440 \ + --hash=sha256:d88e5f7e60f28ae0b0cc6b2f16c43e87cd642a196a86f85e0d8bb6fe016fc16d \ + --hash=sha256:db0fcd827ca61622a01b220aadfbece01939acf53888f2cb98cd93e9b1e2c97e \ + --hash=sha256:df612391feca41c44d20118f3b88d1b86419465cd1f5496859f715ca60ec2210 \ + --hash=sha256:f0c3103fcff20183e593459cfea6e012281c0e76ae3ed8b5565ad1b92eac3990 \ + --hash=sha256:f9c4420a7a864fe1b86ce35befc95b8e39fb852493b81cf798671ddc265de638 \ + --hash=sha256:ffff613aaa9ce6236766e2fc6dc560bb5abde7a2e2416e3db1f9ae395a2b4dd4 + # via argon2-cffi +cffi==2.1.1 \ + --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ + --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ + --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ + --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ + --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ + --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ + --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ + --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ + --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ + --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ + --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ + --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ + --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ + --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ + --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ + --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ + --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ + --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ + --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ + --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ + --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ + --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ + --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ + --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ + --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ + --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ + --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ + --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ + --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ + --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ + --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ + --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ + --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ + --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ + --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ + --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ + --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ + --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ + --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ + --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ + --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ + --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ + --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ + --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ + --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ + --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ + --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ + --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ + --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ + --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ + --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ + --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ + --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ + --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ + --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ + --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ + --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ + --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ + --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ + --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ + --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ + --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ + --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ + --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ + --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ + --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ + --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ + --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ + --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ + --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ + --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ + --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ + --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ + --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ + --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ + --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ + --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ + --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ + --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ + --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ + --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ + --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ + --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ + --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ + --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ + --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ + --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ + --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ + --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ + --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ + --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ + --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ + --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ + --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ + --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ + --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ + --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ + --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ + --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ + --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 + # via argon2-cffi-bindings +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 + # via uvicorn +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via uvicorn +httptools==0.9.0 \ + --hash=sha256:02bc5b3dcb6394b9d825fd62a7bfa0b2943063a3c89abc4492ad45e334a20eb5 \ + --hash=sha256:050f7ab098121873c8f13e35857f97ab60a76185c8302bde9a384939bb7c3b96 \ + --hash=sha256:050f84b7ec46a6efe0e5f521cf8729e3397c1cef4384f62ed8d5d68ca0045776 \ + --hash=sha256:06bfe7fad972a417269d8a5fc53b87e4eca970354abf5e9e24336fd06d64292e \ + --hash=sha256:088de1738e1af624466a01c35d652dbe6fb825be887c76d68aa850621d81db88 \ + --hash=sha256:0adc974916efe1fbf89d0363a86dcb2c746727643e362ff398de1a4b50b6bc77 \ + --hash=sha256:0cc339a807c156d840b54f8bf050ba0fc265eb81692c24bca8535b52fbd797c6 \ + --hash=sha256:0fd73d0bbf700a30dd87e4412adf41cfa71542a533d6b390c7244bbb8a1152bb \ + --hash=sha256:130635fea6e611a6b2026120037965ddb88b3dafd11bb64e264b101a70a76630 \ + --hash=sha256:13873eb8aef5972fcfee614f63d47064312ad4efbfe65ade15b8a3b77f8c8659 \ + --hash=sha256:18d800aaa2d6bff7d889df810d1b19a5fde72b1f6c0ca96e8d9f28a692fe5460 \ + --hash=sha256:1a4050a651e1f2faf05eb028ce9f2168abbcee9e24b209f5c1f2eb96d8c569e4 \ + --hash=sha256:1a7f1df31829c258158be01bb04eb668c4fba7df1ddf2262131a972962e651b6 \ + --hash=sha256:1b01c0fcd6725a8d79a164ecdc4116866282479d68bb3d6d74a909bf994656c4 \ + --hash=sha256:1b95775f6292d72cb452c33e5c0f8b8551807c29a10e3c1671fef7f61361370a \ + --hash=sha256:1f6da814aeecbc6cb8872d6d3e85ed16e8ab1653f9557cea8658725ce212348a \ + --hash=sha256:2095207b75a83c9e947346da9c127fb7e4fb29f41589df2643764f06b750989c \ + --hash=sha256:22ab1b10b06d357f01092e60f5e6856a0d479ed79b0ec2166a339ea26c699be2 \ + --hash=sha256:2319858018eedd0c0b2f950a620413c0a9d1352607be4267eb28209eca8b1e3f \ + --hash=sha256:268d18601feb5367885c6ebf6f402c18fc25a324cee215784adafe0a1eef925f \ + --hash=sha256:26e1d9629f3bf70d23f0d22238152aec51c837a7c9e384cb74f356fdccad7eb3 \ + --hash=sha256:272db0c51e8b71e953c1f2ecbe63402b819680e4564be2ef285cfd4584ee8355 \ + --hash=sha256:289f213d2a3dde2e8312c415ffecec5a01698589ec6249ec4e8fb3b47c0444ba \ + --hash=sha256:29b0d823e3c1e7cd1093a5dc889245db693ef13ada624cd66e2262421ef38867 \ + --hash=sha256:310266a2db1377ffae3bdf6556ab4973f4f94508a8ce37b2f6bb096a89bcefa1 \ + --hash=sha256:3238e198429cb8909ec42951b82d6a33fe0fdfcf86371732f8f09311c5b8ac32 \ + --hash=sha256:34266cec8c1d4e3e91fcca7efe38971d6bdda64a7944f2a46ab576da15173680 \ + --hash=sha256:36fac804b8cfd6b935ae64f71349f833d2b6298404626d017a2c57bb942bc643 \ + --hash=sha256:3af4e45ff455fce5511fdf2653c1ce428ef09c56fe37a83eb4d924c2d474f31e \ + --hash=sha256:3e3201fe4d46e0d15d7ff9fafc94a605da9eb82d2c5b9837f0368acb325481f1 \ + --hash=sha256:45b3002392948dcf578029c89f6318e1289a993a1a5ec38a4161560fab60f811 \ + --hash=sha256:465bc1526debf53a3be92022a16ca0c38f891ea3b5c1587af4f52e44020f8a07 \ + --hash=sha256:48c705bd0b1afb6253ed71eca9f9ba7ac7d47838e5fed1ef7891d67f21ecd4de \ + --hash=sha256:4a4d8c2c7e73ba5967be74d7c3a5ff81fde815ee1b48d9c5c0f14de8463a847b \ + --hash=sha256:4a85401b0c3f893cf5695c1199e8679fbf673f7f78c2f6c11d6b1850f8c7e358 \ + --hash=sha256:4c58dc91aefb31adad500aa68054334f429b840b36dd29e34e834101044cb2ef \ + --hash=sha256:4efbee349138a3fee7a4cc3a95abd2d499fae70dd5bff9fed9138d6f570f4283 \ + --hash=sha256:4fb995082fe41ec410b33c48b54fb1d44abb8a6ee762c31e8c42519e8c3a30a9 \ + --hash=sha256:5042aa1c7e2b1a24c17dab31d8770b63a5101c9abc25f832c6aef6b201e1ca4f \ + --hash=sha256:52fe0176682a25b15370f23f5b0f1366a84771df89144fb0cd979cb72a94b5ca \ + --hash=sha256:5332a020a60bbe32ede4bda1a62b3d56c4831d309cdf0932842c0fca8ad6aaa3 \ + --hash=sha256:563e4568217dc907a91843f38c737be865222c0400a38cdcd0d26ce92b3db271 \ + --hash=sha256:581b27663c6e9f4df68068f32fe6d1cd7647b31fac90237221a66f8821c342eb \ + --hash=sha256:58a1b0ec4cbb930e69669f9771715b2c7898d3cdf064d9811f7a66afef96b544 \ + --hash=sha256:5cc5d3a29f9ec86ce406e5ec09c241dd8dc4d30e838f74f68d728b89131a3acf \ + --hash=sha256:63d38e9a9a10a20fb57593742e63c6b1e78dd7f6ef5472de8e0b1e4cf4f3db26 \ + --hash=sha256:6b1ac7f1bc6c0dbf90684b77571a51a21b2463909fd916ce0ac9bfc4d566dc75 \ + --hash=sha256:6b900073e7b8481ef1aaf4f6c1789d210a1db01a9da8789821578cfeb4c2d540 \ + --hash=sha256:6c12d0393a903b58bc5f5a7406d6c5290acfb8284290d68547ce620c06f7d133 \ + --hash=sha256:6e2780e33a58a93f27cc3bb74a55bae6f9a8278a1dbabdff392940d30d381671 \ + --hash=sha256:6ebd39ee26db460cfe5ab8b71a15d1149b289139a0d3981522757d6af620887e \ + --hash=sha256:6f8b41299b203ce8f627db670cfea82067d9638853dbeaf86dccd93878879b85 \ + --hash=sha256:6f9549ca354a1d6d6167c458a1f1b12147726b968f02dd64b6a5801dba91ae0f \ + --hash=sha256:6ff0145b34610e57c9fae20df4e133c8d54266447387de6fcc0bdabfe4db4569 \ + --hash=sha256:6ff5f0ed70783dcb9562dbd20edca51c3d4d277f128223709e3da6b75986d1d4 \ + --hash=sha256:714bf348f468532d86bed670837e7d5ddff3834dd7f5d3c08066da400c86f088 \ + --hash=sha256:757e3f79cb865a7db94e0db5f4d0ed3284a69e39d53568f433982ea13c60cac1 \ + --hash=sha256:7e32b83bd8c2f8b6fa726ef34e63e21c4d7eddc277d40d4ef7245ea3ed28e5b6 \ + --hash=sha256:805b0f2618e5d4c3e28f45b731eb1a0539691ae4a2f97b4ce014de0bf96a1ff5 \ + --hash=sha256:80eae881cfb69383303e9a4d7961a478025b89c24f38f2e69b30c516fa0d57f2 \ + --hash=sha256:813a32f94991b9627795528053c73a57d2ce3eb98ede89f0e1c7a31095938e81 \ + --hash=sha256:8463b34ebde3f000627e9dbd8a545f995ad49fbf7ff9dd5abc0cd507da98a603 \ + --hash=sha256:8a59c749a73fbdbc8e63b895a3079825fa085d752e75bc0a500042cb8a801e48 \ + --hash=sha256:8d90d10e9b6594c28f27896a68fab97fd784c43804e9fe419dab8e8dcfcf4b02 \ + --hash=sha256:8e1e037bb57dbc549c6fe20370b763ea74bdb09413cdcf857e4f14d9e4e2fb13 \ + --hash=sha256:931f45f84e15daafec5f82cc92e6710569e1f50933f3253d206eab4132bec678 \ + --hash=sha256:995b52f7c260ac7023640221f27472303968753cb6fc6fce1ddfb0e9db59a398 \ + --hash=sha256:9b4da5789d7cf576c7e81f0088c632f6ee3786d87d17f08e90e703c22ce15633 \ + --hash=sha256:a3ed60ea9a7c352c590182c67404599e6b5a0c901e75ae4cceee9a9fd6bfa455 \ + --hash=sha256:a4d1ecad62e83cc65b411ea0125972cf3af98821e8117129947fd1e3a113f8d2 \ + --hash=sha256:ae9bb62a7902e2ab65782447cd3eeb753510feace4e3ea03937a85489b01b16b \ + --hash=sha256:b2ab3aad55d75d0b8df8d8a1b5920baaec9b161112cd5e95984848b4d2cd3dfe \ + --hash=sha256:b2cc6991f16f6d666d48e4b57318104e7b29109e32e2f6b86e9d44c4e6a27f4e \ + --hash=sha256:b5a3f5f70967a1aa2bc47fec42a1e19d2fb38c61700e3ee62b63a4af4f4fd001 \ + --hash=sha256:b68fb053b37c258a473ab67f4965c3b439500dc160fe364667035a6833eaf50a \ + --hash=sha256:b6ee42112d785a913dd63ec0335435a3dddbea5040c151252db815b0095cf066 \ + --hash=sha256:b928ab0ecaa664e8caecc529dcb8bc881b6b35bb2b74bf9a39ae25f982ee8812 \ + --hash=sha256:b9430f65db521db7962ad951571d446171213686f96c998a54dc18ed574821e2 \ + --hash=sha256:b9cd15cb7cf0d5cc41f649fd789aae12c56c3b83eff593f8e095c1d4555ad5c3 \ + --hash=sha256:bb1533541c729ad422f870a780d8b4af924f9817d45b5f580390418cda72eaa2 \ + --hash=sha256:bbf7377fbd41b7c87d47820e25b9876724963681c2a1d6f6ff2adb4db46ac174 \ + --hash=sha256:bca180cbe84e4fba7807eb408a8655295f697928512324517e30a091ede522a8 \ + --hash=sha256:beb2c8a34cc90fb4d862b7284eafdb322030d6a8b2ee5eb6a744f84205beedc3 \ + --hash=sha256:bfdabac0c6d3d6a5be8c2a100a001c92c14a39bbafd5999545a675c493626e64 \ + --hash=sha256:c0e45def4d9ce7073e2226535572442d9d6efb4047c7a5fd8960807e877ce70a \ + --hash=sha256:c0f537e5e8152e8d9cae82804024790cb973061abd3b7ef8f66f46e2b5c7bb51 \ + --hash=sha256:c195a69df0ab2541252ab5b1d76e3c182e5688ac2a9b708e5e6f66aaeda91e9a \ + --hash=sha256:c271bfb832be5c5c020b4e2fcbc1e70a0b990adba6de874b0bba1184b89cdea3 \ + --hash=sha256:c42424213c28804f8d0e20f5692106cfb57bf72e1dbc4092b8481fb2f9e4c707 \ + --hash=sha256:c4fa57d3c31889722f64bfa785545a5e603a893b6f29ac1a41bfa830abeaefd5 \ + --hash=sha256:cb2bb3ac0af7fdab2311b895c9eb95442b45deb14cc949b9e65545e74aa0be69 \ + --hash=sha256:cb3e7a4fd0168e362673a980380bf4fd6ae3b1555150e60c5390b4b10d9c50c4 \ + --hash=sha256:cbbfcd5d15056fbd1edd5e725cf3feeb47c7cbccbe205927ebab422cc229f417 \ + --hash=sha256:cd3e55223a77d6e08d5730ebacb4930ecca5d2ce7c57e7ba10833be7e52903f1 \ + --hash=sha256:ce8e723b4637034b76f5382a30a6b725518c332273e8d62a6c7d46e90837c947 \ + --hash=sha256:d1e329a1866981efe0201d05a374617f6c6cf14434a501d78ab22793d1ab1fa6 \ + --hash=sha256:d20ba5c84cf0592afb2713336f07e2b6ced082e4ae803ceada153a85613efc9f \ + --hash=sha256:d2b095129b9a98eb46a271ee9631089529c4e40354576b4aa74e24de9d2bf2f7 \ + --hash=sha256:d3906b5c549ff2ad2473cb711e1fc65d76715c2726a402108fbf55eab6c6b49d \ + --hash=sha256:d484ebb7e3a3f3597b0f645fbd1b85633674ca808c1f5ba11c2caf7c66f5c8b6 \ + --hash=sha256:db735a23ecb0f0450d2b24e0a05fb00a8a35c9db172919c4d3e023e7c7ee4c9b \ + --hash=sha256:dbc9fd1521e573045d71b6afab7398439c5cc259e8cb9d416fe62d485c4899c6 \ + --hash=sha256:df3867518b205be3648e2fbd522bf380c851b5c2500588047505afdd786b6669 \ + --hash=sha256:e0acbd474d0af4afacc6e66c4273f8a19e25f8af4379fc816388095ea6b01371 \ + --hash=sha256:eacf0f45ca3ff84c01481c60c15da9ee56711f7292f66663df0f57af61e011c2 \ + --hash=sha256:ead1a40543a033a6732a9e1e515944979a19db3737ce77363fc0660e38554344 \ + --hash=sha256:eae4e9c7a0785a1a715de0a74fb822ab40084c060f444f18f075d05e322aa7ef \ + --hash=sha256:ecf7037e491c220cd73987838c1ac3958d787bb098c3be0bfaf7f04204a6162c \ + --hash=sha256:ecfeee649184ffd800955068be9a6b579a0f33fc3c98535d685d5779cb59347f \ + --hash=sha256:edd5aa045fa3cc57143db018dd32ce7962bd5b525d05230709015d7e570100aa \ + --hash=sha256:f0ef48ce353f6b6a52232ba23d0983d4c2c84c84a778899404e34b4718509bf2 \ + --hash=sha256:f1734bd6f588975ffc246211e8b96c11933344087ca280d2cbcbf35cf835d7a9 \ + --hash=sha256:f67db0ba2bedafec15b8e5330d40da1e1c7921559fa715af021252bfef81a6f8 \ + --hash=sha256:f6ac1414556b910a879c108d79736f77e797871f9919ed0d2c3cf8cf3ecca986 \ + --hash=sha256:f78f7ae1c2e5aabf29583fc0d302d8081a663776f84578025662eb6f5d63a921 \ + --hash=sha256:f9489c1d87160c126f73b004742fe8654fa1ce37ed89e9e01330a1c10aaecde4 \ + --hash=sha256:f9ccc9884241efceb4547a92955d128574c864681f11b7ea3ecbde295fafbe8b \ + --hash=sha256:fc1a4f9d18d32a6e0a0a0a382986a60a2126f5144dd08715be7adb8df18e8a46 + # via -r requirements.in +idna==3.20 \ + --hash=sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44 \ + --hash=sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c + # via anyio +psycopg==3.3.6 \ + --hash=sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631 \ + --hash=sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2 + # via -r requirements.in +psycopg-binary==3.3.6 \ + --hash=sha256:05a83ac9fd52b9bca7cb5ab04b3691163170bd16f53defa27216ea3aa07ee781 \ + --hash=sha256:0a52991594ac4db888c7d39bccef331797e30cb31a95cae02cf2607f83a42dc2 \ + --hash=sha256:0bf08b749cc144f33b44a91b78e3f71c60eb07963746a0df5a100b36ce3d7475 \ + --hash=sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372 \ + --hash=sha256:1679a1cb93fbe5a6d1fd58d82cbddcc6fcb8c61446ba7cae6eb2a7b19bc585de \ + --hash=sha256:198a48e68cc99ccac03ba95ac857e73aa66f3bf6be77019fafb0832a05f7ad03 \ + --hash=sha256:1fbd30e537dab22cafdf080608f10148fe2a5f3a61294ddb5113caac8a623840 \ + --hash=sha256:289aadd6a00e151203c081f708348ec89f1e483c9b510ef4ac3981f847f01f79 \ + --hash=sha256:2f122603f36050937982abf9668d8bc4769a79f7c93a65013b1c49f1cab7b56b \ + --hash=sha256:303732e798fe6729f8e12021b9c96107df8e95ecec4dd487c67b98ec2a59435e \ + --hash=sha256:31cd942c23f613276b81a6e6598cefa12960058b0f46e1e874b540c793f6aca5 \ + --hash=sha256:366db6e97e66b37211475f20c4c1324a2dc0dd825e46d4e87f9d599304d276f9 \ + --hash=sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f \ + --hash=sha256:37d40450659401600e6d043ff586c89a71a69f33cbb8bcdba6cdb2569beecdbe \ + --hash=sha256:37e517c146b185f9c0c6e8d0a0ebbdeeeb67896af28466e032bc810d0c7dc7a7 \ + --hash=sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138 \ + --hash=sha256:3c9e663b2e800e3218994cf948c11bcc2844e6491b34aa80d089baf6531827bf \ + --hash=sha256:3f84dab25e0385692ee13274c68678377e0b1a70ab9d14e56264cbf61f60c62d \ + --hash=sha256:4690cf67738f0e0e49a32aeec99bf0e4595cc2b4f1af984a4345394b1dcff91a \ + --hash=sha256:566dd827f17728efdf7d88a5b066f815170f6fdad13967ae952842d90e6aaa9f \ + --hash=sha256:5927b7ba63153cd8e9862987290a2b783a5c590daf2a4ef981700cc3569166d4 \ + --hash=sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6 \ + --hash=sha256:5ea8beeb5541780b4b50b462eeacbc4f594ce3b911dc20c81c75f267876f71d2 \ + --hash=sha256:5f598f19fa9a91540b5cee17932ffd227b7b53a481605bcc4573c0eafa647300 \ + --hash=sha256:612382ac3ed13651c7fa44b5fee9fbf7baaa2ddbc6f500391672682c5f1df9e0 \ + --hash=sha256:6ff05561e4a067d35507dc5c90f1deb2ec1c9703ac5cccc1bc26e08a197f9c5a \ + --hash=sha256:7308c93cf0b19bbaf8e6ff0a6ad50d3c442385739245fe15a8d593bf841734a6 \ + --hash=sha256:79a2a1c3449f6c3409427078ed1cec10de79f3023cb5f2504f0597d350ad46c7 \ + --hash=sha256:7beb3e41c9a1e509f3ed85263386588cbe3e975aa67be21f79f44fd35ffaeefc \ + --hash=sha256:86147cb5d140341c3363fb5bacce31f8d5543902a46699d3c536b101bbceaf9e \ + --hash=sha256:889e42acec10450185e0cdfb396f375e2c1a8d7737c114830a7fde4654f59e30 \ + --hash=sha256:910ace140e3e7b7596898d083f37a8fe90c5c40684252ad4e682364b2cd3deba \ + --hash=sha256:955e3dd94da361e052d2e49acf591017158dc8f8ed2c8a42c2e3943403c39dc2 \ + --hash=sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22 \ + --hash=sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef \ + --hash=sha256:9b2f11794e017ce340934e35de46181c46ef71ec75ea3d85dd75cd836761c01e \ + --hash=sha256:a2e44a342d2aee40508e28a563d8961c39d9bbd8cae36d8578f0a3c6658aab0f \ + --hash=sha256:a4ee3bdd5468a725f2a4d9aab8a74b6d0279f768c8b5d3aeb102c5307ff3d59c \ + --hash=sha256:a5165300324efd5a772c48a88ab3a928513ab3979fca76553e62ee815f7b2b9c \ + --hash=sha256:a9348c5b43a3bb5ef8c2e89d5237c9c87eeafb01d338c84a7aebbc5cd0313299 \ + --hash=sha256:aa73160077345ec21b3f51e8e24b3de2e99586217e497629326eb9b2ea88c52e \ + --hash=sha256:ad1c785e784cfd87e8436c6b7702f2d321fc39601bbaf29bc63a41a867091638 \ + --hash=sha256:b3f75dee0f9afafabe4edc52c4842f1e1878ed2069bd05b22d6fe961e97e4dba \ + --hash=sha256:b599defe9190b17e9907c8b4d114c181e702c87efcd1b8a0ad40971cdcc4634a \ + --hash=sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9 \ + --hash=sha256:b8ece331509f7a975b90501f41e83ad905e4141753fedf3f2711b2bc70a8efbc \ + --hash=sha256:b979a42815410432420275412633960807178b1ce26591a16ce06e78a5bd4bb2 \ + --hash=sha256:be4f9b3c9338ac5dd217c5847e21521b396c8117f78dc420d495a5c49bbef874 \ + --hash=sha256:bf8c8481d026b85dd70c5fa7dde85b2333aed0b32a2602bcd38a900cbd78a49c \ + --hash=sha256:c61617eaae0112ca154da87ffb99b73af2c74067acac28dfb9a4455b019dff2e \ + --hash=sha256:c6d19cb4999d03231e8730a5f66c8f5068bc3b532677eb39dab0f600bff3e312 \ + --hash=sha256:c7753871eb57e6a5f4646f6168590c6653073dea5e9e720b201c8875332df4c8 \ + --hash=sha256:c7f92daa0d2a1c76f07264abddf8cbabd30152a2f09c3270e50f0c7efdf5dcac \ + --hash=sha256:cbd5f73073ed19c378d4c35499db1e3e703a5b1a324e521204065967bfaa7a18 \ + --hash=sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269 \ + --hash=sha256:d636338c8f21b0df2f84657b00bc34f9313f826ef93f1155bc743607e4a0c5eb \ + --hash=sha256:dc75da5a20951049f7b773145f998f69d181adad9c58a0ff36e0cf1d73c10e10 \ + --hash=sha256:e23a66a763fbe83fcc210bc77c27e5a5ea380ebf091c06f34d8561b695e5a40f \ + --hash=sha256:e8cbb54454dbf1bbf2ff08dd7693e8d94ac94b1a20f70f4b3b813d52ecb5cbc1 \ + --hash=sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784 \ + --hash=sha256:f0535693ce476a722b718b002d5d2c27d47e71ca945276ac194409c98e74c492 \ + --hash=sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc \ + --hash=sha256:f21d057f3e5f5491067e5b292498073b73847d48799b099803fef100775fcc52 \ + --hash=sha256:f87dbdc42e78ee0f7ea180c03f8c78e80a949e373066629bd90fefff10552dff \ + --hash=sha256:fa34eb47969297471db7b7f193622c7e3ee839ec05abd05f1fe104d5b1b1dcf4 \ + --hash=sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8 + # via -r requirements.in +psycopg-pool==3.3.3 \ + --hash=sha256:9b9cd6a4fcec47a410f7e82d408540e7f77b478509e91b44c1a5457a13e5ff37 \ + --hash=sha256:df87b5d9d0ad7db37f6cdad4fa8ce113d250f5997f6db38e9a99192fb67f9e1d + # via -r requirements.in +pycparser==3.11 \ + --hash=sha256:51d5a8ba2be0bbe440b99d2112604c95bbbc3c2748a64260186c541e1729cd80 \ + --hash=sha256:d875f09c3507d00e1aba0eecc6dcadc1352f30fff09dc6bff2f1c2935e97c2bc + # via cffi +redis==8.1.0 \ + --hash=sha256:6e1a19beef9225c83efd689c7e6b7da2d5215b1f42cd13b7fc3714d0a09c7b25 \ + --hash=sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb + # via -r requirements.in +starlette==1.7.0 \ + --hash=sha256:67f8e99895493dd2911a03f11314af6ceebeae4e704bb9f43dfc6a9db151c93e \ + --hash=sha256:c79f74ea63cff761804fbbfb182f1e0b440c2d07b164d24700c5a1bab5d6ff5d + # via -r requirements.in +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # anyio + # psycopg-pool +uvicorn==0.54.0 \ + --hash=sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf \ + --hash=sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620 + # via -r requirements.in +uvloop==0.23.0 ; sys_platform != "win32" \ + --hash=sha256:0305871ac712f54b62af73f943dbf21ae3ce80a44bc0f0151424484affa85645 \ + --hash=sha256:090865d8ce7a03986755a3ce711b7dd0d4b44eb14ab74368b717f3fad1180208 \ + --hash=sha256:098a85e1393ef5202767b7e5fb41a32cd8bd81e6ee4af364c179801c4aa3f6d4 \ + --hash=sha256:0efdd55bddbd36bb2fcb842d64c0d5f6407c6958c68088cc25df8c09edc5b5fd \ + --hash=sha256:12634f15e6625f78b3f2922f91404c4d7173487eba11746764153f556e9852dc \ + --hash=sha256:1748321e3c59a14a75404b1ae8d5a8d81c4e201803ea0e14c1b6fd84421024b5 \ + --hash=sha256:19c64108b507cd0bc140e400e3396bacebd9d504956aa7726272bf6de7d9aabb \ + --hash=sha256:1e84575f11873c109cf3962ad0bdf679094466184125f4cadcc41a73febff41f \ + --hash=sha256:24c58ae4a83e93a04c504bcc678125e36a0bfc44af928ad69444880c60f187a5 \ + --hash=sha256:28d160f51ab4da3b187063652e643dea6831072add4adc1e6d62afbe73b6be27 \ + --hash=sha256:2dcff2d69be43e6559e5dad2c5a7a2dbfb60e05a77311b6c4b7a4a8123d86c65 \ + --hash=sha256:31e0cf90bc8fd88784f6802cdba968a51fb1aec1cc3feec74d862b2d371d1330 \ + --hash=sha256:378188efbb1524f2219d05246a3e1e5907217848d2882144dff59585f1b81d55 \ + --hash=sha256:42feced24b9b44b856c633eafb5cc5dec354972da55ce77598db6844c054bc7c \ + --hash=sha256:4448e9124537620f9c25d004c227bb5104440b58955c19bbd312d910af919a63 \ + --hash=sha256:4a08875543bbd4519faf30497506c9cda8a48470467ffdf967c7313c7a5981a8 \ + --hash=sha256:4b8e207c67d207a8608fec57e116511030af3495dc0109b8c333cf9cb412b16f \ + --hash=sha256:4bb7f5d0b62b5afaaaea2b7b60d508921c24b0fe39c22c1438bec1811ffe10ec \ + --hash=sha256:4f1798f56c6f4ba5ac11fa2869e5717926e4470d97a1dd42b4f59219d43b5027 \ + --hash=sha256:514698d3683189031dcbfdc31e87115992e5ce9e1b19fe5359941323f2df800c \ + --hash=sha256:53c2c5d7e2024e46776c2d90e6c637d01102126b61aaf5faa5edaf05f8b5722a \ + --hash=sha256:55d6f4135d914305929fe9e9c44d8b5383a9b3fa1bee3bfcf60ee97e01af07ea \ + --hash=sha256:5a2bbad3a63007f7e9524d4903ba04fee252557c2acd86f9a3d4f91786695254 \ + --hash=sha256:5a3e0f56ec19bfd9ad1605572878dd6ff7f01b325f4fc154812ae70d615c3aff \ + --hash=sha256:5bb9be71d9ee39b4359b832f9569518ec9bc08704194034e79e4958e6bc4d46d \ + --hash=sha256:60ec798c40a1810d282ee046f61ecac1c5675cb898763d9f08d97d53a5e00a81 \ + --hash=sha256:6b3cbc4f96ddfa1fb88a78a69dd851369825b7816d9702eee8c4461505ba172e \ + --hash=sha256:6c7ef4701a96553514b2688e342ef1bf2beae6cfd172d89a76c768292aabf405 \ + --hash=sha256:7337b06a9f9ed9ea3049f04b76f65819db9b19bb832ee598e97b388eadf25e5f \ + --hash=sha256:76345f51367fb1f23e08605c6efb18374f669be5b223658fbab6b17627950507 \ + --hash=sha256:7e35c9bc977760981693e1a7a51493b58ee5a501f9ebb1e547565ee40b6c6208 \ + --hash=sha256:80cac5cb90ed7b9b72a217a1d6982b15b829cdbd0ee6bc19b93e3a9e47fb0ac9 \ + --hash=sha256:8af88fe5c7dd68fe1fec6dea8155caa1a47155d219a750ff34049541cf536a5e \ + --hash=sha256:8fcd721113260ffb5e38bf14a8725b17d431f34209f7d1c7005b667946e630b3 \ + --hash=sha256:93087a845cdfb35753e539354ac9551bdd2ff528c202a98df0ae46e852bcf021 \ + --hash=sha256:93935ab27b6eaef4c3e5489aebc84284f0644592f7ab516df60ee1b27eaf5eb3 \ + --hash=sha256:9bf08e4b6362dd1c08623bbfa2d061e8bac0f1da8fc2007062cfe1dc360a49fa \ + --hash=sha256:a6ac96da66c35bf789bdcde78a88dc7d56b7907d8379648c54adc1c61594575d \ + --hash=sha256:ab17b3a8aa754be0de0e397f7b95f13b14e56f077a4c6ae295e3d4afd199b325 \ + --hash=sha256:b0d106d9314546d69b3df1b5352639aa628530ec3ecef8a98a21942d2a2a64f5 \ + --hash=sha256:b90397a50ad6332ed3e459c648ac20d182cce24a557354363ad85fc9ea4a17cd \ + --hash=sha256:bbbdb8fcd5e7062e546eec1ac78c28bb21ae7df54c18f8e4b06e15a18d661a49 \ + --hash=sha256:bd6f2f81c7b9da99d301c0b16b82044e76fe887086e42e1590ecf520b94dbdac \ + --hash=sha256:be53e1d5f83de43dc175c87612ecc128d444b38e5c56cb3f807f5a73d6887476 \ + --hash=sha256:c3f23f403a273900d57de6ee5ca0614c650f7f58563065dad1a4744498960e53 \ + --hash=sha256:cbe8d03d4efcccdb7fcedecbaa1e1fa02913eaf3a74cb933634a6bc6d2ea9e2a \ + --hash=sha256:ce17bc317d089f361b33521654c13e30eacfd3d2034fd34e613ca9c51c969686 \ + --hash=sha256:d918d6f304a309222a784bbd140b85ec5594d97e4dc0e79f590549d28970663a \ + --hash=sha256:dc61e4f9e37b507069dc7e659ae28bca7adcb04c993c3508214315d12c63f848 \ + --hash=sha256:e095f9e105af76593b4c183bb0bcbdae64bd913a59ec595732dc108b48730ab5 \ + --hash=sha256:e2cba180d6451822763eda8364f342435a873bcfb3849cbd82fdeca248ca65eb \ + --hash=sha256:e49eba8f1e28e7c03648b7a476e1ba05309e087ccdea859fc6dd659564aa8d7e \ + --hash=sha256:f1341c6abcee1c31277cfe28d34e46196f2143ec3d755e6efe7452126e1f626d \ + --hash=sha256:f3fbfe82829d8e381426a289b87e59e585278728361db9ce975b88b51f64f410 \ + --hash=sha256:f50b580fad005a092ed87c5a3a4683459b21d1620497d6a5bccad203bee4c071 \ + --hash=sha256:f5576e8ae1723ece60d8f93c6710abf784714e99388bcf023ba9ca800bc587f6 \ + --hash=sha256:f673d835bdb1a60229cc3609a113fd2c9ce3f4a3c75ad4eaed111180c00199d2 \ + --hash=sha256:f7548ede3ee908cfabc0d068106e303a9a2d811af959cdf6ab85676344cedcda \ + --hash=sha256:fa8ed556fcc87a4091cf61587ef172fa104323dc89ecc085a618ba7ff8629a8f \ + --hash=sha256:fefea5cf8cdda9053b962ca8a90216fb0b1d40907dcb6819382b42e483e6e9f6 \ + --hash=sha256:ff7144d8167e513fe39fbb46bffb4f6f192dfb1f4b0b4e9102e1fd4f212e4747 + # via -r requirements.in diff --git a/scripts/explain_history.py b/scripts/explain_history.py new file mode 100644 index 00000000..0a929edb --- /dev/null +++ b/scripts/explain_history.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""EXPLAIN (ANALYZE, BUFFERS) of the history query on >= 10 000 messages (level 4). + + python scripts/explain_history.py --output evidence/explain-history.txt + +Creates (once, idempotent) a synthetic chat with 20 000 messages plus 30 000 messages in +30 other chats, then explains the EXACT statements of src/messenger/repo.py: + 1. first page (HISTORY_FIRST) + 2. late page via cursor (HISTORY_AFTER) — generic plan forced, as a prepared statement + reused by the driver would get it + 3. the rejected variant `($2 IS NULL OR (created_at, id) > ...)` — generic plan + 4. the rejected OFFSET pagination for the same late page +Runs psql inside the postgres container; no database port is published. +""" +from __future__ import annotations + +import argparse +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) +from messenger.repo import HISTORY_AFTER, HISTORY_FIRST # noqa: E402 (the app's real SQL) + +CHAT = "00000000-0000-4000-8000-00000000e002" +USER = "00000000-0000-4000-8000-00000000e001" +TARGET = 20_000 +LATE = 19_950 # cursor position: the 50 newest messages remain + +SEED = f""" +\\set ON_ERROR_STOP on +BEGIN; +INSERT INTO users (id, username, display_name, password_hash) +VALUES ('{USER}', 'explain_seed', convert_to('Explain seed', 'UTF8'), 'not-a-login') ON CONFLICT DO NOTHING; +INSERT INTO chats (id, title, created_by, created_at) +SELECT ('00000000-0000-4000-8000-0000000' || lpad(to_hex(n), 5, '0'))::uuid, convert_to('explain ' || n, 'UTF8'), + '{USER}', timestamptz '2026-01-01 00:00:00+00' +FROM generate_series(57346::int, 57376::int) AS n -- e002 .. e020: target + 30 noise chats +ON CONFLICT DO NOTHING; +INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) +SELECT id, '{USER}', 0, created_at FROM chats WHERE created_by = '{USER}' ON CONFLICT DO NOTHING; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), '{CHAT}', '{USER}', convert_to('synthetic message ' || g, 'UTF8'), + timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second' +FROM generate_series(1::int, {TARGET}::int) AS g +WHERE NOT EXISTS (SELECT 1 FROM messages WHERE chat_id = '{CHAT}'); +INSERT INTO messages (id, chat_id, sender_id, text, created_at) +SELECT gen_random_uuid(), c.id, '{USER}', convert_to('noise ' || g, 'UTF8'), + timestamptz '2026-01-01 00:00:00+00' + g * interval '1 second' +FROM chats c CROSS JOIN generate_series(1::int, 1000::int) AS g +WHERE c.created_by = '{USER}' AND c.id <> '{CHAT}' + AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id); +COMMIT; +ANALYZE messages; +""" + + +def pg(sql_param: str) -> str: + """Turn psycopg %s placeholders into $1..$n for a PREPARE statement.""" + parts = sql_param.split("%s") + return "".join(p + (f"${i + 1}" if i < len(parts) - 1 else "") for i, p in enumerate(parts)) + + +EXPLAIN = f""" +\\set ON_ERROR_STOP on +\\pset pager off +SELECT count(*) AS messages_total, count(*) FILTER (WHERE chat_id = '{CHAT}') AS messages_in_target_chat FROM messages; +SELECT created_at AS cur_ts, id AS cur_id FROM messages WHERE chat_id = '{CHAT}' +ORDER BY created_at, id OFFSET {LATE - 1} LIMIT 1 \\gset +SET plan_cache_mode = force_generic_plan; +PREPARE first_page(uuid, int) AS {pg(HISTORY_FIRST)}; +PREPARE after_page(uuid, timestamptz, uuid, int) AS {pg(HISTORY_AFTER)}; +PREPARE or_page(uuid, timestamptz, uuid, int) AS + SELECT id, sender_id, text, created_at FROM messages + WHERE chat_id = $1 AND ($2 IS NULL OR (created_at, id) > ($2, $3)) ORDER BY created_at, id LIMIT $4; +PREPARE offset_page(uuid, int, int) AS + SELECT id, sender_id, text, created_at FROM messages WHERE chat_id = $1 ORDER BY created_at, id OFFSET $2 LIMIT $3; +\\echo '=== 1. first page, app query HISTORY_FIRST, generic plan' +EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE first_page('{CHAT}', 51); +\\echo '=== 2. late page (cursor at {LATE}), app query HISTORY_AFTER, generic plan' +EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE after_page('{CHAT}', :'cur_ts', :'cur_id', 51); +\\echo '=== 3. REJECTED: one query with ($2 IS NULL OR ...), same late page, generic plan' +EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE or_page('{CHAT}', :'cur_ts', :'cur_id', 51); +\\echo '=== 4. REJECTED: OFFSET pagination, same late page' +EXPLAIN (ANALYZE, BUFFERS, SUMMARY ON) EXECUTE offset_page('{CHAT}', {LATE}, 51); +""" + + +def psql(script: str) -> str: + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-f", "-"], + cwd=ROOT, input=script, capture_output=True, text=True, encoding="utf-8", check=True) + return result.stdout + + +def run() -> str: + psql(SEED) + return psql(EXPLAIN) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--output", default=None) + args = parser.parse_args() + text = run() + print(text) + if args.output: + Path(args.output).write_text(text, encoding="utf-8", newline="\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..b02f635c --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +"""Create local synthetic credentials for the stand in ./secrets (ignored by Git). + + python scripts/init_secrets.py + +Idempotent: existing files are kept, so re-running never changes passwords of a +database that already has data. Prints only file names, never values. +Delete ./secrets together with the volumes (`docker compose down -v`) to start over. +""" +from __future__ import annotations + +import os +import secrets +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + path = SECRETS / name + if path.exists(): + print(f"keep secrets/{name}") + return + path.write_text(value + "\n", encoding="utf-8", newline="\n") + # Readable by the container users (postgres, valkey, app uid 10001) through the + # compose bind mount. Local synthetic lab credentials only. + os.chmod(path, 0o644) + print(f"created secrets/{name}") + + +def read(name: str) -> str: + return (SECRETS / name).read_text(encoding="utf-8").strip() + + +def main() -> int: + SECRETS.mkdir(exist_ok=True) + write("postgres_password", secrets.token_urlsafe(24)) # [A-Za-z0-9_-]: safe inside URLs + write("valkey_password", secrets.token_urlsafe(24)) + write("database_url", f"postgresql://messenger:{read('postgres_password')}@postgres:5432/messenger") + write("cache_url", f"redis://:{read('valkey_password')}@valkey:6379/0") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..f8a4ff9a --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,6 @@ +@echo off +rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows +rem cannot execute a .sh file directly. Runs the same scripts\restart.sh with Git Bash. +rem python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +"%ProgramFiles%\Git\bin\bash.exe" "%~dp0restart.sh" +exit /b %ERRORLEVEL% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..973048a8 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. +# Restarts the API and ALL state stores (PostgreSQL, Valkey) of compose.yaml. +# Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed. +# Output is suppressed by the test; run it by hand to see the steps. +set -eu +cd "$(dirname "$0")/.." + +# 1. API first: no request is served while the stores restart underneath it. +docker compose stop api +# 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync), then start again. +docker compose restart postgres valkey +# 3. API again; it waits for healthy stores via depends_on and reconnects its pool. +docker compose start api +echo "restart.sh: api, postgres and valkey restarted with volumes preserved" diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..26148a23 --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1,3 @@ +"""In-memory messenger for lab 1 (MTUSI, «Высоконагруженные приложения»).""" + +__version__ = "1.0.0" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..331a1fd2 --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,65 @@ +"""Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker). + +Shutdown sequence on SIGTERM/SIGINT: +1. readiness flips to 503 (`app.state.draining = True`); +2. uvicorn closes the listening socket, so new connections are refused; +3. in-flight requests get up to SHUTDOWN_TIMEOUT_SECONDS (default 8 s < Docker's 10 s) to finish; +4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0. + Data stays in PostgreSQL and Valkey volumes. +""" +from __future__ import annotations + +import logging +import sys +from types import FrameType + +import uvicorn + +from . import __version__, logging_setup +from .app import create_app +from .config import ConfigError, Settings + +log = logging.getLogger("messenger") + + +class GracefulServer(uvicorn.Server): + """uvicorn.Server that marks the app as draining before the normal graceful stop.""" + + def handle_exit(self, sig: int, frame: FrameType | None) -> None: + if not self.should_exit: + log.info("shutdown_requested", extra={"fields": {"signal": sig, "timeout_s": self.config.timeout_graceful_shutdown}}) + self.config.app.state.draining = True # config.app is our Starlette instance + super().handle_exit(sig, frame) + + +def main() -> int: + try: + settings = Settings.from_env() + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + logging_setup.configure(settings.log_level) + app = create_app(settings) + config = uvicorn.Config( + app, + host=settings.host, + port=settings.port, + log_config=None, + access_log=False, + server_header=False, + proxy_headers=False, + lifespan="on", + timeout_graceful_shutdown=settings.shutdown_timeout, + ) + server = GracefulServer(config) + log.info("starting", extra={"fields": { + "version": __version__, "env": settings.env, "host": settings.host, "port": settings.port, + "python": sys.version.split()[0], "storage": "postgresql", + }}) + server.run() + log.info("stopped") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..9d0bf79b --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,332 @@ +"""HTTP layer: Starlette routes, request-id/logging middleware and error mapping.""" +from __future__ import annotations + +import asyncio +import contextlib +import logging +import re +import time +import uuid +from typing import Any + +from starlette.applications import Starlette +from starlette.exceptions import HTTPException +from starlette.requests import Request +from starlette.responses import JSONResponse, Response +from starlette.routing import Route +from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Send + +from . import authhdr, cursor, docs, passwords, repo, validation +from .config import Settings +from .db import Database +from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, PayloadTooLarge, Unauthorized, Unavailable +from .sessions import SessionStore, build_store +from .timefmt import rfc3339 + +log = logging.getLogger("messenger") +access_log = logging.getLogger("messenger.access") + +REQUEST_ID_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +# -------------------------------------------------------------------------- +# Responses +# -------------------------------------------------------------------------- +def error_response(request_id: str, status: int, code: str, message: str, headers: dict[str, str] | None = None) -> JSONResponse: + body = {"error": {"code": code, "message": message, "request_id": request_id}} + return JSONResponse(body, status_code=status, headers={"X-Request-Id": request_id, **(headers or {})}) + + +async def api_error_handler(request: Request, exc: Exception) -> JSONResponse: + assert isinstance(exc, ApiError) + return error_response(request.state.request_id, exc.status, exc.code, exc.message, exc.headers) + + +async def http_exception_handler(request: Request, exc: Exception) -> JSONResponse: + """Unknown routes and wrong methods also answer in the contract error format.""" + assert isinstance(exc, HTTPException) + if exc.status_code == 404: + return error_response(request.state.request_id, 404, "not_found", "Объект не найден") + if exc.status_code == 405: + return error_response(request.state.request_id, 405, "invalid_request", "Метод не поддерживается", exc.headers) + return error_response(request.state.request_id, exc.status_code, "invalid_request", "Неверный запрос") + + +# -------------------------------------------------------------------------- +# Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions) +# -------------------------------------------------------------------------- +class RequestContextMiddleware: + """Assigns X-Request-Id to every response, writes one access-log line, turns crashes into 500.""" + + def __init__(self, app: ASGIApp): + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + incoming = dict(scope["headers"]).get(b"x-request-id", b"").decode("latin-1") + request_id = incoming if REQUEST_ID_RE.fullmatch(incoming) else uuid.uuid4().hex + scope.setdefault("state", {})["request_id"] = request_id + status = 500 + started = False + begin = time.perf_counter() + + async def send_with_id(message: AsgiMessage) -> None: + nonlocal status, started + if message["type"] == "http.response.start": + started = True + status = message["status"] + headers = [(k, v) for k, v in message.get("headers", []) if k.lower() != b"x-request-id"] + headers.append((b"x-request-id", request_id.encode("latin-1"))) + message = {**message, "headers": headers} + await send(message) + + try: + await self.app(scope, receive, send_with_id) + except asyncio.CancelledError: + # uvicorn cancels requests still running when SHUTDOWN_TIMEOUT_SECONDS expires: + # answer 503 (retryable) instead of a generic 500, then let the cancellation proceed. + if not started: + status = 503 + response = error_response(request_id, 503, "unavailable", "Сервер завершает работу") + await response(scope, receive, send) + raise + except Exception: + log.exception("unhandled_error", extra={"fields": {"request_id": request_id}}) + if not started: + status = 500 + response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера") + await response(scope, receive, send) + finally: + access_log.info("request", extra={"fields": { + "request_id": request_id, + "method": scope["method"], + "path": scope["path"], # ids only; query (cursor) and bodies are never logged + "status": status, + "duration_ms": round((time.perf_counter() - begin) * 1000, 3), + }}) + + +# -------------------------------------------------------------------------- +# Request helpers +# -------------------------------------------------------------------------- +def db_of(request: Request) -> Database: + return request.app.state.db + + +def sessions_of(request: Request) -> SessionStore: + return request.app.state.sessions + + +async def authenticate(request: Request) -> str: + """User id of a valid Bearer session, otherwise 401. X-User-Id is ignored since lab 2. + + Store failures are 503 (Unavailable), never 401 and never anonymous access. + """ + token = authhdr.parse_bearer(request.headers.get("authorization")) + user_id = await sessions_of(request).user_id_for(token) + if user_id is None: + raise Unauthorized("Сессия недействительна, истекла или отозвана") + return user_id + + +def path_uuid(request: Request, name: str) -> str: + return validation.parse_uuid(request.path_params[name], name) + + +async def read_json(request: Request) -> dict[str, Any]: + content_type = request.headers.get("content-type", "") + media_type, _, params = content_type.partition(";") + if media_type.strip().lower() != "application/json": + raise InvalidRequest("Content-Type должен быть application/json") + charset = params.strip().lower() + if charset and charset.replace(" ", "") not in {"charset=utf-8", "charset=\"utf-8\""}: + raise InvalidRequest("Поддерживается только кодировка UTF-8") + limit: int = request.app.state.settings.max_body_bytes + declared = request.headers.get("content-length") + if declared is not None and declared.isdigit() and int(declared) > limit: + raise PayloadTooLarge() + chunks: list[bytes] = [] + size = 0 + async for chunk in request.stream(): + size += len(chunk) + if size > limit: + raise PayloadTooLarge() + chunks.append(chunk) + return validation.parse_json_object(b"".join(chunks)) + + +def page_params(request: Request) -> tuple[int, str | None]: + query = request.query_params.multi_items() + limit = validation.parse_limit(validation.single_query_param(query, "limit")) + raw_cursor = validation.single_query_param(query, "cursor") + if raw_cursor is not None and raw_cursor == "": + raise InvalidRequest("cursor не может быть пустым") + return limit, raw_cursor + + +def ok(body: Any, status: int = 200) -> JSONResponse: + return JSONResponse(body, status_code=status) + + +# -------------------------------------------------------------------------- +# Handlers +# -------------------------------------------------------------------------- +async def live(request: Request) -> JSONResponse: + return ok({"status": "ok"}) + + +async def ready(request: Request) -> JSONResponse: + """200 only if the mandatory synchronous path can be served: PostgreSQL and, + with SESSION_STORE=valkey, Valkey answer. 503 while draining after SIGTERM.""" + if request.app.state.draining: + raise Unavailable("Сервер завершает работу") + db_ok, store_ok = await asyncio.gather(db_of(request).ping(), sessions_of(request).ping()) + if not db_ok: + raise Unavailable("PostgreSQL недоступен") + if not store_ok: + raise Unavailable("Хранилище сессий недоступно") + return ok({"status": "ok"}) + + +async def register_user(request: Request) -> JSONResponse: + data = await read_json(request) + validation.check_fields(data, {"username", "display_name", "password"}) + username = validation.username_field(data) + display_name = validation.text_field(data, "display_name", 100) + password = validation.password_field(data) + password_hash = await passwords.hash_password(password) + user = await repo.create_user(db_of(request), username, display_name, password_hash) + log.info("user_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}}) + return ok(user.to_json(), 201) + + +async def create_session(request: Request) -> JSONResponse: + """Login: only `Authorization: Basic`; wrong user and wrong password give the same 401.""" + username, password = authhdr.parse_basic(request.headers.get("authorization")) + found = await repo.get_credentials(db_of(request), username) + user, stored_hash = found if found else (None, None) + if not await passwords.verify_password(stored_hash, password) or user is None: + log.info("login_failed", extra={"fields": {"request_id": request.state.request_id}}) + raise Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE) + ttl = request.app.state.settings.session_ttl_seconds + token, expires_at = await sessions_of(request).create(user.id, ttl) + log.info("session_created", extra={"fields": {"request_id": request.state.request_id, "user_id": user.id}}) + return ok({"token": token, "token_type": "Bearer", "expires_at": rfc3339(expires_at), "user": user.to_json()}, 201) + + +async def revoke_session(request: Request) -> Response: + token = authhdr.parse_bearer(request.headers.get("authorization")) + if not await sessions_of(request).revoke(token): + raise Unauthorized("Сессия недействительна, истекла или отозвана") + log.info("session_revoked", extra={"fields": {"request_id": request.state.request_id}}) + return Response(status_code=204) + + +async def me(request: Request) -> JSONResponse: + user_id = await authenticate(request) + user = await repo.get_user(db_of(request), user_id) + if user is None: + raise Unauthorized("Сессия недействительна, истекла или отозвана") + return ok(user.to_json()) + + +async def create_chat(request: Request) -> JSONResponse: + user_id = await authenticate(request) + data = await read_json(request) + validation.check_fields(data, {"title", "member_ids"}) + title = validation.text_field(data, "title", 100) + member_ids = validation.member_ids_field(data) + chat = await repo.create_chat(db_of(request), user_id, title, member_ids) + log.info("chat_created", extra={"fields": {"request_id": request.state.request_id, "chat_id": chat.id, "members": len(member_ids)}}) + return ok(chat.to_json(), 201) + + +async def list_chats(request: Request) -> JSONResponse: + user_id = await authenticate(request) + limit, raw_cursor = page_params(request) + after = cursor.decode(raw_cursor, cursor.CHATS, user_id) if raw_cursor else None + items, has_more = await repo.list_chats(db_of(request), user_id, after, limit) + next_cursor = cursor.encode(cursor.CHATS, user_id, items[-1].key) if has_more else None + return ok({"items": [c.to_json() for c in items], "next_cursor": next_cursor}) + + +async def get_chat(request: Request) -> JSONResponse: + user_id = await authenticate(request) + chat_id = path_uuid(request, "chat_id") + return ok((await repo.get_chat(db_of(request), user_id, chat_id)).to_json()) + + +async def create_message(request: Request) -> JSONResponse: + user_id = await authenticate(request) + chat_id = path_uuid(request, "chat_id") + db = db_of(request) + await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before the body is read + data = await read_json(request) + validation.check_fields(data, {"text"}) + text = validation.text_field(data, "text", 4000) + # sender_id is derived from the session, never taken from the body. + message = await repo.add_message(db, user_id, chat_id, text) + return ok(message.to_json(), 201) + + +async def list_messages(request: Request) -> JSONResponse: + user_id = await authenticate(request) + chat_id = path_uuid(request, "chat_id") + db = db_of(request) + await repo.ensure_member(db, user_id, chat_id) # 404 for outsiders before cursor details leak + limit, raw_cursor = page_params(request) + after = cursor.decode(raw_cursor, cursor.MESSAGES, chat_id) if raw_cursor else None + items, has_more = await repo.list_messages(db, chat_id, after, limit) + next_cursor = cursor.encode(cursor.MESSAGES, chat_id, items[-1].key) if has_more else None + return ok({"items": [m.to_json() for m in items], "next_cursor": next_cursor}) + + +def create_app(settings: Settings | None = None) -> Starlette: + settings = settings or Settings.from_env() + db = Database(settings) + sessions = build_store(settings, db) + + @contextlib.asynccontextmanager + async def lifespan(app: Starlette): + passwords.configure(settings.hash_concurrency) + await db.open() + log.info("dependencies_configured", extra={"fields": { + "session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds, + "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds}}) + try: + yield + finally: + await sessions.close() + await db.close() + + routes = [ + Route("/health/live", live, methods=["GET"]), + Route("/health/ready", ready, methods=["GET"]), + Route("/api/v1/users", register_user, methods=["POST"]), + Route("/api/v1/users/me", me, methods=["GET"]), + Route("/api/v1/auth/sessions", create_session, methods=["POST"]), + Route("/api/v1/auth/sessions/current", revoke_session, methods=["DELETE"]), + Route("/api/v1/chats", create_chat, methods=["POST"]), + Route("/api/v1/chats", list_chats, methods=["GET"]), + Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), + Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), + ] + if settings.docs_enabled: # Swagger UI for manual experiments, not part of the contract + routes += [ + Route("/docs", docs.swagger_page, methods=["GET"]), + Route("/openapi.json", docs.openapi_json, methods=["GET"]), + ] + app = Starlette( + routes=routes, + exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler}, + lifespan=lifespan, + ) + app.add_middleware(RequestContextMiddleware) + app.state.settings = settings + app.state.db = db + app.state.sessions = sessions + app.state.draining = False + return app diff --git a/src/messenger/authhdr.py b/src/messenger/authhdr.py new file mode 100644 index 00000000..f849b029 --- /dev/null +++ b/src/messenger/authhdr.py @@ -0,0 +1,48 @@ +"""Parsing of the Authorization header. Any malformed value is 401, never 500. + +Basic (RFC 7617) is accepted only by POST /auth/sessions; everything else needs +Bearer. Basic is an encoding, not encryption: plain HTTP is acceptable only for the +local synthetic lab stand (lab 7 adds TLS). +""" +from __future__ import annotations + +import base64 +import binascii +import re + +from .errors import BASIC_CHALLENGE, BEARER_CHALLENGE, Unauthorized +from .validation import USERNAME_RE + +B64_RE = re.compile(r"[A-Za-z0-9+/]+={0,2}") +TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") # token_urlsafe(32) gives 43 characters + + +def _split(header: str | None) -> tuple[str, str]: + if not header: + return "", "" + scheme, _, value = header.strip().partition(" ") + return scheme.lower(), value.strip() + + +def parse_basic(header: str | None) -> tuple[str, str]: + """(username, password) from `Basic base64(username:password)`; the password may contain ':'.""" + invalid = Unauthorized("Неверное имя пользователя или пароль", challenge=BASIC_CHALLENGE) + scheme, value = _split(header) + if scheme != "basic" or B64_RE.fullmatch(value) is None or len(value) > 1024: + raise invalid + try: + decoded = base64.b64decode(value, validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise invalid from None + username, sep, password = decoded.partition(":") # split at the FIRST colon + if not sep or USERNAME_RE.fullmatch(username) is None or not password: + raise invalid + return username, password + + +def parse_bearer(header: str | None) -> str: + scheme, value = _split(header) + if scheme != "bearer" or TOKEN_RE.fullmatch(value) is None: + # Missing header, Basic on a business endpoint, garbage token: one answer. + raise Unauthorized("Требуется действительная Bearer-сессия", challenge=BEARER_CHALLENGE) + return value diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..4cecbcd1 --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,127 @@ +"""Runtime configuration read from environment variables (12-factor style). + +Non-secret settings have safe defaults. Connection strings with passwords are +read from files (`DATABASE_URL_FILE`, `CACHE_URL_FILE` → Docker secrets in +/run/secrets), so they never appear in `docker inspect`, compose files or logs. +Invalid values stop the process at startup with a clear message. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + + +class ConfigError(ValueError): + pass + + +def _int(name: str, default: int, low: int, high: int) -> int: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + try: + value = int(raw.strip()) + except ValueError: + raise ConfigError(f"{name} must be an integer, got {raw!r}") from None + if not low <= value <= high: + raise ConfigError(f"{name} must be in [{low}, {high}], got {value}") + return value + + +def _float(name: str, default: float, low: float, high: float) -> float: + raw = os.environ.get(name) + if raw is None or raw.strip() == "": + return default + try: + value = float(raw.strip()) + except ValueError: + raise ConfigError(f"{name} must be a number, got {raw!r}") from None + if not low <= value <= high: + raise ConfigError(f"{name} must be in [{low}, {high}], got {value}") + return value + + +def _bool(name: str, default: bool) -> bool: + raw = os.environ.get(name, "").strip().lower() + if raw == "": + return default + if raw not in {"true", "false", "1", "0"}: + raise ConfigError(f"{name} must be true/false, got {raw!r}") + return raw in {"true", "1"} + + +def _secret(name: str, required: bool) -> str: + """Value of _FILE (preferred, Docker secret) or (local runs/tests).""" + path = os.environ.get(f"{name}_FILE", "").strip() + if path: + try: + value = Path(path).read_text(encoding="utf-8").strip() + except OSError as exc: + # The message names the file, never its content. + raise ConfigError(f"{name}_FILE={path} cannot be read: {exc.strerror}") from None + else: + value = os.environ.get(name, "").strip() + if required and not value: + raise ConfigError(f"{name}_FILE (or {name}) must be set") + return value + + +@dataclass(frozen=True) +class Settings: + host: str + port: int + env: str + log_level: str + shutdown_timeout: float + max_body_bytes: int + docs_enabled: bool = True + database_url: str = "" + session_store: str = "postgres" + cache_url: str = "" + session_ttl_seconds: int = 3600 + db_pool_min: int = 1 + db_pool_max: int = 10 + db_timeout_seconds: float = 3.0 + cache_timeout_seconds: float = 0.5 + hash_concurrency: int = 2 + + @property + def db_statement_timeout_ms(self) -> int: + # The server stops a statement a bit before the client-side deadline fires. + return max(100, int(self.db_timeout_seconds * 1000) - 500) + + @classmethod + def from_env(cls) -> "Settings": + log_level = os.environ.get("LOG_LEVEL", "info").strip().lower() or "info" + if log_level not in {"debug", "info", "warning", "error"}: + raise ConfigError(f"LOG_LEVEL must be debug/info/warning/error, got {log_level!r}") + session_store = os.environ.get("SESSION_STORE", "postgres").strip().lower() or "postgres" + if session_store == "redis": + session_store = "valkey" # same protocol; the course allows Redis OR Valkey + if session_store not in {"postgres", "valkey"}: + raise ConfigError(f"SESSION_STORE must be postgres or valkey, got {session_store!r}") + db_pool_min = _int("DB_POOL_MIN", 1, 0, 50) + db_pool_max = _int("DB_POOL_MAX", 10, 1, 100) + if db_pool_min > db_pool_max: + raise ConfigError("DB_POOL_MIN must not exceed DB_POOL_MAX") + return cls( + host=os.environ.get("APP_HOST", "0.0.0.0").strip() or "0.0.0.0", + port=_int("APP_PORT", 8080, 1, 65535), + env=os.environ.get("APP_ENV", "lab").strip() or "lab", + log_level=log_level, + # Docker sends SIGKILL 10 s after SIGTERM by default; finish earlier. + shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 9.0), + max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024), + docs_enabled=_bool("DOCS_ENABLED", True), + database_url=_secret("DATABASE_URL", required=True), + session_store=session_store, + cache_url=_secret("CACHE_URL", required=session_store == "valkey"), + # Contract name. Minimum 1 s so the expiry test can use 2 s. + session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600), + db_pool_min=db_pool_min, + db_pool_max=db_pool_max, + db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0), + cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0), + hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16), + ) diff --git a/src/messenger/cursor.py b/src/messenger/cursor.py new file mode 100644 index 00000000..8e2d5ee6 --- /dev/null +++ b/src/messenger/cursor.py @@ -0,0 +1,56 @@ +"""Opaque keyset cursors. + +A cursor encodes the sort key `(created_at, id)` of the last item on the page +plus the resource it belongs to, e.g. `v1|m|||`, +packed in URL-safe base64. The client treats it as an opaque string; the server +rejects cursors of another chat/user (400) and any non-canonical encoding. +""" +from __future__ import annotations + +import base64 +import re + +from .errors import InvalidRequest +from .validation import UUID_RE + +B64_RE = re.compile(r"[A-Za-z0-9_-]{1,512}") +TS_RE = re.compile(r"[0-9]{1,19}") + +# Latest instant both Python datetime and PostgreSQL accept: 9999-12-31T23:59:59.999999Z. +MAX_TS_US = 253402300799999999 + +MESSAGES = "m" +CHATS = "c" + +SortKey = tuple[int, str] + + +def encode(scope: str, owner_id: str, key: SortKey) -> str: + ts, item_id = key + raw = f"v1|{scope}|{owner_id}|{ts}|{item_id}".encode("ascii") + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") + + +def decode(cursor: str, scope: str, owner_id: str) -> SortKey: + """Return the sort key stored in the cursor or raise InvalidRequest.""" + bad = InvalidRequest("Неверный cursor") + if B64_RE.fullmatch(cursor) is None: + raise bad + try: + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)).decode("ascii") + except (ValueError, UnicodeDecodeError): + raise bad from None + parts = raw.split("|") + if len(parts) != 5 or parts[0] != "v1": + raise bad + _, got_scope, got_owner, ts, item_id = parts + if TS_RE.fullmatch(ts) is None or UUID_RE.fullmatch(item_id) is None or item_id != item_id.lower(): + raise bad + key = (int(ts), item_id) + if key[0] > MAX_TS_US: + raise bad # out-of-range time must be 400, not an overflow (500) in datetime/SQL + if encode(got_scope, got_owner, key) != cursor: + raise bad # only the canonical form produced by the server is accepted + if got_scope != scope or got_owner != owner_id: + raise InvalidRequest("cursor относится к другому ресурсу") + return key diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..6c753a6e --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,109 @@ +"""PostgreSQL access: a bounded connection pool and a hard deadline for every unit of work. + +Three independent limits keep a request from hanging when the database misbehaves: + +* pool `timeout` — waiting for a free connection (pool exhausted); +* server `statement_timeout` / `lock_timeout` — PostgreSQL itself cancels a slow + statement or a statement stuck behind a lock; +* client deadline — covers what the server cannot report, e.g. a + paused/frozen PostgreSQL container or a dead network. + +The client deadline needs care: when a psycopg query is cancelled, psycopg sends a +cancel request to the server and waits for it (up to 5 s + 5 s) before giving up. +With a frozen server that would stretch a 3 s deadline to 13 s. Therefore every unit +of work runs as its own task: at the deadline the request stops waiting and gets 503 +at once, while the task finishes its cleanup (cancel, close, return to the pool) in +the background. Any failure becomes 503 `unavailable` — never 401, never a hang. +""" +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from typing import TypeVar + +import psycopg +from psycopg_pool import AsyncConnectionPool, PoolTimeout + +from .config import Settings +from .errors import Unavailable + +log = logging.getLogger("messenger.db") + +T = TypeVar("T") +Work = Callable[[psycopg.AsyncConnection], Awaitable[T]] + +# Errors that mean "the database is unavailable right now", not "the request is wrong". +# QueryCanceled (statement_timeout), LockNotAvailable (lock_timeout) and AdminShutdown +# (server restart) are subclasses of OperationalError. +UNAVAILABLE_ERRORS = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TimeoutError) + + +def _forget(task: asyncio.Task) -> None: + """Done-callback for abandoned tasks: retrieve the result so nothing is logged as 'never retrieved'.""" + if not task.cancelled(): + task.exception() + + +class Database: + def __init__(self, settings: Settings): + self.deadline = settings.db_timeout_seconds + options = ( + f"-c statement_timeout={settings.db_statement_timeout_ms} " + f"-c lock_timeout={max(100, settings.db_statement_timeout_ms // 2)} " + "-c idle_in_transaction_session_timeout=10000" + ) + self.pool = AsyncConnectionPool( + conninfo=settings.database_url, + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_timeout_seconds, # max wait for a free connection + open=False, + name="messenger", + check=AsyncConnectionPool.check_connection, # drop connections broken by a DB restart + kwargs={ + "options": options, + "connect_timeout": max(1, int(settings.db_timeout_seconds)), + "application_name": "messenger-api", + }, + ) + + async def open(self) -> None: + # wait=False: the API starts (liveness 200) even if PostgreSQL is not up yet; + # readiness stays 503 until the database answers. + await self.pool.open(wait=False) + + async def close(self) -> None: + await self.pool.close(timeout=2) + + async def run(self, work: Work[T], deadline: float | None = None) -> T: + """Run `work(conn)` in one transaction (COMMIT on success, ROLLBACK on any exception). + + Domain errors raised by `work` (NotFound, Conflict, ...) propagate unchanged; + database unavailability and the deadline become Unavailable (503). + """ + async def unit() -> T: + async with self.pool.connection() as conn: + async with conn.transaction(): + return await work(conn) + + task = asyncio.ensure_future(unit()) + try: + return await asyncio.wait_for(asyncio.shield(task), deadline or self.deadline) + except UNAVAILABLE_ERRORS as exc: + log.warning("database_unavailable", extra={"fields": {"error": type(exc).__name__}}) + raise Unavailable("База данных временно недоступна") from None + finally: + if not task.done(): + # Deadline or cancelled request: stop waiting now, let psycopg clean up later. + task.cancel() + task.add_done_callback(_forget) + + async def ping(self, timeout: float = 1.0) -> bool: + async def select_one(conn: psycopg.AsyncConnection) -> None: + await conn.execute("SELECT 1") + try: + await self.run(select_one, deadline=timeout) + return True + except Unavailable: + return False diff --git a/src/messenger/docs.py b/src/messenger/docs.py new file mode 100644 index 00000000..cb2ea46e --- /dev/null +++ b/src/messenger/docs.py @@ -0,0 +1,57 @@ +"""Swagger UI for manual experiments: http://127.0.0.1:8080/docs + +The page shows the course contract contracts/openapi.json as is (it is not +generated from our code). Only `servers` is replaced with "/" so that the +"Try it out" requests go to the same host the page was opened from. +Swagger UI itself (JS/CSS) is loaded by the browser from the jsDelivr CDN. +Disable with DOCS_ENABLED=false. +""" +from __future__ import annotations + +import json +from functools import lru_cache +from pathlib import Path + +from starlette.requests import Request +from starlette.responses import HTMLResponse, JSONResponse + +SPEC_PATH = Path(__file__).resolve().parents[2] / "contracts" / "openapi.json" +SWAGGER_UI = "https://cdn.jsdelivr.net/npm/swagger-ui-dist@5.33.0" + +PAGE = f""" + + + + Messenger lab1 — Swagger UI + + + +
+ + + + +""" + + +@lru_cache(maxsize=1) +def _spec() -> dict: + spec = json.loads(SPEC_PATH.read_text(encoding="utf-8")) + spec["servers"] = [{"url": "/", "description": "этот сервер"}] + return spec + + +async def swagger_page(request: Request) -> HTMLResponse: + return HTMLResponse(PAGE) + + +async def openapi_json(request: Request) -> JSONResponse: + return JSONResponse(_spec()) diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..22dbe74a --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,65 @@ +"""Domain errors mapped 1:1 to the contract error codes (contracts/README.md).""" +from __future__ import annotations + +BEARER_CHALLENGE = 'Bearer realm="messenger"' +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' + + +class ApiError(Exception): + status = 500 + code = "internal_error" + default_message = "Внутренняя ошибка сервера" + + def __init__(self, message: str | None = None): + self.message = message or self.default_message + super().__init__(self.message) + + @property + def headers(self) -> dict[str, str]: + return {} + + +class InvalidRequest(ApiError): + status = 400 + code = "invalid_request" + default_message = "Неверный запрос" + + +class Unauthorized(ApiError): + """401 with the challenge of the scheme the endpoint expects (RFC 7235).""" + + status = 401 + code = "unauthorized" + default_message = "Требуется действительная сессия" + + def __init__(self, message: str | None = None, challenge: str = BEARER_CHALLENGE): + super().__init__(message) + self.challenge = challenge + + @property + def headers(self) -> dict[str, str]: + return {"WWW-Authenticate": self.challenge} + + +class NotFound(ApiError): + status = 404 + code = "not_found" + default_message = "Объект не найден" + + +class Conflict(ApiError): + status = 409 + code = "conflict" + default_message = "Конфликт" + + +class PayloadTooLarge(ApiError): + status = 413 + code = "payload_too_large" + default_message = "Тело запроса слишком большое" + + +class Unavailable(ApiError): + status = 503 + code = "unavailable" + default_message = "Сервис временно недоступен" diff --git a/src/messenger/healthcheck.py b/src/messenger/healthcheck.py new file mode 100644 index 00000000..8411514b --- /dev/null +++ b/src/messenger/healthcheck.py @@ -0,0 +1,19 @@ +"""Docker HEALTHCHECK probe without curl/wget in the image: `python -m messenger.healthcheck`.""" +from __future__ import annotations + +import os +import sys +import urllib.request + + +def main() -> int: + port = os.environ.get("APP_PORT", "8080").strip() or "8080" + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/ready", timeout=2) as response: + return 0 if response.status == 200 else 1 + except Exception: + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/logging_setup.py b/src/messenger/logging_setup.py new file mode 100644 index 00000000..388f14de --- /dev/null +++ b/src/messenger/logging_setup.py @@ -0,0 +1,36 @@ +"""JSON-lines logging to stdout. Logs never contain message texts or request bodies.""" +from __future__ import annotations + +import datetime as dt +import json +import logging +import sys + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + entry = { + "ts": dt.datetime.fromtimestamp(record.created, dt.timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z"), + "level": record.levelname.lower(), + "logger": record.name, + "event": record.getMessage(), + } + fields = getattr(record, "fields", None) + if fields: + entry.update(fields) + if record.exc_info: + # Stack traces go only to the server log, never to the HTTP client. + entry["exception"] = self.formatException(record.exc_info) + return json.dumps(entry, ensure_ascii=False) + + +def configure(level: str) -> None: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + root = logging.getLogger() + root.handlers[:] = [handler] + root.setLevel(level.upper()) + # uvicorn's own access log is disabled; our middleware writes one line per request. + for name in ("uvicorn", "uvicorn.error"): + logging.getLogger(name).handlers[:] = [] + logging.getLogger(name).propagate = True diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..10b18e9f --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,76 @@ +"""Versioned SQL migrations: `python -m messenger.migrate` (compose service `migrate`). + +* Files `migrations/NNNN_name.sql` are applied in name order, each in its own + transaction together with its row in `schema_migrations` (DDL is transactional + in PostgreSQL, so a failed migration leaves nothing behind). +* Re-running is safe: applied versions are skipped, so data is never duplicated. +* The SHA-256 of every applied file is stored; if an applied file was edited later, + the run stops instead of silently diverging from the real schema. +* `pg_advisory_lock` serialises concurrent runs (two API replicas starting at once). +""" +from __future__ import annotations + +import hashlib +import sys +import time +from pathlib import Path + +import psycopg + +from .config import ConfigError, _secret + +MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations" +LOCK_ID = 2026_02 # arbitrary application-wide advisory lock id + + +def say(text: str) -> None: + print(f"migrate: {text}", flush=True) + + +def connect(url: str, wait_seconds: float = 60) -> psycopg.Connection: + deadline = time.monotonic() + wait_seconds + while True: + try: + return psycopg.connect(url, autocommit=True, connect_timeout=3, application_name="messenger-migrate") + except psycopg.OperationalError as exc: + if time.monotonic() > deadline: + raise SystemExit(f"migrate: database is not reachable: {type(exc).__name__}") from None + time.sleep(1) + + +def main() -> int: + try: + url = _secret("DATABASE_URL", required=True) + except ConfigError as exc: + print(f"configuration error: {exc}", file=sys.stderr) + return 2 + files = sorted(MIGRATIONS_DIR.glob("*.sql")) + with connect(url) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,)) + try: + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY, checksum text NOT NULL," + " applied_at timestamptz NOT NULL DEFAULT now())") + applied = dict(conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()) + for path in files: + version, sql = path.stem, path.read_bytes() + checksum = hashlib.sha256(sql).hexdigest() + if version in applied: + if applied[version] != checksum: + say(f"{version}: file changed after it was applied; create a new migration instead") + return 1 + say(f"{version}: already applied, skipped") + continue + with conn.transaction(): + conn.execute(sql.decode("utf-8")) + conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", (version, checksum)) + say(f"{version}: applied") + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,)) + say(f"done, {len(files)} migration file(s) checked") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/passwords.py b/src/messenger/passwords.py new file mode 100644 index 00000000..a1fc35ad --- /dev/null +++ b/src/messenger/passwords.py @@ -0,0 +1,53 @@ +"""Password hashing with Argon2id (memory-hard password KDF; also mandatory in lab 7). + +* Parameters follow the OWASP minimum for Argon2id: m=19 MiB, t=2, p=1. Unlike bcrypt + there is no 72-byte limit, so any 12-128 code point UTF-8 password is hashed whole. +* Hashing takes tens of milliseconds of CPU and 19 MiB of RAM. It runs in a worker + thread (argon2-cffi releases the GIL), so the event loop keeps serving other + requests, and a semaphore caps parallel hashes (memory/CPU DoS protection). +* An unknown username is checked against a dummy hash, so "no such user" and + "wrong password" take the same time and return the same 401. +""" +from __future__ import annotations + +import asyncio + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +_hasher = PasswordHasher(time_cost=2, memory_cost=19 * 1024, parallelism=1, hash_len=32, salt_len=16) +_semaphore: asyncio.Semaphore | None = None +_dummy_hash: str | None = None + + +def configure(concurrency: int) -> None: + global _semaphore + _semaphore = asyncio.Semaphore(concurrency) + + +async def _run(func, *args): + if _semaphore is None: + configure(2) + async with _semaphore: + return await asyncio.to_thread(func, *args) + + +async def hash_password(password: str) -> str: + return await _run(_hasher.hash, password) + + +def _verify(stored_hash: str, password: str) -> bool: + try: + return _hasher.verify(stored_hash, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +async def verify_password(stored_hash: str | None, password: str) -> bool: + global _dummy_hash + if stored_hash is None: + if _dummy_hash is None: + _dummy_hash = await hash_password("dummy-password-for-timing") + await _run(_verify, _dummy_hash, password) + return False + return await _run(_verify, stored_hash, password) diff --git a/src/messenger/repo.py b/src/messenger/repo.py new file mode 100644 index 00000000..4ae4d952 --- /dev/null +++ b/src/messenger/repo.py @@ -0,0 +1,244 @@ +"""SQL for users, chats and messages. All statements are parameterised (%s placeholders): +values travel separately from the SQL text, so input can never change the query. + +Free text is stored as UTF-8 BYTEA (PostgreSQL TEXT cannot hold U+0000), so it is +encoded on the way in and decoded on the way out. See docs/DATA_MODEL.md. +""" +from __future__ import annotations + +import datetime as dt +import uuid +from dataclasses import dataclass + +import psycopg +from psycopg import errors as pg_errors + +from .cursor import SortKey +from .db import Database +from .errors import Conflict, InvalidRequest, NotFound +from .timefmt import from_us, rfc3339, to_us + + +def _b(text: str) -> bytes: + return text.encode("utf-8") + + +def _s(raw: bytes | memoryview) -> str: + return bytes(raw).decode("utf-8") + + +@dataclass(slots=True, frozen=True) +class User: + id: str + username: str + display_name: str + created_at: dt.datetime + + def to_json(self) -> dict: + return {"id": self.id, "username": self.username, "display_name": self.display_name, + "created_at": rfc3339(self.created_at)} + + +@dataclass(slots=True, frozen=True) +class Chat: + id: str + title: str + member_ids: tuple[str, ...] + created_at: dt.datetime + + @property + def key(self) -> SortKey: + return (to_us(self.created_at), self.id) + + def to_json(self) -> dict: + return {"id": self.id, "title": self.title, "member_ids": list(self.member_ids), + "created_at": rfc3339(self.created_at)} + + +@dataclass(slots=True, frozen=True) +class Message: + id: str + chat_id: str + sender_id: str + text: str + created_at: dt.datetime + + @property + def key(self) -> SortKey: + return (to_us(self.created_at), self.id) + + def to_json(self) -> dict: + return {"id": self.id, "chat_id": self.chat_id, "sender_id": self.sender_id, "text": self.text, + "created_at": rfc3339(self.created_at)} + + +def _user(row) -> User: + return User(str(row[0]), row[1], _s(row[2]), row[3]) + + +# ------------------------------------------------------------------ users +async def create_user(db: Database, username: str, display_name: str, password_hash: str) -> User: + async def work(conn: psycopg.AsyncConnection) -> User: + cur = await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s, %s, %s, %s) " + "RETURNING id, username, display_name, created_at", + (uuid.uuid4(), username, _b(display_name), password_hash)) + return _user(await cur.fetchone()) + try: + return await db.run(work) + except pg_errors.UniqueViolation: + # Two concurrent registrations: the unique index lets exactly one INSERT commit. + raise Conflict("Имя пользователя уже занято") from None + + +async def get_user(db: Database, user_id: str) -> User | None: + async def work(conn: psycopg.AsyncConnection) -> User | None: + cur = await conn.execute( + "SELECT id, username, display_name, created_at FROM users WHERE id = %s", (user_id,)) + row = await cur.fetchone() + return _user(row) if row else None + return await db.run(work) + + +async def get_credentials(db: Database, username: str) -> tuple[User, str] | None: + async def work(conn: psycopg.AsyncConnection) -> tuple[User, str] | None: + cur = await conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,)) + row = await cur.fetchone() + return (_user(row), row[4]) if row else None + return await db.run(work) + + +# ------------------------------------------------------------------ chats +async def create_chat(db: Database, creator_id: str, title: str, member_ids: list[str]) -> Chat: + """Atomic: the chat row and all memberships commit together or not at all. + + The chat is inserted FIRST; an unknown member then fails the membership INSERT + with a foreign-key violation and the whole transaction (chat row included) is + rolled back. student-tests check that nothing is left behind. + """ + if creator_id not in member_ids: + raise InvalidRequest("member_ids должен включать текущего пользователя") + chat_id = uuid.uuid4() + + async def work(conn: psycopg.AsyncConnection) -> dt.datetime: + cur = await conn.execute( + "INSERT INTO chats (id, title, created_by) VALUES (%s, %s, %s) RETURNING created_at", + (chat_id, _b(title), creator_id)) + (created_at,) = await cur.fetchone() + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id, position, chat_created_at) " + "SELECT %s, m.user_id, m.ord - 1, %s " + "FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)", + (chat_id, created_at, member_ids)) + return created_at + try: + created_at = await db.run(work) + except pg_errors.ForeignKeyViolation: + raise InvalidRequest("member_ids содержит несуществующих пользователей") from None + return Chat(str(chat_id), title, tuple(member_ids), created_at) + + +async def _load_chats(conn: psycopg.AsyncConnection, rows) -> list[Chat]: + if not rows: + return [] + ids = [r[0] for r in rows] + cur = await conn.execute( + "SELECT chat_id, user_id FROM chat_members WHERE chat_id = ANY(%s) ORDER BY chat_id, position", (ids,)) + members: dict[uuid.UUID, list[str]] = {i: [] for i in ids} + for chat_id, user_id in await cur.fetchall(): + members[chat_id].append(str(user_id)) + return [Chat(str(r[0]), _s(r[1]), tuple(members[r[0]]), r[2]) for r in rows] + + +async def get_chat(db: Database, user_id: str, chat_id: str) -> Chat: + """The chat if the user is a member; a missing chat and a foreign chat are both 404.""" + async def work(conn: psycopg.AsyncConnection) -> Chat: + cur = await conn.execute( + "SELECT c.id, c.title, c.created_at FROM chats c " + "JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s", + (user_id, chat_id)) + row = await cur.fetchone() + if row is None: + raise NotFound("Чат не найден") + return (await _load_chats(conn, [row]))[0] + return await db.run(work) + + +async def _check_member(conn: psycopg.AsyncConnection, user_id: str, chat_id: str) -> None: + cur = await conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id)) + if await cur.fetchone() is None: + raise NotFound("Чат не найден") + + +async def ensure_member(db: Database, user_id: str, chat_id: str) -> None: + async def work(conn: psycopg.AsyncConnection) -> None: + await _check_member(conn, user_id, chat_id) + await db.run(work) + + +async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: int) -> tuple[list[Chat], bool]: + async def work(conn: psycopg.AsyncConnection) -> tuple[list[Chat], bool]: + if after is None: + cur = await conn.execute( + "SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id " + "WHERE m.user_id = %s ORDER BY m.chat_created_at, m.chat_id LIMIT %s", + (user_id, limit + 1)) + else: + cur = await conn.execute( + "SELECT c.id, c.title, c.created_at FROM chat_members m JOIN chats c ON c.id = m.chat_id " + "WHERE m.user_id = %s AND (m.chat_created_at, m.chat_id) > (%s, %s) " + "ORDER BY m.chat_created_at, m.chat_id LIMIT %s", + (user_id, from_us(after[0]), after[1], limit + 1)) + rows = await cur.fetchall() + return await _load_chats(conn, rows[:limit]), len(rows) > limit + return await db.run(work) + + +# ------------------------------------------------------------------ messages +async def add_message(db: Database, user_id: str, chat_id: str, text: str) -> Message: + """Insert with a per-chat strictly increasing timestamp. + + `UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one + chat are serialised: a message with a smaller created_at can never commit after + a message with a bigger one, and a reader paging by cursor cannot skip it. + """ + message_id = uuid.uuid4() + + async def work(conn: psycopg.AsyncConnection) -> dt.datetime: + await _check_member(conn, user_id, chat_id) + cur = await conn.execute( + "UPDATE chats SET last_message_at = greatest(clock_timestamp(), " + "last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at", + (chat_id,)) + (created_at,) = await cur.fetchone() + await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", + (message_id, chat_id, user_id, _b(text), created_at)) + return created_at + created_at = await db.run(work) + return Message(str(message_id), chat_id, user_id, text, created_at) + + +# Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR +# would push the cursor into a Filter and late pages would scan the whole prefix. +HISTORY_FIRST = ( + "SELECT id, sender_id, text, created_at FROM messages " + "WHERE chat_id = %s ORDER BY created_at, id LIMIT %s") +HISTORY_AFTER = ( + "SELECT id, sender_id, text, created_at FROM messages " + "WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s") + + +async def list_messages(db: Database, chat_id: str, after: SortKey | None, limit: int) -> tuple[list[Message], bool]: + async def work(conn: psycopg.AsyncConnection) -> list: + if after is None: + cur = await conn.execute(HISTORY_FIRST, (chat_id, limit + 1)) + else: + cur = await conn.execute(HISTORY_AFTER, (chat_id, from_us(after[0]), after[1], limit + 1)) + return await cur.fetchall() + rows = await db.run(work) + items = [Message(str(r[0]), chat_id, str(r[1]), _s(r[2]), r[3]) for r in rows[:limit]] + return items, len(rows) > limit diff --git a/src/messenger/sessions.py b/src/messenger/sessions.py new file mode 100644 index 00000000..53736e44 --- /dev/null +++ b/src/messenger/sessions.py @@ -0,0 +1,160 @@ +"""Server-side sessions behind an opaque Bearer token. + +* Token: `secrets.token_urlsafe(32)` = 256 random bits. Only SHA-256(token) is + stored, so a leaked database/AOF does not contain usable tokens. A fast hash is + enough for a 256-bit random value; passwords need a slow KDF (passwords.py). +* Expiry is absolute: `expires_at = login time + SESSION_TTL_SECONDS`. Every lookup + compares it with the current time, and a restart never extends it. + +Two interchangeable stores (SESSION_STORE): + +* `valkey` (compose default): the ONLY copy of a session is a Valkey key with + `PXAT expires_at`, persisted by AOF with `appendfsync always`. There is no second + copy (no cache over PostgreSQL), so a logout cannot be "forgotten" and a revoked + token cannot come back after Valkey fails and recovers. If Valkey is unreachable + a session cannot be checked: protected requests and logout get 503, never access. +* `postgres`: table `sessions` (migrations/0001_init.sql). +""" +from __future__ import annotations + +import asyncio +import datetime as dt +import hashlib +import json +import logging +import secrets +from typing import Protocol + +from redis import exceptions as redis_errors +from redis.asyncio import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + +from .config import Settings +from .db import Database +from .errors import Unavailable +from .timefmt import UTC, from_us, to_us + +log = logging.getLogger("messenger.sessions") + + +def new_token() -> str: + return secrets.token_urlsafe(32) + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() + + +def now() -> dt.datetime: + return dt.datetime.now(UTC) + + +class SessionStore(Protocol): + name: str + + async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: ... + async def user_id_for(self, token: str) -> str | None: ... + async def revoke(self, token: str) -> bool: ... + async def ping(self) -> bool: ... + async def close(self) -> None: ... + + +class PostgresSessionStore: + name = "postgres" + + def __init__(self, db: Database): + self.db = db + + async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: + token = new_token() + expires_at = now() + dt.timedelta(seconds=ttl_seconds) + + async def work(conn) -> None: + await conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at) VALUES (%s, %s, %s)", + (token_hash(token), user_id, expires_at)) + # Housekeeping: drop this user's expired sessions (bounded by the user's own rows). + await conn.execute("DELETE FROM sessions WHERE user_id = %s AND expires_at <= %s", (user_id, now())) + await self.db.run(work) + return token, expires_at + + async def user_id_for(self, token: str) -> str | None: + async def work(conn): + cur = await conn.execute( + "SELECT user_id FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now())) + return await cur.fetchone() + row = await self.db.run(work) + return str(row[0]) if row else None + + async def revoke(self, token: str) -> bool: + async def work(conn) -> bool: + cur = await conn.execute( + "DELETE FROM sessions WHERE token_hash = %s AND expires_at > %s", (token_hash(token), now())) + return cur.rowcount == 1 + return await self.db.run(work) + + async def ping(self) -> bool: + return True # covered by the database readiness check + + async def close(self) -> None: + return None + + +class ValkeySessionStore: + name = "valkey" + PREFIX = "session:" + + def __init__(self, settings: Settings): + self.timeout = settings.cache_timeout_seconds + self.client = Redis.from_url( + settings.cache_url, + socket_timeout=self.timeout, + socket_connect_timeout=self.timeout, + retry=Retry(NoBackoff(), 0), # no hidden retries: a failure is reported at once + decode_responses=False, + ) + + def _key(self, token: str) -> str: + return self.PREFIX + token_hash(token).hex() + + async def _call(self, coro): + try: + async with asyncio.timeout(self.timeout * 2): + return await coro + except (redis_errors.ConnectionError, redis_errors.TimeoutError, TimeoutError, OSError) as exc: + log.warning("session_store_unavailable", extra={"fields": {"error": type(exc).__name__}}) + raise Unavailable("Хранилище сессий временно недоступно") from None + + async def create(self, user_id: str, ttl_seconds: int) -> tuple[str, dt.datetime]: + token = new_token() + expires_at = now() + dt.timedelta(seconds=ttl_seconds) + value = json.dumps({"user_id": user_id, "expires_at_us": to_us(expires_at)}).encode() + # PXAT = absolute expiry time: survives restarts (AOF) without being extended. + await self._call(self.client.set(self._key(token), value, pxat=to_us(expires_at) // 1000)) + return token, expires_at + + async def user_id_for(self, token: str) -> str | None: + raw = await self._call(self.client.get(self._key(token))) + if raw is None: + return None + data = json.loads(raw) + if from_us(data["expires_at_us"]) <= now(): + return None # do not rely only on Valkey's lazy expiry + return data["user_id"] + + async def revoke(self, token: str) -> bool: + return await self._call(self.client.delete(self._key(token))) == 1 + + async def ping(self) -> bool: + try: + return bool(await self._call(self.client.ping())) + except Unavailable: + return False + + async def close(self) -> None: + await self.client.aclose() + + +def build_store(settings: Settings, db: Database) -> SessionStore: + return ValkeySessionStore(settings) if settings.session_store == "valkey" else PostgresSessionStore(db) diff --git a/src/messenger/timefmt.py b/src/messenger/timefmt.py new file mode 100644 index 00000000..9c33b12f --- /dev/null +++ b/src/messenger/timefmt.py @@ -0,0 +1,22 @@ +"""Conversions between PostgreSQL timestamptz values, cursor integers and RFC 3339 strings.""" +from __future__ import annotations + +import datetime as dt + +UTC = dt.timezone.utc +_EPOCH = dt.datetime(1970, 1, 1, tzinfo=UTC) + + +def to_us(value: dt.datetime) -> int: + """Microseconds since the Unix epoch (exact: timestamptz has microsecond precision).""" + delta = value - _EPOCH + return (delta.days * 86_400 + delta.seconds) * 1_000_000 + delta.microseconds + + +def from_us(us: int) -> dt.datetime: + return _EPOCH + dt.timedelta(microseconds=us) + + +def rfc3339(value: dt.datetime) -> str: + """RFC 3339 UTC with fixed microsecond precision, e.g. 2026-10-12T12:00:00.000001Z.""" + return value.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%S.%fZ") diff --git a/src/messenger/validation.py b/src/messenger/validation.py new file mode 100644 index 00000000..87670955 --- /dev/null +++ b/src/messenger/validation.py @@ -0,0 +1,117 @@ +"""Strict request validation according to contracts/openapi.json + contracts/README.md. + +Written by hand instead of a schema library so that every rule is visible: +extra fields, wrong JSON types, whitespace-only strings and lengths counted in +Unicode code points all produce 400 invalid_request. +""" +from __future__ import annotations + +import json +import re +from typing import Any + +from .errors import InvalidRequest + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +# Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case. +UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") +LIMIT_RE = re.compile(r"[0-9]{1,3}") +SURROGATES_RE = re.compile("[\ud800-\udfff]") + +DEFAULT_LIMIT = 50 +MAX_LIMIT = 100 + + +def _reject_constant(name: str) -> Any: + # Python's json accepts NaN/Infinity, which are not valid JSON. + raise ValueError(f"invalid JSON constant {name}") + + +def parse_json_object(body: bytes) -> dict[str, Any]: + """Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400).""" + try: + text = body.decode("utf-8") + value = json.loads(text, parse_constant=_reject_constant) + except (UnicodeDecodeError, ValueError): + raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None + if not isinstance(value, dict): + raise InvalidRequest("Тело запроса должно быть JSON-объектом") + return value + + +def check_fields(data: dict[str, Any], required: set[str], optional: frozenset[str] = frozenset()) -> None: + extra = set(data) - required - optional + if extra: + raise InvalidRequest("Неизвестные поля: " + ", ".join(sorted(extra))) + missing = required - set(data) + if missing: + raise InvalidRequest("Отсутствуют обязательные поля: " + ", ".join(sorted(missing))) + + +def text_field(data: dict[str, Any], name: str, max_len: int, *, min_len: int = 1) -> str: + value = data[name] + if not isinstance(value, str): + raise InvalidRequest(f"Поле {name} должно быть строкой") + if SURROGATES_RE.search(value): + # "\ud800" escapes decode to lone surrogates that cannot be encoded as UTF-8. + raise InvalidRequest(f"Поле {name} содержит недопустимые символы") + # len() of a Python str is the number of Unicode code points, as the contract requires. + if not min_len <= len(value) <= max_len: + raise InvalidRequest(f"Длина поля {name} должна быть от {min_len} до {max_len} символов") + if value.strip() == "": + raise InvalidRequest(f"Поле {name} не может состоять только из пробелов") + return value + + +def username_field(data: dict[str, Any]) -> str: + value = data["username"] + if not isinstance(value, str) or USERNAME_RE.fullmatch(value) is None: + raise InvalidRequest("username должен соответствовать [a-z0-9_]{3,32}") + return value + + +def password_field(data: dict[str, Any]) -> str: + """12-128 Unicode code points, not whitespace-only; any other character (':' too) is allowed.""" + value = data["password"] + if not isinstance(value, str): + raise InvalidRequest("Поле password должно быть строкой") + if SURROGATES_RE.search(value): + raise InvalidRequest("Поле password содержит недопустимые символы") + if not 12 <= len(value) <= 128: + raise InvalidRequest("Длина пароля должна быть от 12 до 128 символов") + if value.strip() == "": + raise InvalidRequest("Пароль не может состоять только из пробелов") + return value + + +def parse_uuid(value: Any, what: str = "UUID") -> str: + if not isinstance(value, str) or UUID_RE.fullmatch(value) is None: + raise InvalidRequest(f"Неверный {what}") + return value.lower() + + +def member_ids_field(data: dict[str, Any]) -> list[str]: + value = data["member_ids"] + if not isinstance(value, list): + raise InvalidRequest("member_ids должен быть массивом") + if not 1 <= len(value) <= 100: + raise InvalidRequest("member_ids должен содержать от 1 до 100 элементов") + ids = [parse_uuid(item, "UUID участника") for item in value] + if len(set(ids)) != len(ids): + raise InvalidRequest("member_ids содержит дубликаты") + return ids + + +def single_query_param(query: list[tuple[str, str]], name: str) -> str | None: + values = [v for k, v in query if k == name] + if len(values) > 1: + raise InvalidRequest(f"Параметр {name} указан несколько раз") + return values[0] if values else None + + +def parse_limit(raw: str | None) -> int: + if raw is None: + return DEFAULT_LIMIT + if LIMIT_RE.fullmatch(raw) is None or not 1 <= int(raw) <= MAX_LIMIT: + raise InvalidRequest(f"limit должен быть целым числом от 1 до {MAX_LIMIT}") + return int(raw) diff --git a/student-tests/conftest.py b/student-tests/conftest.py new file mode 100644 index 00000000..f09a6ad4 --- /dev/null +++ b/student-tests/conftest.py @@ -0,0 +1,226 @@ +"""Shared helpers for the student tests. + +* `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services. +* `test_*` in this folder — marker `stack`: black-box HTTP against the running compose + stand (BASE_URL, default http://127.0.0.1:8080) plus `docker compose` for database + checks (psql inside the postgres container) and failure injection. Every failure + test restores the stopped/paused service in `finally`. + + python -m pytest student-tests # everything (stand must be up) + python -m pytest student-tests -m "not stack" # unit only +""" +from __future__ import annotations + +import base64 +import contextlib +import json +import os +import subprocess +import sys +import time +import uuid +from pathlib import Path + +import httpx2 as httpx +import pytest +from jsonschema import Draft202012Validator + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8")) +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose + + +def pytest_configure(config): + config.addinivalue_line("markers", "stack: needs the running compose stand (docker compose up -d --wait)") + + +def schema_errors(value, name: str) -> list[str]: + """Validate against components/schemas/ of the course contract (JSON Schema 2020-12).""" + schema = {"$ref": f"#/components/schemas/{name}", "components": SPEC["components"]} + validator = Draft202012Validator(schema, format_checker=Draft202012Validator.FORMAT_CHECKER) + return [e.message for e in validator.iter_errors(value)] + + +def basic(username: str, password: str) -> str: + return "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode() + + +# ------------------------------------------------------------------ docker / psql +def compose(*args: str, check: bool = True, timeout: float = 120) -> subprocess.CompletedProcess: + return subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, text=True, + encoding="utf-8", check=check, timeout=timeout) + + +def psql(sql: str) -> str: + """Run SQL inside the postgres container (no DB port is published) and return unaligned output.""" + result = compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", + "-At", "-v", "ON_ERROR_STOP=1", "-c", sql) + return result.stdout.strip() + + +def wait_ready(timeout: float = 90) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + if httpx.get(BASE_URL + "/health/ready", timeout=2).status_code == 200: + return + except httpx.HTTPError: + pass + time.sleep(0.3) + raise AssertionError("stand did not become ready") + + +def wait_healthy(*services: str, timeout: float = 120) -> None: + """Poll Docker health until every service is healthy again. `compose up --wait` is not used: + it gives up at once on a container that was marked unhealthy while paused.""" + deadline = time.monotonic() + timeout + pending = set(services) + while pending and time.monotonic() < deadline: + for service in list(pending): + ids = compose("ps", "-q", service, check=False).stdout.split() + if ids: + state = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], + capture_output=True, text=True).stdout.strip() + if state == "healthy": + pending.discard(service) + time.sleep(0.5) + assert not pending, f"not healthy: {pending}" + + +def restart_services(*services: str) -> None: + compose("restart", *services) + wait_healthy(*services) + wait_ready() + + +@contextlib.contextmanager +def service_down(service: str, how: str = "stop"): + """Stop or pause a compose service; ALWAYS bring it back and wait for readiness.""" + assert how in {"stop", "pause"} + compose(how, service) + try: + yield + finally: + compose("unpause" if how == "pause" else "start", service) + wait_healthy(service) + wait_ready() + + +@contextlib.contextmanager +def api_with_env(**overrides: str): + """Recreate the api container with environment overrides (e.g. SESSION_TTL_SECONDS=2), + then ALWAYS recreate it with the normal configuration.""" + env = {**os.environ, **overrides} + subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env, + capture_output=True, text=True, check=True, timeout=180) + wait_ready() + try: + yield + finally: + clean = {k: v for k, v in os.environ.items() if k not in overrides} + subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean, + capture_output=True, text=True, check=True, timeout=180) + wait_ready() + + +def api_env(name: str) -> str: + """Current value of an environment variable inside the running api container.""" + return compose("exec", "-T", "api", "printenv", name).stdout.strip() + + +# ------------------------------------------------------------------ HTTP +class Api: + """HTTP helper that checks the invariants every response must satisfy.""" + + def __init__(self, base_url: str = BASE_URL): + self.client = httpx.Client(base_url=base_url, timeout=15) + + def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None, + json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None): + h = dict(headers or {}) + if token is not None: + h["Authorization"] = "Bearer " + token + if json_body is not None: + raw = json.dumps(json_body, ensure_ascii=False).encode("utf-8") + h.setdefault("Content-Type", "application/json") + response = self.client.request(method, path, content=raw, headers=h) + allowed = (expected,) if isinstance(expected, int) else expected + assert response.status_code in allowed, (method, path, response.status_code, response.text[:300]) + request_id = response.headers.get("x-request-id") + assert request_id, "every response must carry X-Request-Id" + if response.status_code >= 400: + body = response.json() + assert schema_errors(body, "Error") == [], response.text + assert body["error"]["request_id"] == request_id + assert "Traceback" not in response.text and "psycopg" not in response.text + if response.status_code == 401: + challenge = "basic" if path == "/api/v1/auth/sessions" else "bearer" + assert response.headers.get("www-authenticate", "").lower().startswith(challenge) + elif schema: + assert schema_errors(response.json(), schema) == [], response.text + return response + + def register(self, label: str = "u", *, password: str = PASSWORD, display_name: str | None = None) -> dict: + username = f"{label}_{uuid.uuid4().hex[:12]}"[:32] + body = {"username": username, "display_name": display_name or f"Студент {label}", "password": password} + user = self.call("POST", "/api/v1/users", 201, json_body=body, schema="User").json() + return {"id": user["id"], "user": user, "username": username, "password": password} + + def login(self, actor: dict) -> str: + session = self.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(actor["username"], actor["password"])}, + schema="Session").json() + assert session["user"] == actor["user"] and session["token_type"] == "Bearer" + return session["token"] + + def user(self, label: str = "u", **kwargs) -> dict: + actor = self.register(label, **kwargs) + actor["token"] = self.login(actor) + return actor + + def chat(self, owner: dict, *members: dict, title: str = "Тестовый чат") -> dict: + ids = [owner["id"], *(m["id"] for m in members)] + return self.call("POST", "/api/v1/chats", 201, token=owner["token"], + json_body={"title": title, "member_ids": ids}, schema="Chat").json() + + def send(self, actor: dict, chat: dict, text: str) -> dict: + return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=actor["token"], + json_body={"text": text}, schema="Message").json() + + def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]: + """Walk all pages; returns (items, number_of_pages).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={cursor}" if cursor else "") + page = self.call("GET", f"/api/v1/chats/{chat['id']}/messages{query}", 200, + token=actor["token"], schema="MessagePage").json() + pages += 1 + items.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return items, pages + assert len(page["items"]) == limit, "a non-final page must be full" + assert len(items) < 100_000, "cursor loop" + + +def _stand_is_up() -> bool: + try: + return httpx.get(BASE_URL + "/health/ready", timeout=3).status_code == 200 + except httpx.HTTPError: + return False + + +def pytest_collection_modifyitems(config, items): + if any("stack" in item.keywords for item in items) and not _stand_is_up(): + skip = pytest.mark.skip(reason=f"stand is not running at {BASE_URL}: docker compose up -d --wait") + for item in items: + if "stack" in item.keywords: + item.add_marker(skip) + + +@pytest.fixture(scope="session") +def api() -> Api: + return Api() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..d99077ec --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,201 @@ +"""Contract behaviour over real HTTP (stand must be running). Lab 1 checks moved to Bearer + new lab 2 auth.""" +from __future__ import annotations + +import base64 +import uuid + +import pytest + +from conftest import PASSWORD, basic + +pytestmark = pytest.mark.stack + + +def key(item: dict) -> tuple[str, int]: + return item["created_at"], uuid.UUID(item["id"]).int + + +# ================================================================== registration & login +def test_register_never_returns_password_and_login_returns_bearer(api): + actor = api.register("alice", display_name="Алиса 👩‍💻") + assert set(actor["user"]) == {"id", "username", "display_name", "created_at"} + session = api.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(actor["username"], PASSWORD)}, schema="Session").json() + assert session["token_type"] == "Bearer" and len(session["token"]) >= 43 + assert actor["username"] not in session["token"] and actor["id"] not in session["token"] + assert api.call("GET", "/api/v1/users/me", 200, token=session["token"], schema="User").json() == actor["user"] + + +def test_duplicate_username_is_409(api): + actor = api.register("dup") + body = {"username": actor["username"], "display_name": "Другой", "password": PASSWORD} + assert api.call("POST", "/api/v1/users", 409, json_body=body).json()["error"]["code"] == "conflict" + + +@pytest.mark.parametrize("password", ["short-11ch", " " * 12, " \t \n " * 3, "я" * 129]) +def test_bad_password_on_registration_is_400(api, password): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": password} + api.call("POST", "/api/v1/users", 400, json_body=body) + + +@pytest.mark.parametrize("password", ["я" * 128, "with:colons:inside", "Twelve chars"]) +def test_edge_passwords_register_and_login(api, password): + actor = api.user("edge", password=password) + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + + +def test_wrong_password_and_unknown_user_give_the_same_401(api): + actor = api.register("victim") + wrong = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(actor["username"], "Wrong-password-1")}) + unknown = api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("nobody_" + uuid.uuid4().hex[:8], PASSWORD)}) + assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"] + assert wrong.headers["www-authenticate"] == 'Basic realm="messenger", charset="UTF-8"' + + +@pytest.mark.parametrize("header", ["Basic é", "Basic !!!", "Basic " + base64.b64encode(b"no_colon").decode(), + "Basic " + base64.b64encode(b"\xff\xfe:x").decode(), "Bearer " + "a" * 43, ""]) +def test_malformed_basic_is_401_not_500(api, header): + # Raw UTF-8 bytes, as curl would send them (httpx refuses non-ASCII str headers). + api.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": header.encode("utf-8")} if header else {}) + + +def test_logout_revokes_only_this_token(api): + actor = api.user("logout") + second = api.login(actor) + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"]) + api.call("GET", "/api/v1/users/me", 401, token=actor["token"]) + api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=actor["token"]) + api.call("GET", "/api/v1/users/me", 200, token=second) # other session of the same user survives + + +def test_x_user_id_basic_and_garbage_do_not_authenticate(api): + actor = api.user("hdr") + for headers in ({"X-User-Id": actor["id"]}, + {"Authorization": basic(actor["username"], PASSWORD)}, + {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}): + api.call("GET", "/api/v1/users/me", 401, headers=headers) + me = api.call("GET", "/api/v1/users/me", 200, token=actor["token"], headers={"X-User-Id": str(uuid.uuid4())}) + assert me.json()["id"] == actor["id"] + + +# ================================================================== validation / errors (from lab 1) +@pytest.mark.parametrize("raw", [b"{", b"", b"[]", b"null", b'{"username": "abc_def", "display_name": NaN, "password": "Twelve-chars"}', + b'{"username": "abc_def", "display_name": "x", "password": "Twelve-chars"}{"a": 1}', + b'{"username": "abc_def", "display_name": "\\ud800", "password": "Twelve-chars"}']) +def test_malformed_json_is_400(api, raw): + api.call("POST", "/api/v1/users", 400, raw=raw, headers={"Content-Type": "application/json"}) + + +@pytest.mark.parametrize("body", [ + {"username": "abc_def", "display_name": "x"}, # no password + {"username": "abc_def", "display_name": "x", "password": "Twelve-chars", "extra": 1}, + {"username": "Abc_def", "display_name": "x", "password": "Twelve-chars"}, + {"username": "abc_def", "display_name": " ", "password": "Twelve-chars"}, + {"username": "abc_def", "display_name": "x" * 101, "password": "Twelve-chars"}, + {"username": 5, "display_name": "x", "password": "Twelve-chars"}, +]) +def test_invalid_registration_fields_are_400(api, body): + api.call("POST", "/api/v1/users", 400, json_body=body) + + +def test_unicode_limits_are_code_points_and_nul_is_stored(api): + actor = api.user("uni", display_name="😀" * 100) + assert actor["user"]["display_name"] == "😀" * 100 + nul = api.user("nul", display_name="A\u0000B") + assert api.call("GET", "/api/v1/users/me", 200, token=nul["token"]).json()["display_name"] == "A\u0000B" + chat = api.chat(actor, title="T\u0000itle") + assert chat["title"] == "T\u0000itle" + assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world" + assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000 + api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=actor["token"], json_body={"text": "я" * 4001}) + texts = [m["text"] for m in api.history(actor, chat, 100)[0]] + assert texts == ["hello\u0000world", "🙂" * 4000] + + +def test_oversized_body_and_wrong_content_type(api): + api.call("POST", "/api/v1/users", 413, json_body={"username": "big_body", "display_name": "x" * 70_000, "password": PASSWORD}) + api.call("POST", "/api/v1/users", 400, raw=b'{"username":"abc_def"}', headers={"Content-Type": "text/plain"}) + + +# ================================================================== chats & membership +def test_chat_validation(api): + alice, bob = api.user("alice"), api.user("bob") + for body in ({"title": "x", "member_ids": [alice["id"], bob["id"], bob["id"]]}, + {"title": "x", "member_ids": [alice["id"], str(uuid.uuid4())]}, + {"title": "x", "member_ids": [bob["id"]]}, + {"title": " ", "member_ids": [alice["id"]]}, + {"title": "x", "member_ids": [alice["id"], "bob"]}): + api.call("POST", "/api/v1/chats", 400, token=alice["token"], json_body=body) + + +def test_foreign_chat_is_indistinguishable_from_missing(api): + alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve") + chat = api.chat(alice, bob) + for chat_id in (chat["id"], str(uuid.uuid4())): + base = f"/api/v1/chats/{chat_id}" + errors = [api.call("GET", base, 404, token=eve["token"]).json(), + api.call("GET", base + "/messages", 404, token=eve["token"]).json(), + api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}).json(), + api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()] + assert len({e["error"]["message"] for e in errors}) == 1 + api.call("GET", "/api/v1/chats/not-a-uuid", 401) + api.call("GET", "/api/v1/chats/not-a-uuid", 400, token=alice["token"]) + api.call("GET", "/api/v1/chats/NOT-A-UUID/messages", 400, token=alice["token"]) + + +def test_chat_list_order_pagination_and_isolation(api): + alice, bob, eve = api.user("alice"), api.user("bob"), api.user("eve") + created = [api.chat(alice, bob, title=f"Чат {i}") for i in range(5)] + api.chat(eve, title="Чат Евы") + walked, cursor = [], None + while True: + query = "?limit=2" + (f"&cursor={cursor}" if cursor else "") + page = api.call("GET", "/api/v1/chats" + query, 200, token=bob["token"], schema="ChatPage").json() + walked += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + assert walked == created + first = api.call("GET", "/api/v1/chats?limit=1", 200, token=alice["token"]).json()["next_cursor"] + api.call("GET", f"/api/v1/chats?limit=1&cursor={first}", 400, token=bob["token"]) # cursor of another user + api.call("GET", "/api/v1/chats?limit=0", 400, token=bob["token"]) + + +# ================================================================== messages & keyset pagination +def test_history_pages_without_loss_or_duplicates(api): + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + sent = [api.send(alice if i % 2 else bob, chat, f"сообщение {i}") for i in range(25)] + items, pages = api.history(bob, chat, 7) + assert pages == 4 and items == sent and items == sorted(items, key=key) + assert sent[0]["sender_id"] == bob["id"] and sent[1]["sender_id"] == alice["id"] + + +@pytest.mark.parametrize("query", ["limit=0", "limit=101", "limit=abc", "limit=1&limit=2", "cursor=", "cursor=garbage", + "cursor=djF8bXx4fDF8Mg", "cursor=" + base64.urlsafe_b64encode(b"v1|m|x|99999999999999999999|y").decode()]) +def test_bad_limit_or_cursor_is_400(api, query): + alice = api.user("alice") + chat = api.chat(alice) + api.call("GET", f"/api/v1/chats/{chat['id']}/messages?{query}", 400, token=alice["token"]) + + +def test_cursor_bound_to_chat(api): + alice = api.user("alice") + a, b = api.chat(alice, title="A"), api.chat(alice, title="B") + for chat in (a, b): + api.send(alice, chat, "1") + api.send(alice, chat, "2") + cursor = api.call("GET", f"/api/v1/chats/{a['id']}/messages?limit=1", 200, token=alice["token"]).json()["next_cursor"] + api.call("GET", f"/api/v1/chats/{b['id']}/messages?cursor={cursor}", 400, token=alice["token"]) + empty = api.chat(alice, title="Empty") + api.call("GET", f"/api/v1/chats/{empty['id']}/messages?cursor=garbage", 400, token=alice["token"]) + + +def test_read_after_write(api): + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + for i in range(5): + message = api.send(alice, chat, f"m{i}") + last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1] + assert last == message diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 00000000..fd065e14 --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,114 @@ +"""Database-level guarantees, checked over HTTP and verified directly in PostgreSQL / Valkey.""" +from __future__ import annotations + +import threading +import time +import uuid + +import pytest + +from conftest import PASSWORD, Api, compose, psql + +pytestmark = pytest.mark.stack + + +def run_concurrently(workers: int, target) -> list: + barrier = threading.Barrier(workers) + results: list = [None] * workers + + def body(index: int) -> None: + local = Api() # one HTTP client (connection) per worker + barrier.wait() # all workers fire at the same moment + try: + results[index] = target(local, index) + except BaseException as exc: # noqa: BLE001 - reported by the test + results[index] = exc + + threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)] + for t in threads: + t.start() + for t in threads: + t.join() + return results + + +def test_registration_race_exactly_one_201_and_seven_409(): + username = "race_" + uuid.uuid4().hex[:10] + + def register(client: Api, _index: int) -> int: + body = {"username": username, "display_name": "Гонка", "password": PASSWORD} + response = client.client.post("/api/v1/users", json=body) + return response.status_code + + statuses = run_concurrently(8, register) + # Real HTTP codes are compared: a timeout, 500 or exception fails the test instead of counting as 409. + assert sorted(statuses, key=str) == [201] + [409] * 7, statuses + assert psql(f"SELECT count(*) FROM users WHERE username = '{username}'") == "1" + + +def _ins_counter() -> int: + return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + +def test_create_chat_rolls_back_after_the_chat_row_was_inserted(api): + """The chat INSERT runs first; the unknown member then breaks the membership INSERT (FK). + + Proof that the first INSERT really happened and was rolled back: PostgreSQL counts + attempted inserts in pg_stat_user_tables.n_tup_ins even for aborted transactions, + while the table contains no such chat and no new membership rows. + """ + alice = api.user("rollback") + title = "rollback-" + uuid.uuid4().hex + members_before = int(psql("SELECT count(*) FROM chat_members")) + attempts_before = _ins_counter() + error = api.call("POST", "/api/v1/chats", 400, token=alice["token"], + json_body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}).json() + assert "несуществующих" in error["error"]["message"] + deadline = time.monotonic() + 20 # statistics are flushed by the API backend asynchronously + while _ins_counter() == attempts_before and time.monotonic() < deadline: + time.sleep(0.5) + assert _ins_counter() > attempts_before, "the chat INSERT was never attempted" + assert psql(f"SELECT count(*) FROM chats WHERE title = convert_to('{title}', 'UTF8')") == "0" + assert int(psql("SELECT count(*) FROM chat_members")) == members_before + + +def test_migrations_rerun_is_a_noop(): + users_before = psql("SELECT count(*) FROM users") + result = compose("run", "--rm", "--no-deps", "migrate") + assert "0001_init: already applied, skipped" in result.stdout + assert psql("SELECT count(*) FROM schema_migrations") == "1" + assert psql("SELECT count(*) FROM users") == users_before + + +def test_100_messages_from_10_clients_keep_order_and_cursor_walk(api): + clients = [api.user(f"w{i}") for i in range(10)] + chat = api.chat(clients[0], *clients[1:]) + + def writer(client: Api, index: int) -> list[dict]: + return [client.send(clients[index], chat, f"c{index}-m{j}") for j in range(10)] + + results = run_concurrently(10, writer) + assert all(isinstance(r, list) for r in results), results + confirmed = [m for batch in results for m in batch] + history, pages = api.history(clients[5], chat, 7) + ids = [m["id"] for m in history] + assert len(confirmed) == 100 and len(history) == 100 and len(set(ids)) == 100 and pages == 15 + assert set(ids) == {m["id"] for m in confirmed} + keys = [(m["created_at"], uuid.UUID(m["id"]).int) for m in history] + assert all(a < b for a, b in zip(keys, keys[1:])), "history is not strictly ordered" + stamps = [m["created_at"] for m in history] + assert len(set(stamps)) == 100, "per-chat clock must give unique timestamps" + for i, client in enumerate(clients): + own = [m["text"] for m in history if m["sender_id"] == client["id"]] + assert own == [f"c{i}-m{j}" for j in range(10)] + + +def test_secrets_are_not_stored_in_plain_text(api): + actor = api.user("store") + stored = psql(f"SELECT password_hash FROM users WHERE id = '{actor['id']}'") + assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") and PASSWORD not in stored + token_hex = actor["token"].encode().hex() + keys = compose("exec", "-T", "valkey", "sh", "-c", + 'REDISCLI_AUTH="$(cat /run/secrets/valkey_password)" valkey-cli --scan --pattern "session:*"').stdout.split() + assert keys and all(len(k) == len("session:") + 64 for k in keys) + assert not any(actor["token"] in k or token_hex in k for k in keys) diff --git a/student-tests/test_explain.py b/student-tests/test_explain.py new file mode 100644 index 00000000..a2503a5f --- /dev/null +++ b/student-tests/test_explain.py @@ -0,0 +1,35 @@ +"""The history index really bounds the work of a late page (>= 10 000 messages, generic plan).""" +from __future__ import annotations + +import re +import sys + +import pytest + +from conftest import ROOT + +pytestmark = pytest.mark.stack + +sys.path.insert(0, str(ROOT / "scripts")) +import explain_history # noqa: E402 + + +def section(text: str, number: int) -> str: + parts = re.split(r"^=== ", text, flags=re.M) + return next(p for p in parts if p.startswith(f"{number}.")) + + +def buffers(plan: str) -> int: + return int(re.search(r"Buffers: shared hit=(\d+)", plan).group(1)) + + +def test_late_page_uses_index_range_not_filter(): + text = explain_history.run() + assert "20000" in text # the target chat holds 20 000 messages + app_late = section(text, 2) + assert "Index Cond: ((chat_id = $1) AND (ROW(created_at, id) > ROW($2, $3)))" in app_late + assert "Rows Removed by Filter" not in app_late and "Sort" not in app_late + rejected = section(text, 3) + assert "Rows Removed by Filter: 19950" in rejected + # The app query touches a handful of pages; the rejected variants touch the whole prefix. + assert buffers(app_late) <= 10 < buffers(rejected) and buffers(app_late) <= 10 < buffers(section(text, 4)) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 00000000..0dce65a6 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,99 @@ +"""Dependency failures with a VALID session: bounded 503, never anonymous access, recovery afterwards. + +Every test stops/pauses a real container and restores it in `finally` (service_down). +Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5. +""" +from __future__ import annotations + +import subprocess +import time + +import pytest + +from conftest import ROOT, restart_services, service_down + +pytestmark = pytest.mark.stack + +DB_BOUND = 3.6 # DB_TIMEOUT_SECONDS + network/HTTP overhead +CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead + + +def timed(api, *args, **kwargs): + started = time.monotonic() + response = api.call(*args, **kwargs) + return response, time.monotonic() - started + + +@pytest.fixture +def prepared(api): + alice, bob = api.user("alice"), api.user("bob") + chat = api.chat(alice, bob) + message = api.send(alice, chat, "до отказа") + return alice, bob, chat, message + + +def check_intact(api, alice, bob, chat, message): + """After recovery the SAME tokens work and confirmed data is unchanged.""" + assert api.call("GET", "/api/v1/users/me", 200, token=alice["token"]).json() == alice["user"] + items = api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json()["items"] + assert items == [message] + + +@pytest.mark.parametrize("how", ["stop", "pause"]) +def test_valkey_failure(api, prepared, how): + alice, bob, chat, message = prepared + with service_down("valkey", how): + response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"]) + assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took + api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"]) + api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected + api.call("GET", "/health/ready", 503) + api.call("GET", "/health/live", 200) + check_intact(api, alice, bob, chat, message) + + +@pytest.mark.parametrize("how", ["stop", "pause"]) +def test_postgres_failure(api, prepared, how): + alice, bob, chat, message = prepared + with service_down("postgres", how): + for path in ("/api/v1/users/me", f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"): + response, took = timed(api, "GET", path, 503, token=alice["token"]) + assert response.json()["error"]["code"] == "unavailable", path + assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause) + api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected + api.call("GET", "/health/ready", 503) + api.call("GET", "/health/live", 200) + check_intact(api, alice, bob, chat, message) + + +def test_locked_table_gives_bounded_503(api, prepared): + """A statement stuck behind a lock is stopped by lock_timeout, not after minutes.""" + alice, bob, chat, message = prepared + locker = subprocess.Popen( + ["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-c", + "BEGIN; LOCK TABLE messages IN ACCESS EXCLUSIVE MODE; SELECT pg_sleep(6); COMMIT;"], + cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + time.sleep(1.5) # let the lock be taken + response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"]) + assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took + finally: + locker.wait(timeout=30) + check_intact(api, alice, bob, chat, message) + + +def test_logout_during_valkey_outage_is_never_lost(api, prepared): + """Valkey is the single source of truth for sessions, so a logout cannot be 'forgotten': + during the outage logout answers 503 (nothing revoked, the client knows it failed); + after recovery logout works, and the revocation survives a Valkey and API restart (AOF).""" + alice, bob, chat, message = prepared + with service_down("valkey", "stop"): + api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"]) + api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) + restart_services("valkey", "api") + for _ in range(10): # repeated checks: the revoked token must never come back + api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) + api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], json_body={"text": "x"}) + api.call("GET", "/api/v1/users/me", 200, token=bob["token"]) # other sessions survived the restart diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py new file mode 100644 index 00000000..245db765 --- /dev/null +++ b/student-tests/test_sessions.py @@ -0,0 +1,64 @@ +"""Session lifetime: expiry with a short SESSION_TTL_SECONDS, no extension by restarts, +and the alternative SESSION_STORE=postgres. The api container is recreated with the +override and ALWAYS recreated back with the normal configuration (api_with_env).""" +from __future__ import annotations + +import datetime as dt +import time + +import pytest + +from conftest import api_env, api_with_env, restart_services, service_down + +pytestmark = pytest.mark.stack + + +def seconds_left(expires_at: str) -> float: + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def login_session(api, actor) -> dict: + from conftest import basic + return api.call("POST", "/api/v1/auth/sessions", 201, + headers={"Authorization": basic(actor["username"], actor["password"])}, schema="Session").json() + + +@pytest.mark.parametrize("store", ["valkey", "postgres"]) +def test_short_ttl_expires(api, store): + with api_with_env(SESSION_TTL_SECONDS="2", SESSION_STORE=store): + assert api_env("SESSION_TTL_SECONDS") == "2" # the contract variable reaches the container + actor = api.register("ttl") + session = login_session(api, actor) + assert 1.0 < seconds_left(session["expires_at"]) <= 2.1 + api.call("GET", "/api/v1/users/me", 200, token=session["token"]) + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3) + api.call("GET", "/api/v1/users/me", 401, token=session["token"]) + api.call("DELETE", "/api/v1/auth/sessions/current", 401, token=session["token"]) + + +def test_restart_does_not_extend_ttl(api): + with api_with_env(SESSION_TTL_SECONDS="12"): + actor = api.register("ttlrestart") + session = login_session(api, actor) + restart_services("valkey", "api") + assert seconds_left(session["expires_at"]) > 0, "restart took longer than the TTL; increase it" + api.call("GET", "/api/v1/users/me", 200, token=session["token"]) + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 0.3) + api.call("GET", "/api/v1/users/me", 401, token=session["token"]) + + +def test_postgres_session_store(api): + """SESSION_STORE=postgres: sessions survive an API restart, logout revokes, and a DB outage + is 503 for a valid token (not 401 and not anonymous access).""" + with api_with_env(SESSION_STORE="postgres"): + assert api_env("SESSION_STORE") == "postgres" + actor = api.user("pgstore") + restart_services("api") + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + with service_down("postgres", "stop"): + assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"]).json()["error"]["code"] == "unavailable" + api.call("GET", "/api/v1/users/me", 401) + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"]) + api.call("GET", "/api/v1/users/me", 401, token=actor["token"]) diff --git a/student-tests/unit/test_unit.py b/student-tests/unit/test_unit.py new file mode 100644 index 00000000..34612867 --- /dev/null +++ b/student-tests/unit/test_unit.py @@ -0,0 +1,134 @@ +"""Pure functions, no services: validation, cursor, Authorization parsing, Argon2id, config.""" +from __future__ import annotations + +import asyncio +import base64 +import datetime as dt + +import pytest + +from messenger import authhdr, cursor, passwords, validation +from messenger.config import ConfigError, Settings +from messenger.errors import InvalidRequest, Unauthorized +from messenger.timefmt import UTC, from_us, rfc3339, to_us + + +def b64(raw: bytes) -> str: + return base64.b64encode(raw).decode() + + +# ------------------------------------------------------------------ password rules +@pytest.mark.parametrize("password", ["Twelve-chars", "я" * 128, "пароль:с:двоеточиями", "😀" * 12, "a b c d e f g"]) +def test_valid_passwords(password): + assert validation.password_field({"password": password}) == password + + +@pytest.mark.parametrize("password", ["short-11ch", "я" * 129, " " * 12, "\t\n " * 5, "\ud800" * 12, None, 123456789012]) +def test_invalid_passwords(password): + with pytest.raises(InvalidRequest): + validation.password_field({"password": password}) + + +def test_text_may_contain_nul_but_not_lone_surrogates(): + assert validation.text_field({"t": "a\u0000b"}, "t", 10) == "a\u0000b" + with pytest.raises(InvalidRequest): + validation.text_field({"t": "a\ud800"}, "t", 10) + + +# ------------------------------------------------------------------ cursor +CHAT = "3f2c7a5e-0b7d-4a35-9a43-8d1f0f3b2c11" + + +def test_cursor_roundtrip_and_binding(): + key = (to_us(dt.datetime(2026, 10, 12, 12, 0, 0, 123456, tzinfo=UTC)), "11111111-1111-4111-8111-111111111111") + token = cursor.encode(cursor.MESSAGES, CHAT, key) + assert cursor.decode(token, cursor.MESSAGES, CHAT) == key + with pytest.raises(InvalidRequest): + cursor.decode(token, cursor.MESSAGES, "00000000-0000-4000-8000-000000000000") # other chat + with pytest.raises(InvalidRequest): + cursor.decode(token, cursor.CHATS, CHAT) # other resource + + +@pytest.mark.parametrize("payload", [ + b"v1|m|" + CHAT.encode() + b"|253402300800000000|11111111-1111-4111-8111-111111111111", # 1 us after 9999-12-31 + b"v1|m|" + CHAT.encode() + b"|9999999999999999999|11111111-1111-4111-8111-111111111111", # int64 edge + b"v1|m|" + CHAT.encode() + b"|-1|11111111-1111-4111-8111-111111111111", + b"v1|m|" + CHAT.encode() + b"|1|not-a-uuid", + b"v1|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-11111111111G", + b"v2|m|" + CHAT.encode() + b"|1|11111111-1111-4111-8111-111111111111", +]) +def test_broken_cursors_are_400_not_500(payload): + token = base64.urlsafe_b64encode(payload).rstrip(b"=").decode() + with pytest.raises(InvalidRequest): + cursor.decode(token, cursor.MESSAGES, CHAT) + + +def test_time_helpers_are_exact(): + moment = dt.datetime(2026, 2, 28, 23, 59, 59, 999999, tzinfo=UTC) + assert from_us(to_us(moment)) == moment + assert rfc3339(moment) == "2026-02-28T23:59:59.999999Z" + assert from_us(cursor.MAX_TS_US).year == 9999 + + +# ------------------------------------------------------------------ Authorization header +def test_basic_is_parsed_and_password_may_contain_colons(): + assert authhdr.parse_basic("Basic " + b64("alice:pa:ss:wörd".encode())) == ("alice", "pa:ss:wörd") + assert authhdr.parse_basic("basic " + b64(b"bob_1:x" * 1)) == ("bob_1", "x") + + +@pytest.mark.parametrize("header", [ + None, "", "Basic", "Basic é", "Basic !!!!", "Basic " + b64(b"no-colon"), "Basic " + b64(b"\xff\xfe:pw"), + "Basic " + b64(b":password"), "Basic " + b64(b"alice:"), "Basic " + b64("Алиса:pw".encode()), + "Bearer " + "a" * 43, "Basic " + "A" * 2000, +]) +def test_bad_basic_is_401_with_basic_challenge(header): + with pytest.raises(Unauthorized) as info: + authhdr.parse_basic(header) + assert info.value.headers["WWW-Authenticate"].startswith("Basic realm=\"messenger\"") + + +@pytest.mark.parametrize("header", [None, "", "Bearer", "Bearer short", "Bearer " + "a" * 43 + "!", + "Basic " + b64(b"alice:password-123"), "Token " + "a" * 43, "Bearer " + "é" * 43]) +def test_bad_bearer_is_401_with_bearer_challenge(header): + with pytest.raises(Unauthorized) as info: + authhdr.parse_bearer(header) + assert info.value.headers["WWW-Authenticate"] == 'Bearer realm="messenger"' + + +def test_bearer_ok(): + token = "A_b-" + "x" * 39 + assert authhdr.parse_bearer("Bearer " + token) == token + + +# ------------------------------------------------------------------ Argon2id +def test_argon2id_hash_and_verify(): + async def scenario(): + stored = await passwords.hash_password("я" * 128) + assert stored.startswith("$argon2id$v=19$m=19456,t=2,p=1$") + assert await passwords.verify_password(stored, "я" * 128) + assert not await passwords.verify_password(stored, "я" * 127) + assert not await passwords.verify_password(None, "anything") # unknown user, dummy hash + assert not await passwords.verify_password("not-a-hash", "anything") + assert await passwords.hash_password("same password!") != await passwords.hash_password("same password!") # salt + asyncio.run(scenario()) + + +# ------------------------------------------------------------------ config +def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch): + url_file = tmp_path / "database_url" + url_file.write_text("postgresql://u:p@db/x\n", encoding="utf-8") + monkeypatch.setenv("DATABASE_URL_FILE", str(url_file)) + monkeypatch.setenv("SESSION_STORE", "redis") + monkeypatch.setenv("CACHE_URL", "redis://:p@valkey:6379/0") + monkeypatch.setenv("SESSION_TTL_SECONDS", "2") + settings = Settings.from_env() + assert settings.database_url == "postgresql://u:p@db/x" + assert settings.session_store == "valkey" and settings.session_ttl_seconds == 2 + monkeypatch.setenv("SESSION_TTL_SECONDS", "0") + with pytest.raises(ConfigError): + Settings.from_env() + monkeypatch.setenv("SESSION_TTL_SECONDS", "2") + monkeypatch.setenv("DATABASE_URL_FILE", str(tmp_path / "missing")) + with pytest.raises(ConfigError) as info: + Settings.from_env() + assert "u:p" not in str(info.value) -- 2.54.0 From 3f12698676ba295450bf86409914bd7d3a129567 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:32:13 +0300 Subject: [PATCH 02/18] =?UTF-8?q?feat:=20HAProxy=20=D0=B8=20N=20API-=D1=80?= =?UTF-8?q?=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA,=20X-Instance-Id,=20=D0=BE=D0=B1?= =?UTF-8?q?=D1=89=D0=B0=D1=8F=20=D0=B8=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE=D1=82?= =?UTF-8?q?=D0=B5=D0=BD=D1=82=D0=BD=D0=BE=D1=81=D1=82=D1=8C,=20drain=20?= =?UTF-8?q?=D0=BF=D1=80=D0=B8=20SIGTERM?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - compose: один масштабируемый сервис api (deploy.replicas=2, без ports и container_name), HAProxy 3.2 LTS по digest — единственная точка входа (127.0.0.1), разовый migrate до старта реплик, CPU/RAM-лимиты у всех сервисов, каждому сервису только его secrets. - HAProxy: server-template + Docker DNS (re-resolve 1 с), активная проверка GET /health/ready каждую 1 с (fall 2 / rise 2), без backup-серверов, retry только conn-failure, ошибки самого LB — в формате контракта с X-Request-Id. - API: X-Instance-Id = hostname контейнера на каждом ответе (и на 500/503); Idempotency-Key обязателен; ключ, сообщение и результат пишутся в ОДНОЙ транзакции (миграция 0002), конкурентный повтор ждёт уникальный индекс; другой body — 409, посторонний — всегда 404; attachment_ids=[] равен отсутствию поля; дубли ключей JSON и тело у POST /auth/sessions — 400. - SIGTERM: readiness 503 + DRAIN_SECONDS=3 обслуживания, затем graceful stop. - scripts: stop-one.sh (+ .cmd для Windows), restore-api.sh, restart.sh перезапускает и LB, lbctl.py — состояние бэкендов HAProxy -> контейнеры. Co-Authored-By: Claude Opus 5.5 --- .env.example | 18 +++- Dockerfile | 2 +- Makefile | 30 +++++-- compose.yaml | 72 +++++++++++++--- infra/haproxy/haproxy.cfg | 89 +++++++++++++++++++ migrations/0002_idempotency_keys.sql | 25 ++++++ scripts/lbctl.py | 123 +++++++++++++++++++++++++++ scripts/restart.sh | 12 +-- scripts/restore-api.sh | 8 ++ scripts/stop-one.cmd | 7 ++ scripts/stop-one.sh | 35 ++++++++ src/messenger/__main__.py | 42 ++++++--- src/messenger/app.py | 77 ++++++++++++++--- src/messenger/config.py | 31 ++++++- src/messenger/repo.py | 87 ++++++++++++++++--- src/messenger/validation.py | 32 ++++++- 16 files changed, 629 insertions(+), 61 deletions(-) create mode 100644 infra/haproxy/haproxy.cfg create mode 100644 migrations/0002_idempotency_keys.sql create mode 100644 scripts/lbctl.py create mode 100755 scripts/restore-api.sh create mode 100644 scripts/stop-one.cmd create mode 100755 scripts/stop-one.sh diff --git a/.env.example b/.env.example index c84d3545..c18efac8 100644 --- a/.env.example +++ b/.env.example @@ -11,6 +11,10 @@ DATABASE_URL_FILE=/run/secrets/database_url # valkey = sessions live only in Valkey (AOF, appendfsync always); postgres = table `sessions`. SESSION_STORE=valkey CACHE_URL_FILE=/run/secrets/cache_url +# Number of interchangeable API replicas behind HAProxy (compose deploy.replicas). +API_REPLICAS=2 +# Idempotency-Key results are kept at least this long (contract: >= 24 h). +IDEMPOTENCY_TTL_SECONDS=86400 # Additional settings of this implementation (defaults shown; all optional): LOG_LEVEL=info DB_POOL_MIN=1 @@ -19,5 +23,17 @@ DB_TIMEOUT_SECONDS=3 CACHE_TIMEOUT_SECONDS=0.5 PASSWORD_HASH_CONCURRENCY=2 DOCS_ENABLED=true -# Host port of the API (compose publishes it on 127.0.0.1 only) +# Graceful stop of a replica: readiness 503 + drain, then up to SHUTDOWN for in-flight requests. +DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=8 +# CPU/RAM limits (docker compose deploy.resources.limits); API_* are per replica. +API_CPUS=1.0 +API_MEMORY=256m +PG_CPUS=2.0 +PG_MEMORY=1g +VALKEY_CPUS=0.5 +LB_CPUS=1.0 +PG_MAX_CONNECTIONS=100 +# Host ports of the load balancer (published on 127.0.0.1 only; API replicas publish nothing) APP_PUBLISH_PORT=8080 +LB_STATS_PORT=8404 diff --git a/Dockerfile b/Dockerfile index 470f37a6..12bb32c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -# Messenger API, lab 2 (PostgreSQL + Valkey). Built by compose.yaml (services api, migrate). +# Messenger API, lab 3 (replicas behind HAProxy, PostgreSQL + Valkey). Built by compose.yaml (services api, migrate). # # Python is interpreted, so there is no separate compile stage; the image # contains only the interpreter, locked dependencies and the source code. diff --git a/Makefile b/Makefile index 6233ac48..58704a5a 100644 --- a/Makefile +++ b/Makefile @@ -30,14 +30,14 @@ test-failover: $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) # --------------------------------------------------------------------------- -# Student solution (lab 2, Python). On Windows without make run the commands from -# REPORT.md directly (public tests there need PYTHONUTF8=1; restart hook: scripts/restart.cmd). -.PHONY: secrets up down logs ps migrate student-test unit-test explain +# Student solution (lab 3, Python). On Windows: PowerShell/Git Bash with GNU make, public +# tests need PYTHONUTF8=1, and the hooks are started through scripts/*.cmd (Git Bash). +.PHONY: secrets up down logs ps migrate student-test unit-test explain lb-status restore rolling-update failover-series secrets: $(PYTHON) scripts/init_secrets.py -# Build, apply migrations (service migrate) and wait until every service is healthy. +# Build, apply migrations ONCE (service migrate), start API replicas and the LB, wait for healthy. up: secrets docker compose up -d --build --wait @@ -46,14 +46,32 @@ down: docker compose down logs: - docker compose logs --tail 50 api + docker compose logs --tail 50 api lb ps: - docker compose ps + docker compose ps -a migrate: docker compose run --rm migrate +# HAProxy view of the replicas (UP/DOWN) mapped to containers and X-Instance-Id. +lb-status: + $(PYTHON) scripts/lbctl.py status + $(PYTHON) scripts/lbctl.py replicas + +# Start replicas stopped by scripts/stop-one.sh again and wait until the LB has them all UP. +restore: + sh scripts/restore-api.sh + $(PYTHON) scripts/lbctl.py wait-up $${API_REPLICAS:-2} + +# Replace every API replica one by one with a freshly built image (no downtime). +rolling-update: + sh scripts/rolling-update.sh + +# Level 4: stop each replica in turn under load, measure RPS/errors/recovery (evidence/failover/). +failover-series: + $(PYTHON) bench/failover_series.py --output evidence/failover + # Own tests: unit + black-box against the running stand (needs requirements*.txt in a venv). student-test: $(PYTHON) -m pytest student-tests -v diff --git a/compose.yaml b/compose.yaml index 1f76ebe6..09abcfd4 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,9 +1,12 @@ -# Lab 2 stand: API + PostgreSQL + Valkey. Start from a clean clone: +# Lab 3 stand: client -> HAProxy (lb) -> API x N (one scalable service `api`) -> PostgreSQL + Valkey. +# From a clean clone (details and Windows notes: REPORT.md, docs/WINDOWS.md): # python scripts/init_secrets.py # once: random local credentials into ./secrets (git-ignored) -# cp .env.example .env # optional overrides -# docker compose up -d --build --wait -# Images are pinned by version AND digest. Passwords are passed only as secret files. -name: mtusi-lab2 +# cp .env.example .env # optional overrides (API_REPLICAS, limits, ...) +# docker compose --env-file .env up -d --build --wait +# Order: postgres/valkey healthy -> migrate (ONE run, exits 0) -> api replicas healthy -> lb. +# Images are pinned by version AND digest. Passwords are passed only as secret files, and +# every service gets only the secrets it needs. Only the LB publishes ports (127.0.0.1 only). +name: mtusi-lab3 x-api-env: &api-env APP_ENV: ${APP_ENV:-lab} @@ -17,6 +20,10 @@ x-api-env: &api-env DB_TIMEOUT_SECONDS: ${DB_TIMEOUT_SECONDS:-3} CACHE_TIMEOUT_SECONDS: ${CACHE_TIMEOUT_SECONDS:-0.5} PASSWORD_HASH_CONCURRENCY: ${PASSWORD_HASH_CONCURRENCY:-2} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + # Graceful stop: 3 s drain (readiness 503, still serving) + up to 8 s for in-flight requests. + DRAIN_SECONDS: ${DRAIN_SECONDS:-3} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-8} DOCS_ENABLED: ${DOCS_ENABLED:-true} services: @@ -26,6 +33,8 @@ services: POSTGRES_USER: messenger POSTGRES_DB: messenger POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + # pg_stat_statements: per-statement counters for the track 5 experiments (bench/track1). + command: ["postgres", "-c", "shared_preload_libraries=pg_stat_statements", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"] secrets: [postgres_password] volumes: - pgdata:/var/lib/postgresql # PostgreSQL 18 image keeps PGDATA under this path @@ -37,6 +46,9 @@ services: start_period: 10s stop_grace_period: 30s # clean shutdown checkpoint instead of crash recovery restart: unless-stopped + deploy: + resources: + limits: {cpus: "${PG_CPUS:-2.0}", memory: "${PG_MEMORY:-1g}"} valkey: image: valkey/valkey:8.1.3-bookworm@sha256:fea8b3e67b15729d4bb70589eb03367bab9ad1ee89c876f54327fc7c6e618571 @@ -61,10 +73,15 @@ services: retries: 30 start_period: 5s restart: unless-stopped + deploy: + resources: + limits: {cpus: "${VALKEY_CPUS:-0.5}", memory: 256m} + # One-shot schema migration: runs ONCE per `up`, before any API replica starts + # (api depends_on service_completed_successfully). Replicas never migrate themselves. migrate: build: . - image: mtusi-messenger:lab2 + image: mtusi-messenger:lab3 command: ["python", "-m", "messenger.migrate"] environment: DATABASE_URL_FILE: /run/secrets/database_url @@ -75,14 +92,24 @@ services: healthcheck: disable: true restart: "no" + deploy: + resources: + limits: {cpus: "0.5", memory: 256m} api: build: . - image: mtusi-messenger:lab2 + image: mtusi-messenger:lab3 environment: *api-env secrets: [database_url, cache_url] - ports: - - "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" # only on localhost of the host + # No `ports` and no container_name: replicas are reachable only through `lb`, + # and `--scale api=N` / deploy.replicas works. X-Instance-Id = container hostname. + expose: ["8080"] + healthcheck: + test: ["CMD", "python", "-m", "messenger.healthcheck"] + interval: 2s + timeout: 2s + retries: 3 + start_period: 5s depends_on: postgres: condition: service_healthy @@ -90,8 +117,33 @@ services: condition: service_healthy migrate: condition: service_completed_successfully - stop_grace_period: 10s + stop_grace_period: 15s # > DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS (11 s) restart: unless-stopped + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-256m}"} + + lb: + image: haproxy:3.2.26-alpine@sha256:254e6370f01cd654e0744edf607c67f33777c1b6a9dc0d4b28bcba7171460122 + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + ports: + - "127.0.0.1:${APP_PUBLISH_PORT:-8080}:8080" # the ONLY public entry point + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # local status page (UP/DOWN of replicas) + healthcheck: + # Config is valid and the process answers; whether any API is ready is the LB's own job. + test: ["CMD-SHELL", "haproxy -c -q -f /usr/local/etc/haproxy/haproxy.cfg && wget -q -O /dev/null http://127.0.0.1:8404/"] + interval: 5s + timeout: 3s + retries: 3 + depends_on: + api: + condition: service_healthy + restart: unless-stopped + deploy: + resources: + limits: {cpus: "${LB_CPUS:-1.0}", memory: 128m} volumes: pgdata: diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..09aff871 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,89 @@ +# HAProxy 3.2 LTS — the single public entry point of the lab 3 stand (compose service `lb`). +# +# Backend discovery: Docker DNS name `api` returns one A record per RUNNING replica of the +# compose service `api` (deploy.replicas / --scale). server-template keeps 8 slots and +# re-resolves every second, so scale up/down, a restarted container with a new IP and a +# full stack restart are picked up without restarting the LB. A stopped container drops +# out of DNS and its slot goes to MAINT. +# +# Readiness: an ACTIVE check `GET /health/ready` (PostgreSQL + Valkey reachable, not draining) +# every 1 s; 2 failures -> DOWN, 2 successes -> UP. Passive errors do not decide anything on +# their own, and there are NO backup/fallback servers without checks: a replica that answers +# /health/live but not /health/ready never gets business traffic. +# Worst case detection of a failed replica: +# inter 1 s + fall 2 x (check timeout 1 s) ~= 3 s (frozen process); +# refused connection / 503 ready: inter 1 s + fastinter 0.5 s ~= 1.5 s. +# Requests that could not even be SENT (connection refused/reset before the request) +# are retried on another replica (retry-on conn-failure + redispatch) - safe for POST, +# because the API has not seen them. Nothing that may have reached the API is retried by +# the LB: an uncertain POST is retried by the CLIENT with the same Idempotency-Key. +# No sticky sessions: sessions live in Valkey, every replica serves every user. + +global + log stdout format raw local0 info + maxconn 4096 + # Runtime API for `show servers state` / `show stat` (scripts and tests, no network port). + stats socket /var/lib/haproxy/admin.sock mode 600 level admin + hard-stop-after 15s + +resolvers docker + nameserver dns1 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 1s + +defaults + mode http + log global + option httplog + option dontlognull + option http-keep-alive + timeout connect 1s + timeout client 30s + timeout server 10s # > DB_TIMEOUT_SECONDS (3 s): the API answers 503 itself first + timeout http-request 10s + timeout http-keep-alive 10s + timeout queue 3s + timeout check 1s + retries 2 + retry-on conn-failure + option redispatch 1 + # Errors generated by the LB itself, in the contract error format with the same request id. + # They carry no X-Instance-Id: no API replica produced them. 502/504 = the outcome of the + # request is unknown (a replica died mid-request / did not answer in time): retry with the + # same Idempotency-Key. 503 = no ready replica at all. + http-error status 502 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика оборвала ответ; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + http-error status 503 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"Нет готовых API-реплик\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + http-error status 504 content-type application/json lf-string "{\"error\":{\"code\":\"unavailable\",\"message\":\"API-реплика не ответила вовремя; повторите с тем же Idempotency-Key\",\"request_id\":\"%[var(txn.request_id)]\"}}" hdr X-Request-Id "%[var(txn.request_id)]" + +frontend public + bind :8080 + # One request id end to end: keep a valid client X-Request-Id, otherwise create one; the + # API echoes it, and errors generated by the LB itself carry the same value. + http-request set-header X-Request-Id %[uuid()] unless { req.hdr(x-request-id) -m reg ^[A-Za-z0-9._:-]{1,128}$ } + http-request set-var(txn.request_id) req.hdr(x-request-id) + log-format "%ci:%cp [%tr] %ft %b/%s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq rid=%[var(txn.request_id)] \"%HM %HPO\"" + default_backend api + +backend api + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server inter 1s fastinter 500ms downinter 1s fall 2 rise 2 check resolvers docker init-addr none resolve-prefer ipv4 maxconn 200 + server-template api 8 api:8080 + +# Local-only status page (127.0.0.1:8404 on the host): which replica is UP/DOWN right now. +frontend stats + bind :8404 + stats enable + stats uri / + stats refresh 2s + stats show-legends diff --git a/migrations/0002_idempotency_keys.sql b/migrations/0002_idempotency_keys.sql new file mode 100644 index 00000000..108dafde --- /dev/null +++ b/migrations/0002_idempotency_keys.sql @@ -0,0 +1,25 @@ +-- 0002 (lab 3): shared idempotency of POST /chats/{chat_id}/messages. +-- Additive only (new table), so the lab 2 API keeps working while this runs: it is safe +-- to apply before a rolling update of the API replicas (see REPORT.md, "Миграции"). +-- +-- Scope of a key (contracts/README.md): current user + HTTP method + chat path + key. +-- The method (POST) and the path template are fixed for this table, so the scope +-- is (user_id, chat_id, key). The key row, the message and the result are written in +-- ONE transaction (repo.post_message): there is no "pending" state that a crash could +-- leave behind, and the stored result is the message row itself. + +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id), + key text NOT NULL, + request_hash bytea NOT NULL, -- SHA-256 of the canonical JSON body + -- Inserted before the message in the same transaction, hence checked at COMMIT. + message_id uuid NOT NULL REFERENCES messages (id) DEFERRABLE INITIALLY DEFERRED, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32), + CONSTRAINT idempotency_keys_message_key UNIQUE (message_id) +); +-- Housekeeping of keys older than IDEMPOTENCY_TTL_SECONDS (>= 24 h). +CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at); diff --git a/scripts/lbctl.py b/scripts/lbctl.py new file mode 100644 index 00000000..755d4c6f --- /dev/null +++ b/scripts/lbctl.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""What the load balancer thinks right now, mapped to real containers. + + python scripts/lbctl.py status # table: HAProxy slot, IP, UP/DOWN/MAINT, container, X-Instance-Id + python scripts/lbctl.py wait-up 2 # wait until exactly 2 API replicas are UP in HAProxy + python scripts/lbctl.py replicas # api containers of this compose project: name, hostname, state + +Reads HAProxy's runtime API (`show servers state api`) through `docker compose exec lb socat`, +so no admin port is published. Container hostname = X-Instance-Id of that replica. +Prints no secrets. Used by student-tests and bench scripts as a library as well. +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# srv_op_state of HAProxy (src/server.h): 0 stopped, 1 starting, 2 running, 3 stopping. +OP_STATE = {0: "DOWN", 1: "STARTING", 2: "UP", 3: "STOPPING"} +ADMIN_RMAINT = 0x20 # slot without a resolved address (no running container behind it) + + +def run(*args: str, timeout: float = 60) -> str: + result = subprocess.run(list(args), cwd=ROOT, capture_output=True, text=True, encoding="utf-8", + timeout=timeout) + if result.returncode != 0: + raise RuntimeError(f"{' '.join(args[:3])}... failed: {result.stderr.strip()[:300]}") + return result.stdout + + +def replicas(all_states: bool = True) -> list[dict]: + """API containers of this compose project (running and stopped).""" + ids = run("docker", "compose", "ps", "-q", *(["-a"] if all_states else []), "api").split() + if not ids: + return [] + data = json.loads(run("docker", "inspect", *ids)) + result = [] + for item in data: + networks = item["NetworkSettings"]["Networks"] or {} + result.append({ + "id": item["Id"][:12], + "name": item["Name"].lstrip("/"), + "hostname": item["Config"]["Hostname"], + "state": item["State"]["Status"], + "health": (item["State"].get("Health") or {}).get("Status", "-"), + "exit_code": item["State"]["ExitCode"], + "started_at": item["State"]["StartedAt"], + "pid": item["State"]["Pid"], + "ips": sorted(n["IPAddress"] for n in networks.values() if n.get("IPAddress")), + }) + return sorted(result, key=lambda r: r["name"]) + + +def lb_servers() -> list[dict]: + raw = run("docker", "compose", "exec", "-T", "lb", "sh", "-c", + 'echo "show servers state api" | socat stdio /var/lib/haproxy/admin.sock') + by_ip = {ip: r for r in replicas() for ip in r["ips"]} + servers = [] + for line in raw.splitlines(): + parts = line.split() + if len(parts) < 7 or not parts[0].isdigit(): + continue + name, addr, op, admin = parts[3], parts[4], int(parts[5]), int(parts[6]) + if admin & ADMIN_RMAINT or addr == "-": + state = "MAINT" + else: + state = OP_STATE.get(op, str(op)) + container = by_ip.get(addr) + servers.append({"slot": name, "addr": addr, "state": state, + "container": container["name"] if container else None, + "instance_id": container["hostname"] if container else None}) + return servers + + +def up_instances() -> set[str]: + return {s["instance_id"] or s["addr"] for s in lb_servers() if s["state"] == "UP"} + + +def wait_up(count: int, timeout: float = 60) -> float: + """Seconds until exactly `count` servers are UP (raises TimeoutError otherwise).""" + started = time.monotonic() + while True: + try: + if len(up_instances()) == count: + return time.monotonic() - started + except RuntimeError: + pass # lb restarting + if time.monotonic() - started > timeout: + raise TimeoutError(f"HAProxy does not report {count} UP API replicas after {timeout:.0f} s") + time.sleep(0.25) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("status") + sub.add_parser("replicas") + wait = sub.add_parser("wait-up") + wait.add_argument("count", type=int) + wait.add_argument("--timeout", type=float, default=60) + args = parser.parse_args() + if args.command == "status": + for s in lb_servers(): + if s["state"] != "MAINT" or s["container"]: + print(f"{s['slot']:6} {s['addr']:15} {s['state']:8} {s['container'] or '-':24} X-Instance-Id={s['instance_id'] or '-'}") + print(f"(other slots: MAINT, no container)") + elif args.command == "replicas": + for r in replicas(): + print(f"{r['name']:24} X-Instance-Id={r['hostname']} {r['state']:8} health={r['health']:9} " + f"pid={r['pid']} started={r['started_at'][:23]} ip={','.join(r['ips']) or '-'}") + else: + took = wait_up(args.count, args.timeout) + print(f"{args.count} API replica(s) UP in HAProxy after {took:.1f} s: {sorted(up_instances())}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh index 973048a8..b6416075 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,15 +1,17 @@ #!/bin/sh # Restart hook for `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. -# Restarts the API and ALL state stores (PostgreSQL, Valkey) of compose.yaml. +# Restarts EVERY process of the stand: LB, all API replicas, PostgreSQL and Valkey. # Volumes, schema and data are preserved: no `down -v`, no FLUSH, no re-seed. # Output is suppressed by the test; run it by hand to see the steps. set -eu cd "$(dirname "$0")/.." -# 1. API first: no request is served while the stores restart underneath it. -docker compose stop api +# 1. Entry point and API first: no request is served while the stores restart underneath. +docker compose stop lb api # 2. Stores: clean shutdown (PostgreSQL checkpoint, Valkey AOF fsync), then start again. docker compose restart postgres valkey -# 3. API again; it waits for healthy stores via depends_on and reconnects its pool. +# 3. API replicas (wait for healthy stores via depends_on), then the LB. HAProxy resolves +# the new replica addresses through Docker DNS by itself - no stale backend list. docker compose start api -echo "restart.sh: api, postgres and valkey restarted with volumes preserved" +docker compose start lb +echo "restart.sh: lb, api (all replicas), postgres and valkey restarted with volumes preserved" diff --git a/scripts/restore-api.sh b/scripts/restore-api.sh new file mode 100755 index 00000000..78a06845 --- /dev/null +++ b/scripts/restore-api.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Start every stopped API replica again (after scripts/stop-one.sh) and wait until Docker +# reports them healthy. The LB picks them up by itself (DNS re-resolution + checks, ~2-3 s); +# `python scripts/lbctl.py wait-up 2` waits for that explicitly. +set -eu +cd "$(dirname "$0")/.." +docker compose up -d --no-recreate --no-deps --wait api +docker compose ps api --format '{{.Name}} {{.Status}}' diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..1292249d --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,7 @@ +@echo off +rem Windows wrapper: tests/resilience.py starts the hook as a program, and Windows +rem cannot execute a .sh file directly. Runs the same scripts\stop-one.sh with Git Bash; +rem TARGET_INSTANCE_ID and STOP_MODE are inherited from the environment. +rem python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd +"%ProgramFiles%\Git\bin\bash.exe" "%~dp0stop-one.sh" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..85bc37ad --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Failover hook for `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`. +# Stops exactly ONE API replica: the container whose X-Instance-Id (= container hostname, +# see src/messenger/config.py) is in TARGET_INSTANCE_ID. DB, Valkey, LB and volumes are +# not touched. The replica is LEFT STOPPED (`docker stop` = no restart by the policy); +# restore it explicitly afterwards: scripts/restore-api.sh. +# STOP_MODE=stop (default) SIGTERM: readiness 503, 3 s drain, graceful exit (~4 s) +# STOP_MODE=kill SIGKILL at once (`docker stop -t 0`): crash of the process +# Returns within 10 s. The test hides the output; run by hand to see it (no secrets printed). +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + *[!A-Za-z0-9._-]*) echo "stop-one: invalid TARGET_INSTANCE_ID" >&2; exit 2 ;; +esac +mode="${STOP_MODE:-stop}" + +target="" +# Only RUNNING containers of THIS compose project's service `api` are candidates. +for id in $(docker compose ps -q api); do + if [ "$(docker inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then + target="$id" + fi +done +if [ -z "$target" ]; then + echo "stop-one: no running api replica with X-Instance-Id=$TARGET_INSTANCE_ID" >&2 + exit 3 +fi +name="$(docker inspect -f '{{.Name}}' "$target")" +case "$mode" in + stop) docker stop --time 15 "$target" >/dev/null ;; + kill) docker stop --time 0 "$target" >/dev/null ;; + *) echo "stop-one: STOP_MODE must be stop or kill" >&2; exit 2 ;; +esac +echo "stop-one: $mode $name (X-Instance-Id=$TARGET_INSTANCE_ID) -> $(docker inspect -f '{{.State.Status}} exit={{.State.ExitCode}}' "$target")" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py index 331a1fd2..5701b666 100644 --- a/src/messenger/__main__.py +++ b/src/messenger/__main__.py @@ -1,16 +1,20 @@ """Entry point: `python -m messenger`. The HTTP server is the main process (PID 1 in Docker). -Shutdown sequence on SIGTERM/SIGINT: -1. readiness flips to 503 (`app.state.draining = True`); -2. uvicorn closes the listening socket, so new connections are refused; -3. in-flight requests get up to SHUTDOWN_TIMEOUT_SECONDS (default 8 s < Docker's 10 s) to finish; +Shutdown sequence on SIGTERM/SIGINT (lab 3: the replica stands behind HAProxy): +1. readiness flips to 503 (`app.state.draining = True`), requests are still served; +2. after DRAIN_SECONDS (compose: 3 s > HAProxy detection 2 x 1 s) the load balancer has + marked this replica DOWN and sends nothing new here; +3. uvicorn closes the listening socket; in-flight requests get up to + SHUTDOWN_TIMEOUT_SECONDS (compose: 8 s) to finish; 4. the lifespan closes the DB pool and the Valkey client; the process exits with code 0. - Data stays in PostgreSQL and Valkey volumes. + DRAIN + SHUTDOWN (11 s) < compose stop_grace_period (15 s), so Docker never needs SIGKILL. +A second signal during the drain skips the rest of it. Data stays in PostgreSQL and Valkey. """ from __future__ import annotations import logging import sys +import threading from types import FrameType import uvicorn @@ -23,12 +27,30 @@ log = logging.getLogger("messenger") class GracefulServer(uvicorn.Server): - """uvicorn.Server that marks the app as draining before the normal graceful stop.""" + """uvicorn.Server that drains (readiness 503, still serving) before the normal graceful stop.""" + + def __init__(self, config: uvicorn.Config, drain_seconds: float): + super().__init__(config) + self.drain_seconds = drain_seconds + self.drain_timer: threading.Timer | None = None def handle_exit(self, sig: int, frame: FrameType | None) -> None: - if not self.should_exit: - log.info("shutdown_requested", extra={"fields": {"signal": sig, "timeout_s": self.config.timeout_graceful_shutdown}}) self.config.app.state.draining = True # config.app is our Starlette instance + if self.drain_timer is None and self.drain_seconds > 0 and not self.should_exit: + log.info("drain_started", extra={"fields": {"signal": sig, "drain_s": self.drain_seconds, + "timeout_s": self.config.timeout_graceful_shutdown}}) + # uvicorn polls should_exit in its main loop, so setting it from a timer thread is safe. + self.drain_timer = threading.Timer(self.drain_seconds, self._stop, args=(sig, frame)) + self.drain_timer.daemon = True + self.drain_timer.start() + return + if self.drain_timer is not None: + self.drain_timer.cancel() # second signal: stop draining now + self._stop(sig, frame) + + def _stop(self, sig: int, frame: FrameType | None) -> None: + if not self.should_exit: + log.info("shutdown_requested", extra={"fields": {"signal": sig}}) super().handle_exit(sig, frame) @@ -51,10 +73,10 @@ def main() -> int: lifespan="on", timeout_graceful_shutdown=settings.shutdown_timeout, ) - server = GracefulServer(config) + server = GracefulServer(config, settings.drain_seconds) log.info("starting", extra={"fields": { "version": __version__, "env": settings.env, "host": settings.host, "port": settings.port, - "python": sys.version.split()[0], "storage": "postgresql", + "python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id, }}) server.run() log.info("stopped") diff --git a/src/messenger/app.py b/src/messenger/app.py index 9d0bf79b..e2bb6831 100644 --- a/src/messenger/app.py +++ b/src/messenger/app.py @@ -3,6 +3,8 @@ from __future__ import annotations import asyncio import contextlib +import hashlib +import json import logging import re import time @@ -19,7 +21,7 @@ from starlette.types import ASGIApp, Message as AsgiMessage, Receive, Scope, Sen from . import authhdr, cursor, docs, passwords, repo, validation from .config import Settings from .db import Database -from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, PayloadTooLarge, Unauthorized, Unavailable +from .errors import BASIC_CHALLENGE, ApiError, InvalidRequest, NotFound, PayloadTooLarge, Unauthorized, Unavailable from .sessions import SessionStore, build_store from .timefmt import rfc3339 @@ -56,10 +58,12 @@ async def http_exception_handler(request: Request, exc: Exception) -> JSONRespon # Middleware (pure ASGI, so it also covers 404/405 and unexpected exceptions) # -------------------------------------------------------------------------- class RequestContextMiddleware: - """Assigns X-Request-Id to every response, writes one access-log line, turns crashes into 500.""" + """Adds X-Request-Id and X-Instance-Id to EVERY response (errors, health and crash 500/503 + included), writes one access-log line, turns crashes into 500.""" - def __init__(self, app: ASGIApp): + def __init__(self, app: ASGIApp, instance_id: str): self.app = app + self.instance_id = instance_id.encode("ascii") async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: if scope["type"] != "http": @@ -77,8 +81,10 @@ class RequestContextMiddleware: if message["type"] == "http.response.start": started = True status = message["status"] - headers = [(k, v) for k, v in message.get("headers", []) if k.lower() != b"x-request-id"] + headers = [(k, v) for k, v in message.get("headers", []) + if k.lower() not in (b"x-request-id", b"x-instance-id")] headers.append((b"x-request-id", request_id.encode("latin-1"))) + headers.append((b"x-instance-id", self.instance_id)) message = {**message, "headers": headers} await send(message) @@ -90,20 +96,21 @@ class RequestContextMiddleware: if not started: status = 503 response = error_response(request_id, 503, "unavailable", "Сервер завершает работу") - await response(scope, receive, send) + await response(scope, receive, send_with_id) raise except Exception: log.exception("unhandled_error", extra={"fields": {"request_id": request_id}}) if not started: status = 500 response = error_response(request_id, 500, "internal_error", "Внутренняя ошибка сервера") - await response(scope, receive, send) + await response(scope, receive, send_with_id) finally: access_log.info("request", extra={"fields": { "request_id": request_id, "method": scope["method"], "path": scope["path"], # ids only; query (cursor) and bodies are never logged "status": status, + "instance_id": self.instance_id.decode("ascii"), "duration_ms": round((time.perf_counter() - begin) * 1000, 3), }}) @@ -166,6 +173,27 @@ def page_params(request: Request) -> tuple[int, str | None]: return limit, raw_cursor +async def reject_body(request: Request) -> None: + """POST /auth/sessions takes credentials only from the Basic header; any body is 400.""" + if request.headers.get("transfer-encoding") or request.headers.get("content-length", "0") != "0": + raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела") + async for chunk in request.stream(): + if chunk: + raise InvalidRequest("Вход принимает только заголовок Authorization: Basic, без тела") + + +def idempotency_key(request: Request) -> str: + values = request.headers.getlist("idempotency-key") + return validation.idempotency_key(values[0] if len(values) == 1 else None) + + +def request_fingerprint(text: str, attachment_ids: list[str]) -> bytes: + """SHA-256 of the canonical body: key order and an absent attachment_ids vs [] do not matter.""" + canonical = json.dumps({"attachment_ids": attachment_ids, "text": text}, + ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + def ok(body: Any, status: int = 200) -> JSONResponse: return JSONResponse(body, status_code=status) @@ -205,6 +233,7 @@ async def register_user(request: Request) -> JSONResponse: async def create_session(request: Request) -> JSONResponse: """Login: only `Authorization: Basic`; wrong user and wrong password give the same 401.""" username, password = authhdr.parse_basic(request.headers.get("authorization")) + await reject_body(request) found = await repo.get_credentials(db_of(request), username) user, stored_hash = found if found else (None, None) if not await passwords.verify_password(stored_hash, password) or user is None: @@ -262,13 +291,23 @@ async def create_message(request: Request) -> JSONResponse: user_id = await authenticate(request) chat_id = path_uuid(request, "chat_id") db = db_of(request) - await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before the body is read + await repo.ensure_member(db, user_id, chat_id) # outsiders get 404 before key/body are looked at + key = idempotency_key(request) data = await read_json(request) - validation.check_fields(data, {"text"}) + validation.check_fields(data, {"text"}, frozenset({"attachment_ids"})) text = validation.text_field(data, "text", 4000) + attachment_ids = validation.attachment_ids_field(data) + if attachment_ids: + # Lab 3 has no attachments: every referenced ID is unknown (lab 4 semantics: 404). + raise NotFound("Вложение не найдено") # sender_id is derived from the session, never taken from the body. - message = await repo.add_message(db, user_id, chat_id, text) - return ok(message.to_json(), 201) + settings = request.app.state.settings + result = await repo.post_message(db, user_id, chat_id, key, request_fingerprint(text, attachment_ids), text, + settings.idempotency_ttl_seconds) + if result.replayed: + log.info("idempotent_replay", extra={"fields": {"request_id": request.state.request_id, + "message_id": result.message.id}}) + return ok(result.message.to_json(), 201) async def list_messages(request: Request) -> JSONResponse: @@ -288,16 +327,30 @@ def create_app(settings: Settings | None = None) -> Starlette: db = Database(settings) sessions = build_store(settings, db) + async def purge_loop() -> None: + """Housekeeping: expired Idempotency-Key rows (older than IDEMPOTENCY_TTL_SECONDS).""" + while True: + await asyncio.sleep(600) + try: + deleted = await repo.purge_idempotency_keys(db, settings.idempotency_ttl_seconds) + if deleted: + log.info("idempotency_keys_purged", extra={"fields": {"deleted": deleted}}) + except ApiError: + pass # database unavailable right now; next round + @contextlib.asynccontextmanager async def lifespan(app: Starlette): passwords.configure(settings.hash_concurrency) await db.open() log.info("dependencies_configured", extra={"fields": { "session_store": sessions.name, "session_ttl_s": settings.session_ttl_seconds, - "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds}}) + "db_pool_max": settings.db_pool_max, "db_timeout_s": settings.db_timeout_seconds, + "instance_id": settings.instance_id, "drain_s": settings.drain_seconds}}) + purge = asyncio.create_task(purge_loop()) try: yield finally: + purge.cancel() await sessions.close() await db.close() @@ -324,7 +377,7 @@ def create_app(settings: Settings | None = None) -> Starlette: exception_handlers={ApiError: api_error_handler, HTTPException: http_exception_handler}, lifespan=lifespan, ) - app.add_middleware(RequestContextMiddleware) + app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id) app.state.settings = settings app.state.db = db app.state.sessions = sessions diff --git a/src/messenger/config.py b/src/messenger/config.py index 4cecbcd1..807020cd 100644 --- a/src/messenger/config.py +++ b/src/messenger/config.py @@ -8,6 +8,8 @@ Invalid values stop the process at startup with a clear message. from __future__ import annotations import os +import re +import socket from dataclasses import dataclass from pathlib import Path @@ -16,6 +18,22 @@ class ConfigError(ValueError): pass +INSTANCE_ID_RE = re.compile(r"[A-Za-z0-9._-]{1,64}") + + +def _instance_id() -> str: + """X-Instance-Id of this process: INSTANCE_ID or the container hostname. + + Docker sets the hostname of a container to its short container ID, so the value + is unique per replica, stable for the life of the process and maps 1:1 to a real + container (`docker inspect`), which scripts/stop-one.sh relies on. + """ + value = os.environ.get("INSTANCE_ID", "").strip() or socket.gethostname() + if INSTANCE_ID_RE.fullmatch(value) is None: + raise ConfigError("INSTANCE_ID must match [A-Za-z0-9._-]{1,64}") + return value + + def _int(name: str, default: int, low: int, high: int) -> int: raw = os.environ.get(name) if raw is None or raw.strip() == "": @@ -85,6 +103,9 @@ class Settings: db_timeout_seconds: float = 3.0 cache_timeout_seconds: float = 0.5 hash_concurrency: int = 2 + instance_id: str = "local" + drain_seconds: float = 0.0 + idempotency_ttl_seconds: int = 86400 @property def db_statement_timeout_ms(self) -> int: @@ -110,8 +131,8 @@ class Settings: port=_int("APP_PORT", 8080, 1, 65535), env=os.environ.get("APP_ENV", "lab").strip() or "lab", log_level=log_level, - # Docker sends SIGKILL 10 s after SIGTERM by default; finish earlier. - shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 9.0), + # DRAIN_SECONDS + SHUTDOWN_TIMEOUT_SECONDS must stay below compose stop_grace_period. + shutdown_timeout=_float("SHUTDOWN_TIMEOUT_SECONDS", 8.0, 0.0, 30.0), max_body_bytes=_int("MAX_BODY_BYTES", 64 * 1024, 1024, 1024 * 1024), docs_enabled=_bool("DOCS_ENABLED", True), database_url=_secret("DATABASE_URL", required=True), @@ -124,4 +145,10 @@ class Settings: db_timeout_seconds=_float("DB_TIMEOUT_SECONDS", 3.0, 0.5, 30.0), cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.5, 0.05, 10.0), hash_concurrency=_int("PASSWORD_HASH_CONCURRENCY", 2, 1, 16), + instance_id=_instance_id(), + # After SIGTERM: readiness is 503 but requests are still served for this long, + # so the load balancer stops routing here before the listener closes. + drain_seconds=_float("DRAIN_SECONDS", 0.0, 0.0, 10.0), + # Contract: a replay within 24 h returns the original result. + idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 86400, 30 * 24 * 3600), ) diff --git a/src/messenger/repo.py b/src/messenger/repo.py index 4ae4d952..71bb0c6d 100644 --- a/src/messenger/repo.py +++ b/src/messenger/repo.py @@ -15,7 +15,7 @@ from psycopg import errors as pg_errors from .cursor import SortKey from .db import Database -from .errors import Conflict, InvalidRequest, NotFound +from .errors import Conflict, InvalidRequest, NotFound, Unavailable from .timefmt import from_us, rfc3339, to_us @@ -198,28 +198,89 @@ async def list_chats(db: Database, user_id: str, after: SortKey | None, limit: i # ------------------------------------------------------------------ messages -async def add_message(db: Database, user_id: str, chat_id: str, text: str) -> Message: +@dataclass(slots=True, frozen=True) +class PostResult: + message: Message + replayed: bool # True: the key was already used with this body, nothing new was written + + +async def _insert_message(conn: psycopg.AsyncConnection, message_id: uuid.UUID, user_id: str, chat_id: str, + text: str) -> dt.datetime: """Insert with a per-chat strictly increasing timestamp. `UPDATE chats ... RETURNING` locks the chat row until COMMIT, so writers of one chat are serialised: a message with a smaller created_at can never commit after a message with a bigger one, and a reader paging by cursor cannot skip it. """ + cur = await conn.execute( + "UPDATE chats SET last_message_at = greatest(clock_timestamp(), " + "last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at", + (chat_id,)) + (created_at,) = await cur.fetchone() + await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", + (message_id, chat_id, user_id, _b(text), created_at)) + return created_at + + +# Claim the key. A row of a concurrent transaction that is not committed yet makes this +# INSERT wait on the unique index until that transaction ends: after COMMIT it is a conflict +# (the key is taken), after ROLLBACK the INSERT goes through. A key older than the TTL is +# re-used as if it were new (ON CONFLICT ... DO UPDATE ... WHERE expired). +CLAIM_KEY = ( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id) " + "VALUES (%s, %s, %s, %s, %s) " + "ON CONFLICT (user_id, chat_id, key) DO UPDATE SET " + "request_hash = EXCLUDED.request_hash, message_id = EXCLUDED.message_id, created_at = clock_timestamp() " + "WHERE idempotency_keys.created_at < clock_timestamp() - make_interval(secs => %s) " + "RETURNING 1") +STORED_RESULT = ( + "SELECT k.request_hash, m.id, m.text, m.created_at FROM idempotency_keys k " + "JOIN messages m ON m.id = k.message_id " + "WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s") + + +async def post_message(db: Database, user_id: str, chat_id: str, key: str, request_hash: bytes, text: str, + ttl_seconds: int) -> PostResult: + """Idempotent send: the key, the message and the stored result commit in ONE transaction. + + * first request -> key row + message + chat clock, all or nothing (no "pending" key is + ever left behind: a crash or 5xx rolls everything back and the client may retry); + * same key, same body (any replica, also concurrently) -> the stored Message, 201, replayed; + * same key, other body -> 409; the membership check comes first, so outsiders always get 404. + """ message_id = uuid.uuid4() - async def work(conn: psycopg.AsyncConnection) -> dt.datetime: + async def work(conn: psycopg.AsyncConnection) -> PostResult: await _check_member(conn, user_id, chat_id) + cur = await conn.execute(CLAIM_KEY, (user_id, chat_id, key, request_hash, message_id, ttl_seconds)) + if await cur.fetchone() is not None: + created_at = await _insert_message(conn, message_id, user_id, chat_id, text) + return PostResult(Message(str(message_id), chat_id, user_id, text, created_at), False) + cur = await conn.execute(STORED_RESULT, (user_id, chat_id, key)) + row = await cur.fetchone() + if row is None: + # The key row vanished between the two statements (expired-key cleanup): retryable. + raise Unavailable("Повторите запрос с тем же Idempotency-Key") + if bytes(row[0]) != request_hash: + raise Conflict("Idempotency-Key уже использован с другим сообщением") + return PostResult(Message(str(row[1]), chat_id, user_id, _s(row[2]), row[3]), True) + return await db.run(work) + + +async def purge_idempotency_keys(db: Database, ttl_seconds: int, batch: int = 5000) -> int: + """Delete expired keys in bounded batches; one replica at a time (advisory xact lock).""" + async def work(conn: psycopg.AsyncConnection) -> int: + cur = await conn.execute("SELECT pg_try_advisory_xact_lock(%s)", (2026_03,)) + if not (await cur.fetchone())[0]: + return 0 cur = await conn.execute( - "UPDATE chats SET last_message_at = greatest(clock_timestamp(), " - "last_message_at + interval '1 microsecond') WHERE id = %s RETURNING last_message_at", - (chat_id,)) - (created_at,) = await cur.fetchone() - await conn.execute( - "INSERT INTO messages (id, chat_id, sender_id, text, created_at) VALUES (%s, %s, %s, %s, %s)", - (message_id, chat_id, user_id, _b(text), created_at)) - return created_at - created_at = await db.run(work) - return Message(str(message_id), chat_id, user_id, text, created_at) + "DELETE FROM idempotency_keys WHERE ctid = ANY(ARRAY(" + "SELECT ctid FROM idempotency_keys " + "WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT %s))", + (ttl_seconds, batch)) + return cur.rowcount + return await db.run(work) # Two statements instead of one with "(%s IS NULL OR ...)": with a generic plan the OR diff --git a/src/messenger/validation.py b/src/messenger/validation.py index 87670955..b338c9e8 100644 --- a/src/messenger/validation.py +++ b/src/messenger/validation.py @@ -16,6 +16,8 @@ USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") # Canonical textual UUID (8-4-4-4-12 hex). Case-insensitive, normalised to lower case. UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") LIMIT_RE = re.compile(r"[0-9]{1,3}") +# Idempotency-Key (contract, lab 3): ASCII [A-Za-z0-9._:-]{1,128}. +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") SURROGATES_RE = re.compile("[\ud800-\udfff]") DEFAULT_LIMIT = 50 @@ -27,11 +29,19 @@ def _reject_constant(name: str) -> Any: raise ValueError(f"invalid JSON constant {name}") +def _unique_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + # {"text":"a","text":"b"} is ambiguous (which one is meant?) -> 400, not "last one wins". + result = dict(pairs) + if len(result) != len(pairs): + raise ValueError("duplicate key in JSON object") + return result + + def parse_json_object(body: bytes) -> dict[str, Any]: """Decode a UTF-8 JSON object or raise InvalidRequest (malformed JSON -> 400).""" try: text = body.decode("utf-8") - value = json.loads(text, parse_constant=_reject_constant) + value = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=_unique_keys) except (UnicodeDecodeError, ValueError): raise InvalidRequest("Тело запроса должно быть корректным JSON в UTF-8") from None if not isinstance(value, dict): @@ -102,6 +112,26 @@ def member_ids_field(data: dict[str, Any]) -> list[str]: return ids +def attachment_ids_field(data: dict[str, Any]) -> list[str]: + """Optional `attachment_ids` (lab 4+). Absent and [] are the same message (contracts/README.md); + lab 3 has no attachments yet, so a non-empty list can only reference unknown objects.""" + value = data.get("attachment_ids", []) + if not isinstance(value, list): + raise InvalidRequest("attachment_ids должен быть массивом") + if len(value) > 10: + raise InvalidRequest("attachment_ids содержит больше 10 элементов") + ids = [parse_uuid(item, "UUID вложения") for item in value] + if len(set(ids)) != len(ids): + raise InvalidRequest("attachment_ids содержит дубликаты") + return ids + + +def idempotency_key(raw: str | None) -> str: + if raw is None or IDEMPOTENCY_KEY_RE.fullmatch(raw) is None: + raise InvalidRequest("Заголовок Idempotency-Key обязателен: [A-Za-z0-9._:-]{1,128}") + return raw + + def single_query_param(query: list[tuple[str, str]], name: str) -> str | None: values = [v for k, v in query if k == name] if len(values) > 1: -- 2.54.0 From 78b95acabf77a29fb4f19a779f78324b68189ee0 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:32:13 +0300 Subject: [PATCH 03/18] =?UTF-8?q?test:=20=D1=81=D1=82=D1=80=D0=BE=D0=B3?= =?UTF-8?q?=D0=B8=D0=B5=20=D1=82=D0=B5=D1=81=D1=82=D1=8B=20lab3=20?= =?UTF-8?q?=E2=80=94=20=D0=B8=D0=B4=D0=B5=D0=BC=D0=BF=D0=BE=D1=82=D0=B5?= =?UTF-8?q?=D0=BD=D1=82=D0=BD=D0=BE=D1=81=D1=82=D1=8C,=20=D0=B1=D0=B0?= =?UTF-8?q?=D0=BB=D0=B0=D0=BD=D1=81=D0=B8=D1=80=D0=BE=D0=B2=D0=BA=D0=B0,?= =?UTF-8?q?=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7=D1=8B=20=D1=80=D0=B5=D0=BF?= =?UTF-8?q?=D0=BB=D0=B8=D0=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test_idempotency: точные коды и тела повторов через разные реплики, 24 одновременных запроса с одним ключом -> ровно одно сообщение и одна строка ключа в БД, 409 при другом body, scope user+chat+key, 404 постороннему, откат после сбоя на втором INSERT (триггер, n_tup_ins), неопределённый POST (клиентский timeout) и lock_timeout с повтором тем же ключом. - test_balancing: X-Instance-Id = реальные контейнеры, нет sticky, LB исключает живую, но неготовую реплику (pg_hba reject), graceful stop каждой реплики под нагрузкой без единой ошибки, SIGKILL — ограниченные ошибки и восстановление, полный restart и scale up/down без рестарта LB, политика compose (лимиты, порты, secrets, digest), миграции один раз. - Входы из замечаний к чужим PR: курсоры с []/{}/числами/пустые/пробел, байт FF, TEXT/text, дубли ключей, Bearer é, UUID в разном регистре, тело при входе, пароль "я:"*64. Тесты lab2 переведены на Idempotency-Key и LB; все отказы восстанавливаются в finally и ждут UP всех реплик в HAProxy. Co-Authored-By: Claude Opus 5.5 --- student-tests/conftest.py | 97 ++++++++-- student-tests/test_api.py | 107 ++++++++++- student-tests/test_balancing.py | 298 ++++++++++++++++++++++++++++++ student-tests/test_db.py | 3 +- student-tests/test_failures.py | 30 ++- student-tests/test_idempotency.py | 281 ++++++++++++++++++++++++++++ student-tests/test_sessions.py | 5 +- student-tests/unit/test_unit.py | 57 ++++++ 8 files changed, 848 insertions(+), 30 deletions(-) create mode 100644 student-tests/test_balancing.py create mode 100644 student-tests/test_idempotency.py diff --git a/student-tests/conftest.py b/student-tests/conftest.py index f09a6ad4..68b9e158 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -2,9 +2,10 @@ * `unit/` — pure functions (validation, cursor, auth headers, Argon2id); no services. * `test_*` in this folder — marker `stack`: black-box HTTP against the running compose - stand (BASE_URL, default http://127.0.0.1:8080) plus `docker compose` for database - checks (psql inside the postgres container) and failure injection. Every failure - test restores the stopped/paused service in `finally`. + stand THROUGH THE LOAD BALANCER (BASE_URL, default http://127.0.0.1:8080) plus + `docker compose` for database checks (psql inside the postgres container), the HAProxy + runtime state (scripts/lbctl.py) and failure injection. Every failure test restores the + stopped/paused service or replica in `finally` and waits until HAProxy sees all replicas UP. python -m pytest student-tests # everything (stand must be up) python -m pytest student-tests -m "not stack" # unit only @@ -27,10 +28,13 @@ from jsonschema import Draft202012Validator ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT / "src")) +sys.path.insert(0, str(ROOT / "scripts")) +import lbctl # noqa: E402 SPEC = json.loads((ROOT / "contracts" / "openapi.json").read_text(encoding="utf-8")) BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") PASSWORD = "Test-password-Ω-12345" # synthetic; 12+ code points, non-ASCII on purpose +REPLICAS = int(os.environ.get("API_REPLICAS", "2")) # replicas the stand normally runs def pytest_configure(config): @@ -74,26 +78,32 @@ def wait_ready(timeout: float = 90) -> None: def wait_healthy(*services: str, timeout: float = 120) -> None: - """Poll Docker health until every service is healthy again. `compose up --wait` is not used: - it gives up at once on a container that was marked unhealthy while paused.""" + """Poll Docker health until EVERY container of every service is healthy again. `compose up + --wait` is not used: it gives up at once on a container marked unhealthy while paused.""" deadline = time.monotonic() + timeout pending = set(services) while pending and time.monotonic() < deadline: for service in list(pending): ids = compose("ps", "-q", service, check=False).stdout.split() if ids: - state = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", ids[0]], - capture_output=True, text=True).stdout.strip() - if state == "healthy": + states = subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", *ids], + capture_output=True, text=True).stdout.split() + if states and all(state == "healthy" for state in states): pending.discard(service) time.sleep(0.5) assert not pending, f"not healthy: {pending}" +def wait_stand(replicas: int = REPLICAS) -> None: + """Readiness through the LB AND every replica back in HAProxy rotation.""" + wait_ready() + lbctl.wait_up(replicas, timeout=90) + + def restart_services(*services: str) -> None: compose("restart", *services) wait_healthy(*services) - wait_ready() + wait_stand() @contextlib.contextmanager @@ -106,7 +116,7 @@ def service_down(service: str, how: str = "stop"): finally: compose("unpause" if how == "pause" else "start", service) wait_healthy(service) - wait_ready() + wait_stand() @contextlib.contextmanager @@ -116,14 +126,33 @@ def api_with_env(**overrides: str): env = {**os.environ, **overrides} subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=env, capture_output=True, text=True, check=True, timeout=180) - wait_ready() + wait_stand() try: yield finally: clean = {k: v for k, v in os.environ.items() if k not in overrides} subprocess.run(["docker", "compose", "up", "-d", "--wait", "api"], cwd=ROOT, env=clean, capture_output=True, text=True, check=True, timeout=180) - wait_ready() + wait_stand() + + +def running_replicas() -> list[dict]: + return [r for r in lbctl.replicas() if r["state"] == "running"] + + +@contextlib.contextmanager +def replica_stopped(replica: dict, mode: str = "stop"): + """Stop ONE api container (stop = SIGTERM + drain, kill = SIGKILL); ALWAYS start it again + and wait until HAProxy has every replica UP.""" + assert mode in {"stop", "kill"} + subprocess.run(["docker", "stop", "--time", "15" if mode == "stop" else "0", replica["id"]], + capture_output=True, text=True, check=True, timeout=60) + try: + yield + finally: + subprocess.run(["docker", "start", replica["id"]], capture_output=True, text=True, check=True, timeout=60) + wait_healthy("api") + wait_stand() def api_env(name: str) -> str: @@ -139,7 +168,10 @@ class Api: self.client = httpx.Client(base_url=base_url, timeout=15) def call(self, method: str, path: str, expected: int | tuple[int, ...], *, token: str | None = None, - json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None): + json_body=None, raw: bytes | None = None, headers: dict | None = None, schema: str | None = None, + lb_error_ok: bool = False): + """lb_error_ok: a 502/503/504 generated by HAProxy itself (no API replica answered) is + accepted without X-Instance-Id; every response of an API replica must carry it.""" h = dict(headers or {}) if token is not None: h["Authorization"] = "Bearer " + token @@ -151,6 +183,9 @@ class Api: assert response.status_code in allowed, (method, path, response.status_code, response.text[:300]) request_id = response.headers.get("x-request-id") assert request_id, "every response must carry X-Request-Id" + if not response.headers.get("x-instance-id"): + assert lb_error_ok and response.status_code in (502, 503, 504), \ + ("missing X-Instance-Id", method, path, response.status_code) if response.status_code >= 400: body = response.json() assert schema_errors(body, "Error") == [], response.text @@ -186,9 +221,18 @@ class Api: return self.call("POST", "/api/v1/chats", 201, token=owner["token"], json_body={"title": title, "member_ids": ids}, schema="Chat").json() - def send(self, actor: dict, chat: dict, text: str) -> dict: - return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=actor["token"], - json_body={"text": text}, schema="Message").json() + def post_message(self, actor: dict, chat: dict, body, expected=201, *, key: str | None = None, + raw: bytes | None = None, headers: dict | None = None, **kwargs): + """POST a message with an Idempotency-Key (a fresh one unless `key` is given).""" + h = {"Idempotency-Key": key or uuid.uuid4().hex, **(headers or {})} + if raw is not None: + h.setdefault("Content-Type", "application/json") + return self.call("POST", f"/api/v1/chats/{chat['id']}/messages", expected, token=actor["token"], + json_body=body if raw is None else None, raw=raw, headers=h, + schema="Message" if expected == 201 else None, **kwargs) + + def send(self, actor: dict, chat: dict, text: str, key: str | None = None) -> dict: + return self.post_message(actor, chat, {"text": text}, key=key).json() def history(self, actor: dict, chat: dict, limit: int) -> tuple[list[dict], int]: """Walk all pages; returns (items, number_of_pages).""" @@ -224,3 +268,24 @@ def pytest_collection_modifyitems(config, items): @pytest.fixture(scope="session") def api() -> Api: return Api() + + +def run_hook(name: str, timeout: float = 180, **env: str) -> subprocess.CompletedProcess: + """Run scripts/ (a POSIX sh hook); on Windows through Git Bash like the .cmd wrappers.""" + script = str(ROOT / "scripts" / name) + if os.name == "nt": + command = [os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe"), script] + else: + command = [script] + return subprocess.run(command, cwd=ROOT, capture_output=True, text=True, encoding="utf-8", check=True, + timeout=timeout, env={**os.environ, **env}) + + +def probe_inside(replica: dict, path: str) -> int: + """HTTP status of `path` asked inside the replica container itself (bypassing the LB).""" + code = ("import urllib.request as u, urllib.error as e\n" + "try: print(u.urlopen('http://127.0.0.1:8080%s', timeout=3).status)\n" + "except e.HTTPError as x: print(x.code)") % path + out = subprocess.run(["docker", "exec", replica["id"], "python", "-c", code], capture_output=True, text=True, + timeout=30).stdout.strip() + return int(out or 0) diff --git a/student-tests/test_api.py b/student-tests/test_api.py index d99077ec..7339580b 100644 --- a/student-tests/test_api.py +++ b/student-tests/test_api.py @@ -108,7 +108,7 @@ def test_unicode_limits_are_code_points_and_nul_is_stored(api): assert chat["title"] == "T\u0000itle" assert api.send(actor, chat, "hello\u0000world")["text"] == "hello\u0000world" assert api.send(actor, chat, "🙂" * 4000)["text"] == "🙂" * 4000 - api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=actor["token"], json_body={"text": "я" * 4001}) + api.post_message(actor, chat, {"text": "я" * 4001}, 400) texts = [m["text"] for m in api.history(actor, chat, 100)[0]] assert texts == ["hello\u0000world", "🙂" * 4000] @@ -136,7 +136,7 @@ def test_foreign_chat_is_indistinguishable_from_missing(api): base = f"/api/v1/chats/{chat_id}" errors = [api.call("GET", base, 404, token=eve["token"]).json(), api.call("GET", base + "/messages", 404, token=eve["token"]).json(), - api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}).json(), + api.call("POST", base + "/messages", 404, token=eve["token"], json_body={"text": "чужое"}, headers={"Idempotency-Key": "k1"}).json(), api.call("GET", base + "/messages?cursor=garbage", 404, token=eve["token"]).json()] assert len({e["error"]["message"] for e in errors}) == 1 api.call("GET", "/api/v1/chats/not-a-uuid", 401) @@ -199,3 +199,106 @@ def test_read_after_write(api): message = api.send(alice, chat, f"m{i}") last = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]).json()["items"][-1] assert last == message + + +# ================================================================== lab 3: inputs found by the reviewer in other PRs +def b64(raw: bytes) -> str: + return base64.urlsafe_b64encode(raw).rstrip(b"=").decode() + + +def bad_cursors(owner_id: str) -> list[str]: + """Wrong types/values inside the cursor, empty and whitespace cursors, out-of-range time.""" + good_uuid = str(uuid.uuid4()) + return ["", " ", "%20", "garbage", b64(b"[]"), b64(b"{}"), b64(b"123"), b64(b"false"), b64(b"null"), + b64(b'{"created_at":1,"id":1}'), b64(b'{"created_at":null,"id":"x"}'), + b64(f"v1|m|{owner_id}|1|1".encode()), b64(f"v1|c|{owner_id}|1|1".encode()), + b64(f"v1|m|{owner_id}|abc|{good_uuid}".encode()), b64(f"v1|m|{owner_id}|-1|{good_uuid}".encode()), + b64(f"v1|m|{owner_id}|9223372036854775807|{good_uuid}".encode()), + b64(f"v1|c|{owner_id}|9223372036854775807|{good_uuid}".encode()), + b64(f"v1|m|{owner_id}|1|{good_uuid.upper()}".encode()), b64(f"v2|m|{owner_id}|1|{good_uuid}".encode())] + + +def test_malformed_cursor_values_are_400_on_both_lists(api): + alice = api.user("cur") + chat = api.chat(alice) + for cursor in bad_cursors(chat["id"]) + bad_cursors(alice["id"]): + quoted = cursor if cursor in ("", "%20") else cursor.replace(" ", "%20") + for path in (f"/api/v1/chats/{chat['id']}/messages", "/api/v1/chats"): + error = api.call("GET", f"{path}?cursor={quoted}", 400, token=alice["token"]).json() + assert error["error"]["code"] == "invalid_request", (path, cursor) + + +@pytest.mark.parametrize("raw", [b"\xff", b'{"text":"first","TEXT":"second"}', b'{"text":"a","text":"b"}', + b'{"text":123}', b'{"text":"valid"}}', b'{"text":"valid"}]', b'{"text":"x"} {}', + b'{"text":"x","sender_id":"11111111-1111-4111-8111-111111111111"}', + b'{"text":"x","attachment_ids":null}', b'{"text":"x","attachment_ids":["nope"]}', + b'{"text":" \n\t "}']) +def test_bad_message_body_is_400_and_nothing_is_stored(api, raw): + alice = api.user("body") + chat = api.chat(alice) + error = api.post_message(alice, chat, None, 400, raw=raw).json() + assert error["error"]["code"] == "invalid_request" + assert api.history(alice, chat, 100)[0] == [] + + +def test_ff_byte_body_on_registration_is_contract_error(api): + response = api.call("POST", "/api/v1/users", 400, raw=b"\xff", headers={"Content-Type": "application/json"}) + assert response.json()["error"]["request_id"] == response.headers["x-request-id"] + + +def test_text_is_stored_verbatim_and_length_counts_raw_text(api): + alice = api.user("verbatim") + chat = api.chat(alice) + padded = " строка\nс пробелами " + assert api.send(alice, chat, padded)["text"] == padded + api.post_message(alice, chat, {"text": " " + "a" * 4000}, 400) # 4001 code points before any trimming + assert [m["text"] for m in api.history(alice, chat, 100)[0]] == [padded] + + +def test_unknown_attachment_ids_are_404(api): + alice = api.user("att") + chat = api.chat(alice) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())]}, 404) + api.post_message(alice, chat, {"text": "x", "attachment_ids": [str(uuid.uuid4())] * 2}, 400) # duplicates + assert api.history(alice, chat, 100)[0] == [] + + +def test_member_ids_are_compared_as_uuids_not_strings(api): + alice, bob = api.user("case"), api.user("case") + api.call("POST", "/api/v1/chats", 400, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"], alice["id"].upper()]}) + api.call("POST", "/api/v1/chats", 400, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"], bob["id"]], "id": str(uuid.uuid4())}) + chat = api.call("POST", "/api/v1/chats", 201, token=alice["token"], + json_body={"title": "x", "member_ids": [alice["id"].upper(), bob["id"]]}, schema="Chat").json() + assert chat["member_ids"] == [alice["id"], bob["id"]] + + +@pytest.mark.parametrize("header", ["Bearer é", "Bearer .", "Bearer a b", "Bearer " + "é" * 43, "bearer\t" + "a" * 43]) +def test_non_ascii_or_garbage_bearer_is_401_not_500(api, header): + api.call("GET", "/api/v1/users/me", 401, headers={"Authorization": header.encode("utf-8")}) + + +def test_login_with_a_body_is_rejected(api): + actor = api.register("login_body") + auth = {"Authorization": basic(actor["username"], actor["password"])} + api.call("POST", "/api/v1/auth/sessions", 400, raw=b'{"extra":1}', headers={**auth, "Content-Type": "application/json"}) + api.call("POST", "/api/v1/auth/sessions", 400, raw=b"x", headers=auth) + api.call("POST", "/api/v1/auth/sessions", 201, headers=auth, schema="Session") + + +def test_password_range_is_12_to_128_code_points(api): + api.call("POST", "/api/v1/users", 400, json_body={"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "x", "password": "a"}) + actor = api.user("pw128", password="я:" * 64) + api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) + + +def test_every_api_response_carries_instance_and_request_id(api): + """X-Instance-Id also on 404 of unknown routes, 405, 401, 400 and health (Api.call asserts it).""" + api.call("GET", "/health/live", 200) + api.call("GET", "/health/ready", 200) + api.call("GET", "/no/such/route", 404) + api.call("PUT", "/api/v1/users", 405) + api.call("GET", "/api/v1/users/me", 401) + response = api.call("GET", "/api/v1/users/me", 401, headers={"X-Request-Id": "client-supplied.id-1"}) + assert response.headers["x-request-id"] == "client-supplied.id-1" # one id from LB log to API log diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 00000000..edf9dd6f --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,298 @@ +"""Load balancer, replicas and graceful shutdown (lab 3, levels 3 and 4). + +Each test that stops, kills, rescales or isolates a replica restores the stand in `finally` +and waits until HAProxy reports every replica UP again (conftest.wait_stand). +""" +from __future__ import annotations + +import contextlib +import json +import subprocess +import threading +import time +import uuid + +import httpx2 as httpx +import pytest + +from conftest import BASE_URL, REPLICAS, Api, compose, lbctl, probe_inside, psql, replica_stopped, run_hook, \ + running_replicas, wait_healthy, wait_stand + +pytestmark = pytest.mark.stack + + +def instances_seen(api: Api, token: str, requests: int = 40) -> list[str]: + return [api.call("GET", "/api/v1/users/me", 200, token=token).headers["x-instance-id"] for _ in range(requests)] + + +# ------------------------------------------------------------------ level 3: replicas, no affinity +def test_one_session_is_served_by_every_real_replica(api): + actor = api.user("rr") + seen = instances_seen(api, actor["token"]) + running = {r["hostname"] for r in running_replicas()} + assert len(running) == REPLICAS >= 2 + assert set(seen) == running, "X-Instance-Id values must be exactly the hostnames of the running containers" + # Round robin on ONE keep-alive client connection: consecutive requests switch replicas, + # so there is no sticky session hiding split state. + switches = sum(1 for a, b in zip(seen, seen[1:]) if a != b) + assert switches >= len(seen) // 2, seen + assert {s["instance_id"] for s in lbctl.lb_servers() if s["state"] == "UP"} == running + + +def test_write_on_one_replica_is_read_on_the_other(api): + alice, bob = api.user("rw"), api.user("rw") + chat = api.chat(alice, bob) + for i in range(10): + sent = api.post_message(alice, chat, {"text": f"m{i}"}) + read = api.call("GET", f"/api/v1/chats/{chat['id']}/messages?limit=100", 200, token=bob["token"]) + assert read.json()["items"][-1] == sent.json() + logout = api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + for _ in range(6): # the revocation is visible on every replica at once (shared Valkey) + r = api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) + assert r.headers["x-instance-id"] + assert logout.headers["x-instance-id"] + + +# ------------------------------------------------------------------ readiness drives the LB +def hba_file() -> str: + return psql("SHOW hba_file") + + +@contextlib.contextmanager +def isolated_from_postgres(replica: dict): + """Make ONE replica unable to reach PostgreSQL (pg_hba reject for its IP + terminate its + connections), while it keeps running and answering /health/live. Always undone.""" + ip = replica["ips"][0] + path = hba_file() + marker = "lab3-test-isolate" + compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '1i host all all {ip}/32 reject # {marker}' {path}") + try: + psql("SELECT pg_reload_conf()") + psql(f"SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '{ip}'") + yield replica + finally: + compose("exec", "-T", "postgres", "sh", "-c", f"sed -i '/{marker}/d' {path}") + psql("SELECT pg_reload_conf()") + wait_stand() + + +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_lb_stops_routing_to_a_replica_that_is_alive_but_not_ready(api, position): + """The reviewer's scenario: one API loses PostgreSQL but keeps running. Its /health/live is + 200, /health/ready 503 -> HAProxy must take it out; no business request may reach it.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + actor = api.user("ready") + with isolated_from_postgres(replica): + started = time.monotonic() + while replica["hostname"] in lbctl.up_instances(): + assert time.monotonic() - started < 5, "HAProxy did not mark the unready replica DOWN within 5 s" + time.sleep(0.1) + detected = time.monotonic() - started + assert probe_inside(replica, "/health/live") == 200 # the process is alive ... + assert probe_inside(replica, "/health/ready") == 503 # ... but not ready + seen = instances_seen(api, actor["token"], 40) + assert replica["hostname"] not in seen and len(seen) == 40 + for i in range(20): # writes keep working, only through the ready replicas + assert api.register(f"r{i}")["id"] + print(f"{replica['name']}: unready replica out of rotation after {detected:.2f} s") + + +# ------------------------------------------------------------------ stopping replicas under traffic +class Traffic: + """Background writers/readers through the LB; uncertain POSTs are retried with the SAME key.""" + + def __init__(self, api: Api, workers: int = 4): + self.alice, self.bob = api.user("load"), api.user("load") + self.chat = api.chat(self.alice, self.bob) + self.workers = workers + self.stop = threading.Event() + self.lock = threading.Lock() + self.events: list[tuple[float, str, int, str | None]] = [] # (time, kind, status, instance) + self.confirmed: list[dict] = [] + self.threads = [threading.Thread(target=self.run, args=(i,), daemon=True) for i in range(workers)] + + def record(self, kind: str, status: int, instance: str | None) -> None: + with self.lock: + self.events.append((time.monotonic(), kind, status, instance)) + + def run(self, index: int) -> None: + client = httpx.Client(base_url=BASE_URL, timeout=3) + auth = {"Authorization": "Bearer " + self.alice["token"]} + n = 0 + key, text = uuid.uuid4().hex, f"w{index}-0" + while not self.stop.is_set(): + try: + r = client.post(f"/api/v1/chats/{self.chat['id']}/messages", json={"text": text}, + headers={**auth, "Idempotency-Key": key}) + self.record("POST", r.status_code, r.headers.get("x-instance-id")) + if r.status_code == 201: + with self.lock: + self.confirmed.append(r.json()) + n += 1 + key, text = uuid.uuid4().hex, f"w{index}-{n}" + # any other outcome: keep key and text, retry the same request + except httpx.HTTPError: + self.record("POST", 0, None) + try: + r = client.get(f"/api/v1/chats/{self.chat['id']}/messages?limit=5", headers=auth) + self.record("GET", r.status_code, r.headers.get("x-instance-id")) + except httpx.HTTPError: + self.record("GET", 0, None) + time.sleep(0.02) + client.close() + + def __enter__(self) -> "Traffic": + for t in self.threads: + t.start() + time.sleep(1.0) + return self + + def __exit__(self, *exc) -> None: + self.stop.set() + for t in self.threads: + t.join(timeout=10) + + def failures(self, since: float = 0.0) -> list[tuple]: + return [e for e in self.events if e[0] >= since and e[2] not in (200, 201)] + + def check_history(self, api: Api) -> None: + history = api.history(self.bob, self.chat, 100)[0] + ids, texts = [m["id"] for m in history], [m["text"] for m in history] + assert len(ids) == len(set(ids)) and len(texts) == len(set(texts)), "duplicate after retry" + assert {m["id"] for m in self.confirmed} <= set(ids), "a confirmed message was lost" + + +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_graceful_stop_of_each_replica_loses_no_request(api, position): + """SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit 0: not one failed request.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + with Traffic(api) as traffic: + started = time.monotonic() + with replica_stopped(replica, "stop"): + stopped_after = time.monotonic() - started + time.sleep(3) + after = [e for e in traffic.events if e[0] > time.monotonic() - 2] + assert after and replica["hostname"] not in {e[3] for e in after} + time.sleep(1) + state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True, + check=True).stdout)[0]["State"] + assert traffic.failures() == [], traffic.failures()[:10] + assert stopped_after < 15, "graceful stop must finish before stop_grace_period (no SIGKILL)" + assert state["Status"] == "running" # restored by replica_stopped + traffic.check_history(api) + print(f"{replica['name']}: stop took {stopped_after:.1f} s, {len(traffic.events)} requests, 0 failed") + + +@pytest.mark.parametrize("position", range(REPLICAS)) +def test_killed_replica_bounded_errors_recovery_and_no_duplicates(api, position): + """SIGKILL: requests in flight on that replica may fail (counted), new ones succeed within + 10 s, the killed replica gets nothing after detection, retries do not duplicate.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[position] + with Traffic(api) as traffic: + killed_at = time.monotonic() + with replica_stopped(replica, "kill"): + time.sleep(4) + failures = traffic.failures(killed_at) + last_failure = max((e[0] for e in failures), default=killed_at) - killed_at + post_ok = [e for e in traffic.events if e[0] > killed_at and e[1] == "POST" and e[2] == 201] + assert post_ok and post_ok[0][0] - killed_at <= 10, "writes did not recover within 10 s" + late = [e for e in traffic.events if e[0] > killed_at + 3.5] + assert late and replica["hostname"] not in {e[3] for e in late} + assert last_failure <= 10 + traffic.check_history(api) + print(f"{replica['name']}: killed; {len(failures)} failed of {len(traffic.events)}, last failure +{last_failure:.2f} s") + + +def test_graceful_shutdown_order_in_logs(api): + """Logs prove the order: drain_started -> (HAProxy marks DOWN) -> shutdown_requested -> exit 0.""" + replica = sorted(running_replicas(), key=lambda r: r["name"])[-1] + since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime(time.time() - 1)) + with replica_stopped(replica, "stop"): + logs = subprocess.run(["docker", "logs", "--since", since, replica["id"]], capture_output=True, + text=True, encoding="utf-8").stdout + lb_logs = compose("logs", "--no-log-prefix", "--since", since, "lb").stdout + state = json.loads(subprocess.run(["docker", "inspect", replica["id"]], capture_output=True, text=True, + check=True).stdout)[0]["State"] + events = [json.loads(line) for line in logs.splitlines() if line.startswith("{")] + names = [e["event"] for e in events] + assert names.index("drain_started") < names.index("shutdown_requested") < names.index("stopped") + t = {e["event"]: e["ts"] for e in events} + assert t["drain_started"] < t["shutdown_requested"] + assert state["ExitCode"] == 0 and not state["OOMKilled"] + assert "is DOWN" in lb_logs and ("503" in lb_logs or "Connection refused" in lb_logs) + + +# ------------------------------------------------------------------ discovery: restart and scaling +def test_full_restart_lb_rediscovers_all_replicas(api): + actor = api.user("restart") + before = {r["hostname"]: r["started_at"] for r in running_replicas()} + run_hook("restart.sh") + wait_healthy("postgres", "valkey", "api") + wait_stand() + after = {r["hostname"]: r["started_at"] for r in running_replicas()} + assert set(after) == set(before) and all(after[h] != before[h] for h in after), "every replica restarted" + assert set(instances_seen(api, actor["token"])) == set(after) # same token, both replicas, no LB restart needed + + +def test_scale_up_and_down_is_picked_up_without_errors(api): + with Traffic(api, workers=2) as traffic: + try: + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS + 1}", "api", + timeout=180) + lbctl.wait_up(REPLICAS + 1, timeout=30) + actor = api.user("scale") + assert len(set(instances_seen(api, actor["token"], 30))) == REPLICAS + 1 + finally: + compose("up", "-d", "--no-deps", "--no-recreate", "--wait", "--scale", f"api={REPLICAS}", "api", + timeout=180) + wait_stand() + time.sleep(1) + assert traffic.failures() == [], traffic.failures()[:10] + traffic.check_history(api) + + +# ------------------------------------------------------------------ level 4: deployment policy +def test_compose_policy_limits_ports_secrets_images(): + config = json.loads(compose("config", "--format", "json").stdout) + services = config["services"] + assert set(services) == {"postgres", "valkey", "migrate", "api", "lb"} + for name, svc in services.items(): + limits = svc.get("deploy", {}).get("resources", {}).get("limits", {}) + assert limits.get("cpus") and limits.get("memory"), f"{name}: CPU/RAM limits" + assert "container_name" not in svc, f"{name}: container_name prevents scaling" + if name in ("api", "migrate"): + assert "build" in svc # built from this repository's source + else: + assert "@sha256:" in svc["image"], f"{name}: image must be pinned by digest" + assert services["api"]["deploy"]["replicas"] >= 2 and not services["api"].get("ports") + published = {name: svc.get("ports", []) for name, svc in services.items() if svc.get("ports")} + assert set(published) == {"lb"} and all(p["host_ip"] == "127.0.0.1" for p in published["lb"]) + secrets = {name: sorted(s["source"] for s in svc.get("secrets", [])) for name, svc in services.items()} + assert secrets == {"postgres": ["postgres_password"], "valkey": ["valkey_password"], + "migrate": ["database_url"], "api": ["cache_url", "database_url"], "lb": []} + volumes = {name: [v["source"] for v in svc.get("volumes", []) if v["type"] == "volume"] + for name, svc in services.items()} + assert volumes == {"postgres": ["pgdata"], "valkey": ["valkeydata"], "migrate": [], "api": [], "lb": []} + + +def test_running_containers_have_limits_and_no_bypass_ports(): + ids = compose("ps", "-q").stdout.split() + for item in json.loads(subprocess.run(["docker", "inspect", *ids], capture_output=True, text=True, + check=True).stdout): + name = item["Name"] + assert item["HostConfig"]["NanoCpus"] > 0 and item["HostConfig"]["Memory"] > 0, name + bindings = item["HostConfig"]["PortBindings"] or {} + if "-lb-" in name: + assert all(b["HostIp"] == "127.0.0.1" for v in bindings.values() for b in v) + else: + assert not bindings, f"{name} publishes ports in bypass of the LB" + + +def test_migrations_ran_once_by_the_migrate_service(): + applied = psql("SELECT version FROM schema_migrations ORDER BY version").split() + assert applied == ["0001_init", "0002_idempotency_keys"] + migrate = compose("ps", "-a", "--format", "json", "migrate").stdout + rows = [json.loads(line) for line in migrate.splitlines() if line.strip()] + assert rows and all(r["State"] == "exited" and r["ExitCode"] == 0 for r in rows) + api_logs = compose("logs", "--no-log-prefix", "api").stdout + assert "migrate:" not in api_logs and "schema_migrations" not in api_logs # replicas never migrate diff --git a/student-tests/test_db.py b/student-tests/test_db.py index fd065e14..3e250c68 100644 --- a/student-tests/test_db.py +++ b/student-tests/test_db.py @@ -76,7 +76,8 @@ def test_migrations_rerun_is_a_noop(): users_before = psql("SELECT count(*) FROM users") result = compose("run", "--rm", "--no-deps", "migrate") assert "0001_init: already applied, skipped" in result.stdout - assert psql("SELECT count(*) FROM schema_migrations") == "1" + assert "0002_idempotency_keys: already applied, skipped" in result.stdout + assert psql("SELECT count(*) FROM schema_migrations") == "2" assert psql("SELECT count(*) FROM users") == users_before diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py index 0dce65a6..899d29b8 100644 --- a/student-tests/test_failures.py +++ b/student-tests/test_failures.py @@ -2,6 +2,11 @@ Every test stops/pauses a real container and restores it in `finally` (service_down). Timing bounds: DB_TIMEOUT_SECONDS=3 (client deadline), CACHE_TIMEOUT_SECONDS=0.5. + +Lab 3: requests go through HAProxy. While the outage is being detected the replicas answer +503 themselves; once every replica fails /health/ready, HAProxy answers 503 in the same +contract format (without X-Instance-Id). Both are "unavailable"; never 200 and never 401 +for a valid token. Liveness/readiness of the processes are checked inside each container. """ from __future__ import annotations @@ -10,7 +15,7 @@ import time import pytest -from conftest import ROOT, restart_services, service_down +from conftest import ROOT, probe_inside, restart_services, running_replicas, service_down pytestmark = pytest.mark.stack @@ -20,10 +25,16 @@ CACHE_BOUND = 1.6 # 2 x CACHE_TIMEOUT_SECONDS + overhead def timed(api, *args, **kwargs): started = time.monotonic() - response = api.call(*args, **kwargs) + response = api.call(*args, lb_error_ok=True, **kwargs) return response, time.monotonic() - started +def processes_alive_but_not_ready() -> None: + for replica in running_replicas(): + assert probe_inside(replica, "/health/live") == 200 + assert probe_inside(replica, "/health/ready") == 503 + + @pytest.fixture def prepared(api): alice, bob = api.user("alice"), api.user("bob") @@ -46,9 +57,9 @@ def test_valkey_failure(api, prepared, how): response, took = timed(api, "GET", "/api/v1/users/me", 503, token=alice["token"]) assert response.json()["error"]["code"] == "unavailable" and took < CACHE_BOUND, took api.call("GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"]) - api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected - api.call("GET", "/health/ready", 503) - api.call("GET", "/health/live", 200) + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access + api.call("GET", "/health/ready", 503, lb_error_ok=True) + processes_alive_but_not_ready() check_intact(api, alice, bob, chat, message) @@ -60,9 +71,9 @@ def test_postgres_failure(api, prepared, how): response, took = timed(api, "GET", path, 503, token=alice["token"]) assert response.json()["error"]["code"] == "unavailable", path assert took < DB_BOUND, (path, took) # bounded even when PostgreSQL is frozen (pause) - api.call("GET", "/api/v1/users/me", 401) # anonymous is still rejected - api.call("GET", "/health/ready", 503) - api.call("GET", "/health/live", 200) + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) # anonymous never gets access + api.call("GET", "/health/ready", 503, lb_error_ok=True) + processes_alive_but_not_ready() check_intact(api, alice, bob, chat, message) @@ -77,6 +88,7 @@ def test_locked_table_gives_bounded_503(api, prepared): time.sleep(1.5) # let the lock be taken response, took = timed(api, "GET", f"/api/v1/chats/{chat['id']}/messages", 503, token=bob["token"]) assert response.json()["error"]["code"] == "unavailable" and took < DB_BOUND, took + assert response.headers.get("x-instance-id"), "a lock is the API's own 503, the replicas stay ready" finally: locker.wait(timeout=30) check_intact(api, alice, bob, chat, message) @@ -88,7 +100,7 @@ def test_logout_during_valkey_outage_is_never_lost(api, prepared): after recovery logout works, and the revocation survives a Valkey and API restart (AOF).""" alice, bob, chat, message = prepared with service_down("valkey", "stop"): - api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"]) + api.call("DELETE", "/api/v1/auth/sessions/current", 503, token=alice["token"], lb_error_ok=True) api.call("GET", "/api/v1/users/me", 200, token=alice["token"]) # logout did not happen api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) api.call("GET", "/api/v1/users/me", 401, token=alice["token"]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..a1e2bccc --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,281 @@ +"""Shared idempotency of POST /chats/{id}/messages (lab 3), through the load balancer. + +Strict assertions: exact HTTP codes for every request, identical bodies for replays, +the number of rows in PostgreSQL (messages AND idempotency_keys), and which replica +answered (X-Instance-Id). Fault injection (trigger, row lock, stopped replica) is always +undone in `finally`. +""" +from __future__ import annotations + +import subprocess +import threading +import time +import uuid + +import httpx2 as httpx +import pytest + +from conftest import BASE_URL, ROOT, Api, psql, replica_stopped, running_replicas + +pytestmark = pytest.mark.stack + + +def messages_in_db(chat: dict) -> int: + return int(psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}'")) + + +def key_rows(actor: dict, key: str) -> int: + return int(psql(f"SELECT count(*) FROM idempotency_keys WHERE user_id = '{actor['id']}' AND key = '{key}'")) + + +@pytest.fixture +def pair(api): + alice, bob = api.user("alice"), api.user("bob") + return alice, bob, api.chat(alice, bob) + + +def test_replay_returns_the_same_201_and_message_from_every_replica(api, pair): + alice, bob, chat = pair + key = "replay-" + uuid.uuid4().hex + first = api.post_message(alice, chat, {"text": "один раз"}, key=key) + served = {first.headers["x-instance-id"]} + for _ in range(20): + again = api.post_message(alice, chat, {"text": "один раз"}, key=key) + assert again.status_code == 201 and again.json() == first.json() + served.add(again.headers["x-instance-id"]) + assert len(served) >= 2, "replays must be answered by different replicas, not a local cache" + assert api.history(bob, chat, 100)[0] == [first.json()] + assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_other_body_with_the_same_key_is_409_and_keeps_the_original(api, pair): + alice, bob, chat = pair + key = uuid.uuid4().hex + original = api.send(alice, chat, "оригинал", key=key) + for body in ({"text": "другой текст"}, {"text": "оригинал "}, {"text": "Оригинал"}): + assert api.post_message(alice, chat, body, 409, key=key).json()["error"]["code"] == "conflict" + assert api.send(alice, chat, "оригинал", key=key) == original + assert api.history(bob, chat, 100)[0] == [original] and key_rows(alice, key) == 1 + + +def test_key_order_and_missing_attachment_ids_are_the_same_body(api, pair): + alice, _, chat = pair + key = uuid.uuid4().hex + first = api.post_message(alice, chat, None, raw='{"text":"порядок"}'.encode(), key=key).json() + for raw in ('{"attachment_ids":[],"text":"порядок"}', '{"text":"порядок","attachment_ids":[]}', + '{ "text" : "\\u043f\\u043e\\u0440\\u044f\\u0434\\u043e\\u043a" }'): + assert api.post_message(alice, chat, None, raw=raw.encode(), key=key).json() == first + assert messages_in_db(chat) == 1 + + +@pytest.mark.parametrize("value", ["", "a" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "*"]) +def test_missing_or_invalid_key_is_400(api, pair, value): + alice, _, chat = pair + raw_value = value.encode("utf-8") + error = api.call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + json_body={"text": "x"}, headers={"Idempotency-Key": raw_value} if value else {}).json() + assert error["error"]["code"] == "invalid_request" + assert messages_in_db(chat) == 0 + + +def test_repeated_key_header_is_400(pair): + alice, _, chat = pair + response = httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "x"}, timeout=10, + headers=[("Authorization", "Bearer " + alice["token"]), + ("Idempotency-Key", "one"), ("Idempotency-Key", "two")]) + assert response.status_code == 400 and response.headers["x-instance-id"] + assert messages_in_db(chat) == 0 + + +def test_longest_key_and_every_allowed_character(api, pair): + alice, _, chat = pair + for key in ("k" * 128, "A-Za-z0-9._:-" + uuid.uuid4().hex): + first = api.send(alice, chat, key[:20], key=key) + assert api.send(alice, chat, key[:20], key=key) == first + assert messages_in_db(chat) == 2 + + +def test_scope_is_user_plus_chat_plus_key(api, pair): + alice, bob, chat = pair + other = api.chat(alice, bob, title="второй") + key = "shared-key" + a1 = api.send(alice, chat, "текст", key=key) + b1 = api.send(bob, chat, "текст", key=key) # another user: independent + a2 = api.send(alice, other, "текст", key=key) # another chat: independent + assert len({a1["id"], b1["id"], a2["id"]}) == 3 + assert messages_in_db(chat) == 2 and messages_in_db(other) == 1 + + +def test_outsider_and_revoked_session_never_see_the_stored_result(api, pair): + alice, bob, chat = pair + eve = api.user("eve") + key = uuid.uuid4().hex + api.send(alice, chat, "секрет", key=key) + for _ in range(3): # a replay checks access again: always 404, never 409 or the stored message + assert api.post_message(eve, chat, {"text": "секрет"}, 404, key=key).json()["error"]["code"] == "not_found" + api.post_message(eve, chat, {"text": "другое"}, 404, key=key) + api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + api.post_message(alice, chat, {"text": "секрет"}, 401, key=key) # session checked again + assert messages_in_db(chat) == 1 + + +def fire_together(workers: int, request) -> list: + barrier = threading.Barrier(workers) + results: list = [None] * workers + + def body(index: int) -> None: + client = httpx.Client(base_url=BASE_URL, timeout=15) # own connection -> LB spreads them + barrier.wait() + try: + results[index] = request(client, index) + except BaseException as exc: # noqa: BLE001 - asserted by the test + results[index] = exc + finally: + client.close() + + threads = [threading.Thread(target=body, args=(i,)) for i in range(workers)] + for t in threads: + t.start() + for t in threads: + t.join() + return results + + +def test_concurrent_same_key_through_different_replicas_creates_exactly_one(pair): + alice, bob, chat = pair + key = "race-" + uuid.uuid4().hex + + def post(client: httpx.Client, _index: int): + r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": "гонка"}, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + return r.status_code, r.headers.get("x-instance-id"), r.json() + + for _ in range(3): # three rounds: the first creates, later rounds replay concurrently + results = fire_together(24, post) + assert all(isinstance(r, tuple) for r in results), results + assert [r[0] for r in results] == [201] * 24, [r[0] for r in results] + assert len({str(r[2]) for r in results}) == 1, "every concurrent replay must return the same Message" + assert len({r[1] for r in results}) >= 2, "requests did not reach two replicas" + assert messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_concurrent_same_key_different_bodies_one_wins(pair): + alice, _, chat = pair + key = "conflict-" + uuid.uuid4().hex + + def post(client: httpx.Client, index: int): + r = client.post(f"/api/v1/chats/{chat['id']}/messages", json={"text": f"вариант {index % 2}"}, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + return r.status_code, r.json() + + results = fire_together(20, post) + winners = [body for status, body in results if status == 201] + losers = [body for status, body in results if status == 409] + assert len(winners) + len(losers) == 20, results + assert winners and len({str(b) for b in winners}) == 1 + assert all(b["error"]["code"] == "conflict" for b in losers) + assert {b["text"] for b in winners} <= {"вариант 0", "вариант 1"} + assert len(losers) == sum(1 for i in range(20) if f"вариант {i % 2}" != winners[0]["text"]) + assert messages_in_db(chat) == 1 + + +def inserts_into_keys() -> int: + return int(psql("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'idempotency_keys'")) + + +def test_failure_after_the_key_insert_rolls_everything_back(api, pair): + """A trigger makes the MESSAGE insert fail AFTER the key row was inserted in the same + transaction. Expected: 500 for the client, no key row and no message left behind + (n_tup_ins proves the key INSERT really ran), and a retry with the same key succeeds.""" + alice, bob, chat = pair + marker = "boom-" + uuid.uuid4().hex + key = uuid.uuid4().hex + psql("CREATE OR REPLACE FUNCTION lab3_test_fail() RETURNS trigger LANGUAGE plpgsql AS $$ " + f"BEGIN IF NEW.text = convert_to('{marker}', 'UTF8') THEN RAISE EXCEPTION 'injected failure'; END IF; " + "RETURN NEW; END $$; " + "CREATE TRIGGER lab3_test_fail BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION lab3_test_fail()") + try: + before = inserts_into_keys() + error = api.post_message(alice, chat, {"text": marker}, 500, key=key) + assert error.json()["error"]["code"] == "internal_error" and "injected" not in error.text + assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0 + deadline = time.monotonic() + 20 # table statistics are flushed asynchronously + while inserts_into_keys() == before and time.monotonic() < deadline: + time.sleep(0.5) + assert inserts_into_keys() > before, "the key INSERT never ran before the failure" + finally: + psql("DROP TRIGGER IF EXISTS lab3_test_fail ON messages; DROP FUNCTION IF EXISTS lab3_test_fail()") + retried = api.send(alice, chat, marker, key=key) # transient failure was not recorded + assert api.send(alice, chat, marker, key=key) == retried + assert api.history(bob, chat, 100)[0] == [retried] and key_rows(alice, key) == 1 + + +def lock_chat_row(chat: dict, seconds: float) -> subprocess.Popen: + sql = (f"BEGIN; SELECT 1 FROM chats WHERE id = '{chat['id']}' FOR UPDATE; " + f"SELECT pg_sleep({seconds}); COMMIT;") + return subprocess.Popen(["docker", "compose", "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", + "messenger", "-c", sql], cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def wait_lock_taken(chat: dict) -> None: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + if psql("SELECT count(*) FROM pg_stat_activity WHERE query LIKE '%pg_sleep%' AND state = 'active' " + "AND pid <> pg_backend_pid()") != "0": + return + time.sleep(0.2) + raise AssertionError("the row lock was not taken") + + +def test_client_timeout_then_retry_with_the_same_key_gives_the_committed_message(api, pair): + """Uncertain outcome: the client gives up after 0.4 s, the server commits later (the row + lock is held 1 s < lock_timeout). The retry must return THAT message, not a second one.""" + alice, bob, chat = pair + key = uuid.uuid4().hex + locker = lock_chat_row(chat, 1.0) + try: + wait_lock_taken(chat) + with pytest.raises(httpx.TimeoutException): + httpx.post(BASE_URL + f"/api/v1/chats/{chat['id']}/messages", json={"text": "неопределённо"}, timeout=0.4, + headers={"Authorization": "Bearer " + alice["token"], "Idempotency-Key": key}) + finally: + locker.wait(timeout=30) + deadline = time.monotonic() + 10 + while messages_in_db(chat) == 0 and time.monotonic() < deadline: + time.sleep(0.2) + stored = psql(f"SELECT id FROM messages WHERE chat_id = '{chat['id']}'") + assert stored, "the server-side transaction was expected to commit after the lock was released" + retried = api.send(alice, chat, "неопределённо", key=key) + assert retried["id"] == stored and messages_in_db(chat) == 1 and key_rows(alice, key) == 1 + + +def test_lock_timeout_is_a_transient_503_that_is_not_recorded(api, pair): + """The row lock outlives lock_timeout: the API answers 503 within its deadline and stores + nothing; after the lock is gone the same key creates the message exactly once.""" + alice, bob, chat = pair + key = uuid.uuid4().hex + locker = lock_chat_row(chat, 4.0) + try: + wait_lock_taken(chat) + started = time.monotonic() + api.post_message(alice, chat, {"text": "после блокировки"}, 503, key=key) + assert time.monotonic() - started < 3.6 + assert key_rows(alice, key) == 0 and messages_in_db(chat) == 0 + finally: + locker.wait(timeout=30) + created = api.send(alice, chat, "после блокировки", key=key) + assert api.send(alice, chat, "после блокировки", key=key) == created + assert api.history(bob, chat, 100)[0] == [created] and key_rows(alice, key) == 1 + + +def test_replay_after_the_serving_replica_is_stopped(api, pair): + alice, bob, chat = pair + key = uuid.uuid4().hex + first = api.post_message(alice, chat, {"text": "до остановки"}, key=key) + served_by = first.headers["x-instance-id"] + replica = next(r for r in running_replicas() if r["hostname"] == served_by) + with replica_stopped(replica): + for _ in range(5): + again = Api().post_message(alice, chat, {"text": "до остановки"}, key=key) + assert again.json() == first.json() and again.headers["x-instance-id"] != served_by + assert messages_in_db(chat) == 1 diff --git a/student-tests/test_sessions.py b/student-tests/test_sessions.py index 245db765..8cbda00f 100644 --- a/student-tests/test_sessions.py +++ b/student-tests/test_sessions.py @@ -57,8 +57,9 @@ def test_postgres_session_store(api): restart_services("api") api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) with service_down("postgres", "stop"): - assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"]).json()["error"]["code"] == "unavailable" - api.call("GET", "/api/v1/users/me", 401) + assert api.call("GET", "/api/v1/users/me", 503, token=actor["token"], + lb_error_ok=True).json()["error"]["code"] == "unavailable" + api.call("GET", "/api/v1/users/me", (401, 503), lb_error_ok=True) api.call("GET", "/api/v1/users/me", 200, token=actor["token"]) api.call("DELETE", "/api/v1/auth/sessions/current", 204, token=actor["token"]) api.call("GET", "/api/v1/users/me", 401, token=actor["token"]) diff --git a/student-tests/unit/test_unit.py b/student-tests/unit/test_unit.py index 34612867..ebbc5b7d 100644 --- a/student-tests/unit/test_unit.py +++ b/student-tests/unit/test_unit.py @@ -132,3 +132,60 @@ def test_settings_read_secret_files_and_contract_names(tmp_path, monkeypatch): with pytest.raises(ConfigError) as info: Settings.from_env() assert "u:p" not in str(info.value) + + +# ------------------------------------------------------------------ lab 3: idempotency, JSON, instance id +@pytest.mark.parametrize("key", ["a", "k" * 128, "A-Za-z0-9._:-", "2026-10-16T10:00:00.1"]) +def test_valid_idempotency_keys(key): + assert validation.idempotency_key(key) == key + + +@pytest.mark.parametrize("key", [None, "", "k" * 129, "ключ", "a b", "a/b", "a,b", "a\tb", "a\x00b", "*", "é"]) +def test_invalid_idempotency_keys(key): + with pytest.raises(InvalidRequest): + validation.idempotency_key(key) + + +@pytest.mark.parametrize("raw", [b'{"text":"a","text":"b"}', b'{"a":{"x":1,"x":2}}', b'\xff', b'{"text":"x"} {}']) +def test_duplicate_keys_and_garbage_json_are_rejected(raw): + with pytest.raises(InvalidRequest): + validation.parse_json_object(raw) + + +def test_attachment_ids_absent_equals_empty_and_is_validated(): + assert validation.attachment_ids_field({"text": "x"}) == [] + assert validation.attachment_ids_field({"attachment_ids": []}) == [] + one = "11111111-1111-4111-8111-111111111111" + assert validation.attachment_ids_field({"attachment_ids": [one.upper()]}) == [one] + for bad in (None, "x", [1], ["nope"], [one, one.upper()], [one] * 11): + with pytest.raises(InvalidRequest): + validation.attachment_ids_field({"attachment_ids": bad}) + + +def test_request_fingerprint_ignores_key_order_and_default_attachments(): + from messenger.app import request_fingerprint + assert request_fingerprint("привет", []) == request_fingerprint("привет", []) + assert request_fingerprint("привет", []) != request_fingerprint("привет ", []) + assert request_fingerprint("a\u0000b", []) != request_fingerprint("ab", []) + one = "11111111-1111-4111-8111-111111111111" + assert request_fingerprint("x", [one]) != request_fingerprint("x", []) + + +def test_instance_id_defaults_to_hostname_and_is_validated(monkeypatch): + import socket + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + monkeypatch.delenv("INSTANCE_ID", raising=False) + assert Settings.from_env().instance_id == socket.gethostname() + monkeypatch.setenv("INSTANCE_ID", "replica-a") + assert Settings.from_env().instance_id == "replica-a" + for bad in ("with space", "x" * 65, "ip/1"): + monkeypatch.setenv("INSTANCE_ID", bad) + with pytest.raises(ConfigError): + Settings.from_env() + + +def test_idempotency_ttl_cannot_be_below_24_hours(monkeypatch): + monkeypatch.setenv("DATABASE_URL", "postgresql://x@y/z") + monkeypatch.setenv("IDEMPOTENCY_TTL_SECONDS", "3600") + with pytest.raises(ConfigError): + Settings.from_env() -- 2.54.0 From c050642ec25104f3e9445ec5d8f8d7876a686c0d Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:39:54 +0300 Subject: [PATCH 04/18] =?UTF-8?q?feat:=20rolling=20update=20=D0=B1=D0=B5?= =?UTF-8?q?=D0=B7=20=D0=BF=D1=80=D0=BE=D1=81=D1=82=D0=BE=D1=8F=20=D0=B8=20?= =?UTF-8?q?=D1=81=D0=B5=D1=80=D0=B8=D1=8F=20=D0=BE=D1=82=D0=BA=D0=B0=D0=B7?= =?UTF-8?q?=D0=BE=D0=B2=20=D1=80=D0=B5=D0=BF=D0=BB=D0=B8=D0=BA=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=B4=20=D0=BD=D0=B0=D0=B3=D1=80=D1=83=D0=B7=D0=BA=D0=BE?= =?UTF-8?q?=D0=B9=20(=D1=83=D1=80=D0=BE=D0=B2=D0=B5=D0=BD=D1=8C=204)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scripts/rolling-update.sh: сборка, однократные миграции новым образом, затем для каждой старой реплики: +1 новая (healthy и UP в HAProxy), graceful stop старой; ревизия образа видна в label и логе "starting". - bench/loadgen.py: фоновая нагрузка через LB (POST с Idempotency-Key + GET), неопределённый POST повторяется тем же ключом, проверка истории на потери и дубли. - bench/failover_series.py: каждая реплика по очереди в режимах stop / kill / pause; RPS, ошибки, время выхода из ротации и возврата. - bench/rolling_under_load.py: rolling update под нагрузкой. Co-Authored-By: Claude Opus 5.5 --- Dockerfile | 6 + bench/failover_series.py | 141 ++++++++++++++++++++++++ bench/loadgen.py | 212 ++++++++++++++++++++++++++++++++++++ bench/rolling_under_load.py | 75 +++++++++++++ compose.yaml | 10 +- scripts/rolling-update.sh | 37 +++++++ src/messenger/__main__.py | 3 +- 7 files changed, 481 insertions(+), 3 deletions(-) create mode 100644 bench/failover_series.py create mode 100644 bench/loadgen.py create mode 100644 bench/rolling_under_load.py create mode 100755 scripts/rolling-update.sh diff --git a/Dockerfile b/Dockerfile index 12bb32c9..f46ac0be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,12 @@ ENV PYTHONPATH=/app/src \ SESSION_TTL_SECONDS=3600 \ DATABASE_URL_FILE=/run/secrets/database_url +# Build revision (scripts/rolling-update.sh passes git commit + time): makes every rolled-out +# image distinguishable in `docker inspect` and in the "starting" log line of each replica. +ARG BUILD_REVISION=dev +LABEL org.opencontainers.image.revision=$BUILD_REVISION +ENV APP_REVISION=$BUILD_REVISION + USER app:app EXPOSE 8080 STOPSIGNAL SIGTERM diff --git a/bench/failover_series.py b/bench/failover_series.py new file mode 100644 index 00000000..b4c5dafa --- /dev/null +++ b/bench/failover_series.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Level 4: stop EACH API replica in turn under load (restoring the previous one first) and +measure RPS, errors and recovery. Modes: graceful `stop` (SIGTERM + drain), `kill` (SIGKILL) +and `pause` (frozen process: the kernel still accepts TCP, nothing answers - the worst case +for detection, only the active health check can find it). + + python bench/failover_series.py --output evidence/failover # ~5 min for 2 replicas x 3 modes + +Timeline of one round (all times from the moment the stop command is issued): + warm-up 6 s -> stop replica -> 12 s with it down -> start it again -> wait until HAProxy + has every replica UP (rejoin time) -> 6 s steady state. +Measured per round: requests/ok/failed and ok-RPS before / during / after, the last failed +request (+s), the first moment after which the stopped replica never answered again +(= removal from rotation), time until the restarted replica is UP in HAProxy and serves +traffic again, plus the history check (no confirmed message lost, no duplicate). +Output: /series.json (summary), /timeline--.csv (per second), +/series.md (table for REPORT.md). No tokens are written. +""" +from __future__ import annotations + +import argparse +import csv +import datetime as dt +import json +import platform +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "bench")) +import lbctl # noqa: E402 +from loadgen import Load # noqa: E402 + + +def docker(*args: str) -> str: + return subprocess.run(["docker", *args], capture_output=True, text=True, check=True, timeout=120).stdout + + +def one_round(replica: dict, mode: str, replicas: int, workers: int, out: Path) -> dict: + load = Load(workers).start() + time.sleep(6) + t_stop = load.now() + if mode == "pause": + docker("pause", replica["id"]) + else: + docker("stop", "--time", "15" if mode == "stop" else "0", replica["id"]) + t_stopped = load.now() + time.sleep(12) + t_start = load.now() + docker("unpause" if mode == "pause" else "start", replica["id"]) + lbctl.wait_up(replicas, timeout=90) + t_up = load.now() + # The restarted container keeps its hostname: wait until it serves requests again. + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if any(e[3] == replica["hostname"] and e[0] > t_up for e in load.events): + break + time.sleep(0.1) + t_serves = load.now() + time.sleep(6) + load.stop() + t_end = load.now() + + failed_after_stop = [e for e in load.events if e[0] >= t_stop and e[2] not in (200, 201)] + served_by_target = [e[0] for e in load.events if e[3] == replica["hostname"] and t_stop <= e[0] < t_start] + out_of_rotation = (max(served_by_target) - t_stop) if served_by_target else 0.0 + rows = load.per_second() + with (out / f"timeline-{mode}-{replica['name']}.csv").open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"]) + writer.writeheader() + for row in rows: + row = dict(row) + row["instances"] = row["instances"].replace(replica["hostname"], "TARGET") + writer.writerow(row) + state = json.loads(docker("inspect", replica["id"]))[0]["State"] + return { + "mode": mode, "replica": replica["name"], "instance_id": replica["hostname"], + "stop_command_seconds": round(t_stopped - t_stop, 2), + "before": load.window(0.5, t_stop), "during": load.window(t_stop, t_start), "after": load.window(t_serves, t_end), + "failed_after_stop": len(failed_after_stop), + "failed_statuses": sorted({e[2] for e in failed_after_stop}), + "last_failure_after_stop_s": round(max((e[0] for e in failed_after_stop), default=t_stop) - t_stop, 3), + "target_last_served_after_stop_s": round(out_of_rotation, 3), + "rejoin_up_in_lb_s": round(t_up - t_start, 2), "rejoin_serving_s": round(t_serves - t_start, 2), + "client_retries_same_key": load.retries, + "history": load.verify_history(), + "restarted_state": f"{state['Status']} health={(state.get('Health') or {}).get('Status')}", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--output", default="evidence/failover") + parser.add_argument("--modes", default="stop,kill,pause") + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + replicas = sorted((r for r in lbctl.replicas() if r["state"] == "running"), key=lambda r: r["name"]) + assert len(replicas) >= 2, "need at least two running replicas" + lbctl.wait_up(len(replicas)) + results = [] + for mode in args.modes.split(","): + for replica in replicas: # each replica in turn; the previous one is already restored + print(f"== {mode} {replica['name']} ...", flush=True) + result = one_round(replica, mode, len(replicas), args.workers, out) + results.append(result) + print(json.dumps({k: result[k] for k in ("failed_after_stop", "last_failure_after_stop_s", + "target_last_served_after_stop_s", "rejoin_serving_s")}), flush=True) + meta = {"date": dt.datetime.now(dt.timezone.utc).isoformat(timespec="seconds"), + "commit": subprocess.run(["git", "rev-parse", "--short", "HEAD"], cwd=ROOT, capture_output=True, + text=True).stdout.strip(), + "host": f"{platform.system()} {platform.release()}, docker {docker('version', '--format', '{{.Server.Version}}').strip()}", + "workload": f"{args.workers} closed-loop workers from the host via 127.0.0.1:8080: POST message " + "(Idempotency-Key, uncertain -> same key) + GET last page; client timeout 3 s", + "lb": "HAProxy: check GET /health/ready inter 1s fastinter 500ms fall 2 rise 2, timeout check 1s; " + "retry-on conn-failure + redispatch", + "api": "DRAIN_SECONDS=3, SHUTDOWN_TIMEOUT_SECONDS=8, stop_grace_period 15s"} + (out / "series.json").write_text(json.dumps({"meta": meta, "rounds": results}, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8") + lines = ["| Режим | Реплика | Запросов во время отказа | Ошибок после stop | ok-RPS до / во время / после | " + "Последняя ошибка, с | Цель ушла из ротации, с | Возврат в LB / обслуживает, с | Потери / дубли |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- |"] + for r in results: + h = r["history"] + lines.append(f"| {r['mode']} | {r['replica']} | {r['during']['requests']} | {r['failed_after_stop']} " + f"{r['failed_statuses'] or ''} | {r['before']['ok_rps']} / {r['during']['ok_rps']} / {r['after']['ok_rps']} | " + f"{r['last_failure_after_stop_s']} | {r['target_last_served_after_stop_s']} | " + f"{r['rejoin_up_in_lb_s']} / {r['rejoin_serving_s']} | " + f"{h['lost_confirmed']} / {h['duplicate_ids'] + h['duplicate_texts']} |") + (out / "series.md").write_text("\n".join([f"Дата {meta['date']}, commit {meta['commit']}. {meta['workload']}.", "", + *lines, ""]), encoding="utf-8") + print("\n".join(lines)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/loadgen.py b/bench/loadgen.py new file mode 100644 index 00000000..97a94de8 --- /dev/null +++ b/bench/loadgen.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Background HTTP traffic through the LB for failover / rolling-update experiments (level 4). + +Closed loop, N worker threads, each: POST a message (Idempotency-Key) + GET the last page. +An uncertain POST (timeout, connection error, 5xx) is retried with the SAME key and body, +exactly as a correct client must do; the history is checked afterwards for losses and +duplicates. Every request is recorded as (t, kind, status, instance, latency). +This is a correctness/availability probe, not a capacity benchmark (see bench/track1 for that). + + python bench/loadgen.py --duration 30 --workers 4 # prints a summary JSON +""" +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import threading +import time +import uuid +from collections import Counter, defaultdict + +import urllib.error +import urllib.request + +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") + + +class Http: + """Minimal stdlib client with keep-alive per thread (http.client under urllib is not).""" + + def __init__(self, base_url: str, timeout: float): + import http.client + from urllib.parse import urlsplit + parts = urlsplit(base_url) + self.host, self.port = parts.hostname, parts.port or 80 + self.timeout = timeout + self.factory = lambda: http.client.HTTPConnection(self.host, self.port, timeout=self.timeout) + self.conn = self.factory() + + def request(self, method: str, path: str, body: bytes | None = None, headers: dict | None = None): + try: + self.conn.request(method, path, body=body, headers=headers or {}) + response = self.conn.getresponse() + data = response.read() + if response.getheader("connection", "").lower() == "close": + self.conn.close() + self.conn = self.factory() + return response.status, dict(response.getheaders()), data + except Exception: + self.conn.close() + self.conn = self.factory() + raise + + +def call(method: str, path: str, body=None, headers=None, timeout=10.0): + data = json.dumps(body).encode() if body is not None else None + h = {"Content-Type": "application/json", **(headers or {})} + req = urllib.request.Request(BASE_URL + path, data=data, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=timeout) as r: + return r.status, json.loads(r.read() or b"null") + except urllib.error.HTTPError as e: + return e.code, json.loads(e.read() or b"null") + + +def make_user(label: str) -> dict: + username = f"{label}_{uuid.uuid4().hex[:12]}" + password = "Load-password-" + uuid.uuid4().hex + status, user = call("POST", "/api/v1/users", {"username": username, "display_name": label, "password": password}) + assert status == 201, (status, user) + cred = base64.b64encode(f"{username}:{password}".encode()).decode() + status, session = call("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) + assert status == 201, (status, session) + return {"id": user["id"], "token": session["token"]} + + +class Load: + def __init__(self, workers: int = 4, timeout: float = 3.0, pause: float = 0.0): + self.workers = workers + self.timeout = timeout + self.pause = pause + self.alice, self.bob = make_user("load"), make_user("load") + status, chat = call("POST", "/api/v1/chats", {"title": "load", "member_ids": [self.alice["id"], self.bob["id"]]}, + headers={"Authorization": "Bearer " + self.alice["token"]}) + assert status == 201, chat + self.chat = chat + self.stop_event = threading.Event() + self.lock = threading.Lock() + self.events: list[tuple[float, str, int, str, float]] = [] + self.confirmed: dict[str, dict] = {} + self.retries = 0 + self.t0 = time.monotonic() + self.threads = [threading.Thread(target=self._worker, args=(i,), daemon=True) for i in range(workers)] + + def now(self) -> float: + return time.monotonic() - self.t0 + + def _record(self, kind: str, status: int, instance: str, latency: float) -> None: + with self.lock: + self.events.append((round(self.now(), 4), kind, status, instance, round(latency * 1000, 2))) + + def _worker(self, index: int) -> None: + http = Http(BASE_URL, self.timeout) + auth = {"Authorization": "Bearer " + self.alice["token"]} + path = f"/api/v1/chats/{self.chat['id']}/messages" + n, key = 0, uuid.uuid4().hex + text = f"w{index}-0-{key[:6]}" + while not self.stop_event.is_set(): + started = time.monotonic() + try: + status, headers, data = http.request("POST", path, json.dumps({"text": text}).encode(), + {**auth, "Idempotency-Key": key, "Content-Type": "application/json"}) + instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-" + except Exception: + status, instance, data = 0, "-", b"" + self._record("POST", status, instance, time.monotonic() - started) + if status == 201: + msg = json.loads(data) + with self.lock: + self.confirmed[msg["id"]] = msg + n += 1 + key = uuid.uuid4().hex + text = f"w{index}-{n}-{key[:6]}" + elif status in (0, 502, 503, 504, 500): + with self.lock: + self.retries += 1 # same key + same text on the next iteration + started = time.monotonic() + try: + status, headers, _ = http.request("GET", path + "?limit=5", None, auth) + instance = headers.get("x-instance-id") or headers.get("X-Instance-Id") or "-" + except Exception: + status, instance = 0, "-" + self._record("GET", status, instance, time.monotonic() - started) + if self.pause: + time.sleep(self.pause) + + def start(self) -> "Load": + self.t0 = time.monotonic() + for t in self.threads: + t.start() + return self + + def stop(self) -> None: + self.stop_event.set() + for t in self.threads: + t.join(timeout=15) + + def verify_history(self) -> dict: + """Walk the whole history with the second user: no confirmed message lost, no duplicate.""" + items, cursor = [], None + while True: + q = "?limit=100" + (f"&cursor={cursor}" if cursor else "") + status, page = call("GET", f"/api/v1/chats/{self.chat['id']}/messages{q}", + headers={"Authorization": "Bearer " + self.bob["token"]}, timeout=30) + assert status == 200, page + items += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + ids = [m["id"] for m in items] + texts = [m["text"] for m in items] + lost = [i for i in self.confirmed if i not in set(ids)] + changed = [m for m in items if m["id"] in self.confirmed and self.confirmed[m["id"]] != m] + return {"history_messages": len(items), "confirmed_messages": len(self.confirmed), + "lost_confirmed": len(lost), "altered_confirmed": len(changed), + "duplicate_ids": len(ids) - len(set(ids)), "duplicate_texts": len(texts) - len(set(texts)), + "unconfirmed_but_stored": len(set(ids) - set(self.confirmed))} + + # ------------------------------------------------------------------ summaries + def window(self, start: float, end: float) -> dict: + ev = [e for e in self.events if start <= e[0] < end] + ok = [e for e in ev if e[2] in (200, 201)] + lat = sorted(e[4] for e in ok) + + def pct(p: float): + return lat[max(0, math.ceil(p * len(lat)) - 1)] if lat else None + span = max(end - start, 1e-9) + return {"seconds": round(span, 2), "requests": len(ev), "ok": len(ok), "failed": len(ev) - len(ok), + "ok_rps": round(len(ok) / span, 1), "error_ratio": round((len(ev) - len(ok)) / len(ev), 5) if ev else None, + "statuses": dict(Counter(e[2] for e in ev)), "instances": dict(Counter(e[3] for e in ok)), + "p50_ms": pct(.5), "p95_ms": pct(.95), "p99_ms": pct(.99)} + + def per_second(self) -> list[dict]: + buckets: dict[int, list] = defaultdict(list) + for e in self.events: + buckets[int(e[0])].append(e) + rows = [] + for second in sorted(buckets): + ev = buckets[second] + rows.append({"second": second, "ok": sum(1 for e in ev if e[2] in (200, 201)), + "failed": sum(1 for e in ev if e[2] not in (200, 201)), + "instances": "|".join(f"{k}:{v}" for k, v in sorted(Counter(e[3] for e in ev).items()))}) + return rows + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--duration", type=float, default=20) + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + load = Load(args.workers).start() + time.sleep(args.duration) + load.stop() + print(json.dumps({"total": load.window(0, load.now()), "history": load.verify_history(), + "client_retries_same_key": load.retries}, ensure_ascii=False, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/bench/rolling_under_load.py b/bench/rolling_under_load.py new file mode 100644 index 00000000..c4fd9ba9 --- /dev/null +++ b/bench/rolling_under_load.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Level 4: scripts/rolling-update.sh while bench/loadgen.py traffic runs through the LB. + + python bench/rolling_under_load.py --output evidence/rolling-update + +Writes /summary.json (errors, RPS, history check, revisions before/after), +/timeline.csv (per second ok/failed/instances) and /script-output.txt. +""" +from __future__ import annotations + +import argparse +import csv +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) +sys.path.insert(0, str(ROOT / "bench")) +import lbctl # noqa: E402 +from loadgen import Load # noqa: E402 + + +def revisions() -> dict[str, str]: + result = {} + for r in lbctl.replicas(): + if r["state"] == "running": + label = subprocess.run(["docker", "inspect", "-f", '{{index .Config.Labels "org.opencontainers.image.revision"}}', + r["id"]], capture_output=True, text=True).stdout.strip() + result[r["name"]] = f"{r['hostname']} revision={label}" + return result + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--output", default="evidence/rolling-update") + parser.add_argument("--workers", type=int, default=4) + args = parser.parse_args() + out = ROOT / args.output + out.mkdir(parents=True, exist_ok=True) + before = revisions() + load = Load(args.workers).start() + time.sleep(5) + t_begin = load.now() + bash = (os.path.join(os.environ.get("ProgramFiles", r"C:\Program Files"), "Git", "bin", "bash.exe") + if os.name == "nt" else "sh") + env = {**os.environ, "PYTHON": sys.executable} + script = subprocess.run([bash, str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env, + capture_output=True, text=True, encoding="utf-8", timeout=900) + t_end = load.now() + time.sleep(5) + load.stop() + (out / "script-output.txt").write_text(f"$ sh scripts/rolling-update.sh (exit {script.returncode})\n" + + script.stdout + script.stderr, encoding="utf-8") + with (out / "timeline.csv").open("w", newline="", encoding="utf-8") as f: + writer = csv.DictWriter(f, fieldnames=["second", "ok", "failed", "instances"]) + writer.writeheader() + writer.writerows(load.per_second()) + summary = {"script_exit_code": script.returncode, "update_seconds": round(t_end - t_begin, 1), + "before": before, "after": revisions(), + "during_update": load.window(t_begin, t_end), "whole_run": load.window(0, load.now()), + "client_retries_same_key": load.retries, "history": load.verify_history(), + "workload": f"{args.workers} closed-loop workers: POST (Idempotency-Key) + GET page via 127.0.0.1:8080"} + (out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps({k: summary[k] for k in ("script_exit_code", "update_seconds", "before", "after", "history")}, + ensure_ascii=False, indent=2)) + print(json.dumps(summary["during_update"], ensure_ascii=False)) + return 0 if script.returncode == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/compose.yaml b/compose.yaml index 09abcfd4..22bf9433 100644 --- a/compose.yaml +++ b/compose.yaml @@ -80,7 +80,10 @@ services: # One-shot schema migration: runs ONCE per `up`, before any API replica starts # (api depends_on service_completed_successfully). Replicas never migrate themselves. migrate: - build: . + build: + context: . + args: + BUILD_REVISION: ${BUILD_REVISION:-dev} image: mtusi-messenger:lab3 command: ["python", "-m", "messenger.migrate"] environment: @@ -97,7 +100,10 @@ services: limits: {cpus: "0.5", memory: 256m} api: - build: . + build: + context: . + args: + BUILD_REVISION: ${BUILD_REVISION:-dev} image: mtusi-messenger:lab3 environment: *api-env secrets: [database_url, cache_url] diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..5a8e4263 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Rolling update of the API without downtime (Compose has no built-in one for a scaled service). +# 1. build the new image from the working tree; +# 2. run migrations ONCE with the new image (they must be additive / compatible with the +# running old version: expand first, contract in a later release); +# 3. for every OLD replica: start one NEW replica (scale N+1, old ones untouched), wait until +# Docker says healthy AND HAProxy has N+1 replicas UP, then stop the old one gracefully +# (SIGTERM -> readiness 503 -> HAProxy DOWN -> drain -> exit) and remove it. +# At every moment at least N replicas serve traffic. Stops on the first error (set -e); +# the old replicas that were not replaced yet keep running. +set -eu +cd "$(dirname "$0")/.." +PY="${PYTHON:-$(command -v python3 || command -v python)}" +# New image = new revision label (git commit + UTC time), visible in docker inspect and logs. +BUILD_REVISION="${BUILD_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date -u +%Y%m%dT%H%M%SZ)}" +export BUILD_REVISION + +replicas="$(docker compose ps -q api | wc -l | tr -d ' ')" +[ "$replicas" -ge 1 ] || { echo "rolling-update: no running api replica" >&2; exit 2; } +old="$(docker compose ps -q api)" +echo "rolling-update: $replicas replica(s) to replace, new revision $BUILD_REVISION" + +docker compose build api +docker compose run --rm --no-deps migrate + +for id in $old; do + name="$(docker inspect -f '{{.Name}}' "$id" | tr -d '/')" + docker compose up -d --no-deps --no-recreate --wait --scale "api=$((replicas + 1))" api + "$PY" scripts/lbctl.py wait-up "$((replicas + 1))" --timeout 60 + docker stop --time 15 "$id" >/dev/null + docker rm "$id" >/dev/null + "$PY" scripts/lbctl.py wait-up "$replicas" --timeout 60 + echo "rolling-update: replaced $name" +done +for id in $(docker compose ps -q api); do + docker inspect -f '{{.Name}} revision={{index .Config.Labels "org.opencontainers.image.revision"}} {{.State.Status}}' "$id" +done diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py index 5701b666..ef22eaf7 100644 --- a/src/messenger/__main__.py +++ b/src/messenger/__main__.py @@ -13,6 +13,7 @@ A second signal during the drain skips the rest of it. Data stays in PostgreSQL from __future__ import annotations import logging +import os import sys import threading from types import FrameType @@ -75,7 +76,7 @@ def main() -> int: ) server = GracefulServer(config, settings.drain_seconds) log.info("starting", extra={"fields": { - "version": __version__, "env": settings.env, "host": settings.host, "port": settings.port, + "version": __version__, "revision": os.environ.get("APP_REVISION", "dev"), "env": settings.env, "host": settings.host, "port": settings.port, "python": sys.version.split()[0], "storage": "postgresql", "instance_id": settings.instance_id, }}) server.run() -- 2.54.0 From 2033440cdbb057951c723e8263e7048edcf17e17 Mon Sep 17 00:00:00 2001 From: Alexey Bodanyuk <144894236+wfa22@users.noreply.github.com> Date: Fri, 16 Oct 2026 13:51:37 +0300 Subject: [PATCH 05/18] =?UTF-8?q?bench:=20=D1=82=D1=80=D0=B5=D0=BA=201=20?= =?UTF-8?q?=E2=80=94=20=D0=BE=D1=80=D0=BA=D0=B5=D1=81=D1=82=D1=80=D0=B0?= =?UTF-8?q?=D1=82=D0=BE=D1=80=20=D1=81=D0=B5=D1=80=D0=B8=D0=B9,=20k6=20ope?= =?UTF-8?q?n-loop,=20py-spy=20sidecar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - bench/track1/run.py: на каждую конфигурацию x повтор — свежий стенд (отдельный compose-проект, пустые volumes), образ API из git-коммита (git archive), ступени constant-arrival-rate; CPU каждого контейнера, ожидание пула (STATS_LOG_SECONDS), pg_stat_database/pg_stat_statements, счётчики HAProxy; raw JSON + results.csv, без токенов. - compose.bench.yaml: cpuset для k6 / LB+Valkey / PostgreSQL / API, чтобы генератор и LB не отнимали CPU у реплик. - bench/track1/profile.py + pyspy/: py-spy с хешем пакета, профиль одной реплики под нагрузкой. - API: STATS_LOG_SECONDS (по умолчанию выключено) — счётчики пула и CPU процесса в лог; API_IMAGE в compose для образа из коммита. Co-Authored-By: Claude Opus 5.5 --- bench/track1/compose.bench.yaml | 30 +++ bench/track1/k6/messages.js | 67 +++++ bench/track1/profile.py | 125 ++++++++++ bench/track1/pyspy/Dockerfile | 6 + bench/track1/pyspy/requirements.txt | 3 + bench/track1/run.py | 371 ++++++++++++++++++++++++++++ compose.yaml | 4 +- src/messenger/app.py | 23 +- src/messenger/config.py | 3 + 9 files changed, 628 insertions(+), 4 deletions(-) create mode 100644 bench/track1/compose.bench.yaml create mode 100644 bench/track1/k6/messages.js create mode 100644 bench/track1/profile.py create mode 100644 bench/track1/pyspy/Dockerfile create mode 100644 bench/track1/pyspy/requirements.txt create mode 100644 bench/track1/run.py diff --git a/bench/track1/compose.bench.yaml b/bench/track1/compose.bench.yaml new file mode 100644 index 00000000..1d52cc41 --- /dev/null +++ b/bench/track1/compose.bench.yaml @@ -0,0 +1,30 @@ +# Track 1 overlay: docker compose -p mtusi-lab3-bench -f compose.yaml -f bench/track1/compose.bench.yaml ... +# Used only by bench/track1/run.py. A separate project (own volumes, own ports) so the +# experiments never touch the main stand's data. Every role is pinned to its own CPU set, +# so the load generator, the LB and PostgreSQL do not steal CPU from the API replicas and +# the "total API CPU" of a configuration is exactly what the limits say: +# k6 0-1 | lb + valkey 2 | postgres 3-4 | api 5-7 (all replicas share these 3 CPUs; per-replica quota API_CPUS) +services: + postgres: + cpuset: "${PG_CPUSET:-3-4}" + valkey: + cpuset: "${VALKEY_CPUSET:-2}" + lb: + cpuset: "${LB_CPUSET:-2}" + migrate: + cpuset: "${API_CPUSET:-5-7}" + api: + cpuset: "${API_CPUSET:-5-7}" + environment: + STATS_LOG_SECONDS: "${STATS_LOG_SECONDS:-2}" + k6: + image: grafana/k6:2.3.0@sha256:9c2dee7f8ed74d317e4027c06a10f169b625638189de8d4555d0b3486a5aeb34 + profiles: ["bench"] + cpuset: "${K6_CPUSET:-0-1}" + environment: + BASE_URL: http://lb:8080 + volumes: + - ./bench/track1/k6:/scripts:ro + deploy: + resources: + limits: {cpus: "2.0", memory: 1g} diff --git a/bench/track1/k6/messages.js b/bench/track1/k6/messages.js new file mode 100644 index 00000000..7b42b565 --- /dev/null +++ b/bench/track1/k6/messages.js @@ -0,0 +1,67 @@ +// Track 1 workload: OPEN-LOOP arrival rate of "send message" requests through the LB. +// Every iteration = one POST /api/v1/chats/{id}/messages with a NEW Idempotency-Key into one +// of USERS chats (user i writes into chat i, shared with user i+1), so writers are spread over +// many chat rows and no single-row lock dominates. constant-arrival-rate starts iterations at +// RATE/s regardless of response time (no coordinated omission); iterations that cannot start +// because all VUs are busy are counted in dropped_iterations - they are failed demand. +// Phases: `warmup` (not reported) then `measure`. Only status 201 counts as success. +// Env: BASE_URL, RATE, WARMUP, DURATION, MAX_VUS, CHATS_FILE. +import http from 'k6/http'; +import { check } from 'k6'; +import exec from 'k6/execution'; +import { Counter, Trend } from 'k6/metrics'; + +const BASE = __ENV.BASE_URL || 'http://lb:8080'; +const RATE = parseInt(__ENV.RATE || '100', 10); +const WARMUP = __ENV.WARMUP || '10s'; +const DURATION = __ENV.DURATION || '20s'; +const MAX_VUS = parseInt(__ENV.MAX_VUS || '400', 10); + +const ok = new Counter('ok_messages'); // 201 only +const okLatency = new Trend('ok_latency', true); // latency of 201 only + +function phase(name, startTime, duration) { + return { + executor: 'constant-arrival-rate', rate: RATE, timeUnit: '1s', duration, startTime, + preAllocatedVUs: Math.min(MAX_VUS, Math.max(20, Math.ceil(RATE / 10))), maxVUs: MAX_VUS, + gracefulStop: '5s', exec: 'send', tags: { phase: name }, + }; +} + +export const options = { + discardResponseBodies: true, + scenarios: { warmup: phase('warmup', '0s', WARMUP), measure: phase('measure', WARMUP, DURATION) }, + summaryTrendStats: ['avg', 'p(50)', 'p(95)', 'p(99)', 'max'], + // Thresholds only create per-phase sub-metrics in the summary; they never fail the run. + thresholds: { + 'http_reqs{phase:measure}': ['count>=0'], + 'http_req_failed{phase:measure}': ['rate>=0'], + 'http_req_duration{phase:measure}': ['max>=0'], + 'ok_messages{phase:measure}': ['count>=0'], + 'ok_latency{phase:measure}': ['max>=0'], + 'dropped_iterations{scenario:measure}': ['count>=0'], + 'iterations{scenario:measure}': ['count>=0'], + }, +}; + +// Users, sessions and chats are created by bench/track1/run.py before k6 starts (so the timed +// phases start at once); the file with the synthetic tokens is mounted read-only and deleted after. +const chats = JSON.parse(open(__ENV.CHATS_FILE || '/data/chats.json')); + +export function send() { + const n = exec.scenario.iterationInTest; + const c = chats[n % chats.length]; + const res = http.post(`${BASE}/api/v1/chats/${c.chat}/messages`, + JSON.stringify({ text: `bench ${exec.vu.idInTest}-${n}` }), + { headers: { 'Content-Type': 'application/json', Authorization: `Bearer ${c.token}`, + 'Idempotency-Key': `${exec.scenario.name}-${exec.vu.idInTest}-${n}-${Date.now()}` }, + tags: { name: 'post_message' }, timeout: '10s' }); + if (check(res, { '201': (r) => r.status === 201 })) { + ok.add(1); + okLatency.add(res.timings.duration); + } +} + +export function handleSummary(data) { + return { stdout: '\n@@SUMMARY@@' + JSON.stringify(data) + '\n' }; +} diff --git a/bench/track1/profile.py b/bench/track1/profile.py new file mode 100644 index 00000000..27c68016 --- /dev/null +++ b/bench/track1/profile.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Where does one API replica spend its CPU under the track 1 workload? (py-spy sidecar) + + python bench/track1/profile.py --label baseline --rate 260 --config replicas=2,api_cpus=1.0 + +Starts a fresh bench stand, runs k6 at a fixed arrival rate and samples ONE replica with +py-spy (sidecar sharing its PID namespace, SYS_PTRACE only for the sidecar, image +bench/track1/pyspy with a hash-pinned py-spy). Output evidence/track1/profile-