From 3410871c411f9d47ff5ba7f1689776cfa0ad5a49 Mon Sep 17 00:00:00 2001 From: kalibr456 <145665723+kalibr456@users.noreply.github.com> Date: Wed, 14 Oct 2026 20:29:41 +0300 Subject: [PATCH 1/2] =?UTF-8?q?lab3:=20=D0=B2=D1=8B=D0=BF=D0=BE=D0=BB?= =?UTF-8?q?=D0=BD=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=B8=20=D0=B4=D0=BE=D0=BA?= =?UTF-8?q?=D0=B0=D0=B7=D0=B0=D1=82=D0=B5=D0=BB=D1=8C=D1=82=D1=81=D0=B2?= =?UTF-8?q?=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 7 + .env.example | 14 +- .gitattributes | 6 + .gitignore | 1 + Dockerfile | 27 ++ REPORT.md | 168 ++++++++++- auth.py | 139 +++++++++ compose.yaml | 125 ++++++++ db.py | 276 +++++++++++++++++ errors.py | 32 ++ evidence/balancing.txt | 16 + evidence/check.txt | 6 + evidence/db.txt | 21 ++ evidence/expiry.txt | 13 + evidence/explain-history.txt | 79 +++++ evidence/explain.txt | 33 +++ evidence/failover-metrics.json | 169 +++++++++++ evidence/failover-metrics.txt | 29 ++ evidence/failover.txt | 33 +++ evidence/failures.txt | 29 ++ evidence/idempotency.txt | 22 ++ evidence/load.json | 17 ++ evidence/load.txt | 22 ++ evidence/persistence.txt | 20 ++ evidence/race.txt | 10 + evidence/redis.txt | 14 + evidence/rolling-update.json | 51 ++++ evidence/rolling-update.txt | 14 + evidence/smoke.txt | 6 + evidence/stack.txt | 32 ++ evidence/student-tests.txt | 25 ++ evidence/summary.md | 55 ++++ infra/haproxy.cfg | 74 +++++ infra/redis.conf | 13 + main.py | 476 ++++++++++++++++++++++++++++++ migrate.py | 8 + migrations/001_init.sql | 33 +++ migrations/002_text_as_bytes.sql | 11 + migrations/003_idempotency.sql | 11 + requirements.txt | 21 ++ scripts/init_secrets.py | 30 ++ scripts/restart.cmd | 9 + scripts/restart.sh | 8 + scripts/stop-one.cmd | 6 + scripts/stop-one.sh | 5 + scripts/stop_one.py | 29 ++ settings.py | 59 ++++ student-tests/check_stack.py | 72 +++++ student-tests/collect_evidence.py | 243 +++++++++++++++ student-tests/common.py | 273 +++++++++++++++++ student-tests/demo_instances.py | 16 + student-tests/explain.py | 28 ++ student-tests/explain.sql | 66 +++++ student-tests/failover_metrics.py | 129 ++++++++ student-tests/loadgen.py | 102 +++++++ student-tests/redis_info.py | 29 ++ student-tests/rolling_update.py | 88 ++++++ student-tests/run_failover.py | 55 ++++ student-tests/run_persistence.py | 47 +++ student-tests/test_api.py | 246 +++++++++++++++ student-tests/test_balancing.py | 86 ++++++ student-tests/test_db.py | 101 +++++++ student-tests/test_expiry.py | 57 ++++ student-tests/test_failures.py | 110 +++++++ student-tests/test_idempotency.py | 174 +++++++++++ student-tests/test_race.py | 57 ++++ 66 files changed, 4266 insertions(+), 17 deletions(-) create mode 100644 .gitattributes create mode 100644 Dockerfile create mode 100644 auth.py create mode 100644 compose.yaml create mode 100644 db.py create mode 100644 errors.py create mode 100644 evidence/balancing.txt create mode 100644 evidence/check.txt create mode 100644 evidence/db.txt create mode 100644 evidence/expiry.txt create mode 100644 evidence/explain-history.txt create mode 100644 evidence/explain.txt create mode 100644 evidence/failover-metrics.json create mode 100644 evidence/failover-metrics.txt create mode 100644 evidence/failover.txt create mode 100644 evidence/failures.txt create mode 100644 evidence/idempotency.txt create mode 100644 evidence/load.json create mode 100644 evidence/load.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/race.txt create mode 100644 evidence/redis.txt create mode 100644 evidence/rolling-update.json create mode 100644 evidence/rolling-update.txt create mode 100644 evidence/smoke.txt create mode 100644 evidence/stack.txt create mode 100644 evidence/student-tests.txt create mode 100644 evidence/summary.md create mode 100644 infra/haproxy.cfg create mode 100644 infra/redis.conf create mode 100644 main.py create mode 100644 migrate.py create mode 100644 migrations/001_init.sql create mode 100644 migrations/002_text_as_bytes.sql create mode 100644 migrations/003_idempotency.sql create mode 100644 requirements.txt create mode 100644 scripts/init_secrets.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.cmd create mode 100644 scripts/stop-one.sh create mode 100644 scripts/stop_one.py create mode 100644 settings.py create mode 100644 student-tests/check_stack.py create mode 100644 student-tests/collect_evidence.py create mode 100644 student-tests/common.py create mode 100644 student-tests/demo_instances.py create mode 100644 student-tests/explain.py create mode 100644 student-tests/explain.sql create mode 100644 student-tests/failover_metrics.py create mode 100644 student-tests/loadgen.py create mode 100644 student-tests/redis_info.py create mode 100644 student-tests/rolling_update.py create mode 100644 student-tests/run_failover.py create mode 100644 student-tests/run_persistence.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_balancing.py create mode 100644 student-tests/test_db.py create mode 100644 student-tests/test_expiry.py create mode 100644 student-tests/test_failures.py create mode 100644 student-tests/test_idempotency.py create mode 100644 student-tests/test_race.py diff --git a/.dockerignore b/.dockerignore index bc16e556..82449c19 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,10 @@ __pycache__ node_modules evidence .DS_Store +tests +student-tests +contracts +scripts +infra +examples +*.md diff --git a/.env.example b/.env.example index d32b9be0..e845ec74 100644 --- a/.env.example +++ b/.env.example @@ -7,7 +7,17 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 + + +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_WAIT_SECONDS=3 +DB_CONNECT_TIMEOUT_SECONDS=3 +DB_STATEMENT_TIMEOUT_MS=5000 +REDIS_POOL_MAX=20 +REDIS_TIMEOUT_MS=500 +SHUTDOWN_DRAIN_SECONDS=3 +SHUTDOWN_TIMEOUT_SECONDS=10 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..3fd80096 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +*.sh text eol=lf +*.sql text eol=lf +*.conf text eol=lf +*.cfg text eol=lf +*.example text eol=lf +*.cmd text eol=crlf diff --git a/.gitignore b/.gitignore index b5a71896..0a6ee0a6 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ data/ backups/ # Runtime/build artifacts .venv/ +venv/ __pycache__/ *.py[cod] node_modules/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2328d213 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,27 @@ +FROM python:3.11.13-slim + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir --only-binary=:all: -r requirements.txt + +COPY main.py settings.py errors.py db.py auth.py migrate.py ./ +COPY migrations/ ./migrations/ + +RUN useradd --create-home --shell /usr/sbin/nologin appuser + +USER appuser + +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + +EXPOSE 8080 + +HEALTHCHECK --interval=10s --timeout=5s --start-period=20s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=4)" + +CMD ["python", "main.py"] diff --git a/REPORT.md b/REPORT.md index c2679005..b970e41c 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,175 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Бабиченко Максим, БВТ2303 +- Целевая оценка; выбранный трек 5, если нужен: 4, трек 5 не выбран +- Commit/PR: <<ссылка на MR в Gitea после публикации>> +- Использованные источники, библиотеки и помощь ИИ: всё из lab2 (FastAPI, Uvicorn, Pydantic, psycopg 3 и psycopg-pool, argon2-cffi, redis-py; точные версии в requirements.txt); образы python:3.11.13-slim, postgres:17.11-alpine, redis:7.4.11-alpine и новый haproxy:3.2.25-alpine; документация HAProxy 3.2 (resolvers, server-template, http-check, retries, observe), документация Docker Compose (deploy.replicas и resources, depends_on с condition, healthcheck), документация PostgreSQL (INSERT ... ON CONFLICT, уникальные ограничения). ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Проверял на Windows 11, все команды в PowerShell. Версии Docker, Compose и ресурсы машины — в `evidence/stack.txt`. Python нужен только для тестов (стандартная библиотека). Перед запуском остановить стенд lab2 (`docker compose down` в его папке), иначе занят порт 8080. + +```powershell +git clone https://gitea.privcloud.tech/Kalibr/mtusi2026lab3.git +cd mtusi2026lab3 +git checkout solution/bvt2303-babichenko +$env:PYTHONUTF8 = "1" # contracts/openapi.json в UTF-8, без этого тесты на Windows падают +Copy-Item .env.example .env +python scripts/init_secrets.py # случайные пароли в secrets/ (папка в .gitignore) +docker compose up -d --build --wait # migrate -> 2 реплики api -> lb; ждём healthy +docker compose ps + +python student-tests/collect_evidence.py # все проверки по очереди (~10 мин), вывод в evidence/, таблицы в этот отчёт +``` + +Отдельные проверки (make на моём Windows нет, ниже то же самое без make): + +| Команда из README | То же в PowerShell | +| --- | --- | +| `make check` | `python scripts/check_contract.py` | +| `make test` | `python tests/smoke.py --base-url http://127.0.0.1:8080` | +| `make test-persistence ACTION_SCRIPT=...` | `python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080` | +| `make test-failover ACTION_SCRIPT=...` | `python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080` | + +После `test-failover` одна реплика остаётся остановленной (так требует тест). Вернуть её: `docker start ` или `docker compose up -d --wait`. На Linux/macOS вместо `.cmd` — `scripts/restart.sh` и `scripts/stop-one.sh`: тест запускает hook без shell, а Python на Windows не умеет запускать `.sh` (WinError 193). + +Миграции применяет одноразовый сервис `migrate` при `docker compose up`, отдельной команды нет. Остановка без потери данных: `docker compose down` (volumes `pgdata`, `redisdata` остаются). Намеренное удаление учебных данных: `docker compose down -v`, затем удалить папку `secrets` и снова `python scripts/init_secrets.py`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +Клиент --HTTP--> lb: HAProxy 3.2, 127.0.0.1:8080 (единственный опубликованный порт) + | round robin, без привязки клиента к реплике + +--> api-1 --+ + +--> api-2 --+--> postgres: пользователи, чаты, сообщения, ключи идемпотентности (volume pgdata) + +--> redis: сессии (AOF, volume redisdata) +migrate (одноразовый): применяет migrations/*.sql до старта api и завершается +сети: edge (lb <-> api) и data (api, migrate <-> postgres, redis; internal, без выхода наружу) +``` + +| Роль | Сервис Compose | Версия | Health / готовность | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| Балансировщик | `lb` | haproxy:3.2.25-alpine | `/lb-health` на 8404 (есть куда отправить запрос) | нет | edge; `127.0.0.1:8080`, статистика `127.0.0.1:8404/stats` | не нужны | +| API ×2 | `api` (из Dockerfile, `deploy.replicas: 2`) | python:3.11.13-slim | `/health/ready` (Docker раз в 5 с, HAProxy раз в 1 с) | нет | edge + data; порт наружу не публикуется | `/run/secrets/database_url`, `/run/secrets/cache_url` | +| Миграции | `migrate` (тот же Dockerfile) | python:3.11.13-slim | завершается с кодом 0 | нет | data | только `/run/secrets/database_url` | +| База данных | `postgres` | postgres:17.11-alpine | `pg_isready` | `pgdata` | data, 5432 | `POSTGRES_PASSWORD_FILE` | +| Сессии | `redis` | redis:7.4.11-alpine | `redis-cli ping` с паролем | `redisdata` (AOF) | data, 6379 | `include /run/secrets/redis_auth` | + +Что перенесено из lab2: весь код API (`main.py`, `db.py`, `auth.py`, `settings.py`, `errors.py`), миграции 001–002, свои тесты, Dockerfile, `compose.yaml`, `infra/redis.conf`, `scripts/init_secrets.py`, restart-скрипты. Сессии и данные уже были в Redis и PostgreSQL, в памяти API ничего не хранится, поэтому реплики взаимозаменяемы. + +Что изменено в lab3: +- `compose.yaml`: две реплики `api` без `ports` и `container_name`, балансировщик `lb`, сервис `migrate`, лимиты CPU/RAM у всех контейнеров, две сети. +- `infra/haproxy.cfg`: список реплик HAProxy берёт из DNS Docker (имя `api` -> IP всех реплик, перечитывается раз в 2 с), поэтому подхватывает остановленные и новые реплики без перезапуска. +- `X-Instance-Id` в каждом ответе (middleware в `main.py`) — hostname контейнера, то есть его короткий ID. По заголовку сразу видно контейнер в `docker ps`, а `scripts/stop_one.py` останавливает именно его. +- Обязательный `Idempotency-Key` у POST сообщения (`main.py`, `db.add_message`, `migrations/003_idempotency.sql`). +- Миграции больше не применяет API: это делает `migrate.py` (сервис `migrate`), а реплика при старте только проверяет, что все миграции её образа применены (`db.wait_for_schema`). +- Остановка по SIGTERM в два шага (класс `Server` в `main.py`). + +Идемпотентность. Ключ и SHA-256 тела запроса хранятся прямо в строке сообщения. Уникальное ограничение `(chat_id, sender_id, idempotency_key)` — это область ключа из контракта: пользователь + чат (метод и путь у ключа всегда `POST .../messages`). Запись — один `INSERT ... ON CONFLICT DO NOTHING`, поэтому сообщение и «запомненный результат» сохраняются атомарно. Если ключ уже есть: то же тело (сравнивается хеш, порядок полей и пробелы в JSON не важны) -> тот же 201 и то же сообщение, другое тело -> 409. Если два запроса с одним ключом пришли одновременно на разные реплики, PostgreSQL заставляет второй `INSERT` ждать, пока завершится первая транзакция, после чего второй видит готовую строку. Повтор снова проверяет сессию (middleware) и членство в чате (404). Ответы 400/404/503 не сохраняются: после ошибки тот же ключ можно использовать снова. Ключ хранится столько же, сколько сообщение (удаления в API нет), то есть не меньше 24 ч из контракта. + +Балансировщик и отказ реплики: +- Проверка готовности: HAProxy раз в 1 с запрашивает `/health/ready` у каждой реплики (при смене состояния раз в 0,5 с, `timeout check 1s`). 2 неудачи подряд -> реплика выведена, 2 удачи -> возвращена. Худший случай для зависшей реплики: около 1 + 1 + 0,5 + 1 = 3,5 с. Кроме того, `observe layer4` выводит реплику сразу после первой ошибки соединения у живого запроса, а DNS Docker убирает остановленный контейнер (HAProxy замечает это за 2–4 с). +- Повторы в HAProxy только при ошибке соединения (запрос ещё не отправлен) и на другую реплику (`retries 2`, `option redispatch`). Уже отправленный POST HAProxy сам не повторяет: это делает клиент с тем же `Idempotency-Key`. +- Аварийный режим: если готовых реплик нет совсем (например, упал Redis или PostgreSQL), запросы всё равно идут в API через резервные записи `fallback` в HAProxy. Клиент получает ответ самого API (503 в формате контракта, с `X-Request-Id`), `/health/live` отвечает 200, регистрация без Redis работает. Пока готова хоть одна реплика, резервные записи не используются. +- SIGTERM: `/health/ready` сразу отвечает 503, но запросы ещё обслуживаются 3 с (`SHUTDOWN_DRAIN_SECONDS`). За это время HAProxy выводит реплику, затем uvicorn закрывает приём и доделывает начатые запросы (не дольше `SHUTDOWN_TIMEOUT_SECONDS=10`). Docker ждёт `stop_grace_period: 15s`. Самый долгий запрос — 8 с (пул 3 с + `statement_timeout` 5 с), поэтому у HAProxy `timeout server 15s`. + +Миграции: сервис `migrate` применяет `migrations/*.sql` один раз (`schema_migrations`, `pg_advisory_lock` на случай двух запусков). `api` стартует только после его успешного завершения (`depends_on: condition: service_completed_successfully`). Реплика с кодом новее схемы не стартует. Миграция 003 только добавляет столбцы и ограничение, у старых сообщений ключ `NULL`, поэтому API из lab2 продолжает работать на новой схеме (проверено: старый API пишет без ключа и читает новые сообщения). + +Соответствие `.env.example`: + +| Переменная | Что делает в этом решении | +| --- | --- | +| APP_HOST, APP_PORT, APP_ENV | где слушает API (0.0.0.0:8080 внутри контейнера), APP_ENV пишется в лог | +| SESSION_TTL_SECONDS | срок жизни сессии, 3600 с | +| DATABASE_URL_FILE, CACHE_URL_FILE | пути к секретам с адресами PostgreSQL и Redis | +| SESSION_STORE=redis | сессии в Redis (как в lab2) | +| API_REPLICAS | число реплик `api` (`deploy.replicas`), по умолчанию 2 | +| IDEMPOTENCY_TTL_SECONDS | не используется: ключ хранится вместе с сообщением бессрочно (не меньше 24 ч) | +| SHUTDOWN_DRAIN_SECONDS, SHUTDOWN_TIMEOUT_SECONDS | остановка: 3 с на выход из балансировки, до 10 с на начатые запросы | +| DB_*, REDIS_* | размеры пулов и таймауты (как в lab2) | ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS через балансировщик | evidence/smoke.txt | +| Skeleton и контракт не изменены | `python scripts/check_contract.py` | PASS | evidence/check.txt | +| 3: весь стенд в compose.yaml, одна команда запуска | `docker compose up -d --build --wait` | migrate Exited (0), api ×2 и lb healthy | evidence/stack.txt | +| 3: ≥2 API-реплики, одна точка входа, общая БД и общее хранилище сессий | `python student-tests/check_stack.py` | 2 реплики api без портов; наружу только lb на 127.0.0.1; один postgres, один redis | evidence/stack.txt | +| 3: запросы одного пользователя обслуживают разные реплики без повторного входа | `test_balancing.py` (test_requests_spread_over_replicas) | 20 запросов с одним токеном, ответы от 2 разных X-Instance-Id | evidence/balancing.txt | +| 3: состояние общее: запись на одной реплике сразу видна на другой, logout действует везде | `test_balancing.py` | записано одной репликой, прочитано другой; после logout обе отвечают 401 | evidence/balancing.txt | +| 3: X-Instance-Id в каждом ответе, это реальный контейнер | `test_balancing.py` (test_instance_header_everywhere) | 200/201/400/401/404/409 с X-Instance-Id = ID контейнера api | evidence/balancing.txt | +| 3: доступ через LB — основной маршрут, без sticky session | все тесты идут на 127.0.0.1:8080; `infra/haproxy.cfg` (`balance roundrobin`, без cookie) | ответы чередуются между репликами | evidence/balancing.txt, evidence/stack.txt | +| 3: остановка любой реплики: не позднее 10 с новые чтения/записи, подтверждённое на месте, ошибки измерены | `python student-tests/run_failover.py` (публичный failover для каждой реплики, `scripts/stop-one.cmd`) | PASS для каждой реплики; видно, какой контейнер остановлен (`docker compose ps` до/после) | evidence/failover.txt | +| 3: повтор POST с тем же ключом не создаёт дубль, в том числе одновременный и через другую реплику | `test_idempotency.py`, `test_balancing.py` (test_idempotent_retry_on_other_replica) | повтор -> тот же 201 и то же сообщение; 20 одновременных запросов через 2 реплики -> 1 сообщение | evidence/idempotency.txt, evidence/balancing.txt | +| 3: неподтверждённую запись можно безопасно повторить | `test_idempotency.py` (test_lost_response_then_retry); публичный failover повторяет POST с тем же ключом | ответ потерян -> повтор 201, сообщение одно; в failover дублей нет | evidence/idempotency.txt, evidence/failover.txt | +| 3: миграции отдельно и согласованно, не гонкой реплик | сервис `migrate`; `test_db.py` | миграции применяет только migrate (Exited 0), в логах api — только проверка схемы; повторный запуск migrate ничего не применяет | evidence/db.txt | +| 3: make test, persistence и failover проходят | smoke, `run_persistence.py`, `run_failover.py` | все PASS; в failover до отказа наблюдались 2 реплики | evidence/smoke.txt, evidence/persistence.txt, evidence/failover.txt | +| 3: hook перезапуска реально перезапускает процессы | `python student-tests/run_persistence.py` | у postgres, redis, обеих api и lb тот же ID контейнера и новое время старта; данные и прежний токен на месте | evidence/persistence.txt | +| 3: Swarm | — | выбран Compose, пункт про сборку образа для узлов Swarm не нужен | — | +| 3: возможности lab1/lab2 сохранены | `test_api.py`, `test_race.py`, `test_db.py`, `test_expiry.py`, `test_failures.py`, `redis_info.py`, `explain.py` через балансировщик | все PASS | evidence/student-tests.txt, race.txt, db.txt, expiry.txt, failures.txt, redis.txt, explain-history.txt | +| 4: health/readiness, таймауты и SIGTERM согласованы; LB исключает недоступную реплику; период проверок и время обнаружения | `infra/haproxy.cfg`, класс `Server` в `main.py`; `failover_metrics.py` | проверка раз в 1 с (0,5 с при смене), 2 неудачи; худший случай ~3,5 с; измеренное время вывода — в таблице ниже | evidence/failover-metrics.txt | +| 4: остановлена каждая реплика по очереди с восстановлением предыдущей; RPS, ошибки, время восстановления | `python student-tests/failover_metrics.py` (docker stop и docker kill, нагрузка 4+1 потока) | таблица ниже; потерь и дублей 0 | evidence/failover-metrics.txt, evidence/failover-metrics.json | +| 4: общая идемпотентность проверена параллельными запросами с одним ключом | `test_idempotency.py` (test_same_key_parallel_through_replicas, test_same_key_different_bodies_parallel) | 20 одновременных запросов: 1 сообщение, 20 одинаковых ответов; с разными телами: одно сохранено, другие 409 | evidence/idempotency.txt | +| 4: CPU/RAM-лимиты у контейнеров | `check_stack.py` | лимиты у всех: api 1 CPU / 512 MiB, postgres 1 CPU / 512 MiB, redis и lb 0,5 CPU / 128 MiB | evidence/stack.txt | +| 4: API не публикует порты в обход LB | `check_stack.py` | у api нет опубликованных портов, у postgres и redis тоже; lb только на 127.0.0.1 | evidence/stack.txt | +| 4: секреты и хранилища общие только где нужно | `check_stack.py` | lb без секретов, migrate только database_url; postgres и redis только во внутренней сети data | evidence/stack.txt | +| 4: нет container_name | `check_stack.py` | ни у одного сервиса нет container_name | evidence/stack.txt | +| 4: rolling update под нагрузкой | `python student-tests/rolling_update.py` | реплики заменены новой версией по одной, ошибки и RPS — ниже | evidence/rolling-update.txt, evidence/rolling-update.json | +| 4: миграции однократные и совместимые с работающей версией | сервис `migrate`, `migrations/003_idempotency.sql`; `test_db.py` | 003 только добавляет, API из lab2 работает на новой схеме; ограничение см. ниже | evidence/db.txt | +| Выбранный трек 5 | — | не выбран | — | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотезы: +1. Плановая остановка реплики (`docker stop`) проходит без ошибок у клиента: по SIGTERM реплика сначала отвечает 503 на readiness и уходит из балансировки, а приём закрывает только через 3 с. +2. При аварии (`docker kill`) ошибки получают только запросы, начатые на убитой реплике, а HAProxy выводит её быстрее, чем за 3,5 с. Клиент повторяет с тем же ключом, и дублей не появляется. +3. Rolling update (новая реплика -> готова -> старая уходит) проходит без ошибок при постоянной нагрузке. + +Методика: стенд из `compose.yaml` с лимитами из таблицы выше; нагрузка — `student-tests/loadgen.py`: 4 потока пишут сообщения без пауз (closed loop), 1 поток читает историю, timeout клиента 2 с, неподтверждённый POST повторяется с тем же ключом. Отказ — через 3 с нагрузки, наблюдение 10 с. Состояние HAProxy читается со страницы статистики раз в 0,1 с. После каждого прогона реплика запускается (`docker start`) и ждём, пока HAProxy вернёт её. Это closed-loop генератор на той же машине, поэтому RPS ограничен и клиентом, и лимитами CPU; это не предельная пропускная способность. + + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-14, commit 0ab3a34 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 58542a53644c | 2 | 128 | 0 | 4.203 | +| 2 | 9737b915ff2f | 2 | 132 | 0 | 4.281 | + +Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа): + +| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок (коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | Вернулась в балансировку после docker start, с | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 58542a53644c | docker stop (SIGTERM) | 4.12 | 1.11 | 0 (-) | 0.0 | 304.7 | 202.2 | 47.0 | 3.25 | +| 58542a53644c | docker kill (SIGKILL) | 0.81 | 0.11 | 2 (502: 2) | 0.05 | 328.7 | 228.2 | 32.0 | 1.55 | +| 9737b915ff2f | docker stop (SIGTERM) | 4.12 | 1.5 | 0 (-) | 0.0 | 288.7 | 232.7 | 32.0 | 4.09 | +| 9737b915ff2f | docker kill (SIGKILL) | 0.83 | 0.11 | 2 (502: 2) | 0.05 | 266.7 | 219.9 | 32.0 | 4.14 | + +Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика после остановки не ответила ни разу. + +Rolling update (`student-tests/rolling_update.py`): версия rolling-20261014-201405 -> rolling-20261014-202804, 2 реплики заменены по одной за 13.48 с под той же нагрузкой. Записей/с до: 284.7, во время замены: 346.3; ошибок: 0 (-); p95 31.0 мс, максимум 78.0 мс; подтверждено 6527 сообщений, потерь и дублей нет. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 308.848 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 15.0 мс, p95 32.0 мс, p99 46.0 мс. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `docker stop` реплики (SIGTERM) под нагрузкой | ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём | ошибок: 0, 0 (по репликам); HAProxy вывел реплику через 1.11, 1.5 с | ошибки прекратились через 0.0 с; потерь 0, дублей 0; реплика вернулась через 4.09 с после `docker start` | +| `docker kill` реплики (SIGKILL, авария) под нагрузкой | единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом | ошибок: 2, 2 (по репликам); HAProxy вывел реплику через 0.11, 0.11 с | ошибки прекратились через 0.05 с; потерь 0, дублей 0; реплика вернулась через 4.14 с после `docker start` | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.203, 4.281 с (hook ждёт остановки ~4 с); потерь нет | +| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше 2 | ошибок: 0, замена заняла 13.48 с | потерь 0, дублей 0 | +| перезапуск всего стенда (`restart` hook: postgres, redis, api, lb) | данные и прежний токен живы | PASS, те же контейнеры с новым временем старта | 8.5 с до готовности; потерь нет | + + +Также проверено (как в lab2, теперь через балансировщик, `evidence/failures.txt`): остановка Redis -> `/health/ready` 503, запросы с токеном 503, без токена 401, регистрация 201; аварийная остановка Redis -> сессия на месте (AOF); остановка PostgreSQL -> 503 без подробностей, анонимов не пускаем; после запуска прежний токен и сообщения на месте. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Выдерживается отказ одной API-реплики (процесса или контейнера). Не выдерживается отказ единственных `lb`, `postgres`, `redis` и всей машины: стенд на одном узле, это SPOF. Реплика БД и Redis, второй балансировщик — не в этой работе. +- HAProxy на время проверок может послать запрос на реплику, которая только что умерла. Такой запрос получает 502 (если он уже был отправлен) или уходит на другую реплику (если упало соединение). 502 на POST сообщения клиент безопасно повторяет с тем же ключом; HAProxy сам POST не повторяет. +- Создание чата и регистрация не идемпотентны по контракту: их повтор после timeout может создать второй объект. +- Миграция 003 добавляет уникальное ограничение: на большой таблице построение индекса заблокировало бы запись в `messages`. Для большой базы нужен `CREATE UNIQUE INDEX CONCURRENTLY` отдельным шагом; на учебной базе это доли секунды. +- Rolling update сделан скриптом (Compose не умеет заменять реплики по одной сам). В Swarm это было бы `update_config: order: start-first`. +- `X-Instance-Id` раскрывает ID контейнера; в lab7 его нужно заменить на случайный alias. +- Ключи идемпотентности хранятся вместе с сообщениями бессрочно (место в индексе растёт вместе с числом сообщений). diff --git a/auth.py b/auth.py new file mode 100644 index 00000000..c21212f8 --- /dev/null +++ b/auth.py @@ -0,0 +1,139 @@ +import base64 +import binascii +import hashlib +import logging +import secrets +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from uuid import UUID + +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError +from redis.backoff import NoBackoff +from redis.retry import Retry + +import settings +from errors import BASIC_CHALLENGE, unauthorized, unavailable + +log = logging.getLogger("messenger.auth") + +hasher = PasswordHasher(time_cost=3, memory_cost=12 * 1024, parallelism=1) + +DUMMY_HASH = hasher.hash("there-is-no-such-user") + +PASSWORD_MIN_LENGTH = 12 +PASSWORD_MAX_LENGTH = 128 + + +def hash_password(password: str) -> str: + return hasher.hash(password) + + +def password_matches(password_hash: str | None, password: str) -> bool: + try: + hasher.verify(password_hash or DUMMY_HASH, password) + except (VerificationError, InvalidHashError): + return False + return password_hash is not None + +def parse_basic(header: str | None) -> tuple[str, str] | None: + """'Basic base64(username:password)' -> (username, password) или None. + + Пароль может содержать двоеточие, поэтому делим по первому двоеточию (RFC 7617). + """ + scheme, _, value = (header or "").strip().partition(" ") + if scheme.lower() != "basic" or not value or len(value) > 1024: + return None + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, UnicodeDecodeError): + return None + username, colon, password = decoded.partition(":") + if not colon or not username or not username.isascii(): + return None + return username, password + + +def parse_bearer(header: str | None) -> str | None: + scheme, _, value = (header or "").strip().partition(" ") + value = value.strip() + if scheme.lower() != "bearer" or not value or len(value) > 512 or " " in value: + return None + return value + + + +KEY_PREFIX = "session:" + +_redis = redis.Redis( + connection_pool=redis.BlockingConnectionPool.from_url( + settings.REDIS_URL or "redis://localhost:6379/0", + max_connections=settings.REDIS_POOL_MAX, + timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_timeout=settings.REDIS_TIMEOUT_MS / 1000, + socket_connect_timeout=settings.REDIS_TIMEOUT_MS / 1000, + retry=Retry(NoBackoff(), 0), + decode_responses=True, + ) +) + + +@dataclass(frozen=True) +class Session: + user_id: UUID + key: str + + +def session_key(token: str) -> str: + return KEY_PREFIX + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def create_session(user_id: UUID) -> tuple[str, datetime]: + """Новая сессия: случайный токен и абсолютное время истечения.""" + token = secrets.token_urlsafe(32) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=settings.SESSION_TTL_SECONDS) + try: + _redis.set(session_key(token), str(user_id), pxat=int(expires_at.timestamp() * 1000)) + except redis.RedisError as exc: + log.warning("Redis is unavailable on login: %s", type(exc).__name__) + raise unavailable() from None + return token, expires_at + + +def check_bearer(header: str | None) -> Session: + token = parse_bearer(header) + if token is None: + raise unauthorized("Bearer token is required") + key = session_key(token) + try: + user_id = _redis.get(key) + except redis.RedisError as exc: + log.warning("Redis is unavailable: %s", type(exc).__name__) + raise unavailable() from None + if user_id is None: + raise unauthorized("Session is invalid or expired") + return Session(UUID(user_id), key) + + +def delete_session(session: Session) -> None: + try: + _redis.delete(session.key) + except redis.RedisError: + raise unavailable() from None + + +def ping_redis() -> None: + try: + _redis.ping() + except redis.RedisError: + raise unavailable() from None + + +def login_failed(): + return unauthorized("Invalid username or password", BASIC_CHALLENGE) + + +def close() -> None: + _redis.close() + _redis.connection_pool.disconnect() diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..71587f10 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,125 @@ +name: messenger-lab3 + +x-app-build: &app-build + context: . + args: + APP_VERSION: ${APP_VERSION:-dev} + +services: + migrate: + build: *app-build + command: ["python", "migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + healthcheck: + disable: true + restart: "no" + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + api: + build: *app-build + env_file: .env + environment: + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url, cache_url] + networks: [edge, data] + depends_on: + migrate: {condition: service_completed_successfully} + redis: {condition: service_healthy} + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 30s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "1.00", memory: 512M} + + lb: + image: haproxy:3.2.25-alpine + ports: + - "127.0.0.1:8080:8080" + - "127.0.0.1:8404:8404" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: [edge] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: ["CMD", "wget", "-q", "-T", "2", "-O", "/dev/null", "http://127.0.0.1:8404/lb-health"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + start_interval: 1s + restart: unless-stopped + stop_grace_period: 15s + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + + postgres: + image: postgres:17.11-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + command: ["postgres", "-c", "fsync=on", "-c", "synchronous_commit=on"] + secrets: [postgres_password] + volumes: ["pgdata:/var/lib/postgresql/data"] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "pg_isready -h 127.0.0.1 -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "1.00", memory: 512M} + + redis: + image: redis:7.4.11-alpine + command: ["redis-server", "/usr/local/etc/redis/redis.conf"] + volumes: + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + - redisdata:/data + secrets: [redis_auth] + networks: [data] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli ping | grep -q PONG"] + interval: 5s + timeout: 3s + retries: 12 + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.50", memory: 128M} + +networks: + edge: {} + data: + internal: true + +volumes: + pgdata: {} + redisdata: {} + +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + redis_auth: {file: ./secrets/redis_auth} diff --git a/db.py b/db.py new file mode 100644 index 00000000..894f2737 --- /dev/null +++ b/db.py @@ -0,0 +1,276 @@ +import logging +import time +from contextlib import contextmanager +from datetime import datetime +from uuid import UUID, uuid4 + +import psycopg +from psycopg import errors as pg_errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout + +import settings +from errors import bad_request, conflict, not_found, unavailable + +log = logging.getLogger("messenger.db") + + +pool = ConnectionPool( + conninfo=settings.DATABASE_URL, + min_size=settings.DB_POOL_MIN, + max_size=settings.DB_POOL_MAX, + timeout=settings.DB_POOL_WAIT_SECONDS, + kwargs={ + "connect_timeout": settings.DB_CONNECT_TIMEOUT_SECONDS, + "row_factory": dict_row, + "options": f"-c statement_timeout={settings.DB_STATEMENT_TIMEOUT_MS}" + " -c idle_in_transaction_session_timeout=10000 -c TimeZone=UTC", + }, + check=ConnectionPool.check_connection, + open=False, +) + + +@contextmanager +def connection(wait_seconds: float | None = None): + try: + with pool.connection(timeout=wait_seconds) as conn: + yield conn + except (psycopg.OperationalError, PoolTimeout) as exc: + log.warning("PostgreSQL is unavailable: %s", type(exc).__name__) + raise unavailable() from None + except psycopg.DataError as exc: + log.warning("PostgreSQL rejected a value: %s", type(exc).__name__) + raise bad_request("Invalid request data") from None + + +TEXT_FIELDS = ("display_name", "title", "text") + + +def to_db(value: str) -> bytes: + return value.encode("utf-8") + + +def from_db(row: dict | None) -> dict | None: + """Строка из БД -> dict для API: байты текстовых полей обратно в str.""" + if row is None: + return None + return {key: bytes(value).decode("utf-8") if key in TEXT_FIELDS else value for key, value in row.items()} + + +def ping() -> None: + with connection(wait_seconds=1) as conn: + conn.execute("SELECT 1") + + +MIGRATIONS_LOCK = 20260202 + + +def migration_files() -> list: + return sorted(settings.MIGRATIONS_DIR.glob("*.sql")) + + +def run_migrations() -> list[str]: + files = migration_files() + applied = [] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATIONS_LOCK,)) + try: + conn.execute("""CREATE TABLE IF NOT EXISTS schema_migrations ( + version text PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT now())""") + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + for path in files: + if path.name in done: + continue + with conn.transaction(): + conn.execute(path.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (path.name,)) + applied.append(path.name) + finally: + conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATIONS_LOCK,)) + return applied + + +def wait_and_migrate(max_wait_seconds: int = 60) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + applied = run_migrations() + log.info("Migrations applied: %s", ", ".join(applied) if applied else "none (schema is up to date)") + return + except psycopg.OperationalError: + if time.monotonic() > deadline: + raise + log.warning("PostgreSQL is not ready yet, retry in 1 s") + time.sleep(1) + + +def missing_migrations() -> list[str]: + expected = [path.name for path in migration_files()] + with psycopg.connect(settings.DATABASE_URL, autocommit=True, + connect_timeout=settings.DB_CONNECT_TIMEOUT_SECONDS) as conn: + try: + done = {row[0] for row in conn.execute("SELECT version FROM schema_migrations")} + except pg_errors.UndefinedTable: + return expected + return [name for name in expected if name not in done] + + +def wait_for_schema(max_wait_seconds: int) -> None: + deadline = time.monotonic() + max_wait_seconds + while True: + try: + missing = missing_migrations() + except psycopg.OperationalError: + missing = None + if missing == []: + log.info("Schema is up to date: %s", ", ".join(path.name for path in migration_files())) + return + if time.monotonic() > deadline: + raise RuntimeError(f"Database schema is not ready, missing migrations: {missing}") + log.warning("Waiting for the migrate service: %s", "PostgreSQL is not ready" if missing is None + else "missing " + ", ".join(missing)) + time.sleep(1) + + + +USER_COLUMNS = "id, username, display_name, created_at" + + +def create_user(username: str, display_name: str, password_hash: str) -> dict: + try: + with connection() as conn: + return from_db(conn.execute( + f"""INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) RETURNING {USER_COLUMNS}""", + (uuid4(), username, to_db(display_name), password_hash), + ).fetchone()) + except pg_errors.UniqueViolation: + raise conflict("Username already exists") from None + + +def get_user(user_id: UUID) -> dict | None: + with connection() as conn: + return from_db(conn.execute(f"SELECT {USER_COLUMNS} FROM users WHERE id = %s", (user_id,)).fetchone()) + + +def get_user_with_password(username: str) -> dict | None: + with connection() as conn: + return from_db(conn.execute( + f"SELECT {USER_COLUMNS}, password_hash FROM users WHERE username = %s", (username,) + ).fetchone()) + + +CHAT_COLUMNS = """c.id, c.title, c.created_at, + ARRAY(SELECT m.user_id FROM chat_members m WHERE m.chat_id = c.id ORDER BY m.user_id) AS member_ids""" + + +def create_chat(title: str, member_ids: list[UUID]) -> dict: + chat_id = uuid4() + members = sorted(member_ids) + try: + with connection() as conn: + with conn.transaction(): + row = conn.execute( + "INSERT INTO chats (id, title) VALUES (%s, %s) RETURNING id, title, created_at", + (chat_id, to_db(title)), + ).fetchone() + with conn.cursor() as cur: + cur.executemany( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + [(chat_id, member_id) for member_id in members], + ) + except pg_errors.ForeignKeyViolation: + raise bad_request("Unknown member") from None + return {**from_db(row), "member_ids": members} + + +def get_chat(chat_id: UUID, user_id: UUID) -> dict: + with connection() as conn: + chat = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + WHERE c.id = %s""", + (user_id, chat_id), + ).fetchone() + if chat is None: + raise not_found("Chat not found") + return from_db(chat) + + +def list_chats(user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """Чаты пользователя после курсора. Возвращает до limit+1 строк: лишняя значит «есть ещё».""" + where, params = "", [user_id] + if after is not None: + where = "WHERE (c.created_at, c.id) > (%s, %s)" + params += list(after) + with connection() as conn: + rows = conn.execute( + f"""SELECT {CHAT_COLUMNS} FROM chats c + JOIN chat_members me ON me.chat_id = c.id AND me.user_id = %s + {where} + ORDER BY c.created_at, c.id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] + + + +MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at" + + +def add_message(chat_id: UUID, sender_id: UUID, text: str, idempotency_key: str, request_hash: bytes) -> dict: + try: + with connection() as conn: + with conn.transaction(): + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, sender_id) + ).fetchone() + if member is None: # проверяется и при повторе: чужой чат — 404, а не сохранённый ответ + raise not_found("Chat not found") + created = conn.execute( + f"""INSERT INTO messages (id, chat_id, sender_id, text, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s, %s, %s) + ON CONFLICT (chat_id, sender_id, idempotency_key) DO NOTHING + RETURNING {MESSAGE_COLUMNS}""", + (uuid4(), chat_id, sender_id, to_db(text), idempotency_key, request_hash), + ).fetchone() + if created is not None: + return from_db(created) + saved = conn.execute( + f"""SELECT {MESSAGE_COLUMNS}, request_hash FROM messages + WHERE chat_id = %s AND sender_id = %s AND idempotency_key = %s""", + (chat_id, sender_id, idempotency_key), + ).fetchone() + except pg_errors.ForeignKeyViolation: + raise not_found("Chat not found") from None + if saved is None: # не должно случиться: конфликт был, значит строка есть; повтор безопасен + raise unavailable() + if bytes(saved.pop("request_hash")) != request_hash: + raise conflict("Idempotency-Key was already used with a different request") + return from_db(saved) + + +def list_messages(chat_id: UUID, user_id: UUID, after: tuple[datetime, UUID] | None, limit: int) -> list[dict]: + """История чата после курсора (keyset-пагинация), до limit+1 строк.""" + where, params = "chat_id = %s", [chat_id] + if after is not None: + where += " AND (created_at, id) > (%s, %s)" + params += list(after) + with connection() as conn: + member = conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", (chat_id, user_id) + ).fetchone() + if member is None: + raise not_found("Chat not found") + rows = conn.execute( + f"""SELECT {MESSAGE_COLUMNS} FROM messages + WHERE {where} + ORDER BY created_at, id + LIMIT %s""", + (*params, limit + 1), + ).fetchall() + return [from_db(row) for row in rows] diff --git a/errors.py b/errors.py new file mode 100644 index 00000000..02a97305 --- /dev/null +++ b/errors.py @@ -0,0 +1,32 @@ +# Что сервер пишет в WWW-Authenticate при 401: какую схему входа он ждёт. +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' # только POST /auth/sessions +BEARER_CHALLENGE = 'Bearer realm="messenger"' # все остальные защищённые запросы + + +class ApiError(Exception): + def __init__(self, status_code: int, code: str, message: str, headers: dict | None = None): + super().__init__(message) + self.status_code = status_code + self.code = code + self.message = message + self.headers = headers or {} + + +def bad_request(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(message: str = "Unauthorized", challenge: str = BEARER_CHALLENGE) -> ApiError: + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found(message: str) -> ApiError: + return ApiError(404, "not_found", message) + + +def conflict(message: str) -> ApiError: + return ApiError(409, "conflict", message) + + +def unavailable(message: str = "Service temporarily unavailable") -> ApiError: + return ApiError(503, "unavailable", message) diff --git a/evidence/balancing.txt b/evidence/balancing.txt new file mode 100644 index 00000000..512dc71b --- /dev/null +++ b/evidence/balancing.txt @@ -0,0 +1,16 @@ +# balancing: balancing, shared sessions and data, X-Instance-Id +# date: 2026-10-14 20:24:12, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/test_balancing.py +# exit code: 0, 1.3 s + +20 x GET /users/me with one token -> replicas: {'9737b915ff2f': 10, '58542a53644c': 10} +PASS: one session works on every replica, no second login (sessions are shared in Redis) +message written by 58542a53644c; read back by ['58542a53644c', '9737b915ff2f'] +PASS: a message written via one replica is read right away via another (shared PostgreSQL) +logout on 9737b915ff2f; old token then rejected by ['58542a53644c', '9737b915ff2f'] +PASS: logout on one replica revokes the session on all of them +first POST on 9737b915ff2f -> 201 a8111033-9f9c-4d32-8260-ec76d289aace; same key on 58542a53644c -> 201 a8111033-9f9c-4d32-8260-ec76d289aace +PASS: Idempotency-Key is shared: a retry through another replica returns the same message +statuses [200, 200, 201, 400, 401, 404, 409]: every answer has X-Instance-Id of a running container ['58542a53644c', '9737b915ff2f']; api containers: ['58542a53644c', '9737b915ff2f'] +PASS: X-Instance-Id is in every response and points to a real container +5 passed, 0 failed diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..dd3db7b7 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,6 @@ +# check: offline contract check (make check) +# date: 2026-10-14 20:24:08, commit: 0ab3a34 + uncommitted changes +# command: python.exe scripts/check_contract.py +# exit code: 0, 0.1 s + +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/db.txt b/evidence/db.txt new file mode 100644 index 00000000..3df70812 --- /dev/null +++ b/evidence/db.txt @@ -0,0 +1,21 @@ +# db: PostgreSQL: rollback, FK, Argon2id, unique keys, migrations +# date: 2026-10-14 20:24:29, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/test_db.py +# exit code: 0, 10.2 s + +chat with unknown member -> 400; rows with this title: 0; chats without members: 0; chats insert counter: 117 -> 118 +PASS: chat row was inserted and then rolled back together with members +direct INSERT of a message from a non-member -> exit code 1: ERROR: insert or update on table "messages" violates foreign key constraint "messages_chat_id_sender_id_fkey" +PASS: the database itself rejects a message from a non-member +stored password: $argon2id$v=19$m=12288,t=3,p=1$... (length 97) +PASS: password is stored as Argon2id hash +unique constraint on users: UNIQUE (username) +PASS: username is unique in the database +applied migrations: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql; running the migrate service once more +after the second run: 001_init.sql,002_text_as_bytes.sql,003_idempotency.sql; log: Migrations applied: none (schema is up to date) +PASS: migrations are applied once, a repeated run does nothing +api replicas that checked the schema at start: 4; api log lines 'Migrations applied': 0; migrate service: Exited (0) 38 seconds ago +PASS: only the migrate service applies migrations, replicas wait for the schema +unique constraint for Idempotency-Key: UNIQUE (chat_id, sender_id, idempotency_key) +PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database +7 passed, 0 failed diff --git a/evidence/expiry.txt b/evidence/expiry.txt new file mode 100644 index 00000000..8f19844f --- /dev/null +++ b/evidence/expiry.txt @@ -0,0 +1,13 @@ +# expiry: session TTL=2 s, then normal TTL +# date: 2026-10-14 20:27:07, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/test_expiry.py +# exit code: 0, 17.1 s + +recreating api with SESSION_TTL_SECONDS=2 +session issued, expires in 2.00 s; Redis TTL of the key: 1700 ms +before expiry: GET /users/me -> 200 +after expiry: GET /users/me -> 401; key exists in Redis: 0 +recreating api with SESSION_TTL_SECONDS from .env +normal API again: new session expires in 3600 s +PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401 +1 passed, 0 failed diff --git a/evidence/explain-history.txt b/evidence/explain-history.txt new file mode 100644 index 00000000..d1b29d05 --- /dev/null +++ b/evidence/explain-history.txt @@ -0,0 +1,79 @@ +=== Data volume + messages_total | messages_in_big_chat +----------------+---------------------- + 48658 | 15000 + + +=== History index + indexdef +---------------------------------------------------------------------------------------- + CREATE INDEX messages_history ON public.messages USING btree (chat_id, created_at, id) + + +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + QUERY PLAN +------------------------------------------------------------------------------------------------ + Limit (actual time=0.007..0.011 rows=51 loops=1) + Buffers: shared hit=4 + -> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1) + Index Cond: (chat_id = 'b246d1e6-d41a-4111-aad6-c0474ab30641'::uuid) + Buffers: shared hit=4 + Planning: + Buffers: shared hit=7 + Planning Time: 0.073 ms + Execution Time: 0.020 ms + + +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ + Limit (actual time=0.007..0.011 rows=51 loops=1) + Buffers: shared hit=4 + -> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1) + Index Cond: ((chat_id = 'b246d1e6-d41a-4111-aad6-c0474ab30641'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'c1c5eccc-7e7c-4a03-a48e-3a8fb237e518'::uuid))) + Buffers: shared hit=4 + Planning Time: 0.044 ms + Execution Time: 0.030 ms + + +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + QUERY PLAN +-------------------------------------------------------------------------------- + Limit (actual time=3.445..3.448 rows=51 loops=1) + Buffers: shared hit=860 + -> Sort (actual time=3.038..3.253 rows=10051 loops=1) + Sort Key: created_at, id + Sort Method: quicksort Memory: 2377kB + Buffers: shared hit=860 + -> Seq Scan on messages (actual time=0.007..1.686 rows=15000 loops=1) + Filter: (chat_id = 'b246d1e6-d41a-4111-aad6-c0474ab30641'::uuid) + Rows Removed by Filter: 33658 + Buffers: shared hit=860 + Planning Time: 0.037 ms + Execution Time: 3.511 ms + + +=== 4. Query 2 without the index (index is dropped only inside this transaction) + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (actual time=1.864..1.868 rows=51 loops=1) + Buffers: shared hit=860 + -> Sort (actual time=1.864..1.865 rows=51 loops=1) + Sort Key: created_at, id + Sort Method: top-N heapsort Memory: 31kB + Buffers: shared hit=860 + -> Seq Scan on messages (actual time=1.115..1.524 rows=5000 loops=1) + Filter: ((chat_id = 'b246d1e6-d41a-4111-aad6-c0474ab30641'::uuid) AND (ROW(created_at, id) > ROW('2026-09-01 02:46:40+00'::timestamp with time zone, 'c1c5eccc-7e7c-4a03-a48e-3a8fb237e518'::uuid))) + Rows Removed by Filter: 43658 + Buffers: shared hit=860 + Planning: + Buffers: shared hit=4 + Planning Time: 0.051 ms + Execution Time: 1.882 ms + + +=== ROLLBACK: test data removed, index is back + explain_users_left +-------------------- + 0 + diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 00000000..a1b5bec9 --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,33 @@ +# explain: EXPLAIN of the history page (full plans: explain-history.txt) +# date: 2026-10-14 20:24:32, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/explain.py +# exit code: 0, 0.7 s + +=== Data volume +=== History index +=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page + Buffers: shared hit=4 + -> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1) + Buffers: shared hit=4 + Buffers: shared hit=7 + Execution Time: 0.020 ms +=== 2. Page after the 10000th message by keyset cursor (this is what the app does) + Buffers: shared hit=4 + -> Index Scan using messages_history on messages (actual time=0.006..0.009 rows=51 loops=1) + Buffers: shared hit=4 + Execution Time: 0.030 ms +=== 3. The same page with OFFSET 10000 (the app does NOT do this) + Buffers: shared hit=860 + Buffers: shared hit=860 + -> Seq Scan on messages (actual time=0.007..1.686 rows=15000 loops=1) + Buffers: shared hit=860 + Execution Time: 3.511 ms +=== 4. Query 2 without the index (index is dropped only inside this transaction) + Buffers: shared hit=860 + Buffers: shared hit=860 + -> Seq Scan on messages (actual time=1.115..1.524 rows=5000 loops=1) + Buffers: shared hit=860 + Buffers: shared hit=4 + Execution Time: 1.882 ms +=== ROLLBACK: test data removed, index is back +PASS: full plans saved to evidence\explain-history.txt diff --git a/evidence/failover-metrics.json b/evidence/failover-metrics.json new file mode 100644 index 00000000..b9a2a43c --- /dev/null +++ b/evidence/failover-metrics.json @@ -0,0 +1,169 @@ +{ + "profile": { + "writers": 4, + "readers": 1, + "client_timeout_seconds": 2, + "warmup_seconds": 3, + "after_seconds": 10, + "replicas": 2, + "client_python": "3.11.1" + }, + "runs": [ + { + "replica": "58542a53644c", + "mode": "stop", + "command_seconds": 4.12, + "lb_excluded_after_seconds": 1.11, + "errors": 0, + "error_codes": {}, + "errors_stopped_after_seconds": 0.0, + "before": { + "seconds": 3.0, + "write_rps": 304.7, + "reads": 46, + "errors": 0, + "error_codes": {}, + "write_p50_ms": 15.0, + "write_p95_ms": 32.0, + "write_max_ms": 63.0 + }, + "after": { + "seconds": 10.0, + "write_rps": 202.2, + "reads": 148, + "errors": 0, + "error_codes": {}, + "write_p50_ms": 16.0, + "write_p95_ms": 47.0, + "write_max_ms": 766.0 + }, + "stopped_replica_answers_after_stop": 0, + "integrity": { + "confirmed_writes": 2938, + "messages_in_history": 2938, + "duplicates": 0, + "lost": 0 + }, + "rejoin_seconds": 3.25 + }, + { + "replica": "58542a53644c", + "mode": "kill", + "command_seconds": 0.81, + "lb_excluded_after_seconds": 0.11, + "errors": 2, + "error_codes": { + "502": 2 + }, + "errors_stopped_after_seconds": 0.05, + "before": { + "seconds": 3.0, + "write_rps": 328.7, + "reads": 49, + "errors": 0, + "error_codes": {}, + "write_p50_ms": 15.0, + "write_p95_ms": 32.0, + "write_max_ms": 78.0 + }, + "after": { + "seconds": 10.0, + "write_rps": 228.2, + "reads": 144, + "errors": 2, + "error_codes": { + "502": 2 + }, + "write_p50_ms": 16.0, + "write_p95_ms": 32.0, + "write_max_ms": 1015.0 + }, + "stopped_replica_answers_after_stop": 0, + "integrity": { + "confirmed_writes": 3274, + "messages_in_history": 3274, + "duplicates": 0, + "lost": 0 + }, + "rejoin_seconds": 1.55 + }, + { + "replica": "9737b915ff2f", + "mode": "stop", + "command_seconds": 4.12, + "lb_excluded_after_seconds": 1.5, + "errors": 0, + "error_codes": {}, + "errors_stopped_after_seconds": 0.0, + "before": { + "seconds": 3.0, + "write_rps": 288.7, + "reads": 49, + "errors": 0, + "error_codes": {}, + "write_p50_ms": 15.0, + "write_p95_ms": 32.0, + "write_max_ms": 63.0 + }, + "after": { + "seconds": 10.0, + "write_rps": 232.7, + "reads": 148, + "errors": 0, + "error_codes": {}, + "write_p50_ms": 16.0, + "write_p95_ms": 32.0, + "write_max_ms": 94.0 + }, + "stopped_replica_answers_after_stop": 0, + "integrity": { + "confirmed_writes": 3194, + "messages_in_history": 3194, + "duplicates": 0, + "lost": 0 + }, + "rejoin_seconds": 4.09 + }, + { + "replica": "9737b915ff2f", + "mode": "kill", + "command_seconds": 0.83, + "lb_excluded_after_seconds": 0.11, + "errors": 2, + "error_codes": { + "502": 2 + }, + "errors_stopped_after_seconds": 0.05, + "before": { + "seconds": 3.0, + "write_rps": 266.7, + "reads": 47, + "errors": 0, + "error_codes": {}, + "write_p50_ms": 16.0, + "write_p95_ms": 32.0, + "write_max_ms": 78.0 + }, + "after": { + "seconds": 10.0, + "write_rps": 219.9, + "reads": 143, + "errors": 2, + "error_codes": { + "502": 2 + }, + "write_p50_ms": 16.0, + "write_p95_ms": 32.0, + "write_max_ms": 1000.0 + }, + "stopped_replica_answers_after_stop": 0, + "integrity": { + "confirmed_writes": 3000, + "messages_in_history": 3000, + "duplicates": 0, + "lost": 0 + }, + "rejoin_seconds": 4.14 + } + ] +} diff --git a/evidence/failover-metrics.txt b/evidence/failover-metrics.txt new file mode 100644 index 00000000..741c978d --- /dev/null +++ b/evidence/failover-metrics.txt @@ -0,0 +1,29 @@ +# failover-metrics: stop/kill every replica under load, metrics +# date: 2026-10-14 20:26:19, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/failover_metrics.py +# exit code: 0, 69.7 s + +api replicas: ['58542a53644c', '9737b915ff2f']; load: 4 writers without pauses + 1 reader, client timeout 2 s, retry with the same Idempotency-Key; 3 s before and 10 s after the failure +--- stop 58542a53644c ... + docker stop: 4.12 s; HAProxy excluded it after 1.11 s; errors: 0 {}, stopped after 0.0 s + write RPS before 304.7 -> after 202.2; p95 after 47.0 ms, max 766.0 ms; answers from the stopped replica after stop: 0 + confirmed 2938, in history 2938, lost 0, duplicates 0; back in the balancer 3.25 s after docker start +--- kill 58542a53644c ... + docker kill: 0.81 s; HAProxy excluded it after 0.11 s; errors: 2 {'502': 2}, stopped after 0.05 s + write RPS before 328.7 -> after 228.2; p95 after 32.0 ms, max 1015.0 ms; answers from the stopped replica after stop: 0 + confirmed 3274, in history 3274, lost 0, duplicates 0; back in the balancer 1.55 s after docker start +--- stop 9737b915ff2f ... + docker stop: 4.12 s; HAProxy excluded it after 1.5 s; errors: 0 {}, stopped after 0.0 s + write RPS before 288.7 -> after 232.7; p95 after 32.0 ms, max 94.0 ms; answers from the stopped replica after stop: 0 + confirmed 3194, in history 3194, lost 0, duplicates 0; back in the balancer 4.09 s after docker start +--- kill 9737b915ff2f ... + docker kill: 0.83 s; HAProxy excluded it after 0.11 s; errors: 2 {'502': 2}, stopped after 0.05 s + write RPS before 266.7 -> after 219.9; p95 after 32.0 ms, max 1000.0 ms; answers from the stopped replica after stop: 0 + confirmed 3000, in history 3000, lost 0, duplicates 0; back in the balancer 4.14 s after docker start + +replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s +58542a53644c stop 4.12 1.11 0 0.0 304.7 202.2 47.0 3.25 +58542a53644c kill 0.81 0.11 2 0.05 328.7 228.2 32.0 1.55 +9737b915ff2f stop 4.12 1.5 0 0.0 288.7 232.7 32.0 4.09 +9737b915ff2f kill 0.83 0.11 2 0.05 266.7 219.9 32.0 4.14 +PASS: every replica was stopped (stop, kill) and restored; no lost or duplicated messages; saved evidence\failover-metrics.json diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..f40e7360 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,33 @@ +# failover: public failover test for every replica (make test-failover) +# date: 2026-10-14 20:25:10, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/run_failover.py +# exit code: 0, 37.8 s + +hook: scripts\stop-one.cmd; api replicas: ['58542a53644c', '9737b915ff2f'] +=== run 1: api containers before +SERVICE CONTAINER ID STATUS +api 58542a53644c Up About a minute (healthy) +api 9737b915ff2f Up About a minute (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 4.203} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 58542a53644c Exited (0) 11 seconds ago +api 9737b915ff2f Up About a minute (healthy) +the hook stopped 58542a53644c (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 58542a53644c -> healthy and back in HAProxy +=== run 2: api containers before +SERVICE CONTAINER ID STATUS +api 58542a53644c Up 1 second (healthy) +api 9737b915ff2f Up About a minute (healthy) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 4.281} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +api containers after +SERVICE CONTAINER ID STATUS +api 58542a53644c Up 18 seconds (healthy) +api 9737b915ff2f Exited (0) 11 seconds ago +the hook stopped 9737b915ff2f (X-Instance-Id of that replica = container ID); other api containers kept running, postgres/redis/lb were not touched +restored: docker start 9737b915ff2f -> healthy and back in HAProxy +PASS: public failover test passed 2 times; every api replica was stopped once and restored diff --git a/evidence/failures.txt b/evidence/failures.txt new file mode 100644 index 00000000..94391a21 --- /dev/null +++ b/evidence/failures.txt @@ -0,0 +1,29 @@ +# failures: Redis and PostgreSQL outages via the balancer +# date: 2026-10-14 20:27:54, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/test_failures.py +# exit code: 0, 47.0 s + +stopping redis +/health/live -> 200, /health/ready -> 503 (3297 ms) +valid token: GET /users/me -> 503 unavailable (3281 ms), session cannot be checked +unknown token -> 503, not 200: nobody is trusted without a check +no token -> 401 +login -> 503: there is nowhere to store the session +registration -> 201: it needs only PostgreSQL +starting redis +after start: the old token works, confirmed messages are in place +PASS: Redis outage gives 503 and does not let anyone in without a session check +killing redis with SIGKILL (docker compose kill redis): no graceful shutdown +right after the crash: GET /users/me -> 503 +starting redis +after start: the session issued before the crash still works (AOF, appendfsync always) +PASS: sessions survive a Redis crash, not only a clean restart +stopping postgres +/health/live -> 200, /health/ready -> 503 (1015 ms) +valid token: users/me, chats, history, new message -> 503 unavailable, no details +no token -> 401, unknown token -> 401: anonymous users are not let in +login -> 503, registration -> 503 +starting postgres +after start: the old token works, confirmed messages are in place +PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out +3 passed, 0 failed diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt new file mode 100644 index 00000000..0076d276 --- /dev/null +++ b/evidence/idempotency.txt @@ -0,0 +1,22 @@ +# idempotency: Idempotency-Key, parallel same key via replicas +# date: 2026-10-14 20:24:15, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/test_idempotency.py +# exit code: 0, 2.3 s + +first -> 201; same JSON written differently -> 201; plain retry -> 201 +PASS: a retry with the same key returns the original 201 and message, no duplicate +20 parallel POSTs with one key: statuses {201: 20}, replicas {'58542a53644c': 10, '9737b915ff2f': 10}, different bodies in answers: 1, messages in history: 1 +PASS: parallel requests with one key through different replicas create exactly one message +20 parallel POSTs, one key, two different bodies: {('body A', 409): 10, ('body B', 201): 10}; saved: ['body B'] +PASS: the first body wins, the same body gets 201, the other body gets 409 +one key: alice in chat 1, bob in chat 1, alice in chat 2 -> 3 messages +PASS: the key is scoped to the current user and chat; another body with a used key -> 409 +missing, empty, 129 chars, space, '/', non-ASCII, '"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201 +PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128} +key used by a 400 and by a 404 first, then valid request -> 201 +PASS: rejected requests (400, 404) do not store a result for the key +same key and body by an outsider -> 404; by alice after logout -> 401 +PASS: a retry checks the session and chat access again, the saved answer is not leaked +POST sent, connection closed without reading the answer; retry -> 201; messages: 1 +PASS: an unconfirmed POST can be safely retried with the same key +8 passed, 0 failed diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..ca0b9c6e --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.031, + "concurrency": 4, + "successful_requests": 9275, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 308.848, + "successful_latency_ms": { + "p50": 15.0, + "p95": 32.0, + "p99": 46.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 00000000..907239c0 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,22 @@ +# load: public load generator, 30 s, concurrency 4 +# date: 2026-10-14 20:26:50, commit: 0ab3a34 + uncommitted changes +# command: python.exe tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +# exit code: 0, 30.3 s + +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.031, + "concurrency": 4, + "successful_requests": 9275, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 308.848, + "successful_latency_ms": { + "p50": 15.0, + "p95": 32.0, + "p99": 46.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..07786c22 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,20 @@ +# persistence: public persistence test + proof of real restarts (make test-persistence) +# date: 2026-10-14 20:28:03, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/run_persistence.py +# exit code: 0, 9.2 s + +before: container, ID, started at + messenger-lab3-api-3 231a6acf25e5 2026-10-14T17:27:05.347335515Z + messenger-lab3-api-4 5ef3550d83e5 2026-10-14T17:27:05.632137055Z + messenger-lab3-lb-1 143799027af8 2026-10-14T17:23:15.482251883Z + messenger-lab3-postgres-1 94f24029f4b4 2026-10-14T17:27:46.840755333Z + messenger-lab3-redis-1 b78c2ab71807 2026-10-14T17:27:26.471429301Z +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after (hook + waiting for readiness took 8.5 s): container, ID, started at, restarted? + messenger-lab3-api-3 231a6acf25e5 2026-10-14T17:28:00.882064128Z same container: True, restarted: True + messenger-lab3-api-4 5ef3550d83e5 2026-10-14T17:28:00.692590504Z same container: True, restarted: True + messenger-lab3-lb-1 143799027af8 2026-10-14T17:28:02.788694335Z same container: True, restarted: True + messenger-lab3-postgres-1 94f24029f4b4 2026-10-14T17:27:56.091073554Z same container: True, restarted: True + messenger-lab3-redis-1 b78c2ab71807 2026-10-14T17:27:56.26191188Z same container: True, restarted: True +PASS: the hook restarted postgres, redis, all api replicas and lb (same containers, new start time); data and the old token survived diff --git a/evidence/race.txt b/evidence/race.txt new file mode 100644 index 00000000..3b107925 --- /dev/null +++ b/evidence/race.txt @@ -0,0 +1,10 @@ +# race: lab2 races via the balancer +# date: 2026-10-14 20:24:18, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/test_race.py +# exit code: 0, 1.1 s + +sent 100 messages from 10 clients in 0.22 s; read 100 in 4 pages, unique ids: 100 +PASS: 100 concurrent messages from 10 clients, no losses and no duplicates +20 parallel registrations of one username: 201 x1, 409 x19 +PASS: concurrent registration created exactly one user +2 passed, 0 failed diff --git a/evidence/redis.txt b/evidence/redis.txt new file mode 100644 index 00000000..d0615e0a --- /dev/null +++ b/evidence/redis.txt @@ -0,0 +1,14 @@ +# redis: Redis as the session store +# date: 2026-10-14 20:24:31, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/redis_info.py +# exit code: 0, 2.4 s + +session key: session:b215258e3e5be3f4...; value is the user id: True +TTL of the key, seconds: 3599 +sessions in Redis: 153 +config appendonly = yes +config appendfsync = always +config maxmemory-policy = noeviction +aof_enabled:1 +aof_last_write_status:ok +without password: NOAUTH Authentication required. diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..4a09e3ef --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,51 @@ +{ + "from_version": "rolling-20261014-201405", + "to_version": "rolling-20261014-202804", + "replicas": 2, + "profile": { + "writers": 4, + "readers": 1, + "client_timeout_seconds": 2 + }, + "steps": [ + { + "old": "231a6acf25e5", + "new": "02b2a36daac5", + "new_ready_seconds": 2.36, + "step_seconds": 6.58 + }, + { + "old": "5ef3550d83e5", + "new": "cb8c881f5046", + "new_ready_seconds": 2.74, + "step_seconds": 6.91 + } + ], + "update_seconds": 13.48, + "before": { + "seconds": 3.0, + "write_rps": 284.7, + "reads": 45, + "errors": 0, + "error_codes": {}, + "write_p50_ms": 15.0, + "write_p95_ms": 32.0, + "write_max_ms": 109.0 + }, + "during": { + "seconds": 13.48, + "write_rps": 346.3, + "reads": 213, + "errors": 0, + "error_codes": {}, + "write_p50_ms": 15.0, + "write_p95_ms": 31.0, + "write_max_ms": 78.0 + }, + "integrity": { + "confirmed_writes": 6527, + "messages_in_history": 6527, + "duplicates": 0, + "lost": 0 + } +} diff --git a/evidence/rolling-update.txt b/evidence/rolling-update.txt new file mode 100644 index 00000000..a53a592d --- /dev/null +++ b/evidence/rolling-update.txt @@ -0,0 +1,14 @@ +# rolling-update: rolling update under load +# date: 2026-10-14 20:28:31, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/rolling_update.py +# exit code: 0, 28.0 s + +old replicas ['231a6acf25e5', '5ef3550d83e5'], version rolling-20261014-201405; building version rolling-20261014-202804 +image built in 5.0 s +migrations: Migrations applied: none (schema is up to date) +replaced 231a6acf25e5 -> 02b2a36daac5: new replica ready and in HAProxy after 2.4 s, step 6.58 s +replaced 5ef3550d83e5 -> cb8c881f5046: new replica ready and in HAProxy after 2.7 s, step 6.91 s +now: replicas ['02b2a36daac5', 'cb8c881f5046'], versions ['rolling-20261014-202804'] +under load: write RPS before 284.7, during the update 346.3; errors 0 {}; p95 31.0 ms, max 78.0 ms; update took 13.5 s +confirmed 6527, in history 6527, lost 0, duplicates 0 +PASS: all 2 replicas replaced one by one under load, 0 errors, no lost or duplicated messages; saved evidence\rolling-update.json diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..82ba7281 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,6 @@ +# smoke: public smoke via the balancer (make test) +# date: 2026-10-14 20:24:11, commit: 0ab3a34 + uncommitted changes +# command: python.exe tests/smoke.py --base-url http://127.0.0.1:8080 +# exit code: 0, 0.4 s + +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 00000000..dcada72c --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1,32 @@ +# stack: stand layout: replicas, ports, limits, networks, secrets +# date: 2026-10-14 20:24:11, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/check_stack.py +# exit code: 0, 2.0 s + +host: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; client Python 3.11.1 +Docker 28.1.1, Docker Compose version v2.35.1-desktop.1; Docker VM: 16 CPU, 15.2 GiB RAM, Docker Desktop + +SERVICE CONTAINER ID STATUS PORTS +api 58542a53644c Up 56 seconds (healthy) 8080/tcp +api 9737b915ff2f Up 56 seconds (healthy) 8080/tcp +lb 143799027af8 Up 54 seconds (healthy) 127.0.0.1:8080->8080/tcp, 127.0.0.1:8404->8404/tcp +migrate edb8550a4469 Exited (0) 19 seconds ago +postgres 94f24029f4b4 Up 56 seconds (healthy) +redis b78c2ab71807 Up 56 seconds (healthy) + +container image cpus memory host ports networks +messenger-lab3-api-3 messenger-lab3-api 1.00 512 MiB - data,edge +messenger-lab3-api-4 messenger-lab3-api 1.00 512 MiB - data,edge +messenger-lab3-lb-1 haproxy:3.2.25-alpine 0.50 128 MiB 127.0.0.1:8080,127.0.0.1:8404 edge +messenger-lab3-postgres-1 postgres:17.11-alpine 1.00 512 MiB - data +messenger-lab3-redis-1 redis:7.4.11-alpine 0.50 128 MiB - data + +secrets of api: ['cache_url', 'database_url'] +secrets of lb: - +secrets of migrate: ['database_url'] +secrets of postgres: ['postgres_password'] +secrets of redis: ['redis_auth'] + +through http://127.0.0.1:8080 one session is served by 2 replicas: ['58542a53644c', '9737b915ff2f'] +/health/ready -> 200 from 58542a53644c +PASS: 2 api replicas behind one entry point, no direct API ports, limits on every container, internal data network, secrets only where needed diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..3efbbc8c --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,25 @@ +# student-tests: lab1/lab2 API tests via the balancer +# date: 2026-10-14 20:24:17, commit: 0ab3a34 + uncommitted changes +# command: python.exe student-tests/test_api.py +# exit code: 0, 2.5 s + +PASS: duplicate username returns 409 +PASS: invalid JSON returns 400 +PASS: wrong and extra fields return 400 +PASS: foreign chat returns 404 +PASS: empty and too long messages return 400, 4000 chars is ok +PASS: pagination works across multiple pages, foreign cursor returns 400 +PASS: chat list pagination +PASS: unicode text is stored as is +PASS: strings with U+0000 are stored and returned as is +PASS: user response has no password or hash +PASS: password length 12..128, blank password is rejected +PASS: Basic login returns opaque Bearer token with expiry +PASS: wrong password and unknown user both return the same 401 +PASS: malformed Basic header returns 401 +PASS: password with colon and unicode works +PASS: business endpoints need Bearer; Basic and X-User-Id give 401 +PASS: X-User-Id does not change the current user +PASS: 401 comes before 400 and 404 +PASS: logout revokes only the current session +19 passed, 0 failed diff --git a/evidence/summary.md b/evidence/summary.md new file mode 100644 index 00000000..b9abf5d1 --- /dev/null +++ b/evidence/summary.md @@ -0,0 +1,55 @@ +# Сводка проверок + +| Шаг | Что проверяет | Результат | Время, с | Файл | +| --- | --- | --- | --- | --- | +| check | offline contract check (make check) | PASS | 0 | evidence/check.txt | +| stack | stand layout: replicas, ports, limits, networks, secrets | PASS | 2 | evidence/stack.txt | +| smoke | public smoke via the balancer (make test) | PASS | 0 | evidence/smoke.txt | +| balancing | balancing, shared sessions and data, X-Instance-Id | PASS | 1 | evidence/balancing.txt | +| idempotency | Idempotency-Key, parallel same key via replicas | PASS | 2 | evidence/idempotency.txt | +| student-tests | lab1/lab2 API tests via the balancer | PASS | 2 | evidence/student-tests.txt | +| race | lab2 races via the balancer | PASS | 1 | evidence/race.txt | +| db | PostgreSQL: rollback, FK, Argon2id, unique keys, migrations | PASS | 10 | evidence/db.txt | +| redis | Redis as the session store | PASS | 2 | evidence/redis.txt | +| explain | EXPLAIN of the history page (full plans: explain-history.txt) | PASS | 1 | evidence/explain.txt | +| failover | public failover test for every replica (make test-failover) | PASS | 38 | evidence/failover.txt | +| failover-metrics | stop/kill every replica under load, metrics | PASS | 70 | evidence/failover-metrics.txt | +| load | public load generator, 30 s, concurrency 4 | PASS | 30 | evidence/load.txt | +| expiry | session TTL=2 s, then normal TTL | PASS | 17 | evidence/expiry.txt | +| failures | Redis and PostgreSQL outages via the balancer | PASS | 47 | evidence/failures.txt | +| persistence | public persistence test + proof of real restarts (make test-persistence) | PASS | 9 | evidence/persistence.txt | +| rolling-update | rolling update under load | PASS | 28 | evidence/rolling-update.txt | + +# Замеры + +Машина: Windows 10, AMD64 Family 26 Model 68 Stepping 0, AuthenticAMD; Docker VM: 16 CPU, 15.2 GiB RAM; клиент Python 3.11.1; дата прогона 2026-10-14, commit 0ab3a34 + uncommitted changes. Лимиты контейнеров: api 1 CPU / 512 MiB каждая, postgres 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB. + +Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики): + +| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток (повторены с тем же ключом) | Первая запись после hook, с | +| --- | --- | --- | --- | --- | --- | +| 1 | 58542a53644c | 2 | 128 | 0 | 4.203 | +| 2 | 9737b915ff2f | 2 | 132 | 0 | 4.281 | + +Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа): + +| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок (коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | Вернулась в балансировку после docker start, с | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 58542a53644c | docker stop (SIGTERM) | 4.12 | 1.11 | 0 (-) | 0.0 | 304.7 | 202.2 | 47.0 | 3.25 | +| 58542a53644c | docker kill (SIGKILL) | 0.81 | 0.11 | 2 (502: 2) | 0.05 | 328.7 | 228.2 | 32.0 | 1.55 | +| 9737b915ff2f | docker stop (SIGTERM) | 4.12 | 1.5 | 0 (-) | 0.0 | 288.7 | 232.7 | 32.0 | 4.09 | +| 9737b915ff2f | docker kill (SIGKILL) | 0.83 | 0.11 | 2 (502: 2) | 0.05 | 266.7 | 219.9 | 32.0 | 4.14 | + +Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика после остановки не ответила ни разу. + +Rolling update (`student-tests/rolling_update.py`): версия rolling-20261014-201405 -> rolling-20261014-202804, 2 реплики заменены по одной за 13.48 с под той же нагрузкой. Записей/с до: 284.7, во время замены: 346.3; ошибок: 0 (-); p95 31.0 мс, максимум 78.0 мс; подтверждено 6527 сообщений, потерь и дублей нет. + +Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency 4, 30 с, 2 реплики): 308.848 успешных POST/с, ошибок 0 (доля 0.0), задержка успешных p50 15.0 мс, p95 32.0 мс, p99 46.0 мс. + +| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| --- | --- | --- | --- | +| `docker stop` реплики (SIGTERM) под нагрузкой | ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём | ошибок: 0, 0 (по репликам); HAProxy вывел реплику через 1.11, 1.5 с | ошибки прекратились через 0.0 с; потерь 0, дублей 0; реплика вернулась через 4.09 с после `docker start` | +| `docker kill` реплики (SIGKILL, авария) под нагрузкой | единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом | ошибок: 2, 2 (по репликам); HAProxy вывел реплику через 0.11, 0.11 с | ошибки прекратились через 0.05 с; потерь 0, дублей 0; реплика вернулась через 4.14 с после `docker start` | +| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, подтверждённые сообщения на месте, дублей нет | PASS 2 раз(а), неудачных попыток: 0, 0 | первая запись после hook через 4.203, 4.281 с (hook ждёт остановки ~4 с); потерь нет | +| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше 2 | ошибок: 0, замена заняла 13.48 с | потерь 0, дублей 0 | +| перезапуск всего стенда (`restart` hook: postgres, redis, api, lb) | данные и прежний токен живы | PASS, те же контейнеры с новым временем старта | 8.5 с до готовности; потерь нет | diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..5768739b --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,74 @@ +# HAProxy — единственная точка входа: клиент -> lb:8080 -> api (2+ реплики). +# Сессии и данные общие (Redis, PostgreSQL), поэтому любой запрос может обслужить +# любая реплика: балансировка round robin, без привязки клиента к реплике (no sticky). + +global + log stdout format raw local0 info + maxconn 4096 + # При остановке контейнера (SIGUSR1, soft stop) дождаться начатых запросов, но не дольше 10 с. + hard-stop-after 10s + +# Список реплик берётся из DNS Docker: имя сервиса "api" -> IP всех запущенных реплик. +# Остановили реплику или добавили новую — HAProxy увидит это без перезапуска. +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s # перечитывать список реплик раз в 2 с + hold obsolete 2s # IP, пропавший из DNS (контейнер остановлен), убрать через 2 с + hold nx 2s + hold other 2s + hold refused 2s + hold timeout 2s + accepted_payload_size 8192 + +defaults + mode http + log global + option httplog + option dontlog-normal # в лог только ошибки, а не каждый запрос + timeout connect 1s # сеть внутри Docker быстрая: 1 с хватает, мёртвый адрес не держит запрос + timeout client 30s + timeout server 15s # дольше самого долгого ответа API (8 с: пул 3 с + statement_timeout 5 с) + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + timeout check 1s + # Ошибка СОЕДИНЕНИЯ (запрос ещё не отправлен) — повторить на другой реплике, это безопасно для любого метода. + # Запрос, который уже ушёл в API, HAProxy сам не повторяет: POST повторяет клиент с тем же Idempotency-Key. + retries 2 + option redispatch 1 + +frontend api_in + bind :8080 + option forwardfor + default_backend api + +backend api + balance roundrobin + # Проверка готовности: /health/ready = API отвечает и видит PostgreSQL и Redis. + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + # До 6 реплик (2 обычно, 3 во время rolling update); пустые слоты стоят в MAINT. + # Проверка раз в 1 с, при смене состояния раз в 0,5 с; 2 неудачи подряд -> DOWN, 2 удачи -> UP. + # observe layer4: первая же ошибка соединения у живого запроса сразу выводит реплику из балансировки. + server-template api 6 api:8080 check inter 1s fastinter 500ms downinter 2s fall 2 rise 2 observe layer4 error-limit 1 on-error mark-down resolvers docker resolve-prefer ipv4 init-addr none + # Аварийный режим (как panic mode у Envoy): если НИ ОДНА реплика не готова (например, упал Redis + # или PostgreSQL), запросы всё равно идут в API. Клиент получает ответ самого API — 503 в формате + # контракта с X-Request-Id, /health/live — 200, а регистрация, которой Redis не нужен, работает. + # Пока готова хоть одна реплика, эти записи не используются. + # allow-dup-ip: те же IP, что у основных слотов (по умолчанию HAProxy запрещает повтор адреса в backend). + server-template fallback 6 api:8080 backup no-check resolvers docker resolve-prefer ipv4 resolve-opts allow-dup-ip init-addr none + option allbackups + +# Статистика HAProxy: http://127.0.0.1:8404/stats (опубликовано только на 127.0.0.1). +# /lb-health — healthcheck контейнера lb: 503, если направить запрос некуда. +frontend stats + bind :8404 + monitor-uri /lb-health + monitor fail if { nbsrv(api) lt 1 } + stats enable + stats uri /stats + stats refresh 2s diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 00000000..33c8c087 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,13 @@ +bind 0.0.0.0 +port 6379 +protected-mode yes +dir /data + +appendonly yes +appendfsync always +save "" + +maxmemory 64mb +maxmemory-policy noeviction + +include /run/secrets/redis_auth diff --git a/main.py b/main.py new file mode 100644 index 00000000..60b20dba --- /dev/null +++ b/main.py @@ -0,0 +1,476 @@ +import base64 +import binascii +import hashlib +import json +import logging +import re +import threading +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime +from uuid import UUID + +import uvicorn +from fastapi import FastAPI, Header, Query, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator +from starlette.concurrency import run_in_threadpool +from starlette.exceptions import HTTPException as StarletteHTTPException + +import auth +import db +import settings +from errors import ApiError, bad_request, unauthorized, unavailable + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") +log = logging.getLogger("messenger") + +DRAINING = threading.Event() + + +class Server(uvicorn.Server): + """Остановка по SIGTERM в два шага. + + 1. /health/ready сразу отвечает 503, а запросы ещё обслуживаются. За SHUTDOWN_DRAIN_SECONDS + HAProxy видит 503 (проверка раз в 0,5-1 с, 2 неудачи) и перестаёт слать сюда новые запросы. + 2. Обычная остановка uvicorn: закрыть приём соединений и доделать начатые запросы + (не дольше SHUTDOWN_TIMEOUT_SECONDS). Без шага 1 запрос, пришедший в момент SIGTERM, + получал бы 502 от балансировщика. + Повторный сигнал останавливает сразу. + """ + + def handle_exit(self, sig, frame): + if DRAINING.is_set(): + return super().handle_exit(sig, frame) + DRAINING.set() + log.info("Signal %s: readiness -> 503, leaving the balancer, stop in %s s", + sig, settings.SHUTDOWN_DRAIN_SECONDS) + threading.Timer(settings.SHUTDOWN_DRAIN_SECONDS, super().handle_exit, (sig, frame)).start() + + +@asynccontextmanager +async def lifespan(app: FastAPI): + settings.check() + # Миграции применяет сервис migrate (migrate.py); реплика только ждёт готовую схему. + await run_in_threadpool(db.wait_for_schema, settings.SCHEMA_WAIT_SECONDS) + db.pool.open() + log.info("Started: instance=%s, version=%s, env=%s, session TTL=%s s, sessions in Redis", + settings.INSTANCE_ID, settings.APP_VERSION, settings.APP_ENV, settings.SESSION_TTL_SECONDS) + yield + db.pool.close() + auth.close() + log.info("Stopped: connections closed, data stays in PostgreSQL and Redis") + + +app = FastAPI(lifespan=lifespan) + + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +class CreateUser(BaseModel): + model_config = ConfigDict(extra="forbid") + + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=auth.PASSWORD_MIN_LENGTH, max_length=auth.PASSWORD_MAX_LENGTH) + + @field_validator("display_name", "password") + @classmethod + def validate_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("value cannot be blank") + return value + + +class User(BaseModel): + id: UUID + username: str + display_name: str + created_at: datetime + + +class Session(BaseModel): + token: str + token_type: str + expires_at: datetime + user: User + + +class CreateChat(BaseModel): + model_config = ConfigDict(extra="forbid") + + title: str = Field(min_length=1, max_length=100) + member_ids: list[UUID] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def validate_title(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title cannot be empty") + return value + + +class Chat(BaseModel): + id: UUID + title: str + member_ids: list[UUID] + created_at: datetime + + +class ChatPage(BaseModel): + items: list[Chat] + next_cursor: str | None + + +class CreateMessage(BaseModel): + model_config = ConfigDict(extra="forbid") + + text: str = Field(max_length=4000) + + @field_validator("text") + @classmethod + def validate_text(cls, value: str) -> str: + if not value.strip(): + raise ValueError("text cannot be empty") + return value + + +class Message(BaseModel): + id: UUID + chat_id: UUID + sender_id: UUID + text: str + created_at: datetime + + +class MessagePage(BaseModel): + items: list[Message] + next_cursor: str | None + + +def error_response( + request: Request, + code: str, + message: str, + status_code: int, + headers: dict | None = None, +): + request_id = request.state.request_id + + return JSONResponse( + status_code=status_code, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + headers={"X-Request-Id": request_id, **(headers or {})}, + ) + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + return error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + + +@app.exception_handler(RequestValidationError) +async def validation_exception_handler( + request: Request, + exc: RequestValidationError, +): + return error_response( + request, + "invalid_request", + "Invalid request data", + 400, + ) + + +@app.exception_handler(StarletteHTTPException) +async def http_exception_handler(request: Request, exc: StarletteHTTPException): + if exc.status_code == 404: + return error_response(request, "not_found", "Not found", 404) + return error_response(request, "invalid_request", "Request cannot be processed", exc.status_code, exc.headers) + + +@app.exception_handler(Exception) +async def general_exception_handler( + request: Request, + exc: Exception, +): + return error_response( + request, + "internal_error", + "Internal server error", + 500, + ) + +PUBLIC_ENDPOINTS = { + ("POST", "/api/v1/users"), + ("POST", "/api/v1/auth/sessions"), +} + + +@app.middleware("http") +async def request_context(request: Request, call_next): + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + started = time.perf_counter() + path = request.url.path + + try: + if path.startswith("/api/") and (request.method, path) not in PUBLIC_ENDPOINTS: + request.state.session = await run_in_threadpool( + auth.check_bearer, request.headers.get("Authorization") + ) + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc.code, exc.message, exc.status_code, exc.headers) + except Exception: + log.exception("Unhandled error, request_id=%s", request_id) + response = error_response(request, "internal_error", "Internal server error", 500) + + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = settings.INSTANCE_ID # какая реплика ответила + # Проверки здоровья приходят каждую секунду от HAProxy и Docker, в лог пишем только ошибки. + level = logging.DEBUG if path.startswith("/health/") and response.status_code == 200 else logging.INFO + log.log( + level, + "%s %s -> %s (%.1f ms) request_id=%s", + request.method, path, response.status_code, + (time.perf_counter() - started) * 1000, request_id, + ) + return response + + +def current_user_id(request: Request) -> UUID: + session = getattr(request.state, "session", None) + if session is None: + raise unauthorized("Bearer token is required") + return session.user_id + + +def encode_cursor( + created_at: datetime, + object_id: UUID, + *, + chat_id: UUID | None = None, + user_id: UUID | None = None, +) -> str: + data = { + "created_at": created_at.isoformat(), + "id": str(object_id), + } + + if chat_id is not None: + data["chat_id"] = str(chat_id) + + if user_id is not None: + data["user_id"] = str(user_id) + + raw = json.dumps(data).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii") + + +def decode_cursor(cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor.encode("ascii")) + data = json.loads(raw.decode("utf-8")) + + created_at = datetime.fromisoformat(data["created_at"]) + + if created_at.tzinfo is None: + raise ValueError("Invalid cursor") + + if not isinstance(data["id"], str): + raise ValueError("Invalid cursor") + + object_id = UUID(data["id"]) + + chat_id = UUID(data["chat_id"]) if "chat_id" in data else None + user_id = UUID(data["user_id"]) if "user_id" in data else None + + return created_at, object_id, chat_id, user_id + + except ( + ValueError, + KeyError, + TypeError, + AttributeError, + json.JSONDecodeError, + binascii.Error, + ): + raise ValueError("Invalid cursor") + + +def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None): + """Курсор -> ключ (created_at, id), с которого продолжать. Чужой или битый курсор — 400.""" + if not cursor: + return None + try: + created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor) + except ValueError: + raise bad_request("Invalid cursor") from None + if cursor_chat_id != chat_id or cursor_user_id != user_id: + raise bad_request("Invalid cursor") + return created_at, object_id + + +@app.get("/health/live") +def health_live(): + return {"status": "ok"} + + +@app.get("/health/ready") +def health_ready(): + if DRAINING.is_set(): + raise unavailable("Instance is shutting down") + db.ping() + auth.ping_redis() + return {"status": "ok"} + + +@app.post("/api/v1/users", response_model=User, status_code=201) +def create_user(user_data: CreateUser): + password_hash = auth.hash_password(user_data.password) # сам пароль нигде не сохраняется + return db.create_user(user_data.username, user_data.display_name, password_hash) + + +@app.post("/api/v1/auth/sessions", response_model=Session, status_code=201) +def create_session( + response: Response, + authorization: str | None = Header(default=None), +): + credentials = auth.parse_basic(authorization) + if credentials is None: + raise auth.login_failed() + username, password = credentials + + found = None + if USERNAME_RE.fullmatch(username) and len(password) <= auth.PASSWORD_MAX_LENGTH: + found = db.get_user_with_password(username) + if not auth.password_matches(found["password_hash"] if found else None, password): + raise auth.login_failed() + + token, expires_at = auth.create_session(found["id"]) + response.headers["Cache-Control"] = "no-store" + user = {key: found[key] for key in ("id", "username", "display_name", "created_at")} + return {"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user} + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def delete_session(request: Request): + current_user_id(request) + auth.delete_session(request.state.session) + return Response(status_code=204) + + +@app.get("/api/v1/users/me", response_model=User) +def get_current_user(request: Request): + user = db.get_user(current_user_id(request)) + if user is None: + raise unauthorized("Session is invalid or expired") + return user + + + +@app.post("/api/v1/chats", response_model=Chat, status_code=201) +def create_chat(request: Request, chat_data: CreateChat): + user_id = current_user_id(request) + + if user_id not in chat_data.member_ids: + raise bad_request("Creator must be a chat member") + + if len(chat_data.member_ids) != len(set(chat_data.member_ids)): + raise bad_request("Duplicate member IDs") + return db.create_chat(chat_data.title, chat_data.member_ids) + + +@app.get("/api/v1/chats", response_model=ChatPage) +def list_chats( + request: Request, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, user_id=user_id) + + rows = db.list_chats(user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], user_id=user_id) + + return ChatPage(items=items, next_cursor=next_cursor) + + +@app.get("/api/v1/chats/{chat_id}", response_model=Chat) +def get_chat(request: Request, chat_id: UUID): + return db.get_chat(chat_id, current_user_id(request)) + + +def request_fingerprint(data: BaseModel) -> bytes: + """SHA-256 тела после разбора: порядок ключей и пробелы в JSON на него не влияют.""" + canonical = json.dumps(data.model_dump(mode="json"), sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +@app.post( + "/api/v1/chats/{chat_id}/messages", + response_model=Message, + status_code=201, +) +def create_message( + request: Request, + message_data: CreateMessage, + chat_id: UUID, + idempotency_key: str | None = Header(default=None), +): + # С lab3 ключ обязателен: повтор того же запроса (после таймаута, через другую реплику) + # вернёт то же сообщение, а не создаст второе. См. db.add_message. + if idempotency_key is None or not IDEMPOTENCY_KEY_RE.fullmatch(idempotency_key): + raise bad_request("Idempotency-Key header is required: 1-128 characters A-Z a-z 0-9 . _ : -") + return db.add_message(chat_id, current_user_id(request), message_data.text, + idempotency_key, request_fingerprint(message_data)) + + +@app.get( + "/api/v1/chats/{chat_id}/messages", + response_model=MessagePage, +) +def list_messages( + request: Request, + chat_id: UUID, + limit: int = Query(default=50, ge=1, le=100), + cursor: str | None = Query(default=None), +): + user_id = current_user_id(request) + after = page_after(cursor, chat_id=chat_id) + + rows = db.list_messages(chat_id, user_id, after, limit) + items = rows[:limit] + + next_cursor = None + if len(rows) > limit: + last = items[-1] + next_cursor = encode_cursor(last["created_at"], last["id"], chat_id=chat_id) + + return MessagePage(items=items, next_cursor=next_cursor) + + +if __name__ == "__main__": + Server(uvicorn.Config( + app, + host=settings.APP_HOST, + port=settings.APP_PORT, + timeout_graceful_shutdown=settings.SHUTDOWN_TIMEOUT_SECONDS, + access_log=False, + )).run() diff --git a/migrate.py b/migrate.py new file mode 100644 index 00000000..e95f414b --- /dev/null +++ b/migrate.py @@ -0,0 +1,8 @@ +import logging + +import db + +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s: %(message)s") + +if __name__ == "__main__": + db.wait_and_migrate() diff --git a/migrations/001_init.sql b/migrations/001_init.sql new file mode 100644 index 00000000..8b981d08 --- /dev/null +++ b/migrations/001_init.sql @@ -0,0 +1,33 @@ + +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE + CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id), + user_id uuid NOT NULL REFERENCES users (id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_by_user ON chat_members (user_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT now(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members (chat_id, user_id) +); + +CREATE INDEX messages_history ON messages (chat_id, created_at, id); diff --git a/migrations/002_text_as_bytes.sql b/migrations/002_text_as_bytes.sql new file mode 100644 index 00000000..73f0c849 --- /dev/null +++ b/migrations/002_text_as_bytes.sql @@ -0,0 +1,11 @@ +ALTER TABLE users DROP CONSTRAINT users_display_name_check; +ALTER TABLE chats DROP CONSTRAINT chats_title_check; +ALTER TABLE messages DROP CONSTRAINT messages_text_check; + +ALTER TABLE users ALTER COLUMN display_name TYPE bytea USING convert_to(display_name, 'UTF8'); +ALTER TABLE chats ALTER COLUMN title TYPE bytea USING convert_to(title, 'UTF8'); +ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8'); + +ALTER TABLE users ADD CONSTRAINT users_display_name_check CHECK (octet_length(display_name) BETWEEN 1 AND 400); +ALTER TABLE chats ADD CONSTRAINT chats_title_check CHECK (octet_length(title) BETWEEN 1 AND 400); +ALTER TABLE messages ADD CONSTRAINT messages_text_check CHECK (octet_length(text) BETWEEN 1 AND 16000); diff --git a/migrations/003_idempotency.sql b/migrations/003_idempotency.sql new file mode 100644 index 00000000..5770e1d7 --- /dev/null +++ b/migrations/003_idempotency.sql @@ -0,0 +1,11 @@ +ALTER TABLE messages ADD COLUMN idempotency_key text + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'); + +ALTER TABLE messages ADD COLUMN request_hash bytea + CHECK (octet_length(request_hash) = 32); + +ALTER TABLE messages ADD CONSTRAINT messages_key_and_hash_together + CHECK ((idempotency_key IS NULL) = (request_hash IS NULL)); + +ALTER TABLE messages ADD CONSTRAINT messages_idempotency_key + UNIQUE (chat_id, sender_id, idempotency_key); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..cd4d141d --- /dev/null +++ b/requirements.txt @@ -0,0 +1,21 @@ +fastapi==0.141.1 +uvicorn==0.54.0 +pydantic==2.13.5 +psycopg==3.3.6 +psycopg-binary==3.3.6 +psycopg-pool==3.3.3 +argon2-cffi==25.1.0 +redis==6.4.0 +annotated-doc==0.0.5 +annotated-types==0.8.0 +anyio==4.15.1 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +click==8.5.0 +h11==0.16.0 +idna==3.20 +pycparser==3.1 +pydantic_core==2.46.5 +starlette==1.7.0 +typing-inspection==0.4.4 +typing_extensions==4.16.0 diff --git a/scripts/init_secrets.py b/scripts/init_secrets.py new file mode 100644 index 00000000..3a422634 --- /dev/null +++ b/scripts/init_secrets.py @@ -0,0 +1,30 @@ +import secrets +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SECRETS = ROOT / "secrets" + + +def write(name: str, value: str) -> None: + with open(SECRETS / name, "w", encoding="utf-8", newline="") as file: + file.write(value) + (SECRETS / name).chmod(0o644) + + +def main() -> None: + if (SECRETS / "postgres_password").exists(): + print("secrets/ already exists, nothing changed.") + return + SECRETS.mkdir(exist_ok=True) + db_password = secrets.token_hex(24) + redis_password = secrets.token_hex(24) + + write("postgres_password", db_password) + write("database_url", f"postgresql://messenger:{db_password}@postgres:5432/messenger") + write("cache_url", f"redis://:{redis_password}@redis:6379/0") + write("redis_auth", f"requirepass {redis_password}\n") + print("Created secrets/: postgres_password, database_url, cache_url, redis_auth (values are not printed).") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..c7418fce --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,9 @@ +@echo off +rem Restart hook for tests/resilience.py on Windows (same as scripts/restart.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Restarts the stores first, then all api replicas, then the balancer. Volumes are kept (no down -v). +cd /d "%~dp0.." +docker compose restart postgres redis || exit /b 1 +docker compose restart api || exit /b 1 +docker compose restart lb || exit /b 1 +exit /b 0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..c2786efa --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Restart hook for tests/resilience.py --mode persistence (Linux/macOS). +# Restarts the stores first, then all api replicas, then the balancer. Volumes are kept (no down -v). +set -eu +cd "$(dirname "$0")/.." +docker compose restart postgres redis +docker compose restart api +docker compose restart lb diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..8ffd0fa2 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,6 @@ +@echo off +rem Hook for tests/resilience.py --mode failover on Windows (same as scripts/stop-one.sh). +rem Python on Windows cannot start .sh files directly (WinError 193), a .cmd file works. +rem Stops only the api replica from TARGET_INSTANCE_ID, the logic is in scripts\stop_one.py. +python "%~dp0stop_one.py" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..cf4d7244 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Hook for tests/resilience.py --mode failover (Linux/macOS): stops the api replica from TARGET_INSTANCE_ID. +# Logic is in scripts/stop_one.py, Windows uses scripts/stop-one.cmd. +set -eu +exec python3 "$(dirname "$0")/stop_one.py" diff --git a/scripts/stop_one.py b/scripts/stop_one.py new file mode 100644 index 00000000..5cdcce70 --- /dev/null +++ b/scripts/stop_one.py @@ -0,0 +1,29 @@ +import os +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +STOP_TIMEOUT_SECONDS = 8 + + +def main() -> int: + target = os.environ.get("TARGET_INSTANCE_ID", "").strip().lower() + if not re.fullmatch(r"[0-9a-f]{12,64}", target): + print("TARGET_INSTANCE_ID must be a container id (X-Instance-Id)", file=sys.stderr) + return 2 + running = subprocess.run(["docker", "compose", "ps", "-q", "api"], cwd=ROOT, + capture_output=True, text=True, check=True).stdout.split() + matches = [container for container in running if container.startswith(target)] + if len(matches) != 1: + print(f"no running api container with id {target}", file=sys.stderr) + return 1 + subprocess.run(["docker", "stop", "--time", str(STOP_TIMEOUT_SECONDS), matches[0]], + cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + print(f"stopped api container {matches[0][:12]}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/settings.py b/settings.py new file mode 100644 index 00000000..b03228dd --- /dev/null +++ b/settings.py @@ -0,0 +1,59 @@ +import os +import socket +from pathlib import Path + + +def env_str(name: str, default: str) -> str: + return os.environ.get(name, default).strip() + + +def env_int(name: str, default: int) -> int: + value = int(env_str(name, str(default))) + if value < 1: + raise ValueError(f"{name} must be >= 1") + return value + + +def read_secret(file_variable: str, plain_variable: str) -> str: + path = env_str(file_variable, "") + if path: + return Path(path).read_text(encoding="utf-8").strip() + return env_str(plain_variable, "") + + +APP_HOST = env_str("APP_HOST", "0.0.0.0") +APP_PORT = env_int("APP_PORT", 8080) +APP_ENV = env_str("APP_ENV", "lab") +APP_VERSION = env_str("APP_VERSION", "dev") + +INSTANCE_ID = env_str("INSTANCE_ID", "") or socket.gethostname() + +SESSION_TTL_SECONDS = env_int("SESSION_TTL_SECONDS", 3600) +SESSION_STORE = env_str("SESSION_STORE", "redis") + +DATABASE_URL = read_secret("DATABASE_URL_FILE", "DATABASE_URL") +DB_POOL_MIN = env_int("DB_POOL_MIN", 1) +DB_POOL_MAX = env_int("DB_POOL_MAX", 10) +DB_POOL_WAIT_SECONDS = env_int("DB_POOL_WAIT_SECONDS", 3) +DB_CONNECT_TIMEOUT_SECONDS = env_int("DB_CONNECT_TIMEOUT_SECONDS", 3) +DB_STATEMENT_TIMEOUT_MS = env_int("DB_STATEMENT_TIMEOUT_MS", 5000) + +REDIS_URL = read_secret("CACHE_URL_FILE", "CACHE_URL") +REDIS_POOL_MAX = env_int("REDIS_POOL_MAX", 20) +REDIS_TIMEOUT_MS = env_int("REDIS_TIMEOUT_MS", 500) + +MIGRATIONS_DIR = Path(env_str("MIGRATIONS_DIR", str(Path(__file__).resolve().parent / "migrations"))) +SCHEMA_WAIT_SECONDS = env_int("SCHEMA_WAIT_SECONDS", 60) + +SHUTDOWN_DRAIN_SECONDS = env_int("SHUTDOWN_DRAIN_SECONDS", 3) +SHUTDOWN_TIMEOUT_SECONDS = env_int("SHUTDOWN_TIMEOUT_SECONDS", 10) + + +def check() -> None: + """Проверка при старте: лучше сразу упасть с понятной ошибкой, чем на первом запросе.""" + if not DATABASE_URL: + raise RuntimeError("PostgreSQL address is not set: DATABASE_URL_FILE") + if not REDIS_URL: + raise RuntimeError("Redis address is not set: CACHE_URL_FILE") + if SESSION_STORE != "redis": + raise RuntimeError("SESSION_STORE: only 'redis' is supported in this solution") diff --git a/student-tests/check_stack.py b/student-tests/check_stack.py new file mode 100644 index 00000000..ed74ea52 --- /dev/null +++ b/student-tests/check_stack.py @@ -0,0 +1,72 @@ + +import json +import platform +import sys + +from common import call, check, compose_output, docker, new_user, seen_instances + + +def main(): + info = json.loads(docker("info", "--format", "{{json .}}")) + print(f"host: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"client Python {platform.python_version()}") + print(f"Docker {docker('version', '--format', '{{.Server.Version}}')}, " + f"{compose_output('version').strip()}; Docker VM: {info.get('NCPU')} CPU, " + f"{int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM, {info.get('OperatingSystem')}") + + print() + print(compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}\t{{.Ports}}").strip()) + migrate = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip() + check(migrate.startswith("Exited (0)"), "migrate service must finish with exit code 0") + + config = json.loads(compose_output("config", "--format", "json")) + services = config["services"] + containers = compose_output("ps", "-q").split() + rows = [json.loads(line) for line in docker("inspect", "--format", "{{json .}}", *containers).splitlines()] + print() + print("container image cpus memory host ports networks") + api_count = 0 + for row in sorted(rows, key=lambda r: r["Name"]): + labels = row["Config"]["Labels"] + service = labels.get("com.docker.compose.service") + limits = row["HostConfig"] + cpus = limits.get("NanoCpus", 0) / 1e9 + memory = limits.get("Memory", 0) // 2**20 + ports = sorted({f"{b['HostIp']}:{b['HostPort']}" for bindings in (row["NetworkSettings"]["Ports"] or {}).values() + for b in (bindings or [])}) + networks = sorted(name.split("_", 1)[-1] for name in row["NetworkSettings"]["Networks"]) + print(f"{row['Name'].lstrip('/'):<28} {row['Config']['Image']:<24} {cpus:>4.2f} {memory:>4} MiB " + f"{','.join(ports) or '-':<23} {','.join(networks)}") + check(cpus > 0 and memory > 0, f"{service}: no CPU/RAM limit") + if service == "api": + api_count += 1 + check(not ports, "api must not publish ports: everything goes through the balancer") + if service == "lb": + check(ports and all(p.startswith("127.0.0.1:") for p in ports), "lb must listen on 127.0.0.1 only") + if service in ("postgres", "redis"): + check(not ports and networks == ["data"], f"{service} must be only in the internal data network") + check(api_count >= 2, "need at least 2 api replicas") + check(all("container_name" not in s for s in services.values()), "container_name prevents scaling") + check(config["networks"]["data"].get("internal") is True, "data network must be internal") + check("data" not in services["lb"]["networks"], "lb must not reach postgres/redis") + + print() + for name, service in sorted(services.items()): + secrets = sorted(s["source"] for s in service.get("secrets", [])) + print(f"secrets of {name}: {secrets or '-'}") + check(not services["lb"].get("secrets"), "lb gets no secrets") + check(sorted(s["source"] for s in services["migrate"]["secrets"]) == ["database_url"], "migrate needs only database_url") + + user = new_user("stack") + served = seen_instances(user["token"]) + print() + print(f"through http://127.0.0.1:8080 one session is served by {len(served)} replicas: {sorted(served)}") + check(len(served) >= 2, "requests are not balanced") + health = call("GET", "/health/ready", 200) + print(f"/health/ready -> {health.status} from {health.instance}") + print(f"PASS: {api_count} api replicas behind one entry point, no direct API ports, limits on every " + f"container, internal data network, secrets only where needed") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/student-tests/collect_evidence.py b/student-tests/collect_evidence.py new file mode 100644 index 00000000..fe91ff4b --- /dev/null +++ b/student-tests/collect_evidence.py @@ -0,0 +1,243 @@ + +import json +import os +import platform +import re +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / "evidence" +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +PY = sys.executable + +STEPS = [ + ("check", "offline contract check (make check)", [PY, "scripts/check_contract.py"]), + ("stack", "stand layout: replicas, ports, limits, networks, secrets", [PY, "student-tests/check_stack.py"]), + ("smoke", "public smoke via the balancer (make test)", [PY, "tests/smoke.py", "--base-url", BASE_URL]), + ("balancing", "balancing, shared sessions and data, X-Instance-Id", [PY, "student-tests/test_balancing.py"]), + ("idempotency", "Idempotency-Key, parallel same key via replicas", [PY, "student-tests/test_idempotency.py"]), + ("student-tests", "lab1/lab2 API tests via the balancer", [PY, "student-tests/test_api.py"]), + ("race", "lab2 races via the balancer", [PY, "student-tests/test_race.py"]), + ("db", "PostgreSQL: rollback, FK, Argon2id, unique keys, migrations", [PY, "student-tests/test_db.py"]), + ("redis", "Redis as the session store", [PY, "student-tests/redis_info.py"]), + ("explain", "EXPLAIN of the history page (full plans: explain-history.txt)", [PY, "student-tests/explain.py"]), + ("failover", "public failover test for every replica (make test-failover)", [PY, "student-tests/run_failover.py"]), + ("failover-metrics", "stop/kill every replica under load, metrics", [PY, "student-tests/failover_metrics.py"]), + ("load", "public load generator, 30 s, concurrency 4", [PY, "tests/load.py", "--base-url", BASE_URL, + "--duration", "30", "--concurrency", "4", + "--output", "evidence/load.json"]), + ("expiry", "session TTL=2 s, then normal TTL", [PY, "student-tests/test_expiry.py"]), + ("failures", "Redis and PostgreSQL outages via the balancer", [PY, "student-tests/test_failures.py"]), + ("persistence", "public persistence test + proof of real restarts (make test-persistence)", + [PY, "student-tests/run_persistence.py"]), + ("rolling-update", "rolling update under load", [PY, "student-tests/rolling_update.py"]), +] + + +def git(*args): + try: + return subprocess.run(["git", *args], cwd=ROOT, capture_output=True, text=True, check=True).stdout.strip() + except (OSError, subprocess.CalledProcessError): + return "" + + +def commit(): + head = git("rev-parse", "--short", "HEAD") or "unknown" + changed = [line for line in git("status", "--porcelain").splitlines() + if not line[3:].startswith(("evidence/", "REPORT.md"))] + return head + (" + uncommitted changes" if changed else "") + + +def run_step(name, description, command): + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8", BASE_URL=BASE_URL) + print(f"=== {name}: {description}", flush=True) + started = time.monotonic() + result = subprocess.run(command, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + seconds = time.monotonic() - started + output = result.stdout.decode("utf-8", "replace").replace("\r\n", "\n") + shown = " ".join(Path(part).name if part == PY else part for part in command) + header = (f"# {name}: {description}\n# date: {time.strftime('%Y-%m-%d %H:%M:%S')}, commit: {commit()}\n" + f"# command: {shown}\n# exit code: {result.returncode}, {seconds:.1f} s\n\n") + (EVIDENCE / f"{name}.txt").write_text(header + output, encoding="utf-8") + tail = [line for line in output.splitlines() if line.strip()][-3:] + for line in tail: + print(" " + line) + print(f" -> {'PASS' if result.returncode == 0 else 'FAIL'} ({seconds:.0f} s), evidence/{name}.txt", flush=True) + return result.returncode == 0, seconds + + +def load_json(name): + path = EVIDENCE / name + return json.loads(path.read_text(encoding="utf-8")) if path.exists() else None + + +def measurements(): + """Markdown-таблицы замеров из evidence/*.json и evidence/*.txt (для REPORT.md и summary.md).""" + lines = [] + try: + info = json.loads(subprocess.run(["docker", "info", "--format", "{{json .}}"], capture_output=True, + text=True, check=True).stdout) + docker_vm = f"{info.get('NCPU')} CPU, {int(info.get('MemTotal', 0)) / 2**30:.1f} GiB RAM" + except (OSError, ValueError, subprocess.CalledProcessError): + docker_vm = "unknown" + lines.append(f"Машина: {platform.system()} {platform.release()}, {platform.processor() or platform.machine()}; " + f"Docker VM: {docker_vm}; клиент Python {platform.python_version()}; дата прогона " + f"{time.strftime('%Y-%m-%d')}, commit {commit()}. Лимиты контейнеров: api 1 CPU / 512 MiB " + f"каждая, postgres 1 CPU / 512 MiB, redis и lb по 0,5 CPU / 128 MiB.") + lines.append("") + + failover = EVIDENCE / "failover.txt" + results = [] + if failover.exists(): + text = failover.read_text(encoding="utf-8") + stopped = re.findall(r"the hook stopped (\w+)", text) + results = [json.loads(m) for m in re.findall(r"^(\{\"observed_instances_before\".*\})$", text, re.M)] + if results: + lines.append("Публичный тест отказа (`make test-failover`, 15 с записи во время остановки реплики):") + lines.append("") + lines.append("| Прогон | Остановлена реплика | Реплик до отказа | Успешных записей | Неудачных попыток " + "(повторены с тем же ключом) | Первая запись после hook, с |") + lines.append("| --- | --- | --- | --- | --- | --- |") + for number, (result, container) in enumerate(zip(results, stopped), start=1): + lines.append(f"| {number} | {container} | {result['observed_instances_before']} | " + f"{result['successful_writes']} | {result['failed_attempts']} | " + f"{result['first_recovery_seconds']} |") + lines.append("") + + metrics = load_json("failover-metrics.json") + if metrics: + lines.append("Остановка каждой реплики под нагрузкой (`student-tests/failover_metrics.py`: 4 потока записи " + "без пауз + 1 поток чтения, timeout клиента 2 с, 3 с до и 10 с после отказа):") + lines.append("") + lines.append("| Реплика | Как остановлена | Команда, с | HAProxy вывел из балансировки через, с | Ошибок " + "(коды) | Ошибки прекратились через, с | Записей/с до | Записей/с после | p95 после, мс | " + "Вернулась в балансировку после docker start, с |") + lines.append("| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |") + for r in metrics["runs"]: + how = "docker stop (SIGTERM)" if r["mode"] == "stop" else "docker kill (SIGKILL)" + codes = ", ".join(f"{k}: {v}" for k, v in r["error_codes"].items()) or "-" + lines.append(f"| {r['replica']} | {how} | {r['command_seconds']} | {r['lb_excluded_after_seconds']} | " + f"{r['errors']} ({codes}) | {r['errors_stopped_after_seconds']} | {r['before']['write_rps']} | " + f"{r['after']['write_rps']} | {r['after']['write_p95_ms']} | {r['rejoin_seconds']} |") + lines.append("") + lines.append("Во всех прогонах: потерянных подтверждённых сообщений 0, дублей 0, остановленная реплика " + "после остановки не ответила ни разу.") + lines.append("") + + rolling = load_json("rolling-update.json") + if rolling: + during = rolling["during"] + codes = ", ".join(f"{k}: {v}" for k, v in during["error_codes"].items()) or "-" + lines.append(f"Rolling update (`student-tests/rolling_update.py`): версия {rolling['from_version']} -> " + f"{rolling['to_version']}, {rolling['replicas']} реплики заменены по одной за " + f"{rolling['update_seconds']} с под той же нагрузкой. Записей/с до: {rolling['before']['write_rps']}, " + f"во время замены: {during['write_rps']}; ошибок: {during['errors']} ({codes}); " + f"p95 {during['write_p95_ms']} мс, максимум {during['write_max_ms']} мс; подтверждено " + f"{rolling['integrity']['confirmed_writes']} сообщений, потерь и дублей нет.") + lines.append("") + + load = load_json("load.json") + if load: + latency = load["successful_latency_ms"] + lines.append(f"Публичный генератор нагрузки (`tests/load.py`, closed loop, concurrency {load['concurrency']}, " + f"{load['requested_duration_seconds']:.0f} с, 2 реплики): {load['successful_rps']} успешных " + f"POST/с, ошибок {load['failed_requests']} (доля {load['error_ratio']}), задержка успешных " + f"p50 {latency['p50']} мс, p95 {latency['p95']} мс, p99 {latency['p99']} мс.") + lines.append("") + + persistence = EVIDENCE / "persistence.txt" + restart_seconds = None + if persistence.exists(): + found = re.search(r"took ([\d.]+) s", persistence.read_text(encoding="utf-8")) + restart_seconds = found.group(1) if found else None + + impacts = [] + if metrics: + for mode, what, expected in ( + ("stop", "`docker stop` реплики (SIGTERM) под нагрузкой", + "ошибок нет: реплика сначала уходит из балансировки (readiness 503), потом закрывает приём"), + ("kill", "`docker kill` реплики (SIGKILL, авария) под нагрузкой", + "единицы 502 на запросы, начатые на убитой реплике; клиент повторяет их с тем же ключом")): + runs = [r for r in metrics["runs"] if r["mode"] == mode] + if runs: + impacts.append( + f"| {what} | {expected} | ошибок: {', '.join(str(r['errors']) for r in runs)} (по репликам); " + f"HAProxy вывел реплику через {', '.join(str(r['lb_excluded_after_seconds']) for r in runs)} с | " + f"ошибки прекратились через {max(r['errors_stopped_after_seconds'] for r in runs)} с; " + f"потерь 0, дублей 0; реплика вернулась через " + f"{max(r['rejoin_seconds'] for r in runs)} с после `docker start` |") + if failover.exists() and results: + impacts.append( + f"| публичный `make test-failover` для каждой реплики | запись продолжается не позже 10 с, " + f"подтверждённые сообщения на месте, дублей нет | PASS {len(results)} раз(а), неудачных попыток: " + f"{', '.join(str(r['failed_attempts']) for r in results)} | первая запись после hook через " + f"{', '.join(str(r['first_recovery_seconds']) for r in results)} с (hook ждёт остановки ~4 с); потерь нет |") + if rolling: + impacts.append( + f"| rolling update под нагрузкой | ошибок нет, реплик всё время не меньше {rolling['replicas']} | " + f"ошибок: {rolling['during']['errors']}, замена заняла {rolling['update_seconds']} с | потерь 0, дублей 0 |") + if restart_seconds: + impacts.append( + f"| перезапуск всего стенда (`restart` hook: postgres, redis, api, lb) | данные и прежний токен живы | " + f"PASS, те же контейнеры с новым временем старта | {restart_seconds} с до готовности; потерь нет |") + if impacts: + lines.append("| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |") + lines.append("| --- | --- | --- | --- |") + lines.extend(impacts) + lines.append("") + return "\n".join(lines).strip() + + +def update_report(block): + report = ROOT / "REPORT.md" + text = report.read_text(encoding="utf-8") + begin, end = "", "" + if begin not in text or end not in text: + print("REPORT.md has no measurement markers, only evidence/summary.md was written") + return + head, rest = text.split(begin, 1) + _, tail = rest.split(end, 1) + report.write_text(f"{head}{begin}\n{block}\n{end}{tail}", encoding="utf-8") + print("REPORT.md: measurement tables updated") + + +def main(): + args = sys.argv[1:] + names = [name for name, _, _ in STEPS] + if args[:1] == ["--from"]: + if len(args) < 2 or args[1] not in names: + raise SystemExit("usage: --from ; steps: " + ", ".join(names)) + selected = names[names.index(args[1]):] + else: + unknown = [a for a in args if a not in names] + if unknown: + raise SystemExit(f"unknown steps {unknown}; steps: " + ", ".join(names)) + selected = args or names + EVIDENCE.mkdir(exist_ok=True) + results = [] + for name, description, command in STEPS: + if name not in selected: + continue + ok, seconds = run_step(name, description, command) + results.append((name, description, ok, seconds)) + if not ok: + print(f"\nSTOP: step '{name}' failed, see evidence/{name}.txt.") + print(f"After fixing: python student-tests/collect_evidence.py --from {name}") + break + block = measurements() + summary = ["# Сводка проверок", "", "| Шаг | Что проверяет | Результат | Время, с | Файл |", + "| --- | --- | --- | --- | --- |"] + summary += [f"| {n} | {d} | {'PASS' if ok else 'FAIL'} | {s:.0f} | evidence/{n}.txt |" for n, d, ok, s in results] + (EVIDENCE / "summary.md").write_text("\n".join(summary) + "\n\n# Замеры\n\n" + block + "\n", encoding="utf-8") + print("evidence/summary.md written") + passed = all(ok for _, _, ok, _ in results) + if passed: + update_report(block) # таблицы строятся из всех файлов evidence/, в том числе от прошлых запусков + raise SystemExit(0 if passed else 1) + + +if __name__ == "__main__": + main() diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 00000000..5ad8f63f --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,273 @@ + +import base64 +import csv +import hashlib +import io +import json +import os +import subprocess +import time +import urllib.error +import urllib.parse +import urllib.request +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +LB_STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv") +PASSWORD = "Test-password-123" + + +class Response: + def __init__(self, status, headers, data): + self.status = status + self.headers = {name.lower(): value for name, value in headers.items()} + self.data = data + try: + self.json = json.loads(data.decode("utf-8")) if data else None + except ValueError: + self.json = None + + @property + def instance(self): + """X-Instance-Id: какая реплика API ответила (ID её контейнера).""" + return self.headers.get("x-instance-id") + + +def check(condition, message): + if not condition: + raise AssertionError(message) + + +def call(method, path, expected=200, *, body=None, raw=None, token=None, headers=None, timeout=30): + """HTTP-запрос к API. expected — код или кортеж кодов, иначе AssertionError. + + Заодно проверяет то, что контракт требует от любого ответа: + X-Request-Id, X-Instance-Id (с lab3), единый формат ошибки, WWW-Authenticate у 401. + """ + all_headers = dict(headers or {}) + data = raw + if body is not None: + data = json.dumps(body).encode("utf-8") + if data is not None: + all_headers.setdefault("Content-Type", "application/json") + if token is not None: + all_headers["Authorization"] = "Bearer " + token + + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + response = Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + response = Response(error.code, error.headers, error.read()) + + expected = expected if isinstance(expected, tuple) else (expected,) + check(response.status in expected, + f"{method} {path}: expected {expected}, got {response.status} {response.data[:200]!r}") + request_id = response.headers.get("x-request-id") + check(request_id, f"{method} {path}: no X-Request-Id header") + check(response.instance, f"{method} {path}: no X-Instance-Id header") + if response.status >= 400: + error = (response.json or {}).get("error") or {} + check(error.get("code") and error.get("message"), f"{method} {path}: wrong error format") + check(error.get("request_id") == request_id, f"{method} {path}: request_id differs from header") + if response.status == 401: + check(response.headers.get("www-authenticate"), f"{method} {path}: 401 without WWW-Authenticate") + return response + + +def try_call(method, path, *, body=None, token=None, headers=None, timeout=2): + """Запрос без проверок и без исключений — для нагрузки во время отказа. + + Возвращает Response или None, если соединение оборвалось / вышел timeout. + """ + all_headers = dict(headers or {}) + data = None + if body is not None: + data = json.dumps(body).encode("utf-8") + all_headers["Content-Type"] = "application/json" + if token is not None: + all_headers["Authorization"] = "Bearer " + token + request = urllib.request.Request(BASE_URL + path, data=data, method=method, headers=all_headers) + try: + with urllib.request.urlopen(request, timeout=timeout) as reply: + return Response(reply.status, reply.headers, reply.read()) + except urllib.error.HTTPError as error: + try: + return Response(error.code, error.headers, error.read()) + except OSError: + return None + except (urllib.error.URLError, OSError): + return None + + +def basic(username, password): + """Значение заголовка Authorization: Basic base64(username:password), пароль в UTF-8.""" + return "Basic " + base64.b64encode(f"{username}:{password}".encode("utf-8")).decode("ascii") + + +def login(username, password, expected=201): + return call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(username, password)}) + + +def new_user(prefix="user", password=PASSWORD): + """Регистрирует пользователя и входит. Возвращает dict: id, username, password, token.""" + username = f"{prefix}_{uuid.uuid4().hex[:10]}" + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": prefix.title(), "password": password}).json + session = login(username, password).json + return {**user, "password": password, "token": session["token"]} + + +def new_chat(owner, *others, title="Test chat"): + member_ids = [owner["id"]] + [user["id"] for user in others] + return call("POST", "/api/v1/chats", 201, token=owner["token"], + body={"title": title, "member_ids": member_ids}).json + + +def new_key(): + return uuid.uuid4().hex + + +def send(user, chat_id, text, expected=201, key=None): + """POST сообщения. С lab3 Idempotency-Key обязателен: если не задан, берём новый случайный.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", expected, token=user["token"], + body={"text": text}, headers={"Idempotency-Key": key or new_key()}) + + +def read_history(user, chat_id, limit=100): + """Вся история чата по страницам. Возвращает (сообщения, число_страниц).""" + items, pages, cursor = [], 0, None + while True: + query = f"?limit={limit}" + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "") + page = call("GET", f"/api/v1/chats/{chat_id}/messages{query}", 200, token=user["token"]).json + items += page["items"] + pages += 1 + cursor = page["next_cursor"] + if cursor is None: + return items, pages + + +def wait_ready(seconds=90): + deadline = time.monotonic() + seconds + while True: + try: + with urllib.request.urlopen(BASE_URL + "/health/ready", timeout=3) as reply: + if reply.status == 200: + return + except (urllib.error.URLError, OSError): + pass + check(time.monotonic() < deadline, f"API is not ready after {seconds} s") + time.sleep(0.5) + + +def compose(*args, env=None): + """docker compose в корне репозитория. Вывод скрыт: в нём могут быть пути к секретам.""" + subprocess.run(["docker", "compose", *args], cwd=ROOT, check=True, env=env, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def compose_output(*args, env=None): + """docker compose и его вывод (для logs, ps).""" + result = subprocess.run(["docker", "compose", *args], cwd=ROOT, capture_output=True, check=True, env=env) + return result.stdout.decode("utf-8", "replace") + result.stderr.decode("utf-8", "replace") + + +def docker(*args, check_exit=True): + """docker , возвращает stdout.""" + result = subprocess.run(["docker", *args], cwd=ROOT, capture_output=True, check=check_exit) + return result.stdout.decode("utf-8", "replace").strip() + + +def api_containers(include_stopped=False): + """Полные ID контейнеров сервиса api (по умолчанию только запущенные).""" + args = ["ps", "-q", "-a", "api"] if include_stopped else ["ps", "-q", "api"] + return sorted(compose_output(*args).split()) + + +def health(container): + """Статус Docker healthcheck контейнера: starting / healthy / unhealthy / ''.""" + return docker("inspect", "-f", "{{if .State.Health}}{{.State.Health.Status}}{{end}}", container, + check_exit=False) + + +def lb_servers(): + """Состояние слотов backend api в HAProxy (страница статистики, CSV): {имя: статус}.""" + with urllib.request.urlopen(LB_STATS_URL, timeout=2) as reply: + rows = list(csv.reader(io.StringIO(reply.read().decode("utf-8", "replace")))) + header = [name.lstrip("# ") for name in rows[0]] + servers = {} + for row in rows[1:]: + item = dict(zip(header, row)) + if item.get("pxname") == "api" and item.get("svname") not in ("FRONTEND", "BACKEND"): + servers[item["svname"]] = item.get("status", "") + return servers + + +def lb_up_count(): + """Сколько основных слотов (api1..api6, не аварийных) HAProxy считает готовыми (UP).""" + return sum(1 for name, status in lb_servers().items() if name.startswith("api") and status.startswith("UP")) + + +def seen_instances(token, requests=20): + """Какие реплики отвечают через балансировщик: X-Instance-Id у серии GET /users/me.""" + return {call("GET", "/api/v1/users/me", 200, token=token).instance for _ in range(requests)} + + +def wait_balanced(replicas, seconds=60): + """Ждёт, пока все replicas контейнеров api healthy и HAProxy держит их в балансировке.""" + deadline = time.monotonic() + seconds + while True: + running = api_containers() + healthy = [container for container in running if health(container) == "healthy"] + try: + up = lb_up_count() + except OSError: + up = 0 + if len(healthy) >= replicas and up >= replicas: + return running + check(time.monotonic() < deadline, + f"after {seconds} s: {len(healthy)} healthy api containers, {up} UP in HAProxy, need {replicas}") + time.sleep(0.5) + + +def redis_cli(*args): + """redis-cli внутри контейнера redis. Пароль читается там же из секрета и не попадает в вывод.""" + command = "REDISCLI_AUTH=\"$(sed -n 's/^requirepass //p' /run/secrets/redis_auth)\" redis-cli " + " ".join(args) + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "sh", "-c", command], + cwd=ROOT, capture_output=True, check=True) + return result.stdout.decode("utf-8", "replace").strip() + + +def session_key(token): + """Ключ сессии в Redis, как его считает auth.py: session:.""" + return "session:" + hashlib.sha256(token.encode("utf-8")).hexdigest() + + +def psql(sql): + """Выполняет SQL в контейнере postgres. Возвращает (код выхода, вывод, текст ошибки). + + Значения в SQL тестов — только сгенерированные самим тестом (uuid), не ввод пользователя. + """ + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-A", "-t", "-v", "ON_ERROR_STOP=1", "-c", sql], + cwd=ROOT, capture_output=True, check=False, + ) + output = result.stdout.decode("utf-8", "replace").strip() + error = result.stderr.decode("utf-8", "replace").strip() + return result.returncode, output, error + + +def run_tests(tests): + """Запускает функции-тесты по очереди. Код выхода 1, если хоть один упал.""" + failed = 0 + for test in tests: + try: + test() + except Exception as exc: # noqa: BLE001 — нужно напечатать любую ошибку теста + failed += 1 + print(f"FAIL: {test.__name__}: {exc}") + print(f"{len(tests) - failed} passed, {failed} failed") + raise SystemExit(1 if failed else 0) diff --git a/student-tests/demo_instances.py b/student-tests/demo_instances.py new file mode 100644 index 00000000..8f97f1c3 --- /dev/null +++ b/student-tests/demo_instances.py @@ -0,0 +1,16 @@ + +from common import call, compose_output, new_user + + +def main(): + user = new_user("demo") + print(f"logged in once as {user['username']}; 8 requests with the same Bearer token:") + for number in range(1, 9): + reply = call("GET", "/api/v1/users/me", 200, token=user["token"]) + print(f" {number}. GET /api/v1/users/me -> {reply.status}, X-Instance-Id: {reply.instance}") + print() + print(compose_output("ps", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip()) + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.py b/student-tests/explain.py new file mode 100644 index 00000000..81416de6 --- /dev/null +++ b/student-tests/explain.py @@ -0,0 +1,28 @@ + +import subprocess + +from common import ROOT + +OUTPUT = ROOT / "evidence" / "explain-history.txt" + + +def main(): + sql = (ROOT / "student-tests" / "explain.sql").read_bytes() + result = subprocess.run( + ["docker", "compose", "exec", "-T", "postgres", + "psql", "-U", "messenger", "-d", "messenger", "-X", "-q", "-v", "ON_ERROR_STOP=1"], + cwd=ROOT, input=sql, capture_output=True, check=False, + ) + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_bytes(result.stdout + result.stderr) + if result.returncode != 0: + print(result.stderr.decode("utf-8", "replace")) + raise SystemExit("FAIL: psql returned an error, see output above") + for line in result.stdout.decode("utf-8", "replace").splitlines(): + if line.startswith("===") or "Buffers" in line or "Execution Time" in line or " Scan " in line: + print(line) + print(f"PASS: full plans saved to {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/explain.sql b/student-tests/explain.sql new file mode 100644 index 00000000..39e9a564 --- /dev/null +++ b/student-tests/explain.sql @@ -0,0 +1,66 @@ + +\pset footer off +BEGIN; + +CREATE TEMP TABLE seed_chats AS + SELECT gen_random_uuid() AS id, n, CASE WHEN n = 0 THEN 15000 ELSE 100 END AS size + FROM generate_series(0, 50) AS n; +CREATE TEMP TABLE seed_user AS SELECT gen_random_uuid() AS id; + +INSERT INTO users (id, username, display_name, password_hash) + SELECT id, 'explain_' || substr(md5(random()::text), 1, 10), convert_to('Explain', 'UTF8'), 'not-a-real-hash' + FROM seed_user; +INSERT INTO chats (id, title) SELECT id, convert_to('explain ' || n, 'UTF8') FROM seed_chats; +INSERT INTO chat_members (chat_id, user_id) SELECT c.id, u.id FROM seed_chats c, seed_user u; +INSERT INTO messages (id, chat_id, sender_id, text, created_at) + SELECT gen_random_uuid(), c.id, u.id, convert_to('message ' || g, 'UTF8'), + timestamptz '2026-09-01 00:00:00+00' + g * interval '1 second' + FROM seed_chats c, seed_user u, generate_series(1, c.size) AS g; +ANALYZE messages; + +SELECT id AS big_chat FROM seed_chats WHERE n = 0 \gset +SELECT created_at AS cursor_time, id AS cursor_id FROM messages + WHERE chat_id = :'big_chat' ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset + +\echo '=== Data volume' +SELECT count(*) AS messages_total, + count(*) FILTER (WHERE chat_id = :'big_chat') AS messages_in_big_chat +FROM messages; + +\echo '' +\echo '=== History index' +SELECT indexdef FROM pg_indexes WHERE indexname = 'messages_history'; + +\echo '' +\echo '=== 1. First page: limit=50, the app asks for 51 rows to know if there is a next page' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 2. Page after the 10000th message by keyset cursor (this is what the app does)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +\echo '' +\echo '=== 3. The same page with OFFSET 10000 (the app does NOT do this)' +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' +ORDER BY created_at, id OFFSET 10000 LIMIT 51; + +\echo '' +\echo '=== 4. Query 2 without the index (index is dropped only inside this transaction)' +DROP INDEX messages_history; +EXPLAIN (ANALYZE, BUFFERS, COSTS OFF) +SELECT id, chat_id, sender_id, text, created_at FROM messages +WHERE chat_id = :'big_chat' AND (created_at, id) > (:'cursor_time'::timestamptz, :'cursor_id'::uuid) +ORDER BY created_at, id LIMIT 51; + +ROLLBACK; +\echo '' +\echo '=== ROLLBACK: test data removed, index is back' +SELECT count(*) AS explain_users_left FROM users WHERE username LIKE 'explain\_%'; diff --git a/student-tests/failover_metrics.py b/student-tests/failover_metrics.py new file mode 100644 index 00000000..19421a15 --- /dev/null +++ b/student-tests/failover_metrics.py @@ -0,0 +1,129 @@ +import json +import platform +import sys +import threading +import time + +from common import ROOT, api_containers, check, docker, lb_up_count, wait_balanced +from loadgen import Load, summary + +WARMUP_SECONDS = 3 +AFTER_SECONDS = 10 +OUTPUT = ROOT / "evidence" / "failover-metrics.json" + + +class LbWatch: + """Раз в 0,1 с спрашивает у HAProxy, сколько реплик в балансировке (UP).""" + + def __init__(self): + self.points, self.stopping = [], threading.Event() + self.thread = threading.Thread(target=self._run, daemon=True) + + def _run(self): + while not self.stopping.is_set(): + try: + self.points.append((time.monotonic(), lb_up_count())) + except OSError: + pass + time.sleep(0.1) + + def start(self): + self.thread.start() + + def stop(self): + self.stopping.set() + self.thread.join(timeout=3) + + def first_drop(self, after, replicas): + return next((t for t, up in self.points if t >= after and up < replicas), None) + + +def measure(container, mode, replicas): + wait_balanced(replicas) + load, watch = Load(), LbWatch() + watch.start() + load.start() + time.sleep(WARMUP_SECONDS) + t0 = time.monotonic() + if mode == "stop": + docker("stop", "--time", "8", container) + else: + docker("kill", container) + command_seconds = time.monotonic() - t0 + time.sleep(max(0.0, AFTER_SECONDS - command_seconds)) + end = t0 + AFTER_SECONDS + load.stop() + watch.stop() + + errors = [e for e in load.window(t0 - WARMUP_SECONDS, end) if not e[2]] + errors_end = max((e[0] + e[5] for e in errors), default=None) + drop = watch.first_drop(t0, replicas) + served_after = [e[4] for e in load.window(t0 + command_seconds, end) if e[2]] + target_hits = sum(1 for instance in served_after if instance and container.startswith(instance)) + integrity = load.verify() + + started = time.monotonic() + docker("start", container) + wait_balanced(replicas) + rejoin_seconds = time.monotonic() - started + + return { + "replica": container[:12], + "mode": mode, + "command_seconds": round(command_seconds, 2), + "lb_excluded_after_seconds": round(drop - t0, 2) if drop else None, + "errors": len(errors), + "error_codes": summary(load, t0 - WARMUP_SECONDS, end)["error_codes"], + "errors_stopped_after_seconds": round(errors_end - t0, 2) if errors_end else 0.0, + "before": summary(load, t0 - WARMUP_SECONDS, t0), + "after": summary(load, t0, end), + "stopped_replica_answers_after_stop": target_hits, + "integrity": integrity, + "rejoin_seconds": round(rejoin_seconds, 2), + } + + +def main(): + modes = sys.argv[1:] or ["stop", "kill"] + check(set(modes) <= {"stop", "kill"}, "usage: python student-tests/failover_metrics.py [stop] [kill]") + containers = api_containers() + replicas = len(containers) + check(replicas >= 2, "need at least 2 running api replicas") + print(f"api replicas: {[c[:12] for c in containers]}; load: 4 writers without pauses + 1 reader, " + f"client timeout 2 s, retry with the same Idempotency-Key; {WARMUP_SECONDS} s before and " + f"{AFTER_SECONDS} s after the failure") + runs = [] + for container in containers: + for mode in modes: + print(f"--- {mode} {container[:12]} ...") + run = measure(container, mode, replicas) + runs.append(run) + print(f" docker {mode}: {run['command_seconds']} s; HAProxy excluded it after " + f"{run['lb_excluded_after_seconds']} s; errors: {run['errors']} {run['error_codes']}, " + f"stopped after {run['errors_stopped_after_seconds']} s") + print(f" write RPS before {run['before']['write_rps']} -> after {run['after']['write_rps']}; " + f"p95 after {run['after']['write_p95_ms']} ms, max {run['after']['write_max_ms']} ms; " + f"answers from the stopped replica after stop: {run['stopped_replica_answers_after_stop']}") + print(f" confirmed {run['integrity']['confirmed_writes']}, in history " + f"{run['integrity']['messages_in_history']}, lost 0, duplicates 0; " + f"back in the balancer {run['rejoin_seconds']} s after docker start") + check(run["stopped_replica_answers_after_stop"] == 0, "the stopped replica still answered") + check(run["after"]["write_rps"] > 0, "no successful writes after the failure") + print() + print("replica mode cmd,s lb_out,s errors errors_end,s rps_before rps_after p95_after,ms rejoin,s") + for r in runs: + print(f"{r['replica']} {r['mode']:<4} {r['command_seconds']:>5} {str(r['lb_excluded_after_seconds']):>8} " + f"{r['errors']:>6} {r['errors_stopped_after_seconds']:>12} {r['before']['write_rps']:>10} " + f"{r['after']['write_rps']:>9} {str(r['after']['write_p95_ms']):>12} {r['rejoin_seconds']:>8}") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2, "warmup_seconds": WARMUP_SECONDS, + "after_seconds": AFTER_SECONDS, "replicas": replicas, "client_python": platform.python_version()}, + "runs": runs, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: every replica was stopped ({', '.join(modes)}) and restored; no lost or duplicated messages; " + f"saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/loadgen.py b/student-tests/loadgen.py new file mode 100644 index 00000000..998548cf --- /dev/null +++ b/student-tests/loadgen.py @@ -0,0 +1,102 @@ +import math +import threading +import time + +from common import check, new_chat, new_key, new_user, read_history, try_call + + +class Load: + def __init__(self, writers=4, timeout=2): + self.alice, self.bob = new_user("load_a"), new_user("load_b") + self.chat = new_chat(self.alice, self.bob, title="load") + self.writers, self.timeout = writers, timeout + self.events = [] # (время, "write"/"read", ok, код или "conn", реплика, задержка в с) + self.confirmed = [] # сообщения, на которые пришёл 201 + self.lock = threading.Lock() + self.stopping = threading.Event() + self.threads = [] + + def _record(self, kind, started, reply, ok): + code = reply.status if reply is not None else "conn" + instance = reply.instance if reply is not None else None + with self.lock: + self.events.append((started, kind, ok, code, instance, time.monotonic() - started)) + + def _writer(self, number): + path = f"/api/v1/chats/{self.chat['id']}/messages" + sent = 0 + key, text = new_key(), f"load-{number}-{sent}" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("POST", path, body={"text": text}, token=self.alice["token"], + headers={"Idempotency-Key": key}, timeout=self.timeout) + ok = reply is not None and reply.status == 201 + self._record("write", started, reply, ok) + if ok: + with self.lock: + self.confirmed.append(reply.json) + sent += 1 + key, text = new_key(), f"load-{number}-{sent}" + else: + time.sleep(0.05) # и повторить тот же text с тем же ключом + + def _reader(self): + path = f"/api/v1/chats/{self.chat['id']}/messages?limit=20" + while not self.stopping.is_set(): + started = time.monotonic() + reply = try_call("GET", path, token=self.bob["token"], timeout=self.timeout) + self._record("read", started, reply, reply is not None and reply.status == 200) + time.sleep(0.05) + + def start(self): + self.threads = [threading.Thread(target=self._writer, args=(n,), daemon=True) for n in range(self.writers)] + self.threads.append(threading.Thread(target=self._reader, daemon=True)) + for thread in self.threads: + thread.start() + + def stop(self): + self.stopping.set() + for thread in self.threads: + thread.join(timeout=self.timeout + 5) + + def window(self, start, end, kind=None): + with self.lock: + return [e for e in self.events if start <= e[0] < end and (kind is None or e[1] == kind)] + + def verify(self): + """Все подтверждённые сообщения есть в истории, id и тексты не повторяются.""" + history, _ = read_history(self.bob, self.chat["id"]) + ids = [message["id"] for message in history] + texts = [message["text"] for message in history] + by_id = {message["id"]: message for message in history} + check(len(ids) == len(set(ids)), "duplicated message ids in history") + check(len(texts) == len(set(texts)), "a retried request created a duplicate message") + lost = [m for m in self.confirmed if by_id.get(m["id"]) != m] + check(not lost, f"{len(lost)} confirmed messages are lost or changed") + return {"confirmed_writes": len(self.confirmed), "messages_in_history": len(history), "duplicates": 0, "lost": 0} + + +def summary(load, start, end): + """Успешные RPS, ошибки и задержки на отрезке [start, end).""" + events = load.window(start, end) + writes = [e for e in events if e[1] == "write"] + ok_writes = sorted(e[5] for e in writes if e[2]) + errors = [e for e in events if not e[2]] + seconds = max(end - start, 1e-6) + + def percentile(p): + return round(1000 * ok_writes[max(0, math.ceil(p * len(ok_writes)) - 1)], 1) if ok_writes else None + + codes = {} + for e in errors: + codes[str(e[3])] = codes.get(str(e[3]), 0) + 1 + return { + "seconds": round(seconds, 2), + "write_rps": round(len(ok_writes) / seconds, 1), + "reads": len([e for e in events if e[1] == "read"]), + "errors": len(errors), + "error_codes": codes, + "write_p50_ms": percentile(0.50), + "write_p95_ms": percentile(0.95), + "write_max_ms": round(1000 * ok_writes[-1], 1) if ok_writes else None, + } diff --git a/student-tests/redis_info.py b/student-tests/redis_info.py new file mode 100644 index 00000000..6535eae2 --- /dev/null +++ b/student-tests/redis_info.py @@ -0,0 +1,29 @@ +import subprocess + +from common import ROOT, login, new_user, redis_cli, session_key + + +def without_password(): + result = subprocess.run(["docker", "compose", "exec", "-T", "redis", "redis-cli", "get", "anything"], + cwd=ROOT, capture_output=True, check=False) + return (result.stdout + result.stderr).decode("utf-8", "replace").strip() + + +def main(): + user = new_user("info") + token = login(user["username"], user["password"]).json["token"] + key = session_key(token) + print(f"session key: {key[:24]}...; value is the user id: {redis_cli('get', key) == user['id']}") + print(f"TTL of the key, seconds: {redis_cli('ttl', key)}") + keys = redis_cli("--scan", "--pattern", "'session:*'").splitlines() + print(f"sessions in Redis: {len(keys)}") + for name in ("appendonly", "appendfsync", "maxmemory-policy"): + print("config", " = ".join(redis_cli("config", "get", name).split())) + for line in redis_cli("info", "persistence").splitlines(): + if line.startswith(("aof_enabled", "aof_last_write_status")): + print(line) + print(f"without password: {without_password()}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/rolling_update.py b/student-tests/rolling_update.py new file mode 100644 index 00000000..f2f3ffbb --- /dev/null +++ b/student-tests/rolling_update.py @@ -0,0 +1,88 @@ +import json +import os +import time + +from common import ROOT, api_containers, check, compose, compose_output, docker, health, lb_up_count, wait_balanced +from loadgen import Load, summary + +OUTPUT = ROOT / "evidence" / "rolling-update.json" + + +def env_of(container): + lines = docker("inspect", "-f", "{{range .Config.Env}}{{println .}}{{end}}", container).splitlines() + return dict(line.split("=", 1) for line in lines if "=" in line) + + +def wait_new_replica(container, up_needed, seconds=90): + deadline = time.monotonic() + seconds + while health(container) != "healthy" or lb_up_count() < up_needed: + check(time.monotonic() < deadline, f"new replica {container[:12]} is not in the balancer after {seconds} s") + time.sleep(0.2) + + +def main(): + old = api_containers() + replicas = len(old) + check(replicas >= 2, "need at least 2 running api replicas") + wait_balanced(replicas) + old_version = env_of(old[0]).get("APP_VERSION") + version = time.strftime("rolling-%Y%m%d-%H%M%S") + env = dict(os.environ, APP_VERSION=version) + + print(f"old replicas {[c[:12] for c in old]}, version {old_version}; building version {version}") + started = time.monotonic() + compose("build", "api", "migrate", env=env) + print(f"image built in {time.monotonic() - started:.1f} s") + migrate_log = compose_output("run", "--rm", "--no-deps", "-T", "migrate", env=env) + print("migrations: " + [line for line in migrate_log.splitlines() if "Migrations applied" in line][-1] + .split("messenger.db: ")[-1]) + + load = Load() + load.start() + time.sleep(3) + t0 = time.monotonic() + steps = [] + for container in old: + step_start = time.monotonic() + before = set(api_containers()) + compose("up", "-d", "--no-deps", "--no-recreate", "--scale", f"api={len(before) + 1}", "api", env=env) + new = sorted(set(api_containers()) - before) + check(len(new) == 1, f"expected one new container, got {len(new)}") + wait_new_replica(new[0], replicas + 1) + ready = time.monotonic() - step_start + docker("stop", "--time", "15", container) + docker("rm", container) + steps.append({"old": container[:12], "new": new[0][:12], "new_ready_seconds": round(ready, 2), + "step_seconds": round(time.monotonic() - step_start, 2)}) + print(f"replaced {container[:12]} -> {new[0][:12]}: new replica ready and in HAProxy after {ready:.1f} s, " + f"step {steps[-1]['step_seconds']} s") + t1 = time.monotonic() + time.sleep(3) + load.stop() + + final = api_containers() + versions = sorted({env_of(c).get("APP_VERSION") for c in final}) + integrity = load.verify() + during = summary(load, t0, t1) + before_stats = summary(load, t0 - 3, t0) + print(f"now: replicas {[c[:12] for c in final]}, versions {versions}") + print(f"under load: write RPS before {before_stats['write_rps']}, during the update {during['write_rps']}; " + f"errors {during['errors']} {during['error_codes']}; p95 {during['write_p95_ms']} ms, " + f"max {during['write_max_ms']} ms; update took {t1 - t0:.1f} s") + print(f"confirmed {integrity['confirmed_writes']}, in history {integrity['messages_in_history']}, " + f"lost 0, duplicates 0") + check(len(final) == replicas and not set(final) & set(old), "old replicas are still running") + check(versions == [version], "not every replica runs the new version") + OUTPUT.parent.mkdir(exist_ok=True) + OUTPUT.write_text(json.dumps({ + "from_version": old_version, "to_version": version, "replicas": replicas, + "profile": {"writers": 4, "readers": 1, "client_timeout_seconds": 2}, + "steps": steps, "update_seconds": round(t1 - t0, 2), + "before": before_stats, "during": during, "integrity": integrity, + }, indent=2) + "\n", encoding="utf-8") + print(f"PASS: all {replicas} replicas replaced one by one under load, " + f"{during['errors']} errors, no lost or duplicated messages; saved {OUTPUT.relative_to(ROOT)}") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_failover.py b/student-tests/run_failover.py new file mode 100644 index 00000000..0f28ec8c --- /dev/null +++ b/student-tests/run_failover.py @@ -0,0 +1,55 @@ +import os +import subprocess +import sys + +from common import BASE_URL, ROOT, api_containers, call, check, compose_output, docker, wait_balanced + +HOOK = ROOT / "scripts" / ("stop-one.cmd" if os.name == "nt" else "stop-one.sh") + + +def ps(): + return compose_output("ps", "-a", "--format", "table {{.Service}}\t{{.ID}}\t{{.Status}}", "api").strip() + + +def aim(wanted): + """Для 2 реплик: последним перед тестом должна ответить реплика из wanted.""" + probe = call("GET", "/health/live", 200).instance + if not any(container.startswith(probe) for container in wanted): + call("GET", "/health/live", 200) # теперь последней ответила другая реплика + + +def main(): + containers = api_containers() + check(len(containers) >= 2, "need at least 2 running api replicas") + print(f"hook: {HOOK.relative_to(ROOT)}; api replicas: {[c[:12] for c in containers]}") + stopped_once = set() + for attempt in range(1, 7): + wait_balanced(len(containers)) + if len(containers) == 2: + aim(set(containers) - stopped_once) + print(f"=== run {attempt}: api containers before") + print(ps()) + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "failover", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public failover test failed") + print("api containers after") + print(ps()) + stopped = [container for container in containers if container not in api_containers()] + check(len(stopped) == 1, f"expected exactly one stopped api container, got {len(stopped)}") + print(f"the hook stopped {stopped[0][:12]} (X-Instance-Id of that replica = container ID); " + f"other api containers kept running, postgres/redis/lb were not touched") + stopped_once.add(stopped[0]) + docker("start", stopped[0]) + wait_balanced(len(containers)) + print(f"restored: docker start {stopped[0][:12]} -> healthy and back in HAProxy") + if stopped_once == set(containers): + break + check(stopped_once == set(containers), "not every replica was stopped by the public test") + print(f"PASS: public failover test passed {attempt} times; every api replica was stopped once and restored") + + +if __name__ == "__main__": + main() diff --git a/student-tests/run_persistence.py b/student-tests/run_persistence.py new file mode 100644 index 00000000..a59562fa --- /dev/null +++ b/student-tests/run_persistence.py @@ -0,0 +1,47 @@ +import os +import subprocess +import sys +import time + +from common import BASE_URL, ROOT, check, compose_output, docker + +HOOK = ROOT / "scripts" / ("restart.cmd" if os.name == "nt" else "restart.sh") + + +def started(): + containers = compose_output("ps", "-q").split() + lines = docker("inspect", "-f", "{{.Name}} {{.Id}} {{.State.StartedAt}}", *containers).splitlines() + return {name.lstrip("/"): (cid, at) for name, cid, at in (line.split() for line in lines)} + + +def main(): + before = started() + print("before: container, ID, started at") + for name, (cid, at) in sorted(before.items()): + print(f" {name:<28} {cid[:12]} {at}") + env = dict(os.environ, PYTHONUTF8="1", PYTHONIOENCODING="utf-8") + begin = time.monotonic() + result = subprocess.run([sys.executable, "tests/resilience.py", "--mode", "persistence", + "--action-script", str(HOOK), "--base-url", BASE_URL], + cwd=ROOT, capture_output=True, env=env) + seconds = time.monotonic() - begin + print((result.stdout + result.stderr).decode("utf-8", "replace").strip()) + check(result.returncode == 0, "public persistence test failed") + after = started() + print(f"after (hook + waiting for readiness took {seconds:.1f} s): container, ID, started at, restarted?") + restarted = [] + for name, (cid, at) in sorted(after.items()): + same_id = before.get(name, ("", ""))[0] == cid + new_start = before.get(name, ("", ""))[1] != at + print(f" {name:<28} {cid[:12]} {at} same container: {same_id}, restarted: {new_start}") + check(same_id, f"{name}: container was recreated, volumes/IDs must stay") + if new_start: + restarted.append(name.split("-")[-2]) + for service in ("postgres", "redis", "api", "lb"): + check(service in restarted, f"{service} was not restarted by the hook") + print("PASS: the hook restarted postgres, redis, all api replicas and lb (same containers, new start time); " + "data and the old token survived") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..4a682052 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,246 @@ +import base64 +import datetime as dt +import uuid + +from common import PASSWORD, basic, call, check, login, new_chat, new_key, new_user, read_history, run_tests, send + + +def parse_time(value): + return dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + + + +def test_duplicate_username(): + body = {"username": "dup_" + uuid.uuid4().hex[:10], "display_name": "First user", "password": PASSWORD} + call("POST", "/api/v1/users", 201, body=body) + call("POST", "/api/v1/users", 409, body=dict(body, display_name="Second user")) + print("PASS: duplicate username returns 409") + + +def test_invalid_json(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", "/api/v1/users", 400, raw=b'{"username":') + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], raw=b"not json", + headers={"Idempotency-Key": new_key()}) + print("PASS: invalid JSON returns 400") + + +def test_wrong_fields(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + call("POST", "/api/v1/users", 400, body={"username": "Bad Name!", "display_name": "x", "password": PASSWORD}) + call("POST", "/api/v1/users", 400, + body={"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "x", "password": PASSWORD, "is_admin": True}) + # Отправителя нельзя подставить: он берётся из сессии. + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], + body={"text": "hi", "sender_id": bob["id"]}, headers={"Idempotency-Key": new_key()}) + print("PASS: wrong and extra fields return 400") + + +def test_foreign_chat(): + alice, bob, eve = new_user("alice"), new_user("bob"), new_user("eve") + chat = new_chat(alice, bob) + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"]) + send(eve, chat["id"], "not my chat", expected=404) + # Несуществующий чат выглядит так же, как чужой. + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 404, token=eve["token"]) + print("PASS: foreign chat returns 404") + + +def test_empty_message(): + alice = new_user("alice") + chat = new_chat(alice) + for text in ("", " ", "\n\t"): + send(alice, chat["id"], text, expected=400) + send(alice, chat["id"], "я" * 4001, expected=400) + send(alice, chat["id"], "я" * 4000) + print("PASS: empty and too long messages return 400, 4000 chars is ok") + + +def test_pagination(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = [send(alice, chat["id"], f"Message {i}").json["id"] for i in range(7)] + + history, pages = read_history(bob, chat["id"], limit=2) + check(pages == 4, f"expected 4 pages, got {pages}") + check([m["id"] for m in history] == sent, "pages overlap, lose or reorder messages") + + other = new_chat(alice) + for i in range(3): + send(alice, other["id"], f"Other {i}") + first = call("GET", f"/api/v1/chats/{other['id']}/messages?limit=1", token=alice["token"]).json + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor={first['next_cursor']}", 400, token=alice["token"]) + call("GET", f"/api/v1/chats/{chat['id']}/messages?cursor=broken", 400, token=alice["token"]) + print("PASS: pagination works across multiple pages, foreign cursor returns 400") + + +def test_chat_list_pagination(): + alice, bob = new_user("alice"), new_user("bob") + created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)] + seen, cursor = [], None + while True: + query = "?limit=1" + (f"&cursor={cursor}" if cursor else "") + page = call("GET", "/api/v1/chats" + query, token=bob["token"]).json + seen += [chat["id"] for chat in page["items"]] + cursor = page["next_cursor"] + if cursor is None: + break + check(seen == created, "chat list pages are wrong") + print("PASS: chat list pagination") + + +def test_unicode(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob, title="Чат 👋") + text = "Привет 👋 你好" + check(send(alice, chat["id"], text).json["text"] == text, "text changed on write") + history, _ = read_history(bob, chat["id"]) + check(history[-1]["text"] == text, "text changed on read") + print("PASS: unicode text is stored as is") + + +def test_nul_character_is_stored(): + name, title, text = "A\u0000B", "Chat\u0000title", "hello\u0000world" + body = {"username": "nul_" + uuid.uuid4().hex[:10], "display_name": name, "password": "pass\u0000word-123"} + user = call("POST", "/api/v1/users", 201, body=body).json + check(user["display_name"] == name, "display_name with U+0000 changed") + token = login(body["username"], body["password"]).json["token"] + check(call("GET", "/api/v1/users/me", token=token).json["display_name"] == name, "display_name changed on read") + chat = call("POST", "/api/v1/chats", 201, token=token, body={"title": title, "member_ids": [user["id"]]}).json + check(chat["title"] == title, "title with U+0000 changed") + check(call("GET", f"/api/v1/chats/{chat['id']}", token=token).json["title"] == title, "title changed on read") + sent = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=token, body={"text": text}, + headers={"Idempotency-Key": new_key()}).json + history = call("GET", f"/api/v1/chats/{chat['id']}/messages", token=token).json["items"] + check(sent["text"] == text and history[-1]["text"] == text, "text with U+0000 changed") + login("nul\u0000user", PASSWORD, expected=401) + print("PASS: strings with U+0000 are stored and returned as is") + + + +def test_user_has_no_password(): + body = {"username": "np_" + uuid.uuid4().hex[:10], "display_name": "No password", "password": PASSWORD} + user = call("POST", "/api/v1/users", 201, body=body).json + check(set(user) == {"id", "username", "display_name", "created_at"}, f"unexpected fields: {set(user)}") + print("PASS: user response has no password or hash") + + +def test_password_rules(): + def register(password, expected): + body = {"username": "pw_" + uuid.uuid4().hex[:10], "display_name": "Password", "password": password} + call("POST", "/api/v1/users", expected, body=body) + + register("x" * 11, 400) + register("x" * 129, 400) + register(" " * 12, 400) + register("x" * 12, 201) + register("я" * 128, 201) + print("PASS: password length 12..128, blank password is rejected") + + +def test_login_gives_token(): + username = "lg_" + uuid.uuid4().hex[:10] + user = call("POST", "/api/v1/users", 201, + body={"username": username, "display_name": "Login", "password": PASSWORD}).json + reply = login(username, PASSWORD) + session = reply.json + check(session["token_type"] == "Bearer", "token_type must be Bearer") + check(session["user"] == user, "session user differs from registered user") + check(len(session["token"]) >= 43, "token is too short") + check(username not in session["token"] and user["id"] not in session["token"], "token is not opaque") + check(reply.headers.get("cache-control") == "no-store", "response with token must not be cached") + left = (parse_time(session["expires_at"]) - dt.datetime.now(dt.timezone.utc)).total_seconds() + check(290 < left <= 3600 + 5, f"unexpected session lifetime: {left:.0f} s") + me = call("GET", "/api/v1/users/me", token=session["token"]).json + check(me == user, "/users/me returned another user") + print("PASS: Basic login returns opaque Bearer token with expiry") + + +def test_wrong_password(): + user = new_user("wp") + wrong = login(user["username"], "Wrong-password-000", expected=401) + unknown = login("nobody_" + uuid.uuid4().hex[:8], PASSWORD, expected=401) + check(wrong.headers["www-authenticate"].lower().startswith("basic"), "login 401 must ask for Basic") + check(wrong.json["error"]["message"] == unknown.json["error"]["message"], "different answers leak usernames") + print("PASS: wrong password and unknown user both return the same 401") + + +def test_malformed_basic(): + no_colon = base64.b64encode(b"just_username").decode() + for header in (None, "Basic", "Basic !!!", "Basic " + no_colon, "Bearer something"): + headers = {} if header is None else {"Authorization": header} + call("POST", "/api/v1/auth/sessions", 401, headers=headers) + print("PASS: malformed Basic header returns 401") + + +def test_password_with_colon_and_unicode(): + password = "пароль:с:двоеточием 👋" + user = new_user("colon", password=password) + call("GET", "/api/v1/users/me", token=user["token"]) + print("PASS: password with colon and unicode works") + + +def test_bearer_required(): + user = new_user("br") + for headers in ({}, {"Authorization": basic(user["username"], user["password"])}, + {"X-User-Id": user["id"]}, {"Authorization": "Bearer " + "A" * 43}, + {"Authorization": "Bearer"}, {"Authorization": "Token " + user["token"]}): + reply = call("GET", "/api/v1/users/me", 401, headers=headers) + check(reply.headers["www-authenticate"].lower().startswith("bearer"), "401 must ask for Bearer") + call("GET", "/api/v1/users/me", 200, headers={"Authorization": "bearer " + user["token"]}) + print("PASS: business endpoints need Bearer; Basic and X-User-Id give 401") + + +def test_x_user_id_ignored(): + alice, bob = new_user("alice"), new_user("bob") + me = call("GET", "/api/v1/users/me", token=alice["token"], headers={"X-User-Id": bob["id"]}).json + check(me["id"] == alice["id"], "X-User-Id changed the current user") + print("PASS: X-User-Id does not change the current user") + + +def test_auth_before_body(): + alice = new_user("alice") + chat = new_chat(alice) + call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, raw=b"not json") + call("GET", f"/api/v1/chats/{uuid.uuid4()}", 401) + print("PASS: 401 comes before 400 and 404") + + +def test_logout(): + user = new_user("lo") + second = login(user["username"], user["password"]).json["token"] + call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + call("GET", "/api/v1/users/me", 401, token=user["token"]) + call("DELETE", "/api/v1/auth/sessions/current", 401, token=user["token"]) + call("GET", "/api/v1/users/me", 200, token=second) # другая сессия того же пользователя жива + again = login(user["username"], user["password"]).json["token"] + check(again != user["token"], "new login must give a new token") + call("GET", "/api/v1/users/me", 200, token=again) + print("PASS: logout revokes only the current session") + + +if __name__ == "__main__": + run_tests([ + test_duplicate_username, + test_invalid_json, + test_wrong_fields, + test_foreign_chat, + test_empty_message, + test_pagination, + test_chat_list_pagination, + test_unicode, + test_nul_character_is_stored, + test_user_has_no_password, + test_password_rules, + test_login_gives_token, + test_wrong_password, + test_malformed_basic, + test_password_with_colon_and_unicode, + test_bearer_required, + test_x_user_id_ignored, + test_auth_before_body, + test_logout, + ]) diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 00000000..b7122786 --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,86 @@ +from collections import Counter + +from common import api_containers, call, check, new_chat, new_key, new_user, run_tests, send + + +def test_requests_spread_over_replicas(): + alice = new_user("alice") + replies = [call("GET", "/api/v1/users/me", 200, token=alice["token"]) for _ in range(20)] + served = Counter(reply.instance for reply in replies) + print(f"20 x GET /users/me with one token -> replicas: {dict(served)}") + check(all(reply.json["id"] == alice["id"] for reply in replies), "another user returned") + check(len(served) >= 2, "all requests went to one replica: balancing does not work") + print("PASS: one session works on every replica, no second login (sessions are shared in Redis)") + + +def test_write_on_one_read_on_other(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + sent = send(alice, chat["id"], "written on one replica") + readers = set() + for _ in range(6): + reply = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]) + check(sent.json in reply.json["items"], "confirmed message is not visible on another replica") + readers.add(reply.instance) + print(f"message written by {sent.instance}; read back by {sorted(readers)}") + check(readers - {sent.instance}, "no read went to another replica") + print("PASS: a message written via one replica is read right away via another (shared PostgreSQL)") + + +def test_logout_seen_by_all_replicas(): + user = new_user("logout") + out = call("DELETE", "/api/v1/auth/sessions/current", 204, token=user["token"]) + after = [call("GET", "/api/v1/users/me", 401, token=user["token"]).instance for _ in range(10)] + print(f"logout on {out.instance}; old token then rejected by {sorted(set(after))}") + check(len(set(after)) >= 2, "only one replica answered") + print("PASS: logout on one replica revokes the session on all of them") + + +def test_idempotent_retry_on_other_replica(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "retry me", key=key) + for _ in range(10): # round robin: следующий запрос уйдёт на другую реплику + retry = send(alice, chat["id"], "retry me", key=key) + if retry.instance != first.instance: + break + print(f"first POST on {first.instance} -> 201 {first.json['id']}; " + f"same key on {retry.instance} -> {retry.status} {retry.json['id']}") + check(retry.instance != first.instance, "retry never reached another replica") + check(retry.json == first.json, "retry on another replica returned a different message") + items = call("GET", f"/api/v1/chats/{chat['id']}/messages", 200, token=bob["token"]).json["items"] + check([m["text"] for m in items].count("retry me") == 1, "retry created a duplicate") + print("PASS: Idempotency-Key is shared: a retry through another replica returns the same message") + + +def test_instance_header_everywhere(): + containers = api_containers() + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + replies = [ + call("GET", "/health/live", 200), + call("GET", "/health/ready", 200), + send(alice, chat["id"], "hello"), + call("POST", f"/api/v1/chats/{chat['id']}/messages", 400, token=alice["token"], body={"text": "no key"}), + call("GET", "/api/v1/users/me", 401), + call("GET", f"/api/v1/chats/{chat['id']}", 404, token=eve["token"]), + call("POST", "/api/v1/users", 409, + body={"username": alice["username"], "display_name": "x", "password": alice["password"]}), + ] + for reply in replies: # call() уже проверил, что заголовок есть; здесь — что это настоящий контейнер + check(any(container.startswith(reply.instance) for container in containers), + f"X-Instance-Id {reply.instance} is not a running api container") + print(f"statuses {[r.status for r in replies]}: every answer has X-Instance-Id of a running container " + f"{sorted({r.instance for r in replies})}; api containers: {[c[:12] for c in containers]}") + print("PASS: X-Instance-Id is in every response and points to a real container") + + +if __name__ == "__main__": + run_tests([ + test_requests_spread_over_replicas, + test_write_on_one_read_on_other, + test_logout_seen_by_all_replicas, + test_idempotent_retry_on_other_replica, + test_instance_header_everywhere, + ]) diff --git a/student-tests/test_db.py b/student-tests/test_db.py new file mode 100644 index 00000000..3d71fcc1 --- /dev/null +++ b/student-tests/test_db.py @@ -0,0 +1,101 @@ +import time +import uuid + +from common import call, check, compose_output, new_user, psql, run_tests + + +def sql_value(sql): + code, output, error = psql(sql) + check(code == 0, f"psql failed: {error}") + return output + + +def test_create_chat_rolls_back(): + alice = new_user("rollback") + title = "rollback-" + uuid.uuid4().hex[:8] + inserted_before = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + + call("POST", "/api/v1/chats", 400, token=alice["token"], + body={"title": title, "member_ids": [alice["id"], str(uuid.uuid4())]}) + + left = sql_value(f"SELECT count(*) FROM chats WHERE title = '{title}'") + orphans = sql_value("SELECT count(*) FROM chats c WHERE NOT EXISTS " + "(SELECT 1 FROM chat_members m WHERE m.chat_id = c.id)") + inserted_after = inserted_before + deadline = time.monotonic() + 15 + while inserted_after == inserted_before and time.monotonic() < deadline: + time.sleep(0.5) + inserted_after = int(sql_value("SELECT n_tup_ins FROM pg_stat_user_tables WHERE relname = 'chats'")) + print(f"chat with unknown member -> 400; rows with this title: {left}; chats without members: {orphans}; " + f"chats insert counter: {inserted_before} -> {inserted_after}") + check(left == "0" and orphans == "0", "chat row survived the failed transaction") + check(inserted_after > inserted_before, "the chat row was never inserted, rollback is not shown") + print("PASS: chat row was inserted and then rolled back together with members") + + +def test_message_needs_membership_in_db(): + code, _, error = psql("INSERT INTO messages (id, chat_id, sender_id, text) " + f"VALUES ('{uuid.uuid4()}', '{uuid.uuid4()}', '{uuid.uuid4()}', convert_to('direct insert', 'UTF8'))") + print(f"direct INSERT of a message from a non-member -> exit code {code}: {error.splitlines()[0] if error else ''}") + check(code != 0 and "foreign key" in error, "database accepted a message from a non-member") + print("PASS: the database itself rejects a message from a non-member") + + +def test_password_is_argon2id(): + user = new_user("hash") + stored = sql_value(f"SELECT password_hash FROM users WHERE username = '{user['username']}'") + print(f"stored password: {stored[:31]}... (length {len(stored)})") + check(stored.startswith("$argon2id$") and user["password"] not in stored, "password is not an Argon2id hash") + print("PASS: password is stored as Argon2id hash") + + +def test_unique_username_constraint(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conrelid = 'users'::regclass AND contype = 'u'") + print(f"unique constraint on users: {constraint}") + check(constraint == "UNIQUE (username)", "no unique constraint on username") + print("PASS: username is unique in the database") + + +def test_migrations_are_not_repeated(): + versions = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + print(f"applied migrations: {versions}; running the migrate service once more") + output = compose_output("run", "--rm", "--no-deps", "-T", "migrate") + again = sql_value("SELECT string_agg(version, ',' ORDER BY version) FROM schema_migrations") + last_log = [line for line in output.splitlines() if "Migrations applied" in line][-1] + print(f"after the second run: {again}; log: {last_log.split('messenger.db: ')[-1]}") + check(again == versions and "none" in last_log, "migrations were applied again") + print("PASS: migrations are applied once, a repeated run does nothing") + + +def test_replicas_do_not_migrate(): + """lab3: миграции применяет только сервис migrate, API-реплики лишь проверяют схему.""" + logs = compose_output("logs", "api").splitlines() + checked = [line for line in logs if "Schema is up to date" in line] + migrated = [line for line in logs if "Migrations applied" in line] + migrate_exit = compose_output("ps", "-a", "--format", "{{.Status}}", "migrate").strip() + print(f"api replicas that checked the schema at start: {len(checked)}; api log lines 'Migrations applied': " + f"{len(migrated)}; migrate service: {migrate_exit}") + check(checked and not migrated, "api replicas must not apply migrations") + check(migrate_exit.startswith("Exited (0)"), "migrate service must finish with exit code 0") + print("PASS: only the migrate service applies migrations, replicas wait for the schema") + + +def test_idempotency_key_is_unique_in_db(): + constraint = sql_value("SELECT pg_get_constraintdef(oid) FROM pg_constraint " + "WHERE conname = 'messages_idempotency_key'") + print(f"unique constraint for Idempotency-Key: {constraint}") + check(constraint == "UNIQUE (chat_id, sender_id, idempotency_key)", "no unique constraint for idempotency keys") + print("PASS: one message per (chat, sender, Idempotency-Key) is enforced by the database") + + +if __name__ == "__main__": + run_tests([ + test_create_chat_rolls_back, + test_message_needs_membership_in_db, + test_password_is_argon2id, + test_unique_username_constraint, + test_migrations_are_not_repeated, + test_replicas_do_not_migrate, + test_idempotency_key_is_unique_in_db, + ]) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 00000000..e6017ce1 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,57 @@ +import datetime as dt +import os +import time + +from common import call, check, compose, login, new_user, redis_cli, run_tests, session_key, wait_ready + +SHORT_TTL = 2 + + +def restart_api(ttl=None): + env = dict(os.environ) + if ttl is None: + env.pop("SESSION_TTL_SECONDS", None) # значение снова из .env + else: + env["SESSION_TTL_SECONDS"] = str(ttl) + compose("up", "-d", "--no-deps", "--wait", "api", env=env) + wait_ready() + + +def seconds_left(expires_at): + moment = dt.datetime.fromisoformat(expires_at.replace("Z", "+00:00")) + return (moment - dt.datetime.now(dt.timezone.utc)).total_seconds() + + +def test_session_expires(): + print(f"recreating api with SESSION_TTL_SECONDS={SHORT_TTL}") + restart_api(SHORT_TTL) + try: + user = new_user("ttl") + session = login(user["username"], user["password"]).json + token = session["token"] + left = seconds_left(session["expires_at"]) + print(f"session issued, expires in {left:.2f} s; Redis TTL of the key: " + f"{redis_cli('pttl', session_key(token))} ms") + check(0 < left <= SHORT_TTL + 0.5, "short TTL instance must issue ~2 second sessions") + + call("GET", "/api/v1/users/me", 200, token=token) + print("before expiry: GET /users/me -> 200") + + time.sleep(max(0.0, seconds_left(session["expires_at"])) + 1.0) + call("GET", "/api/v1/users/me", 401, token=token) + exists = redis_cli("exists", session_key(token)) + print(f"after expiry: GET /users/me -> 401; key exists in Redis: {exists}") + check(exists == "0", "Redis must delete the expired session key") + finally: + print("recreating api with SESSION_TTL_SECONDS from .env") + restart_api() + + user = new_user("ttl_back") + left = seconds_left(login(user["username"], user["password"]).json["expires_at"]) + print(f"normal API again: new session expires in {left:.0f} s") + check(left > 300, "normal TTL was not restored") + print("PASS: session expires after SESSION_TTL_SECONDS and is rejected with 401") + + +if __name__ == "__main__": + run_tests([test_session_expires]) diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 00000000..8eaef386 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,110 @@ +import sys +import time +import uuid + +from common import PASSWORD, basic, call, check, compose, new_chat, new_key, new_user, read_history, run_tests, send, wait_ready + +FAKE_TOKEN = "A" * 43 + + +def timed(method, path, expected, **kwargs): + started = time.monotonic() + reply = call(method, path, expected, **kwargs) + return reply, (time.monotonic() - started) * 1000 + + +def prepare(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + before = send(alice, chat["id"], "confirmed before outage").json + return alice, bob, chat, before + + +def recovered(alice, chat, before): + wait_ready() + history, _ = read_history(alice, chat["id"]) + check(before in history, "message confirmed before the outage is lost or changed") + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the old token works, confirmed messages are in place") + + +def test_redis_down(): + alice, bob, chat, before = prepare() + print("stopping redis") + compose("stop", "redis") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + reply, ms = timed("GET", "/api/v1/users/me", 503, token=alice["token"]) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + print(f"valid token: GET /users/me -> 503 unavailable ({ms:.0f} ms), session cannot be checked") + call("GET", "/api/v1/chats", 503, token=FAKE_TOKEN) + print("unknown token -> 503, not 200: nobody is trusted without a check") + call("GET", "/api/v1/chats", 401) + print("no token -> 401") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + print("login -> 503: there is nowhere to store the session") + call("POST", "/api/v1/users", 201, + body={"username": "rd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("registration -> 201: it needs only PostgreSQL") + finally: + print("starting redis") + compose("start", "redis") + recovered(alice, chat, before) + print("PASS: Redis outage gives 503 and does not let anyone in without a session check") + + +def test_redis_crash(): + alice = new_user("crash") + print("killing redis with SIGKILL (docker compose kill redis): no graceful shutdown") + compose("kill", "redis") + try: + # Обычно 503; 200 возможен, если Docker уже успел поднять Redis по restart policy. + status = call("GET", "/api/v1/users/me", (503, 200), token=alice["token"]).status + print(f"right after the crash: GET /users/me -> {status}") + finally: + print("starting redis") + compose("start", "redis") + wait_ready() + call("GET", "/api/v1/users/me", 200, token=alice["token"]) + print("after start: the session issued before the crash still works (AOF, appendfsync always)") + print("PASS: sessions survive a Redis crash, not only a clean restart") + + +def test_postgres_down(): + alice, bob, chat, before = prepare() + print("stopping postgres") + compose("stop", "postgres") + try: + call("GET", "/health/live", 200) + _, ms = timed("GET", "/health/ready", 503) + print(f"/health/live -> 200, /health/ready -> 503 ({ms:.0f} ms)") + for method, path, body in (("GET", "/api/v1/users/me", None), + ("GET", "/api/v1/chats", None), + ("GET", f"/api/v1/chats/{chat['id']}/messages", None), + ("POST", f"/api/v1/chats/{chat['id']}/messages", {"text": "will not be saved"})): + reply = call(method, path, 503, token=alice["token"], body=body, headers={"Idempotency-Key": new_key()}) + check(reply.json["error"]["code"] == "unavailable", "expected code unavailable") + check("postgres" not in reply.data.decode().lower(), "error must not show database details") + print("valid token: users/me, chats, history, new message -> 503 unavailable, no details") + call("GET", "/api/v1/chats", 401) + call("GET", "/api/v1/chats", 401, token=FAKE_TOKEN) + print("no token -> 401, unknown token -> 401: anonymous users are not let in") + call("POST", "/api/v1/auth/sessions", 503, headers={"Authorization": basic(alice["username"], PASSWORD)}) + call("POST", "/api/v1/users", 503, + body={"username": "pd_" + uuid.uuid4().hex[:10], "display_name": "x", "password": PASSWORD}) + print("login -> 503, registration -> 503") + finally: + print("starting postgres") + compose("start", "postgres") + recovered(alice, chat, before) + print("PASS: PostgreSQL outage gives limited 503 errors and keeps anonymous users out") + + +if __name__ == "__main__": + target = sys.argv[1] if len(sys.argv) > 1 else "all" + tests = {"redis": [test_redis_down, test_redis_crash], "postgres": [test_postgres_down], + "all": [test_redis_down, test_redis_crash, test_postgres_down]} + check(target in tests, "usage: python student-tests/test_failures.py [redis|postgres]") + run_tests(tests[target]) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..407dea3d --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,174 @@ +import json +import socket +import threading +import time +import urllib.parse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor + +from common import BASE_URL, call, check, new_chat, new_key, new_user, read_history, run_tests, send + +PARALLEL = 20 + + +def post_raw(user, chat_id, raw, key): + """POST с телом «как есть» (bytes) — чтобы проверить другой порядок ключей и пробелы.""" + return call("POST", f"/api/v1/chats/{chat_id}/messages", (201, 409, 400), token=user["token"], + raw=raw, headers={"Idempotency-Key": key}) + + +def texts_in(user, chat_id): + history, _ = read_history(user, chat_id) + return [message["text"] for message in history] + + +def test_replay_returns_same_message(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + first = send(alice, chat["id"], "Привет 👋", key=key) + # То же JSON-значение, записанное иначе: пробелы и \u-экранирование. + same = post_raw(alice, chat["id"], b'{ "text" : "\\u041f\\u0440\\u0438\\u0432\\u0435\\u0442 \\ud83d\\udc4b" }', key) + again = send(alice, chat["id"], "Привет 👋", key=key) + print(f"first -> {first.status}; same JSON written differently -> {same.status}; plain retry -> {again.status}") + check(same.status == 201 and same.json == first.json and again.json == first.json, + "retry must return the original 201 and the same message") + check(texts_in(bob, chat["id"]).count("Привет 👋") == 1, "retry created a duplicate") + print("PASS: a retry with the same key returns the original 201 and message, no duplicate") + + +def test_same_key_parallel_through_replicas(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(_): + barrier.wait() # все запросы стартуют одновременно + return send(alice, chat["id"], "one message for 20 requests", key=key) + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + replies = list(pool.map(post, range(PARALLEL))) + bodies = {json.dumps(reply.json, sort_keys=True) for reply in replies} + served = Counter(reply.instance for reply in replies) + count = texts_in(bob, chat["id"]).count("one message for 20 requests") + print(f"{PARALLEL} parallel POSTs with one key: statuses {dict(Counter(r.status for r in replies))}, " + f"replicas {dict(served)}, different bodies in answers: {len(bodies)}, messages in history: {count}") + check(len(served) >= 2, "requests did not go through different replicas") + check(len(bodies) == 1 and count == 1, "parallel retries created more than one message") + print("PASS: parallel requests with one key through different replicas create exactly one message") + + +def test_same_key_different_bodies_parallel(): + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + barrier = threading.Barrier(PARALLEL) + + def post(index): + text = "body A" if index % 2 == 0 else "body B" + barrier.wait() + reply = call("POST", f"/api/v1/chats/{chat['id']}/messages", (201, 409), token=alice["token"], + body={"text": text}, headers={"Idempotency-Key": key}) + return text, reply + + with ThreadPoolExecutor(max_workers=PARALLEL) as pool: + results = list(pool.map(post, range(PARALLEL))) + texts = texts_in(alice, chat["id"]) + winner = [t for t in texts if t in ("body A", "body B")] + print(f"{PARALLEL} parallel POSTs, one key, two different bodies: " + f"{dict(Counter((t, r.status) for t, r in results))}; saved: {winner}") + check(len(winner) == 1, "exactly one message must be created") + for text, reply in results: + expected = 201 if text == winner[0] else 409 + check(reply.status == expected, f"{text}: expected {expected}, got {reply.status}") + print("PASS: the first body wins, the same body gets 201, the other body gets 409") + + +def test_key_scope(): + alice, bob = new_user("alice"), new_user("bob") + chat = new_chat(alice, bob) + other = new_chat(alice, bob) + key = new_key() + a1 = send(alice, chat["id"], "scope", key=key).json + b1 = send(bob, chat["id"], "scope", key=key).json + a2 = send(alice, other["id"], "scope", key=key).json + print(f"one key: alice in chat 1, bob in chat 1, alice in chat 2 -> {len({a1['id'], b1['id'], a2['id']})} messages") + check(len({a1["id"], b1["id"], a2["id"]}) == 3, "a key must be scoped to user + chat") + call("POST", f"/api/v1/chats/{chat['id']}/messages", 409, token=alice["token"], + body={"text": "other text"}, headers={"Idempotency-Key": key}) + print("PASS: the key is scoped to the current user and chat; another body with a used key -> 409") + + +def test_key_validation(): + alice = new_user("alice") + chat = new_chat(alice) + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": "no key"}) + for bad in ("", "a" * 129, "has space", "slash/inside", "ключ", "quote\"", "a+b"): + call("POST", path, 400, token=alice["token"], body={"text": "bad key"}, + headers={"Idempotency-Key": bad.encode("utf-8").decode("latin-1")}) # как есть, байтами UTF-8 + longest = ("Az09._:-" * 16)[:128] + call("POST", path, 201, token=alice["token"], body={"text": "longest key"}, headers={"Idempotency-Key": longest}) + print("missing, empty, 129 chars, space, '/', non-ASCII, '\"', '+' -> 400; 128 chars of A-Z a-z 0-9 . _ : - -> 201") + print("PASS: Idempotency-Key is required and checked: [A-Za-z0-9._:-]{1,128}") + + +def test_failed_request_does_not_burn_key(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + path = f"/api/v1/chats/{chat['id']}/messages" + call("POST", path, 400, token=alice["token"], body={"text": " "}, headers={"Idempotency-Key": key}) + call("POST", path, 404, token=eve["token"], body={"text": "not my chat"}, headers={"Idempotency-Key": key}) + created = call("POST", path, 201, token=alice["token"], body={"text": "valid now"}, headers={"Idempotency-Key": key}) + print(f"key used by a 400 and by a 404 first, then valid request -> {created.status}") + print("PASS: rejected requests (400, 404) do not store a result for the key") + + +def test_replay_checks_session_and_access(): + alice, eve = new_user("alice"), new_user("eve") + chat = new_chat(alice) + key = new_key() + send(alice, chat["id"], "private", key=key) + stolen = call("POST", f"/api/v1/chats/{chat['id']}/messages", 404, token=eve["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + call("DELETE", "/api/v1/auth/sessions/current", 204, token=alice["token"]) + revoked = call("POST", f"/api/v1/chats/{chat['id']}/messages", 401, token=alice["token"], + body={"text": "private"}, headers={"Idempotency-Key": key}) + print(f"same key and body by an outsider -> {stolen.status}; by alice after logout -> {revoked.status}") + print("PASS: a retry checks the session and chat access again, the saved answer is not leaked") + + +def test_lost_response_then_retry(): + """Клиент отправил POST и не дождался ответа (оборвал соединение). Повтор с тем же ключом безопасен.""" + alice = new_user("alice") + chat = new_chat(alice) + key = new_key() + body = json.dumps({"text": "answer was lost"}).encode("utf-8") + url = urllib.parse.urlsplit(BASE_URL) + request = (f"POST /api/v1/chats/{chat['id']}/messages HTTP/1.1\r\nHost: {url.netloc}\r\n" + f"Authorization: Bearer {alice['token']}\r\nIdempotency-Key: {key}\r\n" + f"Content-Type: application/json\r\nContent-Length: {len(body)}\r\nConnection: close\r\n\r\n").encode() + with socket.create_connection((url.hostname, url.port or 80), timeout=5) as connection: + connection.sendall(request + body) # и сразу закрываем, ответ не читаем + time.sleep(1) + retry = call("POST", f"/api/v1/chats/{chat['id']}/messages", 201, token=alice["token"], + body={"text": "answer was lost"}, headers={"Idempotency-Key": key}) + count = texts_in(alice, chat["id"]).count("answer was lost") + print(f"POST sent, connection closed without reading the answer; retry -> {retry.status}; messages: {count}") + check(count == 1, "the retry after a lost answer created a duplicate") + print("PASS: an unconfirmed POST can be safely retried with the same key") + + +if __name__ == "__main__": + run_tests([ + test_replay_returns_same_message, + test_same_key_parallel_through_replicas, + test_same_key_different_bodies_parallel, + test_key_scope, + test_key_validation, + test_failed_request_does_not_burn_key, + test_replay_checks_session_and_access, + test_lost_response_then_retry, + ]) diff --git a/student-tests/test_race.py b/student-tests/test_race.py new file mode 100644 index 00000000..4af44fad --- /dev/null +++ b/student-tests/test_race.py @@ -0,0 +1,57 @@ +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +from common import PASSWORD, call, check, new_chat, new_user, read_history, run_tests, send + +CLIENTS = 10 +MESSAGES_PER_CLIENT = 10 +REGISTRATIONS = 20 + + +def test_concurrent_messages(): + users = [new_user(f"race{i}") for i in range(CLIENTS)] + chat = new_chat(*users, title="race") + barrier = threading.Barrier(CLIENTS) # все клиенты стартуют одновременно + + def client(index): + barrier.wait() + for n in range(MESSAGES_PER_CLIENT): + send(users[index], chat["id"], f"client-{index}-message-{n}") + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=CLIENTS) as pool: + list(pool.map(client, range(CLIENTS))) # list() пробрасывает ошибки из потоков + elapsed = time.monotonic() - started + + history, pages = read_history(users[0], chat["id"], limit=30) + ids = {message["id"] for message in history} + expected = {f"client-{i}-message-{n}" for i in range(CLIENTS) for n in range(MESSAGES_PER_CLIENT)} + print(f"sent {CLIENTS * MESSAGES_PER_CLIENT} messages from {CLIENTS} clients in {elapsed:.2f} s; " + f"read {len(history)} in {pages} pages, unique ids: {len(ids)}") + check(len(history) == CLIENTS * MESSAGES_PER_CLIENT, "wrong number of messages") + check(len(ids) == len(history), "duplicate message ids") + check({message["text"] for message in history} == expected, "lost or changed message texts") + print("PASS: 100 concurrent messages from 10 clients, no losses and no duplicates") + + +def test_concurrent_registration(): + username = "race_" + uuid.uuid4().hex[:10] + barrier = threading.Barrier(REGISTRATIONS) + + def register(index): + barrier.wait() + body = {"username": username, "display_name": f"Client {index}", "password": PASSWORD} + return call("POST", "/api/v1/users", (201, 409), body=body).status + + with ThreadPoolExecutor(max_workers=REGISTRATIONS) as pool: + statuses = list(pool.map(register, range(REGISTRATIONS))) + created, conflicts = statuses.count(201), statuses.count(409) + print(f"{REGISTRATIONS} parallel registrations of one username: 201 x{created}, 409 x{conflicts}") + check(created == 1 and conflicts == REGISTRATIONS - 1, "exactly one registration must win") + print("PASS: concurrent registration created exactly one user") + + +if __name__ == "__main__": + run_tests([test_concurrent_messages, test_concurrent_registration]) -- 2.54.0 From 67a061b22ca4c9b476f588dc040674d6025e5672 Mon Sep 17 00:00:00 2001 From: KALIBR Date: Thu, 15 Oct 2026 15:58:03 +0300 Subject: [PATCH 2/2] =?UTF-8?q?lab3:=20=D0=BF=D1=83=D1=81=D1=82=D0=BE?= =?UTF-8?q?=D0=B9=20cursor=20->=20400?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main.py | 3 +-- student-tests/test_api.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/main.py b/main.py index 60b20dba..47dc4e45 100644 --- a/main.py +++ b/main.py @@ -310,8 +310,7 @@ def decode_cursor(cursor: str): def page_after(cursor: str | None, *, chat_id: UUID | None = None, user_id: UUID | None = None): - """Курсор -> ключ (created_at, id), с которого продолжать. Чужой или битый курсор — 400.""" - if not cursor: + if cursor is None: return None try: created_at, object_id, cursor_chat_id, cursor_user_id = decode_cursor(cursor) diff --git a/student-tests/test_api.py b/student-tests/test_api.py index 4a682052..7c770d9c 100644 --- a/student-tests/test_api.py +++ b/student-tests/test_api.py @@ -77,6 +77,17 @@ def test_pagination(): print("PASS: pagination works across multiple pages, foreign cursor returns 400") +def test_empty_cursor(): + alice = new_user("alice") + chat = new_chat(alice) + send(alice, chat["id"], "one message") + for path in ("/api/v1/chats", f"/api/v1/chats/{chat['id']}/messages"): + for query in ("?cursor=", "?cursor", "?limit=10&cursor="): + call("GET", path + query, 400, token=alice["token"]) + call("GET", path, 200, token=alice["token"]) + print("PASS: empty cursor (?cursor= and ?cursor) returns 400, no cursor returns the first page") + + def test_chat_list_pagination(): alice, bob = new_user("alice"), new_user("bob") created = [new_chat(alice, bob, title=f"Chat {i}")["id"] for i in range(3)] @@ -230,6 +241,7 @@ if __name__ == "__main__": test_foreign_chat, test_empty_message, test_pagination, + test_empty_cursor, test_chat_list_pagination, test_unicode, test_nul_character_is_stored, -- 2.54.0