From b53e5d1077c094dc154dd99072eea1af8ebaa57f Mon Sep 17 00:00:00 2001 From: Enterxx2 Date: Tue, 20 Oct 2026 10:41:56 +0300 Subject: [PATCH] =?UTF-8?q?feat:=20=D0=BA=D0=BE=D0=BD=D1=84=20=D0=BD=D0=B3?= =?UTF-8?q?=D0=B8=D0=BD=D0=BA=D1=81,=20=D1=82=D0=B5=D1=81=D1=82=D1=8B,=20?= =?UTF-8?q?=D0=BA=D0=BB=D1=8E=D1=87=20=D0=B8=D0=B4=D0=BC=D0=BF=D1=82=20etc?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 2 + .env.example | 5 + .gitignore | 1 + Dockerfile | 4 + Makefile | 24 +- REPORT.md | 157 +- compose.yaml | 109 +- evidence/00-environment.txt | 21 + evidence/01-up.txt | 150 ++ evidence/02-check-smoke.txt | 6 + evidence/03-student-tests.txt | 128 ++ evidence/04-balancing-idempotency.txt | 50 + evidence/05-failover-replica-a.txt | 41 + evidence/05-failover-replica-b.txt | 69 + evidence/06-persistence.txt | 37 + evidence/07-expiry.txt | 108 ++ evidence/08-dependency-failures.txt | 24 + evidence/09-migrations.txt | 22 + evidence/10-sigterm.txt | 54 + evidence/11-limits-ports-networks.txt | 33 + evidence/12-load-baseline.txt | 18 + evidence/13-experiment-failover.txt | 22 + evidence/14-experiment-rolling.txt | 18 + evidence/15-clean-clone.txt | 192 ++ evidence/experiment-failover.json | 1625 +++++++++++++++++ evidence/experiment-rolling.json | 482 +++++ evidence/load.json | 17 + infra/nginx/errors.conf | 15 + infra/nginx/nginx.conf | 101 + migrations/0002_message_idempotency.sql | 18 + scripts/check_migrations.sh | 23 +- scripts/check_previous_version.sh | 89 + scripts/experiment.py | 480 +++++ scripts/instances.sh | 17 + scripts/restart.sh | 10 +- scripts/restore.sh | 9 + scripts/rolling-update.sh | 42 + scripts/sigterm_check.py | 135 +- scripts/stop-one.sh | 24 + .../api/middlewares/request_context.py | 24 +- .../api/routers/messages/endpoints.py | 19 +- src/application/api/utils/__init__.py | 14 +- src/application/api/utils/errors.py | 1 + src/config/__init__.py | 12 + src/config/settings.py | 10 + src/domain/__init__.py | 12 +- src/domain/messages/__init__.py | 12 +- src/domain/messages/schemas.py | 18 + src/gateway.py | 71 +- src/infrastructure/dependencies/database.py | 36 +- .../dependencies/service_creator.py | 10 +- .../message_repository/postgres.py | 83 +- .../message_service/message_service.py | 57 +- src/interfaces/message_repository.py | 14 +- src/migrate.py | 24 + student-tests/conftest.py | 4 +- student-tests/helpers.py | 5 + student-tests/test_balancing.py | 74 + student-tests/test_concurrency.py | 3 +- student-tests/test_foreign_chat.py | 6 +- student-tests/test_idempotency.py | 249 +++ student-tests/test_invalid_json.py | 16 +- student-tests/test_messages.py | 10 +- 63 files changed, 5013 insertions(+), 153 deletions(-) create mode 100644 evidence/00-environment.txt create mode 100644 evidence/01-up.txt create mode 100644 evidence/02-check-smoke.txt create mode 100644 evidence/03-student-tests.txt create mode 100644 evidence/04-balancing-idempotency.txt create mode 100644 evidence/05-failover-replica-a.txt create mode 100644 evidence/05-failover-replica-b.txt create mode 100644 evidence/06-persistence.txt create mode 100644 evidence/07-expiry.txt create mode 100644 evidence/08-dependency-failures.txt create mode 100644 evidence/09-migrations.txt create mode 100644 evidence/10-sigterm.txt create mode 100644 evidence/11-limits-ports-networks.txt create mode 100644 evidence/12-load-baseline.txt create mode 100644 evidence/13-experiment-failover.txt create mode 100644 evidence/14-experiment-rolling.txt create mode 100644 evidence/15-clean-clone.txt create mode 100644 evidence/experiment-failover.json create mode 100644 evidence/experiment-rolling.json create mode 100644 evidence/load.json create mode 100644 infra/nginx/errors.conf create mode 100644 infra/nginx/nginx.conf create mode 100644 migrations/0002_message_idempotency.sql create mode 100755 scripts/check_previous_version.sh create mode 100644 scripts/experiment.py create mode 100755 scripts/instances.sh create mode 100755 scripts/restore.sh create mode 100755 scripts/rolling-update.sh create mode 100755 scripts/stop-one.sh create mode 100644 src/migrate.py create mode 100644 student-tests/test_balancing.py create mode 100644 student-tests/test_idempotency.py diff --git a/.dockerignore b/.dockerignore index 08658bae..bebce598 100644 --- a/.dockerignore +++ b/.dockerignore @@ -33,3 +33,5 @@ requirements-dev.txt lab.json .mypy_cache .ruff_cache +infra +.venv-validate diff --git a/.env.example b/.env.example index eb953184..75230adf 100644 --- a/.env.example +++ b/.env.example @@ -6,6 +6,9 @@ APP_ENV=lab LOG_LEVEL=INFO SHUTDOWN_TIMEOUT_SEC=5 +SHUTDOWN_DRAIN_SEC=1 +APP_KEEPALIVE_TIMEOUT_SEC=75 + DB_USER=test DB_PASS=test DB_NAME=test @@ -30,4 +33,6 @@ PASSWORD_HASH_WORKERS=4 # lab3: число API-реплик за балансировщиком и срок хранения Idempotency-Key API_REPLICAS=2 +# Публичный порт балансировщика на 127.0.0.1 (порт для make test-expiry — LB_EXPIRY_PORT=8081) +LB_PORT=8080 IDEMPOTENCY_TTL_SECONDS=86400 diff --git a/.gitignore b/.gitignore index 9918753e..b33ef504 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ coverage/ .pytest_cache/ .mypy_cache/ .ruff_cache/ +.venv-validate/ diff --git a/Dockerfile b/Dockerfile index f999aaae..3c8d7873 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,10 @@ COPY src ./src RUN useradd --system --uid 10001 --no-create-home app USER app +ARG APP_REVISION=dev +ENV APP_REVISION=${APP_REVISION} +LABEL org.opencontainers.image.revision=${APP_REVISION} + EXPOSE 8080 HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ CMD ["python", "src/healthcheck.py"] diff --git a/Makefile b/Makefile index d9e8611b..aa885ce6 100644 --- a/Makefile +++ b/Makefile @@ -4,11 +4,15 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover install up down restart ps logs wait-ready destroy-data student-test concurrency-test test-expiry test-failures test-migrations explain sigterm-check +export BUILDX_NO_DEFAULT_ATTESTATIONS := 1 + +.PHONY: help check validate test load test-persistence test-failover install up down restart restore instances rolling-update ps logs wait-ready destroy-data student-test concurrency-test test-expiry test-failures test-migrations explain sigterm-check experiment-failover experiment-rolling help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @echo "up/down/restart/ps/logs: compose stack; destroy-data: DELETE volumes intentionally" + @echo "instances: API containers <-> X-Instance-Id; restore: start replicas stopped by stop-one.sh; rolling-update: replace replicas one by one" + @echo "experiment-failover / experiment-rolling: load + stop each replica / rolling update, metrics to evidence/" @echo "student-test: own tests; test-migrations: from empty volume; test-expiry: short SESSION_TTL; test-failures: stop Valkey/PostgreSQL; explain: EXPLAIN history" check: @@ -44,11 +48,20 @@ restart: ./scripts/restart.sh @$(MAKE) --no-print-directory wait-ready +restore: + ./scripts/restore.sh + +instances: + ./scripts/instances.sh + +rolling-update: + ./scripts/rolling-update.sh + ps: docker compose ps logs: - docker compose logs -f api + docker compose logs -f api nginx wait-ready: @for i in $$(seq 1 60); do \ @@ -65,7 +78,6 @@ student-test: concurrency-test: BASE_URL="$(BASE_URL)" poetry run pytest student-tests/test_concurrency.py -s -test-expiry: docker compose --profile expiry up -d --build --wait api-short-ttl EXPIRY_BASE_URL=http://localhost:8081 poetry run pytest student-tests/test_expiry.py -s; \ status=$$?; docker compose --profile expiry stop api-short-ttl; exit $$status @@ -83,3 +95,9 @@ explain: sigterm-check: $(PYTHON) scripts/sigterm_check.py --base-url "$(BASE_URL)" + +experiment-failover: + $(PYTHON) scripts/experiment.py failover --base-url "$(BASE_URL)" --output evidence/experiment-failover.json + +experiment-rolling: + $(PYTHON) scripts/experiment.py rolling --base-url "$(BASE_URL)" --output evidence/experiment-rolling.json diff --git a/REPORT.md b/REPORT.md index c2679005..9cb2b5ba 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,160 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Саватеев Артём, БВТ2302 +- Целевая оценка: **4** (весь уровень 3 + весь уровень 4); +- Источники: документация [nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) (`resolve`, `max_fails`, `keepalive_timeout`) и [proxy](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) (`proxy_next_upstream`, повтор неидемпотентных запросов), [Compose services/deploy](https://docs.docker.com/reference/compose-file/services/), исходники uvicorn 0.54 (`Server.shutdown`, h11 keep-alive). Библиотеки — из lab2 (FastAPI, asyncpg, redis-py, argon2-cffi), новых зависимостей нет. Помощь ии ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. + +```sh +cp .env.example .env # затем задать свои DB_PASS и VALKEY_PASS (синтетические) +docker compose --env-file .env config --quiet +make up # = docker compose up -d --build --wait: build → postgres/valkey → + # migrate (однократно) → api ×API_REPLICAS → nginx; ждёт healthy +make check test # стенд доступен только через LB: http://localhost:8080 +python3 -m venv .venv-validate && .venv-validate/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv-validate/bin/python # полная OpenAPI-валидация; не в .venv — + # его Poetry принял бы за окружение проекта (Python 3.9) +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +make instances # какой контейнер остановлен: контейнер ↔ X-Instance-Id ↔ PID ↔ ревизия +make restore # вернуть остановленную реплику (работающие не пересоздаются) +make student-test # собственные тесты (poetry install --no-root один раз) +make rolling-update # выкладка новой сборки по одной реплике без простоя +make down # безопасная остановка, volume с данными сохраняется +make destroy-data # намеренное удаление учебных данных (down -v) +``` + +Действия оператора: создать `.env` с секретами (в Git не попадает), после `stop-one.sh` вручную выполнить `make restore`. Миграции отдельно запускать не нужно — их выполняет сервис `migrate` внутри `make up`/`rolling-update`. Сборку через `make`/скрипты лучше, чем голый `docker compose up --build`: Makefile экспортирует `BUILDX_NO_DEFAULT_ATTESTATIONS=1`, иначе Docker Desktop добавляет к образу provenance-attestation, ID образа меняется на каждой сборке и Compose пересоздаёт все реплики разом. Обновлять работающий стенд — только `make rolling-update`. + +Воспроизводимость проверена копией ровно тех файлов, что попадут в Git (без `.env`), отдельным compose-проектом на пустом volume: build, migrate с нуля, smoke, failover, persistence — [evidence/15-clean-clone.txt](evidence/15-clean-clone.txt). ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +клиент ──► nginx :8080 (127.0.0.1, единственный опубликованный порт) + │ round-robin, без sticky; Docker DNS «api» перечитывается каждые 2 с + ├──► api-1 ─┐ сеть edge: только LB и API + └──► api-2 ─┤ + ├──► postgres (источник истины: пользователи, чаты, сообщения, + │ сессии, ключи идемпотентности; volume postgres_data) + migrate ────┤ + └──► valkey (read-through кэш сессий, не источник истины) + сеть backend: internal, LB туда не подключён +``` + +**Перенесено из lab2:** API, PostgreSQL как источник истины, сессии в PostgreSQL с кэшем в Valkey (logout инвалидирует общий кэш), Argon2id, keyset-пагинация, монотонный `created_at` под advisory lock чата, обработка отказов Valkey/PostgreSQL. Локального состояния, от которого зависит корректность, в API нет: circuit breaker Valkey и пул соединений — только оптимизации процесса. + +**Изменено в lab3:** + +| Что | Решение | Где | +| --- | --- | --- | +| Реплики и LB | `api` с `deploy.replicas: ${API_REPLICAS:-2}`, без `container_name` и портов; nginx 1.28 `upstream api { server api:8080 resolve; }` — новые/остановленные контейнеры подхватываются без reload | [compose.yaml](compose.yaml), [infra/nginx/nginx.conf](infra/nginx/nginx.conf) | +| X-Instance-Id | случайный alias процесса (`secrets.token_hex(6)`), без hostname/IP; в каждом ответе API; процесс пишет его в `/tmp/instance-id` для оператора | [request_context.py](src/application/api/middlewares/request_context.py), [settings.py](src/config/settings.py) | +| X-Request-Id | nginx передаёт `$request_id`, API принимает его (только 32 hex) — одна строка access-лога LB и журнал реплики связаны одним ID | там же, [evidence/04](evidence/04-balancing-idempotency.txt) | +| Idempotency-Key | обязательный заголовок `[A-Za-z0-9._:-]{1,128}` (нет/неверный — 400); таблица `message_idempotency_keys`, PK `(sender_id, chat_id, key)`, SHA-256 канонического тела, TTL `IDEMPOTENCY_TTL_SECONDS` = 24 ч | [0002_message_idempotency.sql](migrations/0002_message_idempotency.sql), [postgres.py](src/infrastructure/repositories/message_repository/postgres.py), [message_service.py](src/infrastructure/services/message_service/message_service.py) | +| Миграции | one-shot сервис `migrate` (`src/migrate.py`) до реплик (`service_completed_successfully`); реплика миграции не применяет, а проверяет версию схемы и не стартует, если схема старее кода | [database.py](src/infrastructure/dependencies/database.py), [migrate.py](src/migrate.py) | +| Graceful shutdown | SIGTERM → drain 1 с (порт закрыт, readiness 503, ответы по открытым keep-alive с `Connection: close`) → graceful uvicorn до 5 с → закрытие пулов | [gateway.py](src/gateway.py) | +| Ошибки LB | 502/503/504, созданные nginx, отдаются как `503 {"error":{"code":"unavailable",…}}` с `X-Request-Id`; ответы реплик проходят без изменений | [infra/nginx/errors.conf](infra/nginx/errors.conf) | + +**Транзакционная граница идемпотентности.** В одной транзакции: `pg_advisory_xact_lock(чат)` → поиск действующего ключа → (повтор: вернуть сохранённое сообщение или 409 при другом теле) → `INSERT` сообщения → `INSERT` ключа. Ключ привязан к чату, поэтому блокировка чата, уже нужная для монотонного `created_at`, сериализует и одновременные повторы с разных реплик: второй ждёт COMMIT первого и находит ключ (READ COMMITTED). Ошибка до COMMIT откатывает и сообщение, и ключ — транзиентный 5xx не фиксируется как результат. Повтор заново проходит проверку сессии и членства. Область «пользователь + метод + путь + ключ» сведена к `(sender_id, chat_id, key)`: метод и путь фиксированы. Порядок ключей и пробелы JSON не влияют (хеш от тела, сериализованного по схеме). Истёкший ключ переиспользуется (`ON CONFLICT … DO UPDATE`) и создаёт новое сообщение. + +**Почему nginx не повторяет POST сам.** `proxy_next_upstream error timeout` без `non_idempotent`: запрос, не дошедший до реплики (connect refused/timeout), nginx отправляет следующей реплике — это безопасно для любого метода. POST, уже отправленный реплике, не повторяется: прозрачный повтор `POST /users`/`/chats` создал бы дубль (контракт это запрещает). Неопределённый POST сообщения повторяет клиент с тем же ключом. 503 приложения тоже не повторяется и не исключает реплику: он означает отказ общей зависимости, другая реплика зависит от той же БД. + +### Health, readiness, таймауты и время обнаружения + +| Механизм | Значение | Назначение | +| --- | --- | --- | +| `/health/live` | 200 пока жив процесс | liveness, проверки через LB | +| `/health/ready` | 200 если не завершается и PostgreSQL отвечает (Valkey не критичен — сессии читаются из БД) | Docker healthcheck API: `interval 5s, timeout 3s, retries 2, start_period 20s, start_interval 1s`; используется `up --wait`, `depends_on` nginx, `restore.sh`, `rolling-update.sh` | +| Порт реплики | открывается только после startup (БД + версия схемы), закрывается первым при SIGTERM | LB не шлёт трафик в неготовую/завершающуюся реплику | +| nginx `proxy_connect_timeout` | 0,5 с | исчезнувший IP (SIGKILL/крах контейнера) | +| nginx `proxy_read_timeout` / `send` | 10 с | зависшая реплика; дольше самого долгого запроса API (ожидание пула 3 с + statement_timeout 5 с) | +| `max_fails=1 fail_timeout=5s` | | после ошибки соединения реплика исключена на 5 с, затем пробный запрос | +| `resolver 127.0.0.11 valid=2s` | | остановленный контейнер пропадает из upstream ≤ 2 с, новый появляется ≤ 2 с | +| keep-alive | nginx → API `keepalive_timeout 60s` < uvicorn `timeout_keep_alive 75 s` | idle-соединение закрывает LB, а не реплика: нет гонки «запрос в закрытое соединение» | +| SIGTERM | drain 1 с + graceful ≤ 5 с < `stop_grace_period 10s` | SIGKILL не прерывает завершение | + +У nginx open source нет активных health checks — проверка пассивная, по реальным запросам. **Максимальное время обнаружения:** остановка процесса/SIGTERM — 0 с (порт закрыт → `connection refused` → тот же запрос сразу уходит на другую реплику, клиент ошибки не видит); крах контейнера — ≤ 0,5 с на запрос (connect timeout), затем исключение на 5 с и удаление из DNS ≤ 2 с; зависание без закрытия порта — ≤ 10 с (read timeout; GET повторяется, POST получает 503 и повторяется клиентом с тем же ключом). Docker healthcheck обнаруживает неготовность за ≤ 5 с × 2 = 10 с, но в Compose он не исключает контейнер из DNS — поэтому балансировку обеспечивает nginx, а не `depends_on`/healthcheck. + +### Ресурсы, сети, секреты + +| Роль | Service | Образ | CPU / RAM | Сети | Порт наружу | Volume | Credentials | +| --- | --- | --- | --- | --- | --- | --- | --- | +| LB | `nginx` | nginx:1.28.0-alpine3.21 | 0,5 / 64M | edge | 127.0.0.1:8080 (+8081 только для `test-expiry`) | — | нет | +| API ×2 | `api` | mtusi-messenger:lab3 (из исходников) | 1,0 / 512M каждая | edge, backend | нет | — | DB_PASS, VALKEY_PASS из `.env` | +| Миграции | `migrate` | тот же | 0,5 / 256M | backend | нет | — | DB_PASS | +| БД | `postgres` | postgres:18.2-alpine3.22 | 1,5 / 512M | backend | нет | `postgres_data` | POSTGRES_PASSWORD = DB_PASS | +| Кэш сессий | `valkey` | valkey/valkey:8.1.4-alpine3.22 | 0,25 / 96M | backend | нет | нет (не источник истины) | VALKEY_PASS | + +Сеть `backend` — `internal`: nginx не видит БД и Valkey (даже не резолвит имя), хранилища не имеют выхода наружу. Порт PostgreSQL больше не публикуется на хост (в lab2 был 127.0.0.1:5432); тесты обращаются к БД через `docker compose exec`. Подтверждение: [evidence/11-limits-ports-networks.txt](evidence/11-limits-ports-networks.txt). 512M на реплику: Argon2id RFC 9106 low-memory — 64 MiB × `PASSWORD_HASH_WORKERS=4`. Пул БД: 2 реплики × `DB_POOL_MAX_SIZE=10` + migrate + реплика при rolling update = 40 ≤ `max_connections=100`. + +### Соответствие .env.example + +| Переменная | Используется | Примечание | +| --- | --- | --- | +| `APP_HOST`, `APP_PORT` | uvicorn | 0.0.0.0:8080 внутри контейнера | +| `APP_ENV`, `LOG_LEVEL` | журнал | | +| `APP_KEEPALIVE_TIMEOUT_SEC=75` | uvicorn `timeout_keep_alive` | новое; > keepalive_timeout nginx | +| `SHUTDOWN_TIMEOUT_SEC=5` | uvicorn `timeout_graceful_shutdown` | | +| `SHUTDOWN_DRAIN_SEC=1` | `DrainingServer` | новое | +| `DB_*` | asyncpg pool | `DB_HOST/DB_PORT` переопределены в compose (`postgres:5432`); `DB_PORT` больше не публикуется | +| `VALKEY_*` | redis-py | `VALKEY_HOST/PORT` переопределены в compose | +| `SESSION_TTL_SECONDS`, `SESSION_CACHE_TTL_SECONDS`, `PASSWORD_HASH_WORKERS` | как в lab2 | `api-short-ttl` переопределяет TTL = 2 | +| `API_REPLICAS=2` | `deploy.replicas` | | +| `IDEMPOTENCY_TTL_SECONDS=86400` | срок ключа | | +| `LB_PORT=8080` (+ `LB_EXPIRY_PORT`, по умолчанию 8081) | публикация nginx | новое | +| `APP_REVISION` | build arg → метка образа и журнал старта | не в `.env`; задаёт `rolling-update.sh` | ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Формат файлов — транскрипт терминала: `$ команда`, затем сырой вывод. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke | `make check test` | PASS lab3 public API scenario | [02](evidence/02-check-smoke.txt) | +| 3: стенд одной командой, ≥2 API, один вход, общие БД и сессии | `make up`, `docker compose ps`, `make instances` | 2 реплики healthy, migrate Exited (0), наружу только nginx | [01](evidence/01-up.txt) | +| 3: одна сессия → разные реплики без повторного входа | `pytest test_balancing.py -s`, access-лог nginx | 20 запросов одним токеном: 10/10 по двум `X-Instance-Id`; logout на одной реплике действует на всех | [04](evidence/04-balancing-idempotency.txt) | +| 3: остановка любой реплики, чтение/запись ≤ 10 с | `make test-failover` для `api-1`, затем для `api-2` | 129 / 128 / 129 успешных записей, **0 неудачных попыток**, первая запись после возврата hook 1,6–2,7 с; `instances` и `docker inspect` показывают реально остановленный контейнер (exit 0, журнал drain/shutdown) | [05a](evidence/05-failover-replica-a.txt), [05b](evidence/05-failover-replica-b.txt) | +| 3: Idempotency-Key без дублей, включая конкурентный повтор и смену реплики | `pytest test_idempotency.py -s` | 20 одновременных POST с одним ключом через обе реплики (10/10): все 201, одно тело, 1 строка в БД; разные тела под одним ключом — один победитель, остальные 409; 400 для нет/неверного ключа; область user+chat; повтор после logout — 401, чужой — 404; истёкший ключ создаёт новое | [04](evidence/04-balancing-idempotency.txt) | +| 3: persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; `StartedAt` всех 5 контейнеров изменился, volume тот же (created 16:10:27Z) | [06](evidence/06-persistence.txt) | +| Регрессия lab2 | `make student-test`, `test-expiry`, `test-failures` | 107 passed; TTL 2 с через LB :8081; отказ Valkey/PostgreSQL как в lab2 | [03](evidence/03-student-tests.txt), [07](evidence/07-expiry.txt), [08](evidence/08-dependency-failures.txt) | +| 4: health/readiness/таймауты/SIGTERM согласованы; LB исключает backend | `make sigterm-check` | exit 0 за 1,53 с; 354 запроса через LB после SIGTERM без ошибок; после выхода реплика не отвечает | [10](evidence/10-sigterm.txt) | +| 4: поочерёдная остановка каждой реплики, RPS/ошибки/восстановление | `make experiment-failover` | см. «Эксперименты»: SIGTERM — 0 ошибок; SIGKILL — 1 и 3 ошибки 503, повтор тем же ключом; потерь/дублей 0 | [13](evidence/13-experiment-failover.txt), [json](evidence/experiment-failover.json) | +| 4: идемпотентность параллельными запросами | `test_parallel_*` | см. строку уровня 3 | [04](evidence/04-balancing-idempotency.txt) | +| 4: лимиты CPU/RAM, API без портов, сети/секреты, нет `container_name` | `docker inspect`, `docker network inspect` | лимиты у всех контейнеров; порты только у nginx; backend internal | [11](evidence/11-limits-ports-networks.txt) | +| 4: rolling update под нагрузкой | `make experiment-rolling` | обе реплики заменены новой ревизией, 0 ошибок, наибольший разрыв между подтверждениями записи 0,098 с | [14](evidence/14-experiment-rolling.txt), [json](evidence/experiment-rolling.json) | +| 4: миграции однократно и совместимы с работающей версией | `make test-migrations`, `scripts/check_previous_version.sh` | с пустого volume, 3 повтора идемпотентны; API до migrate отказывается стартовать; **lab2-код (`aa971fb`) работает на схеме v2**: вход, чат, запись, чтение | [09](evidence/09-migrations.txt) | +| Чистый clone | копия файлов Git, отдельный проект | build → smoke → failover → persistence PASS, затем `down -v` | [15](evidence/15-clean-clone.txt) | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Гипотеза.** При остановке одной из двух реплик (SIGTERM) клиент не видит ошибок: порт закрывается раньше, чем реплика перестаёт отвечать, и nginx переотправляет ещё не доставленные запросы; при SIGKILL теряются только запросы, бывшие в обработке у убитой реплики, а повтор с тем же ключом не создаёт дублей. Пропускная способность при одной реплике падает, так как API ограничена 1 CPU. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +**Методика** ([scripts/experiment.py](scripts/experiment.py)). Стенд из таблицы ресурсов на Apple M4 / Docker Desktop VM 10 CPU, 8 GiB. 4 потока closed loop через LB: `POST` сообщения (неподтверждённый POST повторяется с тем же ключом и телом каждые 50 мс до 201) + `GET ?limit=1`; timeout клиента 5 с; новый чат на прогон (≈ 10–12 тыс. сообщений за прогон). Фазы: 8 с до отказа → отказ (`docker stop` = SIGTERM, затем `docker kill -s SIGKILL`) → 10 с с одной репликой → `scripts/restore.sh` → 8 с. Порядок: `api-1`, `api-2` для каждого сигнала, с восстановлением предыдущей. После прогона история читается целиком: каждое подтверждённое сообщение на месте и не изменено, нет дублей id/текстов. «Время восстановления записи» — от первой ошибки после отказа до первой успешной записи после последней ошибки; «разрыв» — наибольший интервал между подтверждениями записи. Один прогон на сценарий; CPU/RAM — `docker stats` в середине фазы «до». + +| Воздействие | Что ожидаем | Что наблюдали (запись ok RPS до / во время / после) | Ошибки, восстановление, данные | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| SIGTERM `api-1` | 0 ошибок | 382 / 277 / 399 | 0 ошибок; разрыв 0,063 с; реплика в ротации через 0,97 с после restore; потерь/дублей 0 | +| SIGTERM `api-2` | 0 ошибок | 401 / 282 / 362 | 0 ошибок; разрыв 0,071 с; в ротации через 3,7 с; 0 / 0 | +| SIGKILL `api-1` | ошибки только у запросов в обработке | 427 / 281 / 408 | 1 × 503 (POST, оборван убитой репликой), восстановление < 0,01 с, разрыв 0,070 с; повтор ключом создал сообщение один раз; 0 / 0 | +| SIGKILL `api-2` | то же | 387 / 277 / 373 | 3 × 503, восстановление 0,006 с, разрыв 0,061 с; 0 / 0 | +| Rolling update 2 → 3 → 2 → 3 → 2 | 0 ошибок | 420 / 308 / 345 | 0 ошибок; разрыв 0,098 с; старые alias исчезли через 8,1 и 14,9 с от начала, новые вошли на 4,9 и 13,9 с; обе реплики на новой ревизии; 0 / 0 | +| Valkey / PostgreSQL недоступны | обход кэша / ограниченный 503 | как в lab2, через LB | [08](evidence/08-dependency-failures.txt) | +| Рестарт всего стенда | данные и токен сохраняются | PASS | [06](evidence/06-persistence.txt) | + +Нагрузка до отказа: API ≈ 84 % от лимита 1 CPU каждая, PostgreSQL ≈ 28 %, nginx ≈ 9 %, память API ≈ 48 MiB. Во время отказа запись падает на ≈ 30 % (не вдвое: при двух репликах узким местом частично являются клиент и БД). Базовый публичный генератор (`tests/load.py`, 30 с, concurrency 4, только POST): 713 успешных RPS, 0 ошибок, p50/p95/p99 = 4,7 / 8,4 / 22,5 мс — [evidence/12](evidence/12-load-baseline.txt), [load.json](evidence/load.json). + +**Подтверждённое улучшение — drain при SIGTERM.** Без drain uvicorn по SIGTERM сразу закрывает idle keep-alive соединения, и nginx изредка отправлял запрос в уже закрываемое соединение: в пробном rolling update был `upstream prematurely closed connection` (GET nginx повторил; POST в той же ситуации получил бы 503). С drain в итоговых прогонах SIGTERM и rolling update таких событий 0 — все 4 случая «prematurely closed» в журнале nginx пришлись на прогоны SIGKILL (сопоставлено по времени, где это ожидаемо). + +**Ограничения измерения.** Closed loop с 4 потоками страдает coordinated omission и ограничен клиентом на той же машине; по одному прогону на сценарий, без доверительных интервалов; RPS — подтверждения HTTP, а не COMMIT. Числа описывают этот ноутбук, не универсальную ёмкость. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Выдерживается:** отказ/остановка/крах одного API-процесса или контейнера, поочерёдная замена реплик, перезапуск всего стенда с volumes, отказ Valkey. **Не выдерживается:** отказ nginx, PostgreSQL (одна реплика БД — SPOF; API честно отвечает 503) и Docker-host — все контейнеры на одном узле. Отказ процесса ≠ отказ узла: здесь проверен только первый. +- nginx open source — только пассивные проверки: реплика, которая принимает соединения, но не отвечает, обнаруживается за `proxy_read_timeout` (10 с) и снова получает пробный запрос каждые 5 с; healthcheck Docker в Compose её не исключает. Активные проверки дал бы HAProxy/Traefik или nginx Plus. +- `docker stop`/`docker kill` не перезапускаются политикой `unless-stopped` (так и задумано для `stop-one.sh`); автоматического восстановления упавшей реплики Compose не даёт — `make restore` выполняет оператор. +- Истёкшие ключи идемпотентности удаляются только при повторном использовании ключа; фоновой очистки нет (строки маленькие, но таблица растёт). +- Совместимость миграций обеспечивается правилом expand-only и проверена для пары lab2 → lab3 на уровне схемы; поведение API при смешанных версиях (старая реплика не требует ключа и не отдаёт `X-Instance-Id`) во время такой выкладки различается — это переход контракта, а не схемы. +- `api-short-ttl` для `make test-expiry` тоже за LB (порт 8081), но 8081 публикуется nginx постоянно; без профиля `expiry` он отвечает 503. \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index ae16854e..bbbedb72 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,27 +1,100 @@ name: messenger-lab3 -x-api: &api - build: . +x-app: &app + build: + context: . + args: + APP_REVISION: ${APP_REVISION:-dev} image: mtusi-messenger:lab3 - restart: unless-stopped env_file: - .env - environment: &api-network + environment: &app-network DB_HOST: postgres DB_PORT: "5432" VALKEY_HOST: valkey VALKEY_PORT: "6379" + +x-api-resources: &api-resources + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.25" + memory: 128M + +x-api: &api + <<: *app + restart: unless-stopped depends_on: + migrate: + condition: service_completed_successfully postgres: condition: service_healthy valkey: condition: service_started networks: + - edge - backend - - default + healthcheck: + test: ["CMD", "python", "src/healthcheck.py"] + interval: 5s + timeout: 3s + retries: 2 + start_period: 20s + start_interval: 1s stop_grace_period: 10s services: + nginx: + image: nginx:1.28.0-alpine3.21 + restart: unless-stopped + depends_on: + api: + condition: service_healthy + ports: + - "127.0.0.1:${LB_PORT:-8080}:8080" + - "127.0.0.1:${LB_EXPIRY_PORT:-8081}:8081" + volumes: + - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx/errors.conf:/etc/nginx/errors.conf:ro + networks: + - edge + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8090/healthz"] + interval: 5s + timeout: 2s + retries: 3 + stop_grace_period: 10s + deploy: + resources: + limits: + cpus: "0.5" + memory: 64M + + api: + <<: *api + deploy: + <<: *api-resources + replicas: ${API_REPLICAS:-2} + + migrate: + <<: *app + command: ["python", "src/migrate.py"] + restart: "no" + depends_on: + postgres: + condition: service_healthy + networks: + - backend + healthcheck: + disable: true + deploy: + resources: + limits: + cpus: "0.5" + memory: 256M + postgres: image: postgres:18.2-alpine3.22 restart: unless-stopped @@ -37,12 +110,14 @@ services: timeout: 3s retries: 15 start_period: 10s - ports: - - "127.0.0.1:${DB_PORT}:5432" networks: - backend - - default stop_grace_period: 30s + deploy: + resources: + limits: + cpus: "1.5" + memory: 512M valkey: image: valkey/valkey:8.1.4-alpine3.22 @@ -68,23 +143,25 @@ services: retries: 15 networks: - backend - - api: - <<: *api - ports: - - "127.0.0.1:${APP_PORT}:${APP_PORT}" + deploy: + resources: + limits: + cpus: "0.25" + memory: 96M api-short-ttl: <<: *api profiles: - expiry environment: - <<: *api-network + <<: *app-network SESSION_TTL_SECONDS: "2" - ports: - - "127.0.0.1:8081:${APP_PORT}" + deploy: + <<: *api-resources + replicas: 1 networks: + edge: {} backend: internal: true diff --git a/evidence/00-environment.txt b/evidence/00-environment.txt new file mode 100644 index 00000000..3c664976 --- /dev/null +++ b/evidence/00-environment.txt @@ -0,0 +1,21 @@ +$ date -u +%Y-%m-%dT%H:%M:%SZ +2026-10-19T16:59:47Z +$ git log -1 --format="%h %s" && git status --short | wc -l | xargs echo "uncommitted files:" +aa971fb feat: перенос core-логики 2 лабы +uncommitted files: 42 +$ sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize +Apple M4 +10 +17179869184 +$ docker version --format "Docker Engine {{.Server.Version}}, client {{.Client.Version}}" +Docker Engine 29.0.1, client 29.0.1 +$ docker compose version +Docker Compose version v2.40.3-desktop.1 +$ docker info --format "Docker VM: {{.NCPU}} CPU, {{.MemTotal}} bytes RAM, {{.OperatingSystem}}, kernel {{.KernelVersion}}" +Docker VM: 10 CPU, 8217448448 bytes RAM, Docker Desktop, kernel 6.12.54-linuxkit +$ python3 --version +Python 3.9.6 +$ poetry run python --version +/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( +Python 3.12.12 diff --git a/evidence/01-up.txt b/evidence/01-up.txt new file mode 100644 index 00000000..cc0ed860 --- /dev/null +++ b/evidence/01-up.txt @@ -0,0 +1,150 @@ +$ docker compose --env-file .env config --quiet && echo "compose config: OK" +compose config: OK +$ make up +docker compose up -d --build --wait +#1 [internal] load local bake definitions +#1 reading from stdin 1.17kB done +#1 DONE 0.0s + +#2 [api internal] load build definition from Dockerfile +#2 transferring dockerfile: 914B done +#2 DONE 0.0s + +#3 [api internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm +#3 DONE 0.3s + +#4 [migrate internal] load .dockerignore +#4 transferring context: 358B done +#4 DONE 0.0s + +#5 [migrate 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c +#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done +#5 DONE 0.0s + +#6 [api internal] load build context +#6 transferring context: 16.17kB done +#6 DONE 0.0s + +#7 [api 3/7] COPY poetry.lock pyproject.toml ./ +#7 CACHED + +#8 [api 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction +#8 CACHED + +#9 [api 2/7] WORKDIR /app +#9 CACHED + +#10 [api 5/7] COPY migrations ./migrations +#10 CACHED + +#11 [migrate 6/7] COPY src ./src +#11 DONE 0.0s + +#12 [migrate 7/7] RUN useradd --system --uid 10001 --no-create-home app +#12 0.110 useradd warning: app's uid 10001 is greater than SYS_UID_MAX 999 +#12 DONE 0.1s + +#13 [api] exporting to image +#13 exporting layers 0.0s done +#13 exporting manifest sha256:8d0545b5703e6c70c35f1136bdfd37c4ceb4a70d08b5bad21b70d6755df05b5d done +#13 exporting config sha256:28c6ec56ed30844b944958f2749a370cea79cfd768ca3f491807eae51ad534a0 done +#13 naming to docker.io/library/mtusi-messenger:lab3 done +#13 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done +#13 DONE 0.1s + +#14 [migrate] exporting to image +#14 exporting layers 0.0s done +#14 exporting manifest sha256:9802819ef7b235f44ad03fed838cf83fcd4573a342405936538c7c969adfd573 done +#14 exporting config sha256:e2b0c133c10bc42d53160db701c32694ed2cbac54f0522999289f4cf7cef21e1 done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done +#14 DONE 0.1s + +#15 [api] resolving provenance for metadata file +#15 DONE 0.0s + +#16 [migrate] resolving provenance for metadata file +#16 DONE 0.0s + mtusi-messenger:lab3 Built + mtusi-messenger:lab3 Built + Network messenger-lab3_edge Creating + Network messenger-lab3_edge Created + Network messenger-lab3_backend Creating + Network messenger-lab3_backend Created + Container messenger-lab3-postgres-1 Creating + Container messenger-lab3-valkey-1 Creating + Container messenger-lab3-valkey-1 Created + Container messenger-lab3-postgres-1 Created + Container messenger-lab3-migrate-1 Creating + Container messenger-lab3-migrate-1 Created + Container messenger-lab3-api-2 Creating + Container messenger-lab3-api-1 Creating + Container messenger-lab3-api-1 Created + Container messenger-lab3-api-2 Created + Container messenger-lab3-nginx-1 Creating + Container messenger-lab3-nginx-1 Created + Container messenger-lab3-postgres-1 Starting + Container messenger-lab3-valkey-1 Starting + Container messenger-lab3-valkey-1 Started + Container messenger-lab3-postgres-1 Started + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Starting + Container messenger-lab3-migrate-1 Started + Container messenger-lab3-migrate-1 Waiting + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-api-1 Starting + Container messenger-lab3-api-1 Started + Container messenger-lab3-api-2 Starting + Container messenger-lab3-api-2 Started + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-nginx-1 Starting + Container messenger-lab3-nginx-1 Started + Container messenger-lab3-valkey-1 Waiting + Container messenger-lab3-nginx-1 Waiting + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-migrate-1 Waiting + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-valkey-1 Healthy + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-nginx-1 Healthy +$ docker compose ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +messenger-lab3-api-1 mtusi-messenger:lab3 "python src/gateway.…" api 15 seconds ago Up 7 seconds (healthy) 8080/tcp +messenger-lab3-api-2 mtusi-messenger:lab3 "python src/gateway.…" api 15 seconds ago Up 7 seconds (healthy) 8080/tcp +messenger-lab3-migrate-1 mtusi-messenger:lab3 "python src/migrate.…" migrate 15 seconds ago Exited (0) 7 seconds ago +messenger-lab3-nginx-1 nginx:1.28.0-alpine3.21 "/docker-entrypoint.…" nginx 15 seconds ago Up 5 seconds (healthy) 127.0.0.1:8080-8081->8080-8081/tcp +messenger-lab3-postgres-1 postgres:18.2-alpine3.22 "docker-entrypoint.s…" postgres 15 seconds ago Up 14 seconds (healthy) 5432/tcp +messenger-lab3-valkey-1 valkey/valkey:8.1.4-alpine3.22 "docker-entrypoint.s…" valkey 15 seconds ago Up 14 seconds (healthy) 6379/tcp +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy 978f2777b379 95632 dev +messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev +$ docker compose logs migrate +migrate-1 | 2026-10-19 17:00:03.868 | INFO | infrastructure.dependencies.database:create_pool:69 - Установка соединения с базой данных psql +migrate-1 | 2026-10-19 17:00:03.937 | INFO | infrastructure.dependencies.database:create_pool:99 - Соединение с базой данных psql установлено. DB_NAME: test, pool: 2..10 +migrate-1 | 2026-10-19 17:00:03.939 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии +migrate-1 | 2026-10-19 17:00:03.940 | INFO | __main__:main:20 - Миграции применены, migrate завершается +$ docker compose logs api | grep -E "Схема БД|Application started" +api-2 | 2026-10-19 17:00:05.191 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2 +api-2 | 2026-10-19 17:00:05.226 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=389b271f592b, session ttl=3600 s +api-1 | 2026-10-19 17:00:05.106 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2 +api-1 | 2026-10-19 17:00:05.173 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=978f2777b379, session ttl=3600 s +$ curl -s -o /dev/null -w "%{http_code}\n" --max-time 2 http://localhost:8080/health/ready +200 +$ docker compose exec -T nginx sh -c "wget -q -O - --timeout 2 http://api:8080/health/live; echo; nslookup api 127.0.0.11 | tail -4" +{"status":"ok"} +Address: 192.168.48.3 +Name: api +Address: 192.168.48.2 + diff --git a/evidence/02-check-smoke.txt b/evidence/02-check-smoke.txt new file mode 100644 index 00000000..b8aded63 --- /dev/null +++ b/evidence/02-check-smoke.txt @@ -0,0 +1,6 @@ +$ make check +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +$ make test +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/03-student-tests.txt b/evidence/03-student-tests.txt new file mode 100644 index 00000000..ce9bfe14 --- /dev/null +++ b/evidence/03-student-tests.txt @@ -0,0 +1,128 @@ +$ make student-test +BASE_URL="http://localhost:8080" poetry run pytest +/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( +============================= test session starts ============================== +platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python +cachedir: .pytest_cache +rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3 +configfile: pyproject.toml +testpaths: student-tests +plugins: anyio-4.15.1 +collecting ... collected 113 items + +student-tests/test_auth.py::test_login_returns_opaque_bearer_session PASSED [ 0%] +student-tests/test_auth.py::test_wrong_password_and_unknown_user_are_indistinguishable PASSED [ 1%] +student-tests/test_auth.py::test_malformed_basic_is_unauthorized[None] PASSED [ 2%] +student-tests/test_auth.py::test_malformed_basic_is_unauthorized[] PASSED [ 3%] +student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic] PASSED [ 4%] +student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic !!!not-base64!!!] PASSED [ 5%] +student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic bm8tY29sb24taGVyZQ==] PASSED [ 6%] +student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic //46cGFzc3dvcmQ=] PASSED [ 7%] +student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Bearer some-token] PASSED [ 7%] +student-tests/test_auth.py::test_utf8_password_with_colon PASSED [ 8%] +student-tests/test_auth.py::test_basic_is_rejected_on_business_endpoints PASSED [ 9%] +student-tests/test_auth.py::test_logout_revokes_only_current_session PASSED [ 10%] +student-tests/test_auth.py::test_logout_without_token PASSED [ 11%] +student-tests/test_auth.py::test_relogin_after_logout PASSED [ 12%] +student-tests/test_auth.py::test_password_is_not_stored_in_plaintext PASSED [ 13%] +student-tests/test_balancing.py::test_one_session_is_served_by_several_replicas PASSED [ 14%] +student-tests/test_balancing.py::test_write_is_visible_through_every_replica PASSED [ 15%] +student-tests/test_balancing.py::test_logout_applies_to_all_replicas PASSED [ 15%] +student-tests/test_balancing.py::test_every_api_response_has_instance_id PASSED [ 16%] +student-tests/test_concurrency.py::test_parallel_messages_are_not_lost_or_duplicated PASSED [ 17%] +student-tests/test_dependency_failures.py::test_valkey_down_cache_is_bypassed SKIPPED [ 18%] +student-tests/test_dependency_failures.py::test_logout_during_valkey_outage_is_not_resurrected SKIPPED [ 19%] +student-tests/test_dependency_failures.py::test_postgres_down_returns_bounded_503_and_no_anonymous_access SKIPPED [ 20%] +student-tests/test_dependency_failures.py::test_session_survives_postgres_restart SKIPPED [ 21%] +student-tests/test_expiry.py::test_session_expires_after_short_ttl SKIPPED [ 22%] +student-tests/test_expiry.py::test_expires_at_matches_short_ttl SKIPPED [ 23%] +student-tests/test_foreign_chat.py::test_outsider_cannot_access_chat PASSED [ 23%] +student-tests/test_foreign_chat.py::test_foreign_and_missing_chat_are_indistinguishable PASSED [ 24%] +student-tests/test_foreign_chat.py::test_chat_list_contains_only_own_chats PASSED [ 25%] +student-tests/test_health.py::test_health_endpoints_have_request_id PASSED [ 26%] +student-tests/test_health.py::test_unknown_route_uses_error_format PASSED [ 27%] +student-tests/test_idempotency.py::test_replay_returns_same_message PASSED [ 28%] +student-tests/test_idempotency.py::test_replay_ignores_json_formatting PASSED [ 29%] +student-tests/test_idempotency.py::test_same_key_with_other_body_is_conflict PASSED [ 30%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers0] PASSED [ 30%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers1] PASSED [ 31%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers2] PASSED [ 32%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers3] PASSED [ 33%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers4] PASSED [ 34%] +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers5] PASSED [ 35%] +student-tests/test_idempotency.py::test_key_boundaries_are_accepted[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 36%] +student-tests/test_idempotency.py::test_key_boundaries_are_accepted[Az09._:-] PASSED [ 37%] +student-tests/test_idempotency.py::test_key_scope_is_user_and_chat PASSED [ 38%] +student-tests/test_idempotency.py::test_replay_rechecks_session_and_access PASSED [ 38%] +student-tests/test_idempotency.py::test_expired_key_can_be_reused PASSED [ 39%] +student-tests/test_idempotency.py::test_parallel_replays_through_replicas_create_one_message PASSED [ 40%] +student-tests/test_idempotency.py::test_parallel_same_key_with_different_bodies PASSED [ 41%] +student-tests/test_invalid_json.py::test_malformed_json_on_register[{"username": "abc",] PASSED [ 42%] +student-tests/test_invalid_json.py::test_malformed_json_on_register[not json at all] PASSED [ 43%] +student-tests/test_invalid_json.py::test_malformed_json_on_register[{"text": "\xff\xfe"}] PASSED [ 44%] +student-tests/test_invalid_json.py::test_malformed_json_on_register[] PASSED [ 45%] +student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[{"username": "abc",] PASSED [ 46%] +student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[not json at all] PASSED [ 46%] +student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[{"text": "\xff\xfe"}] PASSED [ 47%] +student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[] PASSED [ 48%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body0] PASSED [ 49%] +student-tests/test_invalid_json.py::test_invalid_user_payload[alice] PASSED [ 50%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body2] PASSED [ 51%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body3] PASSED [ 52%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body4] PASSED [ 53%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body5] PASSED [ 53%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body6] PASSED [ 54%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body7] PASSED [ 55%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body8] PASSED [ 56%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body9] PASSED [ 57%] +student-tests/test_invalid_json.py::test_invalid_user_payload[body10] PASSED [ 58%] +student-tests/test_invalid_json.py::test_invalid_message_payload[body0] PASSED [ 59%] +student-tests/test_invalid_json.py::test_invalid_message_payload[body1] PASSED [ 60%] +student-tests/test_invalid_json.py::test_invalid_message_payload[body2] PASSED [ 61%] +student-tests/test_invalid_json.py::test_invalid_chat_payload PASSED [ 61%] +student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[] PASSED [ 62%] +student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[ ] PASSED [ 63%] +student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[ \n\t ] PASSED [ 64%] +student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[\u3000] PASSED [ 65%] +student-tests/test_messages.py::test_message_length_is_counted_in_code_points PASSED [ 66%] +student-tests/test_messages.py::test_unicode_is_preserved PASSED [ 67%] +student-tests/test_messages.py::test_nul_character_is_rejected PASSED [ 68%] +student-tests/test_messages.py::test_sender_is_taken_from_session PASSED [ 69%] +student-tests/test_pagination.py::test_walk_message_pages[7-3-3] PASSED [ 69%] +student-tests/test_pagination.py::test_walk_message_pages[6-3-2] PASSED [ 70%] +student-tests/test_pagination.py::test_walk_message_pages[5-1-5] PASSED [ 71%] +student-tests/test_pagination.py::test_walk_message_pages[3-100-1] PASSED [ 72%] +student-tests/test_pagination.py::test_limit_can_change_between_pages PASSED [ 73%] +student-tests/test_pagination.py::test_new_message_appears_after_cursor PASSED [ 74%] +student-tests/test_pagination.py::test_cursor_is_bound_to_chat PASSED [ 75%] +student-tests/test_pagination.py::test_invalid_limit_or_cursor[params0] PASSED [ 76%] +student-tests/test_pagination.py::test_invalid_limit_or_cursor[params1] PASSED [ 76%] +student-tests/test_pagination.py::test_invalid_limit_or_cursor[params2] PASSED [ 77%] +student-tests/test_pagination.py::test_invalid_limit_or_cursor[params3] PASSED [ 78%] +student-tests/test_pagination.py::test_invalid_limit_or_cursor[params4] PASSED [ 79%] +student-tests/test_pagination.py::test_invalid_limit_or_cursor[params5] PASSED [ 80%] +student-tests/test_pagination.py::test_walk_chat_list_pages PASSED [ 81%] +student-tests/test_pagination.py::test_chat_cursor_is_bound_to_user PASSED [ 82%] +student-tests/test_transactions.py::test_same_username_race_creates_exactly_one_user PASSED [ 83%] +student-tests/test_transactions.py::test_failed_chat_creation_is_rolled_back PASSED [ 84%] +student-tests/test_transactions.py::test_successful_chat_creation_commits_all_members PASSED [ 84%] +student-tests/test_users.py::test_duplicate_username_is_conflict PASSED [ 85%] +student-tests/test_users.py::test_registration_never_returns_password PASSED [ 86%] +student-tests/test_users.py::test_invalid_password_is_rejected[None] PASSED [ 87%] +student-tests/test_users.py::test_invalid_password_is_rejected[] PASSED [ 88%] +student-tests/test_users.py::test_invalid_password_is_rejected[\u043a\u043e\u0440\u043e\u0442\u043a\u0438\u0439] PASSED [ 89%] +student-tests/test_users.py::test_invalid_password_is_rejected[xxxxxxxxxxx] PASSED [ 90%] +student-tests/test_users.py::test_invalid_password_is_rejected[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 91%] +student-tests/test_users.py::test_invalid_password_is_rejected[123456789012] PASSED [ 92%] +student-tests/test_users.py::test_password_length_is_counted_in_code_points PASSED [ 92%] +student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers0] PASSED [ 93%] +student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers1] PASSED [ 94%] +student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers2] PASSED [ 95%] +student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers3] PASSED [ 96%] +student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers4] PASSED [ 97%] +student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers5] PASSED [ 98%] +student-tests/test_users.py::test_x_user_id_gives_no_access PASSED [ 99%] +student-tests/test_users.py::test_current_user PASSED [100%] + +======================= 107 passed, 6 skipped in 14.98s ======================== diff --git a/evidence/04-balancing-idempotency.txt b/evidence/04-balancing-idempotency.txt new file mode 100644 index 00000000..5cd85938 --- /dev/null +++ b/evidence/04-balancing-idempotency.txt @@ -0,0 +1,50 @@ +$ BASE_URL=http://localhost:8080 poetry run pytest student-tests/test_balancing.py student-tests/test_idempotency.py student-tests/test_concurrency.py -s +/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( +============================= test session starts ============================== +platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python +cachedir: .pytest_cache +rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3 +configfile: pyproject.toml +plugins: anyio-4.15.1 +collecting ... collected 21 items + +student-tests/test_balancing.py::test_one_session_is_served_by_several_replicas +INSTANCES FOR ONE SESSION: {'389b271f592b': 10, '978f2777b379': 10} +PASSED +student-tests/test_balancing.py::test_write_is_visible_through_every_replica PASSED +student-tests/test_balancing.py::test_logout_applies_to_all_replicas PASSED +student-tests/test_balancing.py::test_every_api_response_has_instance_id PASSED +student-tests/test_idempotency.py::test_replay_returns_same_message PASSED +student-tests/test_idempotency.py::test_replay_ignores_json_formatting PASSED +student-tests/test_idempotency.py::test_same_key_with_other_body_is_conflict PASSED +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers0] PASSED +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers1] PASSED +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers2] PASSED +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers3] PASSED +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers4] PASSED +student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers5] PASSED +student-tests/test_idempotency.py::test_key_boundaries_are_accepted[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED +student-tests/test_idempotency.py::test_key_boundaries_are_accepted[Az09._:-] PASSED +student-tests/test_idempotency.py::test_key_scope_is_user_and_chat PASSED +student-tests/test_idempotency.py::test_replay_rechecks_session_and_access PASSED +student-tests/test_idempotency.py::test_expired_key_can_be_reused PASSED +student-tests/test_idempotency.py::test_parallel_replays_through_replicas_create_one_message +PARALLEL SAME KEY: {"requests": 20, "statuses": {"201": 20}, "instances": {"389b271f592b": 10, "978f2777b379": 10}, "distinct_bodies": 1, "messages_in_history": 1, "rows_in_db": 1} +PASSED +student-tests/test_idempotency.py::test_parallel_same_key_with_different_bodies PASSED +student-tests/test_concurrency.py::test_parallel_messages_are_not_lost_or_duplicated +CONCURRENCY RESULT: {"clients": 10, "messages_sent": 100, "status_201": 100, "messages_read": 100, "unique_ids": 100, "unique_texts": 100, "pages_read": 4, "send_seconds": 0.227} +PASSED + +============================== 21 passed in 3.79s ============================== +$ docker compose logs --no-log-prefix nginx | grep "\"uri\":\"/api/v1/users/me\",\"status\":200" | tail -6 +{"time":"2026-10-19T17:01:21+00:00","request_id":"d754e7abc1b3b0e09ab4bf0d9fd50270","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.002","instance":"389b271f592b"} +{"time":"2026-10-19T17:01:21+00:00","request_id":"aeaf60de5c114ed306169370ede170b6","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.002,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"} +{"time":"2026-10-19T17:01:21+00:00","request_id":"7caafb063b53d8985d17356cd62d812d","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.002,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"389b271f592b"} +{"time":"2026-10-19T17:01:21+00:00","request_id":"a8e4fb898cd5ff5e56bdd1669067a60f","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"} +{"time":"2026-10-19T17:01:21+00:00","request_id":"fb8e6ba34f06d739933b9872b9aadd89","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.000","instance":"389b271f592b"} +{"time":"2026-10-19T17:01:21+00:00","request_id":"e166ba8766667300b8bcc65996ab6827","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"} +$ rid=$(docker compose logs --no-log-prefix nginx | grep "\"uri\":\"/api/v1/users/me\",\"status\":200" | tail -1 | sed -E "s/.*\"request_id\":\"([0-9a-f]+)\".*/\1/"); echo "request_id=$rid"; docker compose logs api | grep "$rid" +request_id=e166ba8766667300b8bcc65996ab6827 +api-1 | 2026-10-19 17:01:21.285 | INFO | application.api.middlewares.request_context:_log_request:70 - GET /api/v1/users/me -> 200 | 0.6 ms | request_id=e166ba8766667300b8bcc65996ab6827 diff --git a/evidence/05-failover-replica-a.txt b/evidence/05-failover-replica-a.txt new file mode 100644 index 00000000..b0e6d7cd --- /dev/null +++ b/evidence/05-failover-replica-a.txt @@ -0,0 +1,41 @@ +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy 978f2777b379 95632 dev +messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 2.693} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 exited/unhealthy - 0 dev +messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev +$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api) +/messenger-lab3-api-1 status=exited exit=0 finished=2026-10-19T17:01:38.389851544Z +/messenger-lab3-api-2 status=running exit=0 finished=0001-01-01T00:00:00Z +$ docker compose logs api | grep -E "SIGTERM|Application stopped|Finished server process" +api-1 | 2026-10-19 17:01:37.194 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=978f2777b379: порт закрыт, keep-alive соединения закрываются после ответа +api-1 | 2026-10-19 17:01:38.301 | INFO | __main__:lifespan:54 - Application stopped, instance=978f2777b379, connections closed +api-1 | INFO: Finished server process [1] +$ docker compose logs --no-log-prefix nginx | grep -E "\[(warn|error)\]" | tail -4 +2026/10/19 17:00:46 [warn] 30#30: *43 a client request body is buffered to a temporary file /var/cache/nginx/client_temp/0000000001, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551b-c8fe-7b27-bef6-22c34e5e5367/messages HTTP/1.1", host: "localhost:8080" +2026/10/19 17:00:46 [warn] 30#30: *43 a client request body is buffered to a temporary file /var/cache/nginx/client_temp/0000000002, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551b-c8fe-7b27-bef6-22c34e5e5367/messages HTTP/1.1", host: "localhost:8080" +2026/10/19 17:01:38 [error] 30#30: *803 connect() failed (111: Connection refused) while connecting to upstream, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages HTTP/1.1", upstream: "http://192.168.48.2:8080/api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages", host: "localhost:8080" +2026/10/19 17:01:38 [warn] 30#30: *803 upstream server temporarily disabled while connecting to upstream, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages HTTP/1.1", upstream: "http://192.168.48.2:8080/api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages", host: "localhost:8080" +$ make restore +./scripts/restore.sh + Container messenger-lab3-api-2 Running + Container messenger-lab3-api-1 Starting + Container messenger-lab3-api-1 Started + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy b0b074af6d9c 18674 dev +messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev diff --git a/evidence/05-failover-replica-b.txt b/evidence/05-failover-replica-b.txt new file mode 100644 index 00000000..f92f38c8 --- /dev/null +++ b/evidence/05-failover-replica-b.txt @@ -0,0 +1,69 @@ +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy b0b074af6d9c 18674 dev +messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.616} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 exited/unhealthy - 0 dev +messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev +$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api) +/messenger-lab3-api-1 status=exited exit=0 finished=2026-10-19T17:02:10.511955087Z +/messenger-lab3-api-2 status=running exit=0 finished=0001-01-01T00:00:00Z +$ make restore +./scripts/restore.sh + Container messenger-lab3-api-2 Running + Container messenger-lab3-api-1 Starting + Container messenger-lab3-api-1 Started + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev +messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev +$ curl -s -o /dev/null -w "shift round-robin: %{http_code}\n" http://localhost:8080/health/live +shift round-robin: 200 +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 1.618} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev +messenger-lab3-api-2 exited/unhealthy - 0 dev +$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api) +/messenger-lab3-api-1 status=running exit=0 finished=2026-10-19T17:02:10.511955087Z +/messenger-lab3-api-2 status=exited exit=0 finished=2026-10-19T17:02:43.134886046Z +$ docker compose logs api-2 2>/dev/null | grep -E "SIGTERM|Application stopped|Finished server process" || docker compose logs api | grep -E "SIGTERM|Application stopped" | tail -2 +api-1 | 2026-10-19 17:02:09.336 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=b0b074af6d9c: порт закрыт, keep-alive соединения закрываются после ответа +api-1 | 2026-10-19 17:02:10.443 | INFO | __main__:lifespan:54 - Application stopped, instance=b0b074af6d9c, connections closed +$ make restore +./scripts/restore.sh + Container messenger-lab3-api-1 Running + Container messenger-lab3-api-2 Starting + Container messenger-lab3-api-2 Started + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev +messenger-lab3-api-2 running/healthy 774df521af21 22387 dev +$ docker logs messenger-lab3-api-2 2>&1 | grep -E "instance=389b271f592b|Finished server process" +2026-10-19 17:00:05.226 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=389b271f592b, session ttl=3600 s +2026-10-19 17:02:41.948 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=389b271f592b: порт закрыт, keep-alive соединения закрываются после ответа +2026-10-19 17:02:43.068 | INFO | __main__:lifespan:54 - Application stopped, instance=389b271f592b, connections closed +INFO: Finished server process [1] diff --git a/evidence/06-persistence.txt b/evidence/06-persistence.txt new file mode 100644 index 00000000..d6fcacde --- /dev/null +++ b/evidence/06-persistence.txt @@ -0,0 +1,37 @@ +$ cat scripts/restart.sh +#!/bin/sh +set -eu +# Перезапуск всего стенда с сохранением volumes: хранилища (PostgreSQL, Valkey), +# затем все API-реплики и балансировщик. Никаких down -v, flush или повторного +# seed; migrate не запускается — схема уже в volume. Готовность ждёт тест. +cd "$(dirname "$0")/.." +ENV_FILE="${ENV_FILE:-.env}" + +docker compose --env-file "$ENV_FILE" restart --no-deps postgres valkey +docker compose --env-file "$ENV_FILE" restart --no-deps api +docker compose --env-file "$ENV_FILE" restart --no-deps nginx +$ docker inspect -f "{{.Name}} started={{.State.StartedAt}}" $(docker compose ps -q) +/messenger-lab3-api-1 started=2026-10-19T17:02:39.217903586Z +/messenger-lab3-api-2 started=2026-10-19T17:02:57.444596886Z +/messenger-lab3-nginx-1 started=2026-10-19T17:00:06.004414293Z +/messenger-lab3-postgres-1 started=2026-10-19T16:59:57.066060636Z +/messenger-lab3-valkey-1 started=2026-10-19T16:59:57.064188719Z +$ docker volume inspect -f "{{.Name}} created={{.CreatedAt}}" messenger-lab3_postgres_data +messenger-lab3_postgres_data created=2026-10-19T16:10:27Z +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +$ docker inspect -f "{{.Name}} started={{.State.StartedAt}}" $(docker compose ps -q) +/messenger-lab3-api-1 started=2026-10-19T17:03:17.125014673Z +/messenger-lab3-api-2 started=2026-10-19T17:03:17.151381881Z +/messenger-lab3-nginx-1 started=2026-10-19T17:03:17.57774484Z +/messenger-lab3-postgres-1 started=2026-10-19T17:03:15.504400631Z +/messenger-lab3-valkey-1 started=2026-10-19T17:03:15.535674256Z +$ docker volume inspect -f "{{.Name}} created={{.CreatedAt}}" messenger-lab3_postgres_data +messenger-lab3_postgres_data created=2026-10-19T16:10:27Z +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy bfb3f7bbb85d 23878 dev +messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev diff --git a/evidence/07-expiry.txt b/evidence/07-expiry.txt new file mode 100644 index 00000000..43c6b2e6 --- /dev/null +++ b/evidence/07-expiry.txt @@ -0,0 +1,108 @@ +$ make test-expiry +docker compose --profile expiry up -d --build --wait api-short-ttl +#1 [internal] load local bake definitions +#1 reading from stdin 1.20kB done +#1 DONE 0.0s + +#2 [api-short-ttl internal] load build definition from Dockerfile +#2 transferring dockerfile: 914B done +#2 DONE 0.0s + +#3 [api-short-ttl internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm +#3 DONE 0.1s + +#4 [api-short-ttl internal] load .dockerignore +#4 transferring context: 358B done +#4 DONE 0.0s + +#5 [api-short-ttl 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c +#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done +#5 DONE 0.0s + +#6 [api-short-ttl internal] load build context +#6 transferring context: 7.39kB done +#6 DONE 0.0s + +#7 [migrate 5/7] COPY migrations ./migrations +#7 CACHED + +#8 [migrate 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction +#8 CACHED + +#9 [migrate 2/7] WORKDIR /app +#9 CACHED + +#10 [migrate 6/7] COPY src ./src +#10 CACHED + +#11 [migrate 3/7] COPY poetry.lock pyproject.toml ./ +#11 CACHED + +#12 [api-short-ttl 7/7] RUN useradd --system --uid 10001 --no-create-home app +#12 CACHED + +#13 [migrate] exporting to image +#13 exporting layers done +#13 exporting manifest sha256:9802819ef7b235f44ad03fed838cf83fcd4573a342405936538c7c969adfd573 done +#13 exporting config sha256:e2b0c133c10bc42d53160db701c32694ed2cbac54f0522999289f4cf7cef21e1 done +#13 naming to docker.io/library/mtusi-messenger:lab3 done +#13 unpacking to docker.io/library/mtusi-messenger:lab3 done +#13 DONE 0.0s + +#14 [api-short-ttl] exporting to image +#14 exporting layers done +#14 exporting manifest sha256:036b7dde37a28a3eeec751ce6ada2034e1a9a1d76cd9310dbd3cd157824ea087 done +#14 exporting config sha256:26fb210928fa7201fedcce1e64d506079d863ddc42532dc4d25f1f77d673e3b3 done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 done +#14 DONE 0.0s + +#15 [migrate] resolving provenance for metadata file +#15 DONE 0.0s + +#16 [api-short-ttl] resolving provenance for metadata file +#16 DONE 0.0s + mtusi-messenger:lab3 Built + mtusi-messenger:lab3 Built + Container messenger-lab3-valkey-1 Running + Container messenger-lab3-postgres-1 Running + Container messenger-lab3-migrate-1 Recreate + Container messenger-lab3-migrate-1 Recreated + Container messenger-lab3-api-short-ttl-1 Creating + Container messenger-lab3-api-short-ttl-1 Created + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Starting + Container messenger-lab3-migrate-1 Started + Container messenger-lab3-migrate-1 Waiting + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-api-short-ttl-1 Starting + Container messenger-lab3-api-short-ttl-1 Started + Container messenger-lab3-api-short-ttl-1 Waiting + Container messenger-lab3-migrate-1 Waiting + Container messenger-lab3-postgres-1 Waiting + Container messenger-lab3-valkey-1 Waiting + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-valkey-1 Healthy + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-api-short-ttl-1 Healthy +EXPIRY_BASE_URL=http://localhost:8081 poetry run pytest student-tests/test_expiry.py -s; \ + status=$?; docker compose --profile expiry stop api-short-ttl; exit $status +/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( +============================= test session starts ============================== +platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python +cachedir: .pytest_cache +rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3 +configfile: pyproject.toml +plugins: anyio-4.15.1 +collecting ... collected 2 items + +student-tests/test_expiry.py::test_session_expires_after_short_ttl PASSED +student-tests/test_expiry.py::test_expires_at_matches_short_ttl PASSED + +============================== 2 passed in 3.54s =============================== + Container messenger-lab3-api-short-ttl-1 Stopping + Container messenger-lab3-api-short-ttl-1 Stopped diff --git a/evidence/08-dependency-failures.txt b/evidence/08-dependency-failures.txt new file mode 100644 index 00000000..10aa143c --- /dev/null +++ b/evidence/08-dependency-failures.txt @@ -0,0 +1,24 @@ +$ make test-failures +BASE_URL="http://localhost:8080" RUN_FAILURE_TESTS=1 poetry run pytest student-tests/test_dependency_failures.py -s +/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020 + warnings.warn( +============================= test session starts ============================== +platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python +cachedir: .pytest_cache +rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3 +configfile: pyproject.toml +plugins: anyio-4.15.1 +collecting ... collected 4 items + +student-tests/test_dependency_failures.py::test_valkey_down_cache_is_bypassed PASSED +student-tests/test_dependency_failures.py::test_logout_during_valkey_outage_is_not_resurrected PASSED +student-tests/test_dependency_failures.py::test_postgres_down_returns_bounded_503_and_no_anonymous_access PASSED +student-tests/test_dependency_failures.py::test_session_survives_postgres_restart PASSED + +============================== 4 passed in 11.79s ============================== +$ docker compose ps --format "{{.Name}}\t{{.Status}}" +messenger-lab3-api-1 Up 12 hours (healthy) +messenger-lab3-api-2 Up 12 hours (healthy) +messenger-lab3-nginx-1 Up 12 hours (healthy) +messenger-lab3-postgres-1 Up Less than a second (health: starting) +messenger-lab3-valkey-1 Up 8 seconds (healthy) diff --git a/evidence/09-migrations.txt b/evidence/09-migrations.txt new file mode 100644 index 00000000..efbf83b6 --- /dev/null +++ b/evidence/09-migrations.txt @@ -0,0 +1,22 @@ +$ make test-migrations +./scripts/check_migrations.sh +2026-10-20 04:55:31.041 | INFO | infrastructure.dependencies.database:create_pool:69 - Установка соединения с базой данных psql +2026-10-20 04:55:31.118 | INFO | infrastructure.dependencies.database:create_pool:99 - Соединение с базой данных psql установлено. DB_NAME: test, pool: 2..10 +пустой volume: таблиц в public = 0 +API до migrate: В БД не применены миграции [1, 2]: сначала запустите migrate +2026-10-20 04:55:31.132 | INFO | infrastructure.dependencies.database:apply_migrations:152 - Применена миграция 0001_init.sql +2026-10-20 04:55:31.134 | INFO | infrastructure.dependencies.database:apply_migrations:152 - Применена миграция 0002_message_idempotency.sql +2026-10-20 04:55:31.134 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии +запуск 1: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7 +2026-10-20 04:55:31.135 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии +запуск 2: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7 +2026-10-20 04:55:31.136 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии +запуск 3: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7 +2026-10-20 04:55:31.136 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2 +API после migrate: схема совместима +$ BUILDX_NO_DEFAULT_ATTESTATIONS=1 ./scripts/check_previous_version.sh +previous version image: mtusi-messenger:compat-aa971fb (aa971fb feat: перенос core-логики 2 лабы) +schema on stand: 0001_init, 0002_message_idempotency +{"ready": true, "register": 201, "login": 201, "create_chat": 201, "send_message_without_key": 201, "read_history": 200, "history_matches": true} +startup log: ['Схема БД в актуальной версии'] +PASS: previous API version works on the current schema (expand-only migration) diff --git a/evidence/10-sigterm.txt b/evidence/10-sigterm.txt new file mode 100644 index 00000000..e94e1388 --- /dev/null +++ b/evidence/10-sigterm.txt @@ -0,0 +1,54 @@ +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy bfb3f7bbb85d 23878 dev +messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev +$ make sigterm-check +python3 scripts/sigterm_check.py --base-url "http://localhost:8080" +Журнал реплики после SIGTERM: +2026-10-20 04:55:46.580 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=bfb3f7bbb85d: порт закрыт, keep-alive соединения закрываются после ответа +INFO: Shutting down +INFO: Waiting for application shutdown. +2026-10-20 04:55:47.701 | INFO | __main__:lifespan:54 - Application stopped, instance=bfb3f7bbb85d, connections closed +INFO: Application shutdown complete. +INFO: Finished server process [1] +{ + "container": "messenger-lab3-api-1", + "instance": "bfb3f7bbb85d", + "exit_code": 0, + "shutdown_seconds": 1.53, + "limit_seconds": 10, + "instances_before": [ + "6d2f6cd2b04a", + "bfb3f7bbb85d" + ], + "probes_ok_after_sigterm": 354, + "instances_after_sigterm": [ + "6d2f6cd2b04a", + "bfb3f7bbb85d" + ], + "last_target_response_after_sigterm_seconds": 0.133, + "target_responses_after_exit": 0, + "errors_after_sigterm": 0, + "error_kinds": [] +} +PASS: SIGTERM завершил messenger-lab3-api-1 за 1.53 с, LB обслужил 354 запросов без ошибок +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 exited/unhealthy - 0 dev +messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev +$ make restore +./scripts/restore.sh + Container messenger-lab3-api-2 Running + Container messenger-lab3-api-1 Starting + Container messenger-lab3-api-1 Started + Container messenger-lab3-api-1 Waiting + Container messenger-lab3-api-2 Waiting + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy 7535b2ed4434 69183 dev +messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev diff --git a/evidence/11-limits-ports-networks.txt b/evidence/11-limits-ports-networks.txt new file mode 100644 index 00000000..ace189a8 --- /dev/null +++ b/evidence/11-limits-ports-networks.txt @@ -0,0 +1,33 @@ +$ docker inspect -f "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} restart={{.HostConfig.RestartPolicy.Name}} ports={{json .HostConfig.PortBindings}}" $(docker compose ps -aq) +/messenger-lab3-api-1 cpus=1000000000 memory=536870912 restart=unless-stopped ports={} +/messenger-lab3-api-2 cpus=1000000000 memory=536870912 restart=unless-stopped ports={} +/messenger-lab3-api-short-ttl-1 cpus=1000000000 memory=536870912 restart=unless-stopped ports={} +/messenger-lab3-migrate-1 cpus=500000000 memory=268435456 restart=no ports={} +/messenger-lab3-nginx-1 cpus=500000000 memory=67108864 restart=unless-stopped ports={"8080/tcp":[{"HostIp":"127.0.0.1","HostPort":"8080"}],"8081/tcp":[{"HostIp":"127.0.0.1","HostPort":"8081"}]} +/messenger-lab3-postgres-1 cpus=1500000000 memory=536870912 restart=unless-stopped ports={} +/messenger-lab3-valkey-1 cpus=250000000 memory=100663296 restart=unless-stopped ports={} +$ for n in messenger-lab3_edge messenger-lab3_backend; do docker network inspect -f "{{.Name}} internal={{.Internal}} containers={{range .Containers}}{{.Name}} {{end}}" $n; done +messenger-lab3_edge internal=false containers=messenger-lab3-api-2 messenger-lab3-api-1 messenger-lab3-nginx-1 +messenger-lab3_backend internal=true containers=messenger-lab3-api-2 messenger-lab3-valkey-1 messenger-lab3-postgres-1 messenger-lab3-api-1 +$ grep -n "container_name" compose.yaml || echo "container_name: not used" +container_name: not used +$ docker compose config | grep -E "^\s+(published|target):" | sort | uniq -c + 1 published: "8080" + 1 published: "8081" + 1 target: /etc/nginx/errors.conf + 1 target: /etc/nginx/nginx.conf + 1 target: /var/lib/postgresql + 1 target: 8080 + 1 target: 8081 +$ docker compose exec -T nginx sh -c "wget -q -O - --timeout 2 http://postgres:5432 2>&1 | head -1; nslookup postgres 127.0.0.11 2>&1 | tail -2" +wget: bad address 'postgres:5432' +** server can't find postgres: NXDOMAIN + +$ curl -s --max-time 2 http://localhost:5432 >/dev/null 2>&1 && echo "5432 open" || echo "host 127.0.0.1:5432: closed" +host 127.0.0.1:5432: closed +$ docker stats --no-stream --format "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" $(docker compose ps -q) +messenger-lab3-api-1 0.25% 45.73MiB / 512MiB +messenger-lab3-api-2 0.21% 45.42MiB / 512MiB +messenger-lab3-nginx-1 0.03% 2.59MiB / 64MiB +messenger-lab3-postgres-1 1.15% 30.5MiB / 512MiB +messenger-lab3-valkey-1 0.22% 11.04MiB / 96MiB diff --git a/evidence/12-load-baseline.txt b/evidence/12-load-baseline.txt new file mode 100644 index 00000000..3fa9f8d1 --- /dev/null +++ b/evidence/12-load-baseline.txt @@ -0,0 +1,18 @@ +$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.024, + "concurrency": 4, + "successful_requests": 21418, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 713.369, + "successful_latency_ms": { + "p50": 4.748, + "p95": 8.395, + "p99": 22.531 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/13-experiment-failover.txt b/evidence/13-experiment-failover.txt new file mode 100644 index 00000000..df9e6f16 --- /dev/null +++ b/evidence/13-experiment-failover.txt @@ -0,0 +1,22 @@ +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy 7535b2ed4434 69183 dev +messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev +$ make experiment-failover +python3 scripts/experiment.py failover --base-url "http://localhost:8080" --output evidence/experiment-failover.json +== SIGTERM messenger-lab3-api-1 (instance 7535b2ed4434) +{"scenario": "SIGTERM messenger-lab3-api-1", "rps_ok_write": {"before": 381.6, "during": 276.9, "after": 398.9}, "rps_ok_read": {"before": 381.6, "during": 276.8, "after": 399.0}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.063}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.971, "instances_seen_from_action": {"6d2f6cd2b04a": [-10.128, 21.291], "7535b2ed4434": [-10.128, 0.032], "eb1dace786c6": [14.257, 21.289]}, "integrity": {"messages_in_history": 10740, "confirmed_unique": 10740, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}} +== SIGTERM messenger-lab3-api-2 (instance 6d2f6cd2b04a) +{"scenario": "SIGTERM messenger-lab3-api-2", "rps_ok_write": {"before": 401.3, "during": 282.3, "after": 362.4}, "rps_ok_read": {"before": 401.1, "during": 282.4, "after": 362.5}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.071}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 3.736, "instances_seen_from_action": {"6d2f6cd2b04a": [-10.122, 0.079], "eb1dace786c6": [-10.121, 21.19], "577316c523cb": [16.921, 21.191]}, "integrity": {"messages_in_history": 10687, "confirmed_unique": 10687, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}} +== SIGKILL messenger-lab3-api-1 (instance eb1dace786c6) +{"scenario": "SIGKILL messenger-lab3-api-1", "rps_ok_write": {"before": 427.4, "during": 281.4, "after": 408.2}, "rps_ok_read": {"before": 427.3, "during": 281.3, "after": 408.5}, "errors": {"before": 0, "during": 1, "after": 0}, "recovery": {"write_errors": 1, "error_kinds": {"HTTP 503": 1}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.07}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.79, "instances_seen_from_action": {"eb1dace786c6": [-9.73, 0.024], "577316c523cb": [-9.73, 20.11], "506b547ba744": [12.893, 20.111]}, "integrity": {"messages_in_history": 10837, "confirmed_unique": 10837, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}} +== SIGKILL messenger-lab3-api-2 (instance 577316c523cb) +{"scenario": "SIGKILL messenger-lab3-api-2", "rps_ok_write": {"before": 387.4, "during": 276.7, "after": 373.2}, "rps_ok_read": {"before": 387.1, "during": 276.8, "after": 373.3}, "errors": {"before": 0, "during": 3, "after": 0}, "recovery": {"write_errors": 3, "error_kinds": {"HTTP 503": 3}, "write_recovery_seconds": 0.006, "max_write_gap_seconds": 0.061}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.0, "instances_seen_from_action": {"506b547ba744": [-9.94, 20.189], "577316c523cb": [-9.939, 0.028], "1c20e7f6dbde": [11.911, 20.19]}, "integrity": {"messages_in_history": 10212, "confirmed_unique": 10212, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}} +saved: evidence/experiment-failover.json +PASS: прогонов 4, подтверждённые сообщения целы, дублей нет +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy 506b547ba744 76140 dev +messenger-lab3-api-2 running/healthy 1c20e7f6dbde 77892 dev diff --git a/evidence/14-experiment-rolling.txt b/evidence/14-experiment-rolling.txt new file mode 100644 index 00000000..d1ee87d2 --- /dev/null +++ b/evidence/14-experiment-rolling.txt @@ -0,0 +1,18 @@ +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-1 running/healthy 506b547ba744 76140 dev +messenger-lab3-api-2 running/healthy 1c20e7f6dbde 77892 dev +$ make experiment-rolling +python3 scripts/experiment.py rolling --base-url "http://localhost:8080" --output evidence/experiment-rolling.json +== rolling update to revision exp-1792472327 +{"scenario": "rolling update", "rps_ok_write": {"before": 420.1, "during": 308.0, "after": 345.4}, "rps_ok_read": {"before": 419.9, "during": 307.9, "after": 345.5}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.098}, "target_responses_while_stopped": null, "restored_replica_in_rotation_seconds": null, "instances_seen_from_action": {"506b547ba744": [-9.228, 8.14], "1c20e7f6dbde": [-9.227, 14.911], "27601bb40a21": [4.875, 24.355], "e7e986ee1b58": [13.885, 24.355]}, "integrity": {"messages_in_history": 11682, "confirmed_unique": 11682, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}} +saved: evidence/experiment-rolling.json +PASS: прогонов 1, подтверждённые сообщения целы, дублей нет +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +messenger-lab3-api-3 running/healthy 27601bb40a21 79887 exp-1792472327 +messenger-lab3-api-4 running/healthy e7e986ee1b58 80346 exp-1792472327 +$ docker compose logs --no-log-prefix nginx --since 3m | grep -cE "prematurely closed" || true +4 diff --git a/evidence/15-clean-clone.txt b/evidence/15-clean-clone.txt new file mode 100644 index 00000000..dccdb543 --- /dev/null +++ b/evidence/15-clean-clone.txt @@ -0,0 +1,192 @@ +$ echo "COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME (отдельный проект: рабочий стенд не затрагивается)" +COMPOSE_PROJECT_NAME=lab3-clean (отдельный проект: рабочий стенд не затрагивается) +$ cp .env.example .env && sed -i "" "s/^DB_PASS=.*/DB_PASS=$(openssl rand -hex 16)/; s/^VALKEY_PASS=.*/VALKEY_PASS=$(openssl rand -hex 16)/; s/^LB_PORT=.*/LB_PORT=18080/" .env && echo "LB_EXPIRY_PORT=18081" >> .env && echo ".env created from .env.example with random synthetic secrets" +.env created from .env.example with random synthetic secrets +$ docker compose --env-file .env config --quiet && echo "compose config: OK" +compose config: OK +$ make up +docker compose up -d --build --wait +#1 [internal] load local bake definitions +#1 reading from stdin 1.47kB done +#1 DONE 0.0s + +#2 [api internal] load build definition from Dockerfile +#2 transferring dockerfile: 914B done +#2 DONE 0.0s + +#3 [api internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm +#3 DONE 0.0s + +#4 [api internal] load .dockerignore +#4 transferring context: 358B done +#4 DONE 0.0s + +#5 [api 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c +#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done +#5 DONE 0.0s + +#6 [api internal] load build context +#6 transferring context: 184.06kB 0.0s done +#6 DONE 0.0s + +#7 [api 2/7] WORKDIR /app +#7 CACHED + +#8 [api 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction +#8 CACHED + +#9 [api 3/7] COPY poetry.lock pyproject.toml ./ +#9 CACHED + +#10 [migrate 5/7] COPY migrations ./migrations +#10 CACHED + +#11 [migrate 6/7] COPY src ./src +#11 DONE 0.0s + +#12 [api 7/7] RUN useradd --system --uid 10001 --no-create-home app +#12 0.118 useradd warning: app's uid 10001 is greater than SYS_UID_MAX 999 +#12 DONE 0.1s + +#13 [migrate] exporting to image +#13 exporting layers 0.0s done +#13 exporting manifest sha256:a99a66062da552c44bf2543747fc3ac215605d7487b93882f0248f46ae71dbf5 done +#13 exporting config sha256:53e2ac5d5830524705f0737d78b69d4f9c1ca9670b8a24788c8717a7d772898f done +#13 naming to docker.io/library/mtusi-messenger:lab3 done +#13 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done +#13 DONE 0.1s + +#14 [api] exporting to image +#14 exporting layers 0.0s done +#14 exporting manifest sha256:47e4d0732ca8d624529f0f775abf4cd8457445931de74053dec5bbab49de873b done +#14 exporting config sha256:d6557e51b469086507b892558ee5960f31b77943d8c8fddc381e955e4eb3718a done +#14 naming to docker.io/library/mtusi-messenger:lab3 done +#14 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done +#14 DONE 0.1s + +#15 [migrate] resolving provenance for metadata file +#15 DONE 0.0s + +#16 [api] resolving provenance for metadata file +#16 DONE 0.0s + mtusi-messenger:lab3 Built + mtusi-messenger:lab3 Built + Network lab3-clean_backend Creating + Network lab3-clean_backend Created + Network lab3-clean_edge Creating + Network lab3-clean_edge Created + Volume lab3-clean_postgres_data Creating + Volume lab3-clean_postgres_data Created + Container lab3-clean-valkey-1 Creating + Container lab3-clean-postgres-1 Creating + Container lab3-clean-valkey-1 Created + Container lab3-clean-postgres-1 Created + Container lab3-clean-migrate-1 Creating + Container lab3-clean-migrate-1 Created + Container lab3-clean-api-2 Creating + Container lab3-clean-api-1 Creating + Container lab3-clean-api-2 Created + Container lab3-clean-api-1 Created + Container lab3-clean-nginx-1 Creating + Container lab3-clean-nginx-1 Created + Container lab3-clean-valkey-1 Starting + Container lab3-clean-postgres-1 Starting + Container lab3-clean-postgres-1 Started + Container lab3-clean-postgres-1 Waiting + Container lab3-clean-valkey-1 Started + Container lab3-clean-postgres-1 Healthy + Container lab3-clean-migrate-1 Starting + Container lab3-clean-migrate-1 Started + Container lab3-clean-migrate-1 Waiting + Container lab3-clean-postgres-1 Waiting + Container lab3-clean-postgres-1 Healthy + Container lab3-clean-migrate-1 Exited + Container lab3-clean-api-1 Starting + Container lab3-clean-api-1 Started + Container lab3-clean-api-2 Starting + Container lab3-clean-api-2 Started + Container lab3-clean-api-1 Waiting + Container lab3-clean-api-2 Waiting + Container lab3-clean-api-1 Healthy + Container lab3-clean-api-2 Healthy + Container lab3-clean-nginx-1 Starting + Container lab3-clean-nginx-1 Started + Container lab3-clean-postgres-1 Waiting + Container lab3-clean-valkey-1 Waiting + Container lab3-clean-nginx-1 Waiting + Container lab3-clean-api-1 Waiting + Container lab3-clean-api-2 Waiting + Container lab3-clean-migrate-1 Waiting + Container lab3-clean-valkey-1 Healthy + Container lab3-clean-postgres-1 Healthy + Container lab3-clean-migrate-1 Exited + Container lab3-clean-api-1 Healthy + Container lab3-clean-api-2 Healthy + Container lab3-clean-nginx-1 Healthy +$ make check test BASE_URL=http://localhost:18080 +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +python3 tests/smoke.py --base-url "http://localhost:18080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:18080 +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 2.118} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +$ make instances +./scripts/instances.sh +CONTAINER STATE INSTANCE PID REVISION +lab3-clean-api-1 exited/unhealthy - 0 dev +lab3-clean-api-2 running/healthy 4a226e142873 83992 dev +$ make restore +./scripts/restore.sh + Container lab3-clean-api-2 Running + Container lab3-clean-api-1 Starting + Container lab3-clean-api-1 Started + Container lab3-clean-api-1 Waiting + Container lab3-clean-api-2 Waiting + Container lab3-clean-api-1 Healthy + Container lab3-clean-api-2 Healthy +$ make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18080 +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +$ docker compose ps -a +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +lab3-clean-api-1 mtusi-messenger:lab3 "python src/gateway.…" api 43 seconds ago Up 6 seconds (healthy) 8080/tcp +lab3-clean-api-2 mtusi-messenger:lab3 "python src/gateway.…" api 43 seconds ago Up 6 seconds (healthy) 8080/tcp +lab3-clean-migrate-1 mtusi-messenger:lab3 "python src/migrate.…" migrate 43 seconds ago Exited (0) 35 seconds ago +lab3-clean-nginx-1 nginx:1.28.0-alpine3.21 "/docker-entrypoint.…" nginx 43 seconds ago Up 5 seconds (healthy) 127.0.0.1:18080->8080/tcp, 127.0.0.1:18081->8081/tcp +lab3-clean-postgres-1 postgres:18.2-alpine3.22 "docker-entrypoint.s…" postgres 44 seconds ago Up 8 seconds (healthy) 5432/tcp +lab3-clean-valkey-1 valkey/valkey:8.1.4-alpine3.22 "docker-entrypoint.s…" valkey 44 seconds ago Up 8 seconds (healthy) 6379/tcp +$ docker compose --profile expiry down -v + Container lab3-clean-nginx-1 Stopping + Container lab3-clean-nginx-1 Stopped + Container lab3-clean-nginx-1 Removing + Container lab3-clean-nginx-1 Removed + Container lab3-clean-api-2 Stopping + Container lab3-clean-api-1 Stopping + Container lab3-clean-api-1 Stopped + Container lab3-clean-api-1 Removing + Container lab3-clean-api-1 Removed + Container lab3-clean-api-2 Stopped + Container lab3-clean-api-2 Removing + Container lab3-clean-api-2 Removed + Container lab3-clean-valkey-1 Stopping + Container lab3-clean-migrate-1 Stopping + Container lab3-clean-migrate-1 Stopped + Container lab3-clean-migrate-1 Removing + Container lab3-clean-migrate-1 Removed + Container lab3-clean-postgres-1 Stopping + Container lab3-clean-postgres-1 Stopped + Container lab3-clean-postgres-1 Removing + Container lab3-clean-postgres-1 Removed + Container lab3-clean-valkey-1 Stopped + Container lab3-clean-valkey-1 Removing + Container lab3-clean-valkey-1 Removed + Volume lab3-clean_postgres_data Removing + Network lab3-clean_edge Removing + Network lab3-clean_backend Removing + Volume lab3-clean_postgres_data Removed + Network lab3-clean_edge Removed + Network lab3-clean_backend Removed diff --git a/evidence/experiment-failover.json b/evidence/experiment-failover.json new file mode 100644 index 00000000..f1cc3d05 --- /dev/null +++ b/evidence/experiment-failover.json @@ -0,0 +1,1625 @@ +{ + "mode": "failover", + "started": "2026-10-20T07:56:31+0300", + "profile": { + "workers": 4, + "loop": "closed: POST message (retry same key until 201) + GET limit=1", + "client_timeout_seconds": 5, + "phases_seconds": { + "before": 8, + "outage": 10, + "after": 8 + } + }, + "runs": [ + { + "scenario": "SIGTERM messenger-lab3-api-1", + "target_instance": "7535b2ed4434", + "workers": 4, + "events_seconds": { + "action_start": 10.137, + "action_end": 11.524, + "restore_start": 21.525, + "restore_end": 23.423, + "end": 31.428 + }, + "instances_before": [ + "6d2f6cd2b04a", + "7535b2ed4434" + ], + "instances_after": [ + "6d2f6cd2b04a", + "eb1dace786c6" + ], + "target_responses_while_stopped": 0, + "restored_replica_in_rotation_seconds": 0.971, + "instances_seen_from_action": { + "6d2f6cd2b04a": [ + -10.128, + 21.291 + ], + "7535b2ed4434": [ + -10.128, + 0.032 + ], + "eb1dace786c6": [ + 14.257, + 21.289 + ] + }, + "phases": { + "before": { + "seconds": 10.137, + "write": { + "ok": 3868, + "errors": 0, + "ok_rps": 381.6, + "latency_ms": { + "p50": 5.0, + "p95": 9.5, + "p99": 18.4 + } + }, + "read": { + "ok": 3868, + "errors": 0, + "ok_rps": 381.6, + "latency_ms": { + "p50": 3.8, + "p95": 7.2, + "p99": 22.3 + } + } + }, + "during": { + "seconds": 13.286, + "write": { + "ok": 3679, + "errors": 0, + "ok_rps": 276.9, + "latency_ms": { + "p50": 6.8, + "p95": 13.1, + "p99": 28.4 + } + }, + "read": { + "ok": 3678, + "errors": 0, + "ok_rps": 276.8, + "latency_ms": { + "p50": 5.4, + "p95": 10.9, + "p99": 25.6 + } + } + }, + "after": { + "seconds": 8.005, + "write": { + "ok": 3193, + "errors": 0, + "ok_rps": 398.9, + "latency_ms": { + "p50": 4.9, + "p95": 8.7, + "p99": 18.6 + } + }, + "read": { + "ok": 3194, + "errors": 0, + "ok_rps": 399.0, + "latency_ms": { + "p50": 3.8, + "p95": 6.5, + "p99": 14.5 + } + } + } + }, + "recovery": { + "write_errors": 0, + "error_kinds": {}, + "write_recovery_seconds": 0.0, + "max_write_gap_seconds": 0.063 + }, + "integrity": { + "messages_in_history": 10740, + "confirmed_unique": 10740, + "confirmed_missing_or_changed": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "retried_writes_confirmed_as_replay": 0 + }, + "container_stats_before_action": [ + "messenger-lab3-nginx-1\t9.28%\t3.09MiB / 64MiB", + "messenger-lab3-postgres-1\t27.64%\t67.91MiB / 512MiB", + "messenger-lab3-api-1\t83.48%\t48.34MiB / 512MiB", + "messenger-lab3-api-2\t85.91%\t46.54MiB / 512MiB" + ], + "action_output": [ + "messenger-lab3-api-1" + ], + "restore_output_tail": [], + "timeline": [ + { + "t": 0, + "ok": 608, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 302, + "7535b2ed4434": 306 + } + }, + { + "t": 1, + "ok": 790, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 397, + "7535b2ed4434": 393 + } + }, + { + "t": 2, + "ok": 619, + "errors": 0, + "instances": { + "7535b2ed4434": 310, + "6d2f6cd2b04a": 309 + } + }, + { + "t": 3, + "ok": 867, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 434, + "7535b2ed4434": 433 + } + }, + { + "t": 4, + "ok": 875, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 437, + "7535b2ed4434": 438 + } + }, + { + "t": 5, + "ok": 966, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 483, + "7535b2ed4434": 483 + } + }, + { + "t": 6, + "ok": 647, + "errors": 0, + "instances": { + "7535b2ed4434": 323, + "6d2f6cd2b04a": 324 + } + }, + { + "t": 7, + "ok": 732, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 366, + "7535b2ed4434": 366 + } + }, + { + "t": 8, + "ok": 687, + "errors": 0, + "instances": { + "7535b2ed4434": 344, + "6d2f6cd2b04a": 343 + } + }, + { + "t": 9, + "ok": 839, + "errors": 0, + "instances": { + "7535b2ed4434": 419, + "6d2f6cd2b04a": 420 + } + }, + { + "t": 10, + "ok": 607, + "errors": 0, + "instances": { + "7535b2ed4434": 64, + "6d2f6cd2b04a": 543 + } + }, + { + "t": 11, + "ok": 431, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 431 + } + }, + { + "t": 12, + "ok": 567, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 567 + } + }, + { + "t": 13, + "ok": 664, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 664 + } + }, + { + "t": 14, + "ok": 659, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 659 + } + }, + { + "t": 15, + "ok": 664, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 664 + } + }, + { + "t": 16, + "ok": 620, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 620 + } + }, + { + "t": 17, + "ok": 482, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 482 + } + }, + { + "t": 18, + "ok": 567, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 567 + } + }, + { + "t": 19, + "ok": 661, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 661 + } + }, + { + "t": 20, + "ok": 548, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 548 + } + }, + { + "t": 21, + "ok": 470, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 470 + } + }, + { + "t": 22, + "ok": 358, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 358 + } + }, + { + "t": 23, + "ok": 542, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 542 + } + }, + { + "t": 24, + "ok": 748, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 491, + "eb1dace786c6": 257 + } + }, + { + "t": 25, + "ok": 904, + "errors": 0, + "instances": { + "eb1dace786c6": 452, + "6d2f6cd2b04a": 452 + } + }, + { + "t": 26, + "ok": 904, + "errors": 0, + "instances": { + "eb1dace786c6": 452, + "6d2f6cd2b04a": 452 + } + }, + { + "t": 27, + "ok": 721, + "errors": 0, + "instances": { + "eb1dace786c6": 362, + "6d2f6cd2b04a": 359 + } + }, + { + "t": 28, + "ok": 525, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 263, + "eb1dace786c6": 262 + } + }, + { + "t": 29, + "ok": 898, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 450, + "eb1dace786c6": 448 + } + }, + { + "t": 30, + "ok": 915, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 457, + "eb1dace786c6": 458 + } + }, + { + "t": 31, + "ok": 395, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 198, + "eb1dace786c6": 197 + } + } + ] + }, + { + "scenario": "SIGTERM messenger-lab3-api-2", + "target_instance": "6d2f6cd2b04a", + "workers": 4, + "events_seconds": { + "action_start": 10.128, + "action_end": 11.576, + "restore_start": 21.577, + "restore_end": 23.313, + "end": 31.319 + }, + "instances_before": [ + "6d2f6cd2b04a", + "eb1dace786c6" + ], + "instances_after": [ + "577316c523cb", + "eb1dace786c6" + ], + "target_responses_while_stopped": 0, + "restored_replica_in_rotation_seconds": 3.736, + "instances_seen_from_action": { + "6d2f6cd2b04a": [ + -10.122, + 0.079 + ], + "eb1dace786c6": [ + -10.121, + 21.19 + ], + "577316c523cb": [ + 16.921, + 21.191 + ] + }, + "phases": { + "before": { + "seconds": 10.128, + "write": { + "ok": 4064, + "errors": 0, + "ok_rps": 401.3, + "latency_ms": { + "p50": 5.0, + "p95": 8.4, + "p99": 22.1 + } + }, + "read": { + "ok": 4062, + "errors": 0, + "ok_rps": 401.1, + "latency_ms": { + "p50": 3.8, + "p95": 6.2, + "p99": 12.9 + } + } + }, + "during": { + "seconds": 13.185, + "write": { + "ok": 3722, + "errors": 0, + "ok_rps": 282.3, + "latency_ms": { + "p50": 6.7, + "p95": 12.6, + "p99": 34.7 + } + }, + "read": { + "ok": 3723, + "errors": 0, + "ok_rps": 282.4, + "latency_ms": { + "p50": 5.4, + "p95": 9.9, + "p99": 17.3 + } + } + }, + "after": { + "seconds": 8.006, + "write": { + "ok": 2901, + "errors": 0, + "ok_rps": 362.4, + "latency_ms": { + "p50": 5.3, + "p95": 10.5, + "p99": 19.2 + } + }, + "read": { + "ok": 2902, + "errors": 0, + "ok_rps": 362.5, + "latency_ms": { + "p50": 4.2, + "p95": 7.9, + "p99": 14.7 + } + } + } + }, + "recovery": { + "write_errors": 0, + "error_kinds": {}, + "write_recovery_seconds": 0.0, + "max_write_gap_seconds": 0.071 + }, + "integrity": { + "messages_in_history": 10687, + "confirmed_unique": 10687, + "confirmed_missing_or_changed": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "retried_writes_confirmed_as_replay": 0 + }, + "container_stats_before_action": [ + "messenger-lab3-nginx-1\t9.92%\t3.406MiB / 64MiB", + "messenger-lab3-postgres-1\t29.31%\t75.47MiB / 512MiB", + "messenger-lab3-api-1\t83.72%\t47.36MiB / 512MiB", + "messenger-lab3-api-2\t84.84%\t48.22MiB / 512MiB" + ], + "action_output": [ + "messenger-lab3-api-2" + ], + "restore_output_tail": [], + "timeline": [ + { + "t": 0, + "ok": 897, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 449, + "eb1dace786c6": 448 + } + }, + { + "t": 1, + "ok": 743, + "errors": 0, + "instances": { + "eb1dace786c6": 372, + "6d2f6cd2b04a": 371 + } + }, + { + "t": 2, + "ok": 911, + "errors": 0, + "instances": { + "eb1dace786c6": 455, + "6d2f6cd2b04a": 456 + } + }, + { + "t": 3, + "ok": 881, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 440, + "eb1dace786c6": 441 + } + }, + { + "t": 4, + "ok": 837, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 419, + "eb1dace786c6": 418 + } + }, + { + "t": 5, + "ok": 863, + "errors": 0, + "instances": { + "eb1dace786c6": 431, + "6d2f6cd2b04a": 432 + } + }, + { + "t": 6, + "ok": 542, + "errors": 0, + "instances": { + "eb1dace786c6": 272, + "6d2f6cd2b04a": 270 + } + }, + { + "t": 7, + "ok": 765, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 383, + "eb1dace786c6": 382 + } + }, + { + "t": 8, + "ok": 785, + "errors": 0, + "instances": { + "eb1dace786c6": 393, + "6d2f6cd2b04a": 392 + } + }, + { + "t": 9, + "ok": 796, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 398, + "eb1dace786c6": 398 + } + }, + { + "t": 10, + "ok": 666, + "errors": 0, + "instances": { + "6d2f6cd2b04a": 86, + "eb1dace786c6": 580 + } + }, + { + "t": 11, + "ok": 427, + "errors": 0, + "instances": { + "eb1dace786c6": 427 + } + }, + { + "t": 12, + "ok": 493, + "errors": 0, + "instances": { + "eb1dace786c6": 493 + } + }, + { + "t": 13, + "ok": 649, + "errors": 0, + "instances": { + "eb1dace786c6": 649 + } + }, + { + "t": 14, + "ok": 651, + "errors": 0, + "instances": { + "eb1dace786c6": 651 + } + }, + { + "t": 15, + "ok": 643, + "errors": 0, + "instances": { + "eb1dace786c6": 643 + } + }, + { + "t": 16, + "ok": 570, + "errors": 0, + "instances": { + "eb1dace786c6": 570 + } + }, + { + "t": 17, + "ok": 444, + "errors": 0, + "instances": { + "eb1dace786c6": 444 + } + }, + { + "t": 18, + "ok": 637, + "errors": 0, + "instances": { + "eb1dace786c6": 637 + } + }, + { + "t": 19, + "ok": 648, + "errors": 0, + "instances": { + "eb1dace786c6": 648 + } + }, + { + "t": 20, + "ok": 656, + "errors": 0, + "instances": { + "eb1dace786c6": 656 + } + }, + { + "t": 21, + "ok": 550, + "errors": 0, + "instances": { + "eb1dace786c6": 550 + } + }, + { + "t": 22, + "ok": 431, + "errors": 0, + "instances": { + "eb1dace786c6": 431 + } + }, + { + "t": 23, + "ok": 463, + "errors": 0, + "instances": { + "eb1dace786c6": 463 + } + }, + { + "t": 24, + "ok": 623, + "errors": 0, + "instances": { + "eb1dace786c6": 623 + } + }, + { + "t": 25, + "ok": 648, + "errors": 0, + "instances": { + "eb1dace786c6": 648 + } + }, + { + "t": 26, + "ok": 566, + "errors": 0, + "instances": { + "eb1dace786c6": 566 + } + }, + { + "t": 27, + "ok": 823, + "errors": 0, + "instances": { + "eb1dace786c6": 427, + "577316c523cb": 396 + } + }, + { + "t": 28, + "ok": 548, + "errors": 0, + "instances": { + "eb1dace786c6": 274, + "577316c523cb": 274 + } + }, + { + "t": 29, + "ok": 941, + "errors": 0, + "instances": { + "eb1dace786c6": 470, + "577316c523cb": 471 + } + }, + { + "t": 30, + "ok": 977, + "errors": 0, + "instances": { + "eb1dace786c6": 490, + "577316c523cb": 487 + } + }, + { + "t": 31, + "ok": 300, + "errors": 0, + "instances": { + "577316c523cb": 151, + "eb1dace786c6": 149 + } + } + ] + }, + { + "scenario": "SIGKILL messenger-lab3-api-1", + "target_instance": "eb1dace786c6", + "workers": 4, + "events_seconds": { + "action_start": 9.738, + "action_end": 9.957, + "restore_start": 19.957, + "restore_end": 21.841, + "end": 29.849 + }, + "instances_before": [ + "577316c523cb", + "eb1dace786c6" + ], + "instances_after": [ + "506b547ba744", + "577316c523cb" + ], + "target_responses_while_stopped": 0, + "restored_replica_in_rotation_seconds": 0.79, + "instances_seen_from_action": { + "eb1dace786c6": [ + -9.73, + 0.024 + ], + "577316c523cb": [ + -9.73, + 20.11 + ], + "506b547ba744": [ + 12.893, + 20.111 + ] + }, + "phases": { + "before": { + "seconds": 9.738, + "write": { + "ok": 4162, + "errors": 0, + "ok_rps": 427.4, + "latency_ms": { + "p50": 4.6, + "p95": 7.6, + "p99": 14.6 + } + }, + "read": { + "ok": 4161, + "errors": 0, + "ok_rps": 427.3, + "latency_ms": { + "p50": 3.6, + "p95": 5.8, + "p99": 11.7 + } + } + }, + "during": { + "seconds": 12.103, + "write": { + "ok": 3406, + "errors": 1, + "ok_rps": 281.4, + "latency_ms": { + "p50": 6.9, + "p95": 12.7, + "p99": 22.9 + } + }, + "read": { + "ok": 3404, + "errors": 0, + "ok_rps": 281.3, + "latency_ms": { + "p50": 5.6, + "p95": 10.1, + "p99": 16.4 + } + } + }, + "after": { + "seconds": 8.008, + "write": { + "ok": 3269, + "errors": 0, + "ok_rps": 408.2, + "latency_ms": { + "p50": 4.7, + "p95": 8.1, + "p99": 21.3 + } + }, + "read": { + "ok": 3271, + "errors": 0, + "ok_rps": 408.5, + "latency_ms": { + "p50": 3.6, + "p95": 6.1, + "p99": 15.5 + } + } + } + }, + "recovery": { + "write_errors": 1, + "error_kinds": { + "HTTP 503": 1 + }, + "write_recovery_seconds": 0.0, + "max_write_gap_seconds": 0.07 + }, + "integrity": { + "messages_in_history": 10837, + "confirmed_unique": 10837, + "confirmed_missing_or_changed": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "retried_writes_confirmed_as_replay": 0 + }, + "container_stats_before_action": [ + "messenger-lab3-nginx-1\t9.89%\t2.816MiB / 64MiB", + "messenger-lab3-postgres-1\t27.37%\t80.08MiB / 512MiB", + "messenger-lab3-api-1\t84.41%\t48.93MiB / 512MiB", + "messenger-lab3-api-2\t84.27%\t47.33MiB / 512MiB" + ], + "action_output": [ + "messenger-lab3-api-1" + ], + "restore_output_tail": [], + "timeline": [ + { + "t": 0, + "ok": 826, + "errors": 0, + "instances": { + "eb1dace786c6": 413, + "577316c523cb": 413 + } + }, + { + "t": 1, + "ok": 645, + "errors": 0, + "instances": { + "577316c523cb": 322, + "eb1dace786c6": 323 + } + }, + { + "t": 2, + "ok": 963, + "errors": 0, + "instances": { + "eb1dace786c6": 481, + "577316c523cb": 482 + } + }, + { + "t": 3, + "ok": 828, + "errors": 0, + "instances": { + "eb1dace786c6": 414, + "577316c523cb": 414 + } + }, + { + "t": 4, + "ok": 899, + "errors": 0, + "instances": { + "eb1dace786c6": 450, + "577316c523cb": 449 + } + }, + { + "t": 5, + "ok": 852, + "errors": 0, + "instances": { + "577316c523cb": 426, + "eb1dace786c6": 426 + } + }, + { + "t": 6, + "ok": 704, + "errors": 0, + "instances": { + "eb1dace786c6": 352, + "577316c523cb": 352 + } + }, + { + "t": 7, + "ok": 918, + "errors": 0, + "instances": { + "eb1dace786c6": 459, + "577316c523cb": 459 + } + }, + { + "t": 8, + "ok": 975, + "errors": 0, + "instances": { + "eb1dace786c6": 487, + "577316c523cb": 488 + } + }, + { + "t": 9, + "ok": 853, + "errors": 1, + "instances": { + "eb1dace786c6": 365, + "577316c523cb": 488 + } + }, + { + "t": 10, + "ok": 621, + "errors": 0, + "instances": { + "577316c523cb": 621 + } + }, + { + "t": 11, + "ok": 618, + "errors": 0, + "instances": { + "577316c523cb": 618 + } + }, + { + "t": 12, + "ok": 474, + "errors": 0, + "instances": { + "577316c523cb": 474 + } + }, + { + "t": 13, + "ok": 569, + "errors": 0, + "instances": { + "577316c523cb": 569 + } + }, + { + "t": 14, + "ok": 633, + "errors": 0, + "instances": { + "577316c523cb": 633 + } + }, + { + "t": 15, + "ok": 548, + "errors": 0, + "instances": { + "577316c523cb": 548 + } + }, + { + "t": 16, + "ok": 592, + "errors": 0, + "instances": { + "577316c523cb": 592 + } + }, + { + "t": 17, + "ok": 459, + "errors": 0, + "instances": { + "577316c523cb": 459 + } + }, + { + "t": 18, + "ok": 570, + "errors": 0, + "instances": { + "577316c523cb": 570 + } + }, + { + "t": 19, + "ok": 612, + "errors": 0, + "instances": { + "577316c523cb": 612 + } + }, + { + "t": 20, + "ok": 487, + "errors": 0, + "instances": { + "577316c523cb": 487 + } + }, + { + "t": 21, + "ok": 580, + "errors": 0, + "instances": { + "577316c523cb": 580 + } + }, + { + "t": 22, + "ok": 742, + "errors": 0, + "instances": { + "577316c523cb": 568, + "506b547ba744": 174 + } + }, + { + "t": 23, + "ok": 605, + "errors": 0, + "instances": { + "577316c523cb": 303, + "506b547ba744": 302 + } + }, + { + "t": 24, + "ok": 959, + "errors": 0, + "instances": { + "577316c523cb": 480, + "506b547ba744": 479 + } + }, + { + "t": 25, + "ok": 921, + "errors": 0, + "instances": { + "506b547ba744": 461, + "577316c523cb": 460 + } + }, + { + "t": 26, + "ok": 745, + "errors": 0, + "instances": { + "577316c523cb": 373, + "506b547ba744": 372 + } + }, + { + "t": 27, + "ok": 889, + "errors": 0, + "instances": { + "506b547ba744": 446, + "577316c523cb": 443 + } + }, + { + "t": 28, + "ok": 748, + "errors": 0, + "instances": { + "577316c523cb": 375, + "506b547ba744": 373 + } + }, + { + "t": 29, + "ok": 839, + "errors": 0, + "instances": { + "577316c523cb": 419, + "506b547ba744": 420 + } + } + ] + }, + { + "scenario": "SIGKILL messenger-lab3-api-2", + "target_instance": "577316c523cb", + "workers": 4, + "events_seconds": { + "action_start": 9.946, + "action_end": 10.207, + "restore_start": 20.209, + "restore_end": 22.126, + "end": 30.136 + }, + "instances_before": [ + "506b547ba744", + "577316c523cb" + ], + "instances_after": [ + "1c20e7f6dbde", + "506b547ba744" + ], + "target_responses_while_stopped": 0, + "restored_replica_in_rotation_seconds": 0.0, + "instances_seen_from_action": { + "506b547ba744": [ + -9.94, + 20.189 + ], + "577316c523cb": [ + -9.939, + 0.028 + ], + "1c20e7f6dbde": [ + 11.911, + 20.19 + ] + }, + "phases": { + "before": { + "seconds": 9.946, + "write": { + "ok": 3853, + "errors": 0, + "ok_rps": 387.4, + "latency_ms": { + "p50": 4.8, + "p95": 9.1, + "p99": 34.8 + } + }, + "read": { + "ok": 3850, + "errors": 0, + "ok_rps": 387.1, + "latency_ms": { + "p50": 3.7, + "p95": 7.0, + "p99": 19.5 + } + } + }, + "during": { + "seconds": 12.18, + "write": { + "ok": 3370, + "errors": 3, + "ok_rps": 276.7, + "latency_ms": { + "p50": 6.8, + "p95": 12.8, + "p99": 26.8 + } + }, + "read": { + "ok": 3372, + "errors": 0, + "ok_rps": 276.8, + "latency_ms": { + "p50": 5.5, + "p95": 11.0, + "p99": 28.6 + } + } + }, + "after": { + "seconds": 8.01, + "write": { + "ok": 2989, + "errors": 0, + "ok_rps": 373.2, + "latency_ms": { + "p50": 5.0, + "p95": 10.1, + "p99": 35.1 + } + }, + "read": { + "ok": 2990, + "errors": 0, + "ok_rps": 373.3, + "latency_ms": { + "p50": 3.8, + "p95": 7.6, + "p99": 22.2 + } + } + } + }, + "recovery": { + "write_errors": 3, + "error_kinds": { + "HTTP 503": 3 + }, + "write_recovery_seconds": 0.006, + "max_write_gap_seconds": 0.061 + }, + "integrity": { + "messages_in_history": 10212, + "confirmed_unique": 10212, + "confirmed_missing_or_changed": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "retried_writes_confirmed_as_replay": 0 + }, + "container_stats_before_action": [ + "messenger-lab3-nginx-1\t9.12%\t2.887MiB / 64MiB", + "messenger-lab3-postgres-1\t27.19%\t90.89MiB / 512MiB", + "messenger-lab3-api-1\t80.43%\t47.9MiB / 512MiB", + "messenger-lab3-api-2\t80.26%\t50.77MiB / 512MiB" + ], + "action_output": [ + "messenger-lab3-api-2" + ], + "restore_output_tail": [], + "timeline": [ + { + "t": 0, + "ok": 925, + "errors": 0, + "instances": { + "506b547ba744": 462, + "577316c523cb": 463 + } + }, + { + "t": 1, + "ok": 665, + "errors": 0, + "instances": { + "506b547ba744": 333, + "577316c523cb": 332 + } + }, + { + "t": 2, + "ok": 856, + "errors": 0, + "instances": { + "577316c523cb": 428, + "506b547ba744": 428 + } + }, + { + "t": 3, + "ok": 641, + "errors": 0, + "instances": { + "577316c523cb": 321, + "506b547ba744": 320 + } + }, + { + "t": 4, + "ok": 868, + "errors": 0, + "instances": { + "577316c523cb": 434, + "506b547ba744": 434 + } + }, + { + "t": 5, + "ok": 874, + "errors": 0, + "instances": { + "506b547ba744": 437, + "577316c523cb": 437 + } + }, + { + "t": 6, + "ok": 753, + "errors": 0, + "instances": { + "577316c523cb": 378, + "506b547ba744": 375 + } + }, + { + "t": 7, + "ok": 875, + "errors": 0, + "instances": { + "506b547ba744": 439, + "577316c523cb": 436 + } + }, + { + "t": 8, + "ok": 649, + "errors": 0, + "instances": { + "506b547ba744": 325, + "577316c523cb": 324 + } + }, + { + "t": 9, + "ok": 622, + "errors": 3, + "instances": { + "577316c523cb": 309, + "506b547ba744": 313 + } + }, + { + "t": 10, + "ok": 532, + "errors": 0, + "instances": { + "506b547ba744": 532 + } + }, + { + "t": 11, + "ok": 653, + "errors": 0, + "instances": { + "506b547ba744": 653 + } + }, + { + "t": 12, + "ok": 361, + "errors": 0, + "instances": { + "506b547ba744": 361 + } + }, + { + "t": 13, + "ok": 626, + "errors": 0, + "instances": { + "506b547ba744": 626 + } + }, + { + "t": 14, + "ok": 618, + "errors": 0, + "instances": { + "506b547ba744": 618 + } + }, + { + "t": 15, + "ok": 649, + "errors": 0, + "instances": { + "506b547ba744": 649 + } + }, + { + "t": 16, + "ok": 651, + "errors": 0, + "instances": { + "506b547ba744": 651 + } + }, + { + "t": 17, + "ok": 461, + "errors": 0, + "instances": { + "506b547ba744": 461 + } + }, + { + "t": 18, + "ok": 516, + "errors": 0, + "instances": { + "506b547ba744": 516 + } + }, + { + "t": 19, + "ok": 611, + "errors": 0, + "instances": { + "506b547ba744": 611 + } + }, + { + "t": 20, + "ok": 436, + "errors": 0, + "instances": { + "506b547ba744": 436 + } + }, + { + "t": 21, + "ok": 487, + "errors": 0, + "instances": { + "506b547ba744": 451, + "1c20e7f6dbde": 36 + } + }, + { + "t": 22, + "ok": 769, + "errors": 0, + "instances": { + "1c20e7f6dbde": 385, + "506b547ba744": 384 + } + }, + { + "t": 23, + "ok": 614, + "errors": 0, + "instances": { + "1c20e7f6dbde": 307, + "506b547ba744": 307 + } + }, + { + "t": 24, + "ok": 804, + "errors": 0, + "instances": { + "506b547ba744": 402, + "1c20e7f6dbde": 402 + } + }, + { + "t": 25, + "ok": 717, + "errors": 0, + "instances": { + "1c20e7f6dbde": 358, + "506b547ba744": 359 + } + }, + { + "t": 26, + "ok": 827, + "errors": 0, + "instances": { + "506b547ba744": 415, + "1c20e7f6dbde": 412 + } + }, + { + "t": 27, + "ok": 735, + "errors": 0, + "instances": { + "1c20e7f6dbde": 369, + "506b547ba744": 366 + } + }, + { + "t": 28, + "ok": 648, + "errors": 0, + "instances": { + "1c20e7f6dbde": 326, + "506b547ba744": 322 + } + }, + { + "t": 29, + "ok": 851, + "errors": 0, + "instances": { + "506b547ba744": 427, + "1c20e7f6dbde": 424 + } + }, + { + "t": 30, + "ok": 130, + "errors": 0, + "instances": { + "506b547ba744": 65, + "1c20e7f6dbde": 65 + } + } + ] + } + ] +} diff --git a/evidence/experiment-rolling.json b/evidence/experiment-rolling.json new file mode 100644 index 00000000..b5a905d1 --- /dev/null +++ b/evidence/experiment-rolling.json @@ -0,0 +1,482 @@ +{ + "mode": "rolling", + "started": "2026-10-20T09:50:22+0300", + "profile": { + "workers": 4, + "loop": "closed: POST message (retry same key until 201) + GET limit=1", + "client_timeout_seconds": 5, + "phases_seconds": { + "before": 8, + "outage": 10, + "after": 8 + } + }, + "runs": [ + { + "scenario": "rolling update", + "target_instance": null, + "workers": 4, + "events_seconds": { + "action_start": 10.323, + "action_end": 26.786, + "end": 34.791 + }, + "instances_before": [ + "109172df7747", + "f6e244c650a2" + ], + "instances_after": [ + "500528cb388e", + "de52b5578b6b" + ], + "target_responses_while_stopped": null, + "restored_replica_in_rotation_seconds": null, + "instances_seen_from_action": { + "109172df7747": [ + -10.3, + 14.813 + ], + "f6e244c650a2": [ + -10.296, + 8.226 + ], + "500528cb388e": [ + 6.241, + 24.466 + ], + "de52b5578b6b": [ + 13.791, + 24.468 + ] + }, + "phases": { + "before": { + "seconds": 10.323, + "write": { + "ok": 3687, + "errors": 0, + "ok_rps": 357.2, + "latency_ms": { + "p50": 5.4, + "p95": 10.3, + "p99": 28.9 + } + }, + "read": { + "ok": 3684, + "errors": 0, + "ok_rps": 356.9, + "latency_ms": { + "p50": 4.2, + "p95": 7.8, + "p99": 19.9 + } + } + }, + "during": { + "seconds": 16.463, + "write": { + "ok": 5071, + "errors": 0, + "ok_rps": 308.0, + "latency_ms": { + "p50": 5.4, + "p95": 16.9, + "p99": 38.6 + } + }, + "read": { + "ok": 5073, + "errors": 0, + "ok_rps": 308.1, + "latency_ms": { + "p50": 4.2, + "p95": 12.3, + "p99": 31.4 + } + } + }, + "after": { + "seconds": 8.005, + "write": { + "ok": 2800, + "errors": 0, + "ok_rps": 349.8, + "latency_ms": { + "p50": 4.8, + "p95": 10.9, + "p99": 48.3 + } + }, + "read": { + "ok": 2801, + "errors": 0, + "ok_rps": 349.9, + "latency_ms": { + "p50": 3.7, + "p95": 8.0, + "p99": 47.3 + } + } + } + }, + "recovery": { + "write_errors": 0, + "error_kinds": {}, + "write_recovery_seconds": 0.0, + "max_write_gap_seconds": 0.299 + }, + "integrity": { + "messages_in_history": 11558, + "confirmed_unique": 11558, + "confirmed_missing_or_changed": 0, + "duplicate_ids": 0, + "duplicate_texts": 0, + "retried_writes_confirmed_as_replay": 0 + }, + "container_stats_before_action": [ + "messenger-lab3-nginx-1\t10.92%\t3.227MiB / 64MiB", + "messenger-lab3-postgres-1\t30.04%\t54.82MiB / 512MiB", + "messenger-lab3-api-1\t85.53%\t52.1MiB / 512MiB", + "messenger-lab3-api-2\t84.90%\t54.01MiB / 512MiB" + ], + "action_output": [ + "[09:50:33] build revision exp-1792479022", + "[09:50:34] migrate", + "[09:50:36] scale api to 3, wait healthy", + "[09:50:41] graceful stop messenger-lab3-api-1", + "[09:50:42] scale api to 3, wait healthy", + "[09:50:47] graceful stop messenger-lab3-api-2", + "[09:50:49] done: all API replicas run revision exp-1792479022" + ], + "restore_output_tail": [], + "timeline": [ + { + "t": 0, + "ok": 705, + "errors": 0, + "instances": { + "109172df7747": 354, + "f6e244c650a2": 351 + } + }, + { + "t": 1, + "ok": 522, + "errors": 0, + "instances": { + "f6e244c650a2": 263, + "109172df7747": 259 + } + }, + { + "t": 2, + "ok": 733, + "errors": 0, + "instances": { + "109172df7747": 367, + "f6e244c650a2": 366 + } + }, + { + "t": 3, + "ok": 771, + "errors": 0, + "instances": { + "109172df7747": 385, + "f6e244c650a2": 386 + } + }, + { + "t": 4, + "ok": 725, + "errors": 0, + "instances": { + "109172df7747": 363, + "f6e244c650a2": 362 + } + }, + { + "t": 5, + "ok": 803, + "errors": 0, + "instances": { + "109172df7747": 401, + "f6e244c650a2": 402 + } + }, + { + "t": 6, + "ok": 449, + "errors": 0, + "instances": { + "109172df7747": 224, + "f6e244c650a2": 225 + } + }, + { + "t": 7, + "ok": 777, + "errors": 0, + "instances": { + "109172df7747": 390, + "f6e244c650a2": 387 + } + }, + { + "t": 8, + "ok": 803, + "errors": 0, + "instances": { + "f6e244c650a2": 402, + "109172df7747": 401 + } + }, + { + "t": 9, + "ok": 811, + "errors": 0, + "instances": { + "109172df7747": 405, + "f6e244c650a2": 406 + } + }, + { + "t": 10, + "ok": 677, + "errors": 0, + "instances": { + "109172df7747": 339, + "f6e244c650a2": 338 + } + }, + { + "t": 11, + "ok": 271, + "errors": 0, + "instances": { + "f6e244c650a2": 135, + "109172df7747": 136 + } + }, + { + "t": 12, + "ok": 432, + "errors": 0, + "instances": { + "f6e244c650a2": 217, + "109172df7747": 215 + } + }, + { + "t": 13, + "ok": 422, + "errors": 0, + "instances": { + "109172df7747": 211, + "f6e244c650a2": 211 + } + }, + { + "t": 14, + "ok": 533, + "errors": 0, + "instances": { + "109172df7747": 267, + "f6e244c650a2": 266 + } + }, + { + "t": 15, + "ok": 737, + "errors": 0, + "instances": { + "f6e244c650a2": 369, + "109172df7747": 368 + } + }, + { + "t": 16, + "ok": 683, + "errors": 0, + "instances": { + "109172df7747": 276, + "f6e244c650a2": 277, + "500528cb388e": 130 + } + }, + { + "t": 17, + "ok": 635, + "errors": 0, + "instances": { + "500528cb388e": 212, + "109172df7747": 212, + "f6e244c650a2": 211 + } + }, + { + "t": 18, + "ok": 841, + "errors": 0, + "instances": { + "f6e244c650a2": 153, + "500528cb388e": 345, + "109172df7747": 343 + } + }, + { + "t": 19, + "ok": 782, + "errors": 0, + "instances": { + "109172df7747": 391, + "500528cb388e": 391 + } + }, + { + "t": 20, + "ok": 254, + "errors": 0, + "instances": { + "109172df7747": 128, + "500528cb388e": 126 + } + }, + { + "t": 21, + "ok": 767, + "errors": 0, + "instances": { + "500528cb388e": 384, + "109172df7747": 383 + } + }, + { + "t": 22, + "ok": 761, + "errors": 0, + "instances": { + "109172df7747": 380, + "500528cb388e": 381 + } + }, + { + "t": 23, + "ok": 795, + "errors": 0, + "instances": { + "500528cb388e": 397, + "109172df7747": 398 + } + }, + { + "t": 24, + "ok": 924, + "errors": 0, + "instances": { + "109172df7747": 322, + "500528cb388e": 321, + "de52b5578b6b": 281 + } + }, + { + "t": 25, + "ok": 681, + "errors": 0, + "instances": { + "500528cb388e": 320, + "109172df7747": 41, + "de52b5578b6b": 320 + } + }, + { + "t": 26, + "ok": 232, + "errors": 0, + "instances": { + "de52b5578b6b": 115, + "500528cb388e": 117 + } + }, + { + "t": 27, + "ok": 252, + "errors": 0, + "instances": { + "de52b5578b6b": 126, + "500528cb388e": 126 + } + }, + { + "t": 28, + "ok": 633, + "errors": 0, + "instances": { + "de52b5578b6b": 317, + "500528cb388e": 316 + } + }, + { + "t": 29, + "ok": 940, + "errors": 0, + "instances": { + "500528cb388e": 470, + "de52b5578b6b": 470 + } + }, + { + "t": 30, + "ok": 814, + "errors": 0, + "instances": { + "de52b5578b6b": 407, + "500528cb388e": 407 + } + }, + { + "t": 31, + "ok": 626, + "errors": 0, + "instances": { + "de52b5578b6b": 313, + "500528cb388e": 313 + } + }, + { + "t": 32, + "ok": 878, + "errors": 0, + "instances": { + "de52b5578b6b": 439, + "500528cb388e": 439 + } + }, + { + "t": 33, + "ok": 740, + "errors": 0, + "instances": { + "500528cb388e": 370, + "de52b5578b6b": 370 + } + }, + { + "t": 34, + "ok": 707, + "errors": 0, + "instances": { + "500528cb388e": 353, + "de52b5578b6b": 354 + } + } + ], + "revisions_after": [ + "CONTAINER STATE INSTANCE PID REVISION", + "messenger-lab3-api-3 running/healthy 500528cb388e 60335 exp-1792479022", + "messenger-lab3-api-4 running/healthy de52b5578b6b 60779 exp-1792479022" + ] + } + ] +} diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..ccd61341 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.024, + "concurrency": 4, + "successful_requests": 21418, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 713.369, + "successful_latency_ms": { + "p50": 4.748, + "p95": 8.395, + "p99": 22.531 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/infra/nginx/errors.conf b/infra/nginx/errors.conf new file mode 100644 index 00000000..495a21c8 --- /dev/null +++ b/infra/nginx/errors.conf @@ -0,0 +1,15 @@ +# Ошибки, созданные самим балансировщиком, — в формате ошибок контракта. +# Ответы реплик (включая их 503) проходят без изменений: proxy_intercept_errors off. +# X-Instance-Id здесь нет: ответ создан LB, а не API-репликой. + +location @unavailable { + default_type application/json; + add_header X-Request-Id $request_id always; + return 503 '{"error":{"code":"unavailable","message":"Нет доступной API-реплики","request_id":"$request_id"}}'; +} + +location @too_large { + default_type application/json; + add_header X-Request-Id $request_id always; + return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}'; +} diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 00000000..1845986f --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,101 @@ +worker_processes 1; +pid /tmp/nginx.pid; +error_log /dev/stderr warn; + +events { + worker_connections 1024; +} + +http { + resolver 127.0.0.11 valid=2s ipv6=off; + resolver_timeout 1s; + + log_format upstream_json escape=json + '{"time":"$time_iso8601","request_id":"$request_id",' + '"method":"$request_method","uri":"$uri","status":$status,' + '"request_time":$request_time,"upstream_addr":"$upstream_addr",' + '"upstream_status":"$upstream_status",' + '"upstream_response_time":"$upstream_response_time",' + '"instance":"$upstream_http_x_instance_id"}'; + access_log /dev/stdout upstream_json; + + server_tokens off; + client_max_body_size 1m; + client_body_buffer_size 64k; + client_header_timeout 10s; + client_body_timeout 10s; + send_timeout 10s; + keepalive_timeout 65s; + + upstream api { + zone api 64k; + server api:8080 resolve max_fails=1 fail_timeout=5s; + keepalive 32; + # Меньше APP_KEEPALIVE_TIMEOUT_SEC (75 с): idle-соединение первым + # закрывает nginx, а не реплика, — нет гонки «запрос в закрытое соединение» + keepalive_timeout 60s; + } + + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_set_header Host $host; + # Один ID на LB и реплику: API принимает $request_id и возвращает его + proxy_set_header X-Request-Id $request_id; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # connect внутри bridge-сети завершает ядро за доли мс даже у занятой реплики. + # Самый долгий запрос API: ожидание пула 3 с + statement_timeout 5 с + proxy_connect_timeout 500ms; + proxy_send_timeout 10s; + proxy_read_timeout 10s; + + # Повтор на другой реплике только при ошибке соединения/таймауте. + # POST, уже отправленный реплике, nginx не повторяет (non_idempotent выключен): + # повтор сообщения делает клиент с тем же Idempotency-Key. + # 503 приложения (общая зависимость недоступна) не повторяется и не исключает + # реплику: другая реплика зависит от той же БД. + proxy_next_upstream error timeout; + proxy_next_upstream_tries 3; + proxy_next_upstream_timeout 5s; + + server { + listen 8080; + + location / { + proxy_pass http://api; + } + + error_page 502 503 504 = @unavailable; + error_page 413 = @too_large; + + include /etc/nginx/errors.conf; + } + + server { + listen 8081; + + # Реплика api-short-ttl только для make test-expiry (профиль expiry): + # адрес резолвится на каждый запрос, без профиля — 503 от LB + location / { + set $api_short_ttl http://api-short-ttl:8080; + proxy_pass $api_short_ttl; + } + + error_page 502 503 504 = @unavailable; + error_page 413 = @too_large; + + include /etc/nginx/errors.conf; + } + + # Liveness самого балансировщика для healthcheck контейнера; наружу не публикуется + server { + listen 127.0.0.1:8090; + access_log off; + + location = /healthz { + default_type text/plain; + return 200 "ok\n"; + } + } +} diff --git a/migrations/0002_message_idempotency.sql b/migrations/0002_message_idempotency.sql new file mode 100644 index 00000000..ffc709e2 --- /dev/null +++ b/migrations/0002_message_idempotency.sql @@ -0,0 +1,18 @@ +-- Idempotency-Key отправки сообщения (lab3). Область по контракту: +-- пользователь + метод + путь + ключ. Метод и путь фиксированы +-- (POST /api/v1/chats/{chat_id}/messages), поэтому ключ — (sender_id, chat_id, key). +-- Только добавление таблицы (expand): предыдущая версия API работает на новой схеме. +CREATE TABLE message_idempotency_keys ( + sender_id uuid NOT NULL, + chat_id uuid NOT NULL, + idempotency_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + PRIMARY KEY (sender_id, chat_id, idempotency_key), + CONSTRAINT message_idempotency_keys_key_format + CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT message_idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32), + CONSTRAINT message_idempotency_keys_expiry_order CHECK (expires_at > created_at) +); diff --git a/scripts/check_migrations.sh b/scripts/check_migrations.sh index 6154ee0b..d38967ea 100755 --- a/scripts/check_migrations.sh +++ b/scripts/check_migrations.sh @@ -1,17 +1,19 @@ #!/bin/sh set -eu -# Миграции с нуля и повторно: отдельный compose-проект с собственным пустым -# volume. Рабочий стенд и его данные не затрагиваются; временный volume -# этого проекта удаляется в конце. +# Миграции с нуля и повторно, отказ API на непримененной схеме: отдельный +# compose-проект с собственным пустым volume. Рабочий стенд и его данные +# не затрагиваются; временный volume этого проекта удаляется в конце. cd "$(dirname "$0")/.." PROJECT=messenger-lab3-migrations-check -# Свой порт на хосте: основной стенд уже занимает DB_PORT -export DB_PORT="${MIGRATIONS_CHECK_DB_PORT:-55432}" +# Публикуется только порт LB: свои порты, чтобы не конфликтовать с основным стендом +export LB_PORT="${MIGRATIONS_CHECK_LB_PORT:-58080}" +export LB_EXPIRY_PORT="${MIGRATIONS_CHECK_LB_EXPIRY_PORT:-58081}" COMPOSE="docker compose --env-file ${ENV_FILE:-.env} -p $PROJECT" trap '$COMPOSE down -v --remove-orphans >/dev/null 2>&1' EXIT $COMPOSE up -d --wait postgres >/dev/null 2>&1 -$COMPOSE run --rm --no-deps -T api python - <<'EOF' +# Тот же образ и сеть, что у сервиса migrate; apply_migrations трижды подряд +$COMPOSE run --rm --no-deps -T migrate python - <<'EOF' import asyncio import sys @@ -27,6 +29,13 @@ async def main() -> None: "SELECT count(*) FROM pg_tables WHERE schemaname = 'public';" ) print(f"пустой volume: таблиц в public = {tables}") + # Так стартует API-реплика: без migrate она отказывается работать + try: + await db.check_migrations() + except RuntimeError as error: + print(f"API до migrate: {error}") + else: + raise SystemExit("FAIL: API стартовал бы на пустой схеме") for attempt in (1, 2, 3): await db.apply_migrations() async with db.acquire() as conn: @@ -40,6 +49,8 @@ async def main() -> None: f"запуск {attempt}: schema_migrations = " f"{[(r['version'], r['name']) for r in versions]}, таблиц = {tables}" ) + await db.check_migrations() + print("API после migrate: схема совместима") await db.close() diff --git a/scripts/check_previous_version.sh b/scripts/check_previous_version.sh new file mode 100755 index 00000000..1df75afb --- /dev/null +++ b/scripts/check_previous_version.sh @@ -0,0 +1,89 @@ +#!/bin/sh +set -eu +# Совместимость текущей схемы с предыдущей версией API — условие rolling update: +# миграции применяются до замены реплик, и старые реплики продолжают работать. +# Образ собирается из коммита PREVIOUS_REVISION (по умолчанию lab2-код aa971fb), +# запускается одним процессом в сети backend рабочего стенда (наружу порт +# не публикуется) и выполняет вход, создание чата, запись и чтение сообщения. +# Создаются только синтетические данные; volumes не затрагиваются. +cd "$(dirname "$0")/.." +ENV_FILE="${ENV_FILE:-.env}" +REVISION="${PREVIOUS_REVISION:-aa971fb}" +IMAGE="mtusi-messenger:compat-$REVISION" +NETWORK=messenger-lab3_backend + +git archive "$REVISION" | docker build --quiet --tag "$IMAGE" - >/dev/null +echo "previous version image: $IMAGE ($(git log -1 --format='%h %s' "$REVISION"))" +echo "schema on stand: $(docker compose --env-file "$ENV_FILE" exec -T postgres sh -c \ + 'psql -X -q -tA -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT string_agg(name, \$\$, \$\$ ORDER BY version) FROM schema_migrations;"')" + +docker run --rm -i --network "$NETWORK" --env-file "$ENV_FILE" \ + -e DB_HOST=postgres -e DB_PORT=5432 -e VALKEY_HOST=valkey -e VALKEY_PORT=6379 \ + "$IMAGE" sh -c 'python src/gateway.py >/tmp/api.log 2>&1 & exec python -' <<'EOF' +import base64 +import json +import time +import urllib.error +import urllib.request +import uuid + +BASE = "http://127.0.0.1:8080" + + +def call(method, path, body=None, headers=None): + data = None if body is None else json.dumps(body).encode() + request = urllib.request.Request(BASE + path, data=data, method=method) + request.add_header("Content-Type", "application/json") + for name, value in (headers or {}).items(): + request.add_header(name, value) + try: + with urllib.request.urlopen(request, timeout=5) as response: + return response.status, json.loads(response.read() or b"null") + except urllib.error.HTTPError as error: + return error.code, None + + +for _ in range(60): + try: + if call("GET", "/health/ready")[0] == 200: + break + except OSError: + pass + time.sleep(0.5) +else: + print(open("/tmp/api.log").read()[-2000:]) + raise SystemExit("FAIL: previous version did not become ready on current schema") + +username = "compat_" + uuid.uuid4().hex[:12] +password = "Compat-" + uuid.uuid4().hex +status_user, user = call( + "POST", "/api/v1/users", + {"username": username, "display_name": "Compat", "password": password}, +) +credentials = base64.b64encode(f"{username}:{password}".encode()).decode() +status_login, session = call( + "POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + credentials} +) +bearer = {"Authorization": "Bearer " + session["token"]} +status_chat, chat = call( + "POST", "/api/v1/chats", {"title": "compat", "member_ids": [user["id"]]}, bearer +) +path = f"/api/v1/chats/{chat['id']}/messages" +status_send, message = call("POST", path, {"text": "старая версия пишет"}, bearer) +status_read, page = call("GET", path, headers=bearer) +result = { + "ready": True, + "register": status_user, + "login": status_login, + "create_chat": status_chat, + "send_message_without_key": status_send, + "read_history": status_read, + "history_matches": page["items"] == [message], +} +print(json.dumps(result, ensure_ascii=False)) +log = open("/tmp/api.log").read() +print("startup log:", [line.split(" - ", 1)[-1] for line in log.splitlines() if "миграц" in line.lower() or "Схема" in line]) +if any(code >= 400 for code in result.values() if isinstance(code, int)) or not result["history_matches"]: + raise SystemExit("FAIL: previous version is not compatible with current schema") +print("PASS: previous API version works on the current schema (expand-only migration)") +EOF diff --git a/scripts/experiment.py b/scripts/experiment.py new file mode 100644 index 00000000..0eaf17c4 --- /dev/null +++ b/scripts/experiment.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +"""Нагрузка через LB во время отказа/замены API-реплик; метрики и проверка данных. + +failover — для каждой API-реплики по очереди: нагрузка → остановка этой реплики + (docker stop = SIGTERM, затем docker kill = SIGKILL) → восстановление + (scripts/restore.sh) → нагрузка после восстановления. +rolling — нагрузка во время scripts/rolling-update.sh (поочерёдная замена реплик). + +Клиент ведёт себя как настоящий: неподтверждённый POST повторяет с тем же +Idempotency-Key и телом. После прогона вся история читается заново: каждое +подтверждённое сообщение на месте и не изменено, дублей нет. +Python 3.9+, стандартная библиотека; tokens не печатаются и не сохраняются. +""" + +from __future__ import annotations + +import argparse +import json +import math +import os +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client # noqa: E402 + +REQUEST_TIMEOUT_SEC = 5 +RETRY_PAUSE_SEC = 0.05 +STATS_FORMAT = "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" + + +@dataclass +class Attempt: + """Одна HTTP-попытка клиента; время — секунды от начала прогона""" + + op: str + started: float + finished: float + status: int | None + error: str | None + instance: str | None + retry: bool + + @property + def ok(self) -> bool: + return self.status in (200, 201) + + +class Recorder: + def __init__(self) -> None: + self.started = time.monotonic() + self.lock = threading.Lock() + self.attempts: list[Attempt] = [] + self.confirmed: list[dict] = [] + self.events: dict[str, float] = {} + + def now(self) -> float: + return time.monotonic() - self.started + + def mark(self, name: str) -> None: + self.events[name] = round(self.now(), 3) + + def add(self, attempt: Attempt, message: dict | None = None) -> None: + with self.lock: + self.attempts.append(attempt) + if message is not None: + self.confirmed.append(message) + + +def request( + base: str, method: str, path: str, token: str, body: dict | None, key: str | None +) -> tuple[int | None, str | None, str | None, dict | None]: + headers = {"Authorization": "Bearer " + token, "Accept": "application/json"} + data = None + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode() + headers["Content-Type"] = "application/json" + if key is not None: + headers["Idempotency-Key"] = key + req = urllib.request.Request(base + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as response: + payload = json.loads(response.read()) + return response.status, None, response.headers.get("X-Instance-Id"), payload + except urllib.error.HTTPError as error: + error.read() + return error.code, None, error.headers.get("X-Instance-Id"), None + except OSError as error: + return None, type(error).__name__, None, None + + +def worker( + base: str, + token: str, + chat_id: str, + worker_id: int, + recorder: Recorder, + stop: threading.Event, +) -> None: + """Запись с повтором по тому же ключу до подтверждения, затем чтение""" + path = f"/api/v1/chats/{chat_id}/messages" + index = 0 + while not stop.is_set(): + key = uuid.uuid4().hex + body = {"text": f"exp-{worker_id}-{index}-{key[:8]}"} + retry = False + while not stop.is_set(): + started = recorder.now() + status, error, instance, payload = request( + base, "POST", path, token, body, key + ) + attempt = Attempt( + "write", started, recorder.now(), status, error, instance, retry + ) + recorder.add(attempt, payload if status == 201 else None) + if status == 201: + break + retry = True + time.sleep(RETRY_PAUSE_SEC) + started = recorder.now() + status, error, instance, _ = request( + base, "GET", path + "?limit=1", token, None, None + ) + recorder.add( + Attempt("read", started, recorder.now(), status, error, instance, False) + ) + index += 1 + + +def percentile(values: list[float], p: float) -> float | None: + if not values: + return None + ordered = sorted(values) + return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1) + + +def phase_stats(attempts: list[Attempt], start: float, end: float) -> dict: + inside = [a for a in attempts if start <= a.finished < end] + duration = max(end - start, 1e-9) + result: dict = {"seconds": round(duration, 3)} + for op in ("write", "read"): + ops = [a for a in inside if a.op == op] + ok = [a for a in ops if a.ok] + result[op] = { + "ok": len(ok), + "errors": len(ops) - len(ok), + "ok_rps": round(len(ok) / duration, 1), + "latency_ms": { + "p50": percentile([a.finished - a.started for a in ok], 0.50), + "p95": percentile([a.finished - a.started for a in ok], 0.95), + "p99": percentile([a.finished - a.started for a in ok], 0.99), + }, + } + return result + + +def recovery_stats(attempts: list[Attempt], action_start: float, end: float) -> dict: + """ + Время восстановления записи: от первой ошибки после воздействия до первой + успешной записи после последней ошибки; без ошибок — 0. Отдельно — + наибольший разрыв между подтверждениями записи (задержка без ошибок). + """ + window = sorted( + (a for a in attempts if a.op == "write" and a.finished >= action_start - 1), + key=lambda a: a.finished, + ) + errors = [a for a in window if not a.ok and a.finished >= action_start] + kinds: dict[str, int] = {} + for a in errors: + kind = a.error or f"HTTP {a.status}" + kinds[kind] = kinds.get(kind, 0) + 1 + if errors: + last_error = errors[-1].finished + first_ok_after = next( + (a.finished for a in window if a.ok and a.finished > last_error), end + ) + recovery = first_ok_after - errors[0].finished + else: + recovery = 0.0 + oks = [a.finished for a in window if a.ok and a.finished <= end] + gap = max((b - a for a, b in zip(oks, oks[1:])), default=None) + return { + "write_errors": len(errors), + "error_kinds": kinds, + "write_recovery_seconds": round(recovery, 3), + "max_write_gap_seconds": None if gap is None else round(gap, 3), + } + + +def timeline(attempts: list[Attempt], end: float) -> list[dict]: + rows = [] + for second in range(int(end) + 1): + inside = [a for a in attempts if second <= a.finished < second + 1] + instances: dict[str, int] = {} + for a in inside: + if a.ok and a.instance: + instances[a.instance] = instances.get(a.instance, 0) + 1 + rows.append( + { + "t": second, + "ok": sum(a.ok for a in inside), + "errors": sum(not a.ok for a in inside), + "instances": instances, + } + ) + return rows + + +def read_history(client: Client, actor: dict, chat_id: str) -> list[dict]: + path = f"/api/v1/chats/{chat_id}/messages?limit=100" + items: list[dict] = [] + while True: + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + items.extend(page["items"]) + if page["next_cursor"] is None: + return items + path = f"/api/v1/chats/{chat_id}/messages?limit=100&cursor=" + quote( + page["next_cursor"], safe="" + ) + + +def integrity(client: Client, actor: dict, chat_id: str, recorder: Recorder) -> dict: + history = read_history(client, actor, chat_id) + by_id = {m["id"]: m for m in history} + texts = [m["text"] for m in history] + confirmed = {m["id"]: m for m in recorder.confirmed} + return { + "messages_in_history": len(history), + "confirmed_unique": len(confirmed), + "confirmed_missing_or_changed": sum( + by_id.get(i) != m for i, m in confirmed.items() + ), + "duplicate_ids": len(history) - len(by_id), + "duplicate_texts": len(texts) - len(set(texts)), + "retried_writes_confirmed_as_replay": len(recorder.confirmed) - len(confirmed), + } + + +def run(command: list[str], env: dict | None = None) -> str: + result = subprocess.run( + command, cwd=ROOT, env=env, check=True, capture_output=True, text=True + ) + return result.stdout.strip() + + +def api_containers() -> list[str]: + ids = run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).split() + names = [run(["docker", "inspect", "-f", "{{.Name}}", i]).lstrip("/") for i in ids] + return sorted(names) + + +def instance_of(container: str) -> str: + return run(["docker", "exec", container, "cat", "/tmp/instance-id"]) + + +def container_stats() -> list[str]: + names = ["messenger-lab3-nginx-1", "messenger-lab3-postgres-1", *api_containers()] + output = run(["docker", "stats", "--no-stream", "--format", STATS_FORMAT, *names]) + return output.splitlines() + + +def scenario( + args: argparse.Namespace, + label: str, + action: list[str], + restore: list[str] | None, + target: str | None = None, + env: dict | None = None, +) -> dict: + client = Client(args.base_url, timeout=REQUEST_TIMEOUT_SEC) + alice, _, _, chat = client.bootstrap() + base = client.base + recorder = Recorder() + stop = threading.Event() + threads = [ + threading.Thread( + target=worker, + args=(base, alice["token"], chat["id"], n, recorder, stop), + daemon=True, + ) + for n in range(args.workers) + ] + for thread in threads: + thread.start() + time.sleep(args.before / 2) + stats = container_stats() + time.sleep(args.before / 2) + + instances_before = sorted( + {a.instance for a in recorder.attempts if a.ok and a.instance} + ) + recorder.mark("action_start") + action_output = run(action, env) + recorder.mark("action_end") + restore_output = "" + if restore is not None: + time.sleep(args.outage) + recorder.mark("restore_start") + restore_output = run(restore) + recorder.mark("restore_end") + time.sleep(args.after) + stop.set() + for thread in threads: + thread.join(timeout=REQUEST_TIMEOUT_SEC + 1) + recorder.mark("end") + + events = recorder.events + end = events["end"] + after_start = events.get("restore_end", events["action_end"]) + attempts = recorder.attempts + seen_after = [ + a for a in attempts if a.ok and a.finished >= after_start and a.instance + ] + back_in_rotation = next( + ( + round(a.finished - after_start, 3) + for a in sorted(seen_after, key=lambda a: a.finished) + if a.instance not in instances_before + ), + None, + ) + served = sorted( + (a for a in attempts if a.ok and a.instance), key=lambda a: a.finished + ) + instances_seen: dict[str, list[float]] = {} + for a in served: + offset = round(a.finished - events["action_start"], 3) + instances_seen.setdefault(a.instance, [offset, offset])[1] = offset + phases = { + "before": phase_stats(attempts, 0, events["action_start"]), + "during": phase_stats(attempts, events["action_start"], after_start), + "after": phase_stats(attempts, after_start, end), + } + target_after_stop = None + if target is not None: + target_after_stop = sum( + 1 + for a in attempts + if a.ok + and a.instance == target + and a.finished > events["action_end"] + and a.finished < events.get("restore_start", end) + ) + return { + "scenario": label, + "target_instance": target, + "workers": args.workers, + "events_seconds": events, + "instances_before": instances_before, + "instances_after": sorted({a.instance for a in seen_after}), + "target_responses_while_stopped": target_after_stop, + "restored_replica_in_rotation_seconds": back_in_rotation + if restore is not None + else None, + # alias → [первый, последний] успешный ответ, секунды от начала воздействия + "instances_seen_from_action": instances_seen, + "phases": phases, + "recovery": recovery_stats(attempts, events["action_start"], end), + "integrity": integrity(client, alice, chat["id"], recorder), + "container_stats_before_action": stats, + "action_output": action_output.splitlines(), + "restore_output_tail": restore_output.splitlines()[-2:], + "timeline": timeline(attempts, end), + } + + +def failover(args: argparse.Namespace) -> list[dict]: + results = [] + for signal in args.signals: + for container in api_containers(): + target = instance_of(container) + if signal == "SIGTERM": + action = ["docker", "stop", "--timeout", "10", container] + else: + action = ["docker", "kill", "--signal", "SIGKILL", container] + label = f"{signal} {container}" + print(f"== {label} (instance {target})", flush=True) + result = scenario( + args, label, action, ["./scripts/restore.sh"], target=target + ) + print(json.dumps(brief(result), ensure_ascii=False), flush=True) + results.append(result) + return results + + +def rolling(args: argparse.Namespace) -> list[dict]: + env = dict(os.environ) + env.setdefault("APP_REVISION", f"exp-{int(time.time())}") + print(f"== rolling update to revision {env['APP_REVISION']}", flush=True) + result = scenario( + args, "rolling update", ["./scripts/rolling-update.sh"], None, env=env + ) + result["revisions_after"] = run(["./scripts/instances.sh"]).splitlines() + print(json.dumps(brief(result), ensure_ascii=False), flush=True) + return [result] + + +def brief(result: dict) -> dict: + """Краткая строка прогона для терминала; полная версия — в --output""" + return { + "scenario": result["scenario"], + "rps_ok_write": {k: v["write"]["ok_rps"] for k, v in result["phases"].items()}, + "rps_ok_read": {k: v["read"]["ok_rps"] for k, v in result["phases"].items()}, + "errors": { + k: v["write"]["errors"] + v["read"]["errors"] + for k, v in result["phases"].items() + }, + "recovery": result["recovery"], + "target_responses_while_stopped": result["target_responses_while_stopped"], + "restored_replica_in_rotation_seconds": result[ + "restored_replica_in_rotation_seconds" + ], + "instances_seen_from_action": result["instances_seen_from_action"], + "integrity": result["integrity"], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("mode", choices=["failover", "rolling"]) + parser.add_argument("--base-url", default=None) + parser.add_argument("--workers", type=int, default=4) + parser.add_argument("--before", type=float, default=8, help="нагрузка до отказа") + parser.add_argument("--outage", type=float, default=10, help="реплика остановлена") + parser.add_argument("--after", type=float, default=8, help="после восстановления") + parser.add_argument( + "--signals", + nargs="+", + default=["SIGTERM", "SIGKILL"], + choices=["SIGTERM", "SIGKILL"], + ) + parser.add_argument("--output", default=None, help="JSON с полными результатами") + args = parser.parse_args() + started = time.strftime("%Y-%m-%dT%H:%M:%S%z") + results = failover(args) if args.mode == "failover" else rolling(args) + report = { + "mode": args.mode, + "started": started, + "profile": { + "workers": args.workers, + "loop": "closed: POST message (retry same key until 201) + GET limit=1", + "client_timeout_seconds": REQUEST_TIMEOUT_SEC, + "phases_seconds": { + "before": args.before, + "outage": args.outage, + "after": args.after, + }, + }, + "runs": results, + } + if args.output: + destination = ROOT / args.output + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + print(f"saved: {args.output}") + failed = [ + r["scenario"] + for r in results + if r["integrity"]["confirmed_missing_or_changed"] + or r["integrity"]["duplicate_ids"] + or r["integrity"]["duplicate_texts"] + or r["target_responses_while_stopped"] + ] + if failed: + raise SystemExit(f"FAIL: потеря/дубль или отказ не изолирован: {failed}") + print(f"PASS: прогонов {len(results)}, подтверждённые сообщения целы, дублей нет") + + +if __name__ == "__main__": + main() diff --git a/scripts/instances.sh b/scripts/instances.sh new file mode 100755 index 00000000..eb9d4a3b --- /dev/null +++ b/scripts/instances.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu +# Операторский вид API-реплик: контейнер ↔ alias X-Instance-Id ↔ ревизия образа. +# Нужен, чтобы показать, какой реальный контейнер обслужил запрос или остановлен: +# заголовок сам по себе не доказывает разные процессы. +cd "$(dirname "$0")/.." +ENV_FILE="${ENV_FILE:-.env}" + +printf '%-24s %-28s %-14s %-8s %s\n' CONTAINER STATE INSTANCE PID REVISION +for container in $(docker compose --env-file "$ENV_FILE" ps -a -q api); do + name=$(docker inspect -f '{{.Name}}' "$container") + state=$(docker inspect -f '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$container") + pid=$(docker inspect -f '{{.State.Pid}}' "$container") + revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container") + alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || echo -) + printf '%-24s %-28s %-14s %-8s %s\n' "${name#/}" "$state" "$alias" "$pid" "$revision" +done diff --git a/scripts/restart.sh b/scripts/restart.sh index a80b1276..e3d95996 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,9 +1,11 @@ #!/bin/sh set -eu -# Перезапуск API и всех используемых хранилищ (PostgreSQL, Valkey) с сохранением -# volumes: никаких down -v, flush или повторного seed. Готовность ждёт тест. +# Перезапуск всего стенда с сохранением volumes: хранилища (PostgreSQL, Valkey), +# затем все API-реплики и балансировщик. Никаких down -v, flush или повторного +# seed; migrate не запускается — схема уже в volume. Готовность ждёт тест. cd "$(dirname "$0")/.." ENV_FILE="${ENV_FILE:-.env}" -docker compose --env-file "$ENV_FILE" restart postgres valkey -docker compose --env-file "$ENV_FILE" restart api +docker compose --env-file "$ENV_FILE" restart --no-deps postgres valkey +docker compose --env-file "$ENV_FILE" restart --no-deps api +docker compose --env-file "$ENV_FILE" restart --no-deps nginx diff --git a/scripts/restore.sh b/scripts/restore.sh new file mode 100755 index 00000000..8583c478 --- /dev/null +++ b/scripts/restore.sh @@ -0,0 +1,9 @@ +#!/bin/sh +set -eu +# Возврат API-реплик к числу API_REPLICAS после stop-one.sh: запуск остановленных +# контейнеров и ожидание healthy. Работающие реплики не пересоздаются +# (--no-recreate), хранилища, migrate и LB не перезапускаются. +cd "$(dirname "$0")/.." +ENV_FILE="${ENV_FILE:-.env}" + +docker compose --env-file "$ENV_FILE" up -d --no-deps --no-recreate --wait --wait-timeout 60 api diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..06f02c96 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,42 @@ +#!/bin/sh +set -eu +# Поочерёдная замена API-реплик новой сборкой без остановки обслуживания. +# 1. build нового образа (ревизия видна в метке образа и журнале старта); +# 2. migrate однократно: миграции только расширяют схему, старые реплики +# продолжают работать на новой схеме; +# 3. для каждой старой реплики: +1 новая → ждать healthy → пауза на обновление +# DNS в nginx → graceful stop старой (SIGTERM, LB уводит трафик) → удалить. +# Во время замены реплик на одну больше, чем API_REPLICAS: ресурс заложен заранее. +cd "$(dirname "$0")/.." +ENV_FILE="${ENV_FILE:-.env}" +COMPOSE="docker compose --env-file $ENV_FILE" +APP_REVISION="${APP_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date +%s)}" +export APP_REVISION +# ID образа зависит только от содержимого (без provenance-attestation) +export BUILDX_NO_DEFAULT_ATTESTATIONS=1 +# Больше resolver valid=2s в nginx: новая реплика уже в upstream до остановки старой +LB_DNS_REFRESH_SEC=3 +STOP_TIMEOUT_SEC=10 + +log() { + echo "[$(date +%H:%M:%S)] $*" +} + +log "build revision $APP_REVISION" +$COMPOSE build --quiet api +log "migrate" +$COMPOSE run --rm --no-deps migrate >/dev/null + +old_containers=$($COMPOSE ps -q api) +for container in $old_containers; do + name=$(docker inspect -f '{{.Name}}' "$container") + running=$($COMPOSE ps -q api | wc -l | tr -d ' ') + log "scale api to $((running + 1)), wait healthy" + $COMPOSE up -d --no-deps --no-recreate --wait --wait-timeout 60 \ + --scale "api=$((running + 1))" api + sleep "$LB_DNS_REFRESH_SEC" + log "graceful stop ${name#/}" + docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null + docker rm "$container" >/dev/null +done +log "done: all API replicas run revision $APP_REVISION" diff --git a/scripts/sigterm_check.py b/scripts/sigterm_check.py index 9fa7b1e2..5ee65ef9 100644 --- a/scripts/sigterm_check.py +++ b/scripts/sigterm_check.py @@ -1,8 +1,11 @@ #!/usr/bin/env python3 -"""SIGTERM: процесс перестаёт принимать запросы и завершается не дольше чем за 10 секунд. +"""SIGTERM одной API-реплики за балансировщиком. -Во время проверки фоновый клиент непрерывно шлёт GET /health/live: до сигнала -ответы 200, после — отказ в соединении; ответов 5xx быть не должно. +Реплика перестаёт принимать соединения, дожидается текущих запросов и +завершается не дольше чем за 10 секунд; LB уводит трафик на другие реплики. +Во время проверки фоновые клиенты непрерывно шлют GET /health/live через LB: +ошибок (5xx/обрыв) быть не должно, после выхода реплики её X-Instance-Id +в ответах не появляется. """ import argparse @@ -20,106 +23,124 @@ DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url SHUTDOWN_LIMIT_SEC = 10 PROBE_INTERVAL_SEC = 0.02 +PROBERS = 4 class Prober(threading.Thread): - """Фоновый клиент: каждое обращение — новое соединение""" + """Фоновый клиент через LB: время ответа и обслужившая реплика""" def __init__(self, url: str) -> None: super().__init__(daemon=True) self.url = url self.stop = threading.Event() - self.ok: list[float] = [] - self.refused: list[float] = [] - self.server_errors = 0 + self.ok: list[tuple[float, str]] = [] + self.errors: list[tuple[float, str]] = [] def run(self) -> None: while not self.stop.is_set(): try: with urllib.request.urlopen(self.url, timeout=2) as response: - if response.status == 200: - self.ok.append(time.monotonic()) + self.ok.append( + (time.monotonic(), response.headers.get("X-Instance-Id", "")) + ) except urllib.error.HTTPError as error: - if error.code >= 500: - self.server_errors += 1 - except OSError: - self.refused.append(time.monotonic()) + self.errors.append((time.monotonic(), f"HTTP {error.code}")) + except OSError as error: + self.errors.append((time.monotonic(), type(error).__name__)) time.sleep(PROBE_INTERVAL_SEC) +def docker(*args: str) -> str: + result = subprocess.run( + ["docker", *args], check=True, capture_output=True, text=True + ) + return result.stdout.strip() + + +def first_api_container() -> str: + ids = docker("compose", "--env-file", ".env", "ps", "-q", "api").split() + if not ids: + raise SystemExit("FAIL: нет запущенных API-реплик") + return docker("inspect", "-f", "{{.Name}}", ids[0]).lstrip("/") + + def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--base-url", default=os.environ.get("BASE_URL") or DEFAULT_BASE_URL ) - parser.add_argument( - "--container", default=os.environ.get("CONTAINER", "messenger-lab3-api-1") - ) + parser.add_argument("--container", default=os.environ.get("CONTAINER")) args = parser.parse_args() + os.chdir(ROOT) + container = args.container or first_api_container() + instance = docker("exec", container, "cat", "/tmp/instance-id") logs = subprocess.Popen( - ["docker", "logs", "--follow", "--since", "0s", args.container], + ["docker", "logs", "--follow", "--since", "0s", container], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, ) waiter = subprocess.Popen( - ["docker", "wait", args.container], stdout=subprocess.PIPE, text=True + ["docker", "wait", container], stdout=subprocess.PIPE, text=True ) - prober = Prober(args.base_url.rstrip("/") + "/health/live") - prober.start() + probers = [ + Prober(args.base_url.rstrip("/") + "/health/live") for _ in range(PROBERS) + ] + for prober in probers: + prober.start() time.sleep(1) - if not prober.ok: - raise SystemExit("FAIL: API не отвечает до отправки SIGTERM") + before = {alias for prober in probers for _, alias in prober.ok} + if instance not in before: + raise SystemExit("FAIL: целевая реплика не обслуживает запросы до SIGTERM") sent_at = time.monotonic() - subprocess.run( - ["docker", "kill", "--signal", "SIGTERM", args.container], - check=True, - capture_output=True, - ) + docker("kill", "--signal", "SIGTERM", container) exit_code = int(waiter.communicate(timeout=SHUTDOWN_LIMIT_SEC + 5)[0].strip()) exited_at = time.monotonic() - time.sleep(0.5) - prober.stop.set() - prober.join() + time.sleep(1) + for prober in probers: + prober.stop.set() + prober.join() + logs.terminate() shutdown_log = logs.communicate(timeout=5)[0] - shutdown_sec = exited_at - sent_at - ok_after = [t for t in prober.ok if t > sent_at] - first_refused = min((t for t in prober.refused if t > sent_at), default=None) - print("Логи контейнера после SIGTERM:") + ok_after = [(t, a) for p in probers for t, a in p.ok if t > sent_at] + errors_after = [(t, e) for p in probers for t, e in p.errors if t > sent_at] + target_after_exit = [t for t, a in ok_after if a == instance and t > exited_at] + last_target = max((t for t, a in ok_after if a == instance), default=None) + print("Журнал реплики после SIGTERM:") print( "\n".join(line for line in shutdown_log.splitlines() if "/health/" not in line) ) - print( - json.dumps( - { - "container": args.container, - "exit_code": exit_code, - "shutdown_seconds": round(shutdown_sec, 3), - "limit_seconds": SHUTDOWN_LIMIT_SEC, - "probes_ok_before_sigterm": len(prober.ok) - len(ok_after), - "probes_ok_after_sigterm": len(ok_after), - "first_refused_after_sigterm_seconds": None - if first_refused is None - else round(first_refused - sent_at, 3), - "probes_refused_after_sigterm": len(prober.refused), - "server_errors_5xx": prober.server_errors, - }, - indent=2, - ) - ) + summary = { + "container": container, + "instance": instance, + "exit_code": exit_code, + "shutdown_seconds": round(exited_at - sent_at, 3), + "limit_seconds": SHUTDOWN_LIMIT_SEC, + "instances_before": sorted(before), + "probes_ok_after_sigterm": len(ok_after), + "instances_after_sigterm": sorted({a for _, a in ok_after}), + "last_target_response_after_sigterm_seconds": None + if last_target is None + else round(last_target - sent_at, 3), + "target_responses_after_exit": len(target_after_exit), + "errors_after_sigterm": len(errors_after), + "error_kinds": sorted({e for _, e in errors_after}), + } + print(json.dumps(summary, indent=2, ensure_ascii=False)) if ( exit_code != 0 - or shutdown_sec > SHUTDOWN_LIMIT_SEC - or prober.server_errors - or first_refused is None + or exited_at - sent_at > SHUTDOWN_LIMIT_SEC + or errors_after + or target_after_exit + or not ok_after ): raise SystemExit("FAIL: завершение по SIGTERM не соответствует требованиям") print( - "PASS: SIGTERM завершил процесс за %.2f с, новые соединения отклоняются" - % shutdown_sec + "PASS: SIGTERM завершил %s за %.2f с, LB обслужил %d запросов без ошибок" + % (container, exited_at - sent_at, len(ok_after)) ) diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..8fe66bc5 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu +# Остановка ОДНОЙ API-реплики, обслужившей тест: TARGET_INSTANCE_ID → контейнер. +# Каждый процесс API пишет свой alias (X-Instance-Id) в /tmp/instance-id. +# docker stop: SIGTERM → graceful shutdown; restart-политика unless-stopped +# не поднимает вручную остановленный контейнер, он остаётся остановленным. +# БД, LB и volumes не затрагиваются. Восстановление — scripts/restore.sh. +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +cd "$(dirname "$0")/.." +ENV_FILE="${ENV_FILE:-.env}" +# Меньше 10 с лимита hook; больше drain + graceful shutdown (1 + 5 с) +STOP_TIMEOUT_SEC=8 + +for container in $(docker compose --env-file "$ENV_FILE" ps -q api); do + alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || true) + if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then + name=$(docker inspect -f '{{.Name}}' "$container") + docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null + echo "stopped ${name#/} (instance $alias)" >&2 + exit 0 + fi +done +echo "no running API replica with instance $TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/src/application/api/middlewares/request_context.py b/src/application/api/middlewares/request_context.py index 4ae099c0..389a95be 100644 --- a/src/application/api/middlewares/request_context.py +++ b/src/application/api/middlewares/request_context.py @@ -1,20 +1,26 @@ +import re import time import uuid from fastapi import Request, Response from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint -from application.api.utils import REQUEST_ID_HEADER, error_response -from config import LOGGER +from application.api.utils import INSTANCE_ID_HEADER, REQUEST_ID_HEADER, error_response +from config import INSTANCE_ID, LOGGER from domain.errors import AppError HEALTH_PATH_PREFIX = "/health/" +INCOMING_REQUEST_ID = re.compile(r"^[0-9a-f]{32}$") + class RequestContextMiddleware(BaseHTTPMiddleware): """ - X-Request-Id в каждом ответе, журнал запросов и защита от необработанных ошибок. + X-Request-Id и X-Instance-Id в каждом ответе, журнал запросов и защита + от необработанных ошибок. + X-Request-Id берётся у балансировщика (nginx выставляет $request_id), + поэтому одна строка access-лога LB и журнал реплики связаны одним ID. В лог попадают метод, путь, статус и длительность — без тел запросов, query-параметров и текста сообщений. """ @@ -22,7 +28,7 @@ class RequestContextMiddleware(BaseHTTPMiddleware): async def dispatch( self, request: Request, call_next: RequestResponseEndpoint ) -> Response: - request_id = uuid.uuid4().hex + request_id = self._request_id(request) request.state.request_id = request_id started = time.perf_counter() try: @@ -34,9 +40,19 @@ class RequestContextMiddleware(BaseHTTPMiddleware): request_id, error.status_code, error.code, error.message ) response.headers[REQUEST_ID_HEADER] = request_id + response.headers[INSTANCE_ID_HEADER] = INSTANCE_ID + if getattr(request.app.state, "is_draining", False): + response.headers["Connection"] = "close" self._log_request(request, response, request_id, started) return response + @staticmethod + def _request_id(request: Request) -> str: + incoming = request.headers.get(REQUEST_ID_HEADER, "") + if INCOMING_REQUEST_ID.fullmatch(incoming): + return incoming + return uuid.uuid4().hex + @staticmethod def _log_request( request: Request, response: Response, request_id: str, started: float diff --git a/src/application/api/routers/messages/endpoints.py b/src/application/api/routers/messages/endpoints.py index 0353fd96..9d6ece56 100644 --- a/src/application/api/routers/messages/endpoints.py +++ b/src/application/api/routers/messages/endpoints.py @@ -1,10 +1,12 @@ from typing import Annotated from uuid import UUID -from fastapi import APIRouter, Body, Depends, Query, status +from fastapi import APIRouter, Body, Depends, Header, Query, status from application.api.dependencies import get_current_user from domain import ( + IDEMPOTENCY_KEY_MAX_LENGTH, + IDEMPOTENCY_KEY_PATTERN, CreateMessageSchema, MessagePageSchema, MessageSchema, @@ -26,10 +28,23 @@ messages_router = APIRouter( async def send_message( chat_id: UUID, body: Annotated[CreateMessageSchema, Body(...)], + idempotency_key: Annotated[ + str, + Header( + alias="Idempotency-Key", + min_length=1, + max_length=IDEMPOTENCY_KEY_MAX_LENGTH, + pattern=IDEMPOTENCY_KEY_PATTERN, + ), + ], current_user: UserSchema = Depends(get_current_user), service: MessageService = Depends(ServiceCreator.get_message_service), ) -> MessageSchema: - return await service.send_message(current_user, chat_id, body) + """ + Idempotency-Key обязателен: повтор того же тела возвращает тот же 201 + и то же сообщение, другое тело с тем же ключом — 409. Нет/неверный ключ — 400. + """ + return await service.send_message(current_user, chat_id, body, idempotency_key) @messages_router.get( diff --git a/src/application/api/utils/__init__.py b/src/application/api/utils/__init__.py index 0e10f357..56a88832 100644 --- a/src/application/api/utils/__init__.py +++ b/src/application/api/utils/__init__.py @@ -1,3 +1,13 @@ -from .errors import REQUEST_ID_HEADER, error_response, get_request_id +from .errors import ( + INSTANCE_ID_HEADER, + REQUEST_ID_HEADER, + error_response, + get_request_id, +) -__all__ = ["REQUEST_ID_HEADER", "error_response", "get_request_id"] +__all__ = [ + "REQUEST_ID_HEADER", + "INSTANCE_ID_HEADER", + "error_response", + "get_request_id", +] diff --git a/src/application/api/utils/errors.py b/src/application/api/utils/errors.py index 5c2fc3ce..9425e740 100644 --- a/src/application/api/utils/errors.py +++ b/src/application/api/utils/errors.py @@ -6,6 +6,7 @@ from fastapi.responses import JSONResponse from domain.errors import ErrorBodySchema, ErrorCode, ErrorSchema REQUEST_ID_HEADER = "X-Request-Id" +INSTANCE_ID_HEADER = "X-Instance-Id" def get_request_id(request: Request) -> str: diff --git a/src/config/__init__.py b/src/config/__init__.py index 5a74d2de..8b2e39b1 100644 --- a/src/config/__init__.py +++ b/src/config/__init__.py @@ -2,7 +2,9 @@ from .log_config import LOGGER from .settings import ( APP_ENV, APP_HOST, + APP_KEEPALIVE_TIMEOUT_SEC, APP_PORT, + APP_REVISION, DB_COMMAND_TIMEOUT_SEC, DB_CONNECT_TIMEOUT_SEC, DB_HOST, @@ -13,10 +15,14 @@ from .settings import ( DB_PORT, DB_STARTUP_TIMEOUT_SEC, DB_USER, + IDEMPOTENCY_TTL_SECONDS, + INSTANCE_ID, + INSTANCE_ID_FILE, LOG_LEVEL, PASSWORD_HASH_WORKERS, SESSION_CACHE_TTL_SECONDS, SESSION_TTL_SECONDS, + SHUTDOWN_DRAIN_SEC, SHUTDOWN_TIMEOUT_SEC, VALKEY_HOST, VALKEY_PASS, @@ -31,8 +37,11 @@ __all__ = [ "APP_HOST", "APP_PORT", "APP_ENV", + "APP_REVISION", + "APP_KEEPALIVE_TIMEOUT_SEC", "LOG_LEVEL", "SHUTDOWN_TIMEOUT_SEC", + "SHUTDOWN_DRAIN_SEC", "DB_USER", "DB_PASS", "DB_NAME", @@ -52,4 +61,7 @@ __all__ = [ "SESSION_TTL_SECONDS", "SESSION_CACHE_TTL_SECONDS", "PASSWORD_HASH_WORKERS", + "IDEMPOTENCY_TTL_SECONDS", + "INSTANCE_ID", + "INSTANCE_ID_FILE", ] diff --git a/src/config/settings.py b/src/config/settings.py index 45717ec5..54dc3fdf 100644 --- a/src/config/settings.py +++ b/src/config/settings.py @@ -1,4 +1,6 @@ import os +import secrets +import tempfile from pathlib import Path from envparse import Env @@ -12,9 +14,13 @@ if ENV_FILE.is_file(): APP_HOST = env.str("APP_HOST") APP_PORT = env.int("APP_PORT") APP_ENV = env.str("APP_ENV") +APP_REVISION = env.str("APP_REVISION", default="dev") +# Больше keepalive_timeout балансировщика: idle-соединение закрывает LB, а не API +APP_KEEPALIVE_TIMEOUT_SEC = env.int("APP_KEEPALIVE_TIMEOUT_SEC") LOG_LEVEL = env.str("LOG_LEVEL") SHUTDOWN_TIMEOUT_SEC = env.int("SHUTDOWN_TIMEOUT_SEC") +SHUTDOWN_DRAIN_SEC = env.float("SHUTDOWN_DRAIN_SEC") DB_USER = env.str("DB_USER") DB_PASS = env.str("DB_PASS") @@ -37,3 +43,7 @@ VALKEY_RETRY_AFTER_SEC = env.float("VALKEY_RETRY_AFTER_SEC") SESSION_TTL_SECONDS = env.int("SESSION_TTL_SECONDS") SESSION_CACHE_TTL_SECONDS = env.int("SESSION_CACHE_TTL_SECONDS") PASSWORD_HASH_WORKERS = env.int("PASSWORD_HASH_WORKERS") +IDEMPOTENCY_TTL_SECONDS = env.int("IDEMPOTENCY_TTL_SECONDS") + +INSTANCE_ID = secrets.token_hex(6) +INSTANCE_ID_FILE = Path(tempfile.gettempdir()) / "instance-id" diff --git a/src/domain/__init__.py b/src/domain/__init__.py index c5bd3a15..6d804f46 100644 --- a/src/domain/__init__.py +++ b/src/domain/__init__.py @@ -1,7 +1,14 @@ from .auth import ActiveSessionSchema, BasicCredentialsSchema, SessionSchema from .chats import ChatPageSchema, ChatSchema, CreateChatSchema from .health import HealthSchema -from .messages import CreateMessageSchema, MessagePageSchema, MessageSchema +from .messages import ( + IDEMPOTENCY_KEY_MAX_LENGTH, + IDEMPOTENCY_KEY_PATTERN, + CreateMessageSchema, + IdempotencyKeySchema, + MessagePageSchema, + MessageSchema, +) from .pagination import PageCursorSchema, PageKey from .users import CreateUserSchema, UserCredentialsSchema, UserSchema @@ -16,6 +23,9 @@ __all__ = [ "ChatSchema", "ChatPageSchema", "CreateMessageSchema", + "IdempotencyKeySchema", + "IDEMPOTENCY_KEY_PATTERN", + "IDEMPOTENCY_KEY_MAX_LENGTH", "MessageSchema", "MessagePageSchema", "HealthSchema", diff --git a/src/domain/messages/__init__.py b/src/domain/messages/__init__.py index 2fa9747b..ef929a7c 100644 --- a/src/domain/messages/__init__.py +++ b/src/domain/messages/__init__.py @@ -1,7 +1,17 @@ -from .schemas import CreateMessageSchema, MessagePageSchema, MessageSchema +from .schemas import ( + IDEMPOTENCY_KEY_MAX_LENGTH, + IDEMPOTENCY_KEY_PATTERN, + CreateMessageSchema, + IdempotencyKeySchema, + MessagePageSchema, + MessageSchema, +) __all__ = [ + "IDEMPOTENCY_KEY_PATTERN", + "IDEMPOTENCY_KEY_MAX_LENGTH", "CreateMessageSchema", + "IdempotencyKeySchema", "MessageSchema", "MessagePageSchema", ] diff --git a/src/domain/messages/schemas.py b/src/domain/messages/schemas.py index a2adfbc2..32ee7af2 100644 --- a/src/domain/messages/schemas.py +++ b/src/domain/messages/schemas.py @@ -7,6 +7,9 @@ from domain.utils import NotBlankStr, UtcDateTime MessageText = Annotated[NotBlankStr, Field(min_length=1, max_length=4000)] +IDEMPOTENCY_KEY_PATTERN = r"^[A-Za-z0-9._:-]{1,128}$" +IDEMPOTENCY_KEY_MAX_LENGTH = 128 + class CreateMessageSchema(BaseModel): """Запрос отправки сообщения: отправитель берётся из текущей сессии""" @@ -33,3 +36,18 @@ class MessagePageSchema(BaseModel): items: list[MessageSchema] next_cursor: str | None + + +class IdempotencyKeySchema(BaseModel): + """ + Ключ повторной отправки сообщения. + + request_hash — SHA-256 канонического тела: повтор с тем же ключом и тем же + телом возвращает исходное сообщение, с другим телом — конфликт. + """ + + model_config = ConfigDict(frozen=True) + + key: str + request_hash: bytes + ttl_seconds: int diff --git a/src/gateway.py b/src/gateway.py index 535ed6c3..c076c954 100644 --- a/src/gateway.py +++ b/src/gateway.py @@ -1,3 +1,5 @@ +import asyncio +import socket from collections.abc import AsyncIterator from contextlib import asynccontextmanager @@ -10,11 +12,16 @@ from application.api.middlewares import RequestContextMiddleware from config import ( APP_ENV, APP_HOST, + APP_KEEPALIVE_TIMEOUT_SEC, APP_PORT, + APP_REVISION, DB_STARTUP_TIMEOUT_SEC, + INSTANCE_ID, + INSTANCE_ID_FILE, LOGGER, PASSWORD_HASH_WORKERS, SESSION_TTL_SECONDS, + SHUTDOWN_DRAIN_SEC, SHUTDOWN_TIMEOUT_SEC, ) from infrastructure import PasswordHasher, cache_manager, database_pool_manager @@ -22,29 +29,35 @@ from infrastructure import PasswordHasher, cache_manager, database_pool_manager @asynccontextmanager async def lifespan(app: FastAPI) -> AsyncIterator[None]: + """ + uvicorn открывает порт только после startup: реплика без БД или со старой + схемой не принимает соединений, и LB отправляет запросы другим репликам. + """ await database_pool_manager.create_pool(DB_STARTUP_TIMEOUT_SEC) - await database_pool_manager.apply_migrations() + await database_pool_manager.check_migrations() cache_manager.create_client() password_hasher = PasswordHasher(PASSWORD_HASH_WORKERS) app.state.database = database_pool_manager app.state.cache = cache_manager app.state.password_hasher = password_hasher + INSTANCE_ID_FILE.write_text(INSTANCE_ID) app.state.is_ready = True LOGGER.info( - f"Application started, env={APP_ENV}, session ttl={SESSION_TTL_SECONDS} s" + f"Application started, env={APP_ENV}, revision={APP_REVISION}, " + f"instance={INSTANCE_ID}, session ttl={SESSION_TTL_SECONDS} s" ) yield app.state.is_ready = False await cache_manager.close() await database_pool_manager.close() password_hasher.close() - LOGGER.info("Application stopped, connections closed") + LOGGER.info(f"Application stopped, instance={INSTANCE_ID}, connections closed") app = FastAPI( title="mtuci-labs-messanger", description="мессенджер-конкурент-максу", - version="1.2.0", + version="1.3.0", lifespan=lifespan, ) @@ -55,17 +68,59 @@ app.include_router(health_endpoints) app.include_router(api_endpoints) -def main(host: str, port: int, shutdown_timeout: int) -> None: - """Функция для запуска api""" - uvicorn.run( +class DrainingServer(uvicorn.Server): + """ + SIGTERM: сначала drain, затем штатный graceful shutdown uvicorn. + """ + + def __init__(self, config: uvicorn.Config, drain_seconds: float) -> None: + super().__init__(config) + self._drain_seconds = drain_seconds + + async def shutdown(self, sockets: list[socket.socket] | None = None) -> None: + for server in getattr(self, "servers", []): + server.close() + app.state.is_ready = False + app.state.is_draining = True + LOGGER.info( + f"SIGTERM: drain {self._drain_seconds} с, instance={INSTANCE_ID}: " + "порт закрыт, keep-alive соединения закрываются после ответа" + ) + await asyncio.sleep(self._drain_seconds) + await super().shutdown(sockets) + + +def main( + host: str, + port: int, + shutdown_timeout: int, + keepalive_timeout: int, + drain_seconds: float, +) -> None: + """ + Функция для запуска api. + + SIGTERM: drain_seconds без приёма новых соединений, затем uvicorn ждёт + текущих запросов не дольше shutdown_timeout, и lifespan закрывает пулы. + Сумма меньше stop_grace_period контейнера (10 с). + """ + config = uvicorn.Config( app, host=host, port=port, timeout_graceful_shutdown=shutdown_timeout, + timeout_keep_alive=keepalive_timeout, access_log=False, server_header=False, ) + DrainingServer(config, drain_seconds).run() if __name__ == "__main__": - main(host=APP_HOST, port=APP_PORT, shutdown_timeout=SHUTDOWN_TIMEOUT_SEC) + main( + host=APP_HOST, + port=APP_PORT, + shutdown_timeout=SHUTDOWN_TIMEOUT_SEC, + keepalive_timeout=APP_KEEPALIVE_TIMEOUT_SEC, + drain_seconds=SHUTDOWN_DRAIN_SEC, + ) diff --git a/src/infrastructure/dependencies/database.py b/src/infrastructure/dependencies/database.py index 86ba2376..3b22ea03 100644 --- a/src/infrastructure/dependencies/database.py +++ b/src/infrastructure/dependencies/database.py @@ -106,8 +106,10 @@ class DatabasePoolManager: """ Применение миграций NNNN_name.sql по порядку версий. + Вызывается однократно отдельным процессом (src/migrate.py, сервис migrate) + до запуска API-реплик, а не каждой репликой при старте. Каждая миграция и запись о ней в schema_migrations — одна транзакция. - Advisory lock не даёт двум процессам API применять миграции одновременно; + Advisory lock — страховка от двух одновременных запусков migrate; уже применённые версии пропускаются, поэтому повторный запуск безопасен. """ async with self.acquire() as conn: @@ -133,8 +135,7 @@ class DatabasePoolManager: "SELECT version FROM schema_migrations;" ) } - for path in sorted(directory.glob("*.sql")): - version = int(path.name.split("_", 1)[0]) + for version, path in self._migration_files(directory): if version in applied: continue async with conn.transaction(): @@ -155,6 +156,35 @@ class DatabasePoolManager: ) LOGGER.info("Схема БД в актуальной версии") + async def check_migrations(self, directory: Path = MIGRATIONS_DIR) -> None: + """ + Схема БД не старше кода: все миграции из образа уже применены. + + API миграции не применяет. Если migrate ещё не отработал, реплика не + стартует и не начинает слушать порт, балансировщик её не использует + + """ + expected = {version for version, _ in self._migration_files(directory)} + async with self.acquire() as conn: + try: + records = await conn.fetch("SELECT version FROM schema_migrations;") + except asyncpg.UndefinedTableError: + records = [] + applied = {record["version"] for record in records} + missing = sorted(expected - applied) + if missing: + raise RuntimeError( + f"В БД не применены миграции {missing}: сначала запустите migrate" + ) + LOGGER.info(f"Схема БД совместима: версия {max(applied, default=0)}") + + @staticmethod + def _migration_files(directory: Path) -> list[tuple[int, Path]]: + """Файлы NNNN_name.sql по возрастанию версии""" + return sorted( + (int(path.name.split("_", 1)[0]), path) for path in directory.glob("*.sql") + ) + @asynccontextmanager async def acquire(self) -> AsyncIterator[asyncpg.Connection]: """ diff --git a/src/infrastructure/dependencies/service_creator.py b/src/infrastructure/dependencies/service_creator.py index 7e42b54e..f96a92f4 100644 --- a/src/infrastructure/dependencies/service_creator.py +++ b/src/infrastructure/dependencies/service_creator.py @@ -1,6 +1,10 @@ from fastapi import Depends -from config import SESSION_CACHE_TTL_SECONDS, SESSION_TTL_SECONDS +from config import ( + IDEMPOTENCY_TTL_SECONDS, + SESSION_CACHE_TTL_SECONDS, + SESSION_TTL_SECONDS, +) from infrastructure.dependencies.cache import CacheManager, get_cache from infrastructure.dependencies.database import DatabasePoolManager, get_database from infrastructure.repositories import ( @@ -90,4 +94,6 @@ class ServiceCreator: chat_repository: ChatRepository = Depends(_get_chat_repository), message_repository: MessageRepository = Depends(_get_message_repository), ) -> MessageService: - return MessageService(chat_repository, message_repository) + return MessageService( + chat_repository, message_repository, IDEMPOTENCY_TTL_SECONDS + ) diff --git a/src/infrastructure/repositories/message_repository/postgres.py b/src/infrastructure/repositories/message_repository/postgres.py index f8aa343c..f04301ad 100644 --- a/src/infrastructure/repositories/message_repository/postgres.py +++ b/src/infrastructure/repositories/message_repository/postgres.py @@ -2,7 +2,8 @@ from uuid import UUID import asyncpg -from domain import MessageSchema, PageKey +from domain import IdempotencyKeySchema, MessageSchema, PageKey +from domain.errors import ConflictError from infrastructure.utils import Pool, repository_logs, transaction from interfaces import MessageRepository @@ -15,11 +16,22 @@ class PostgresMessageRepository(MessageRepository): self._pool = pool @repository_logs - async def create(self, chat_id: UUID, sender_id: UUID, text: str) -> MessageSchema: + async def create( + self, + chat_id: UUID, + sender_id: UUID, + text: str, + idempotency: IdempotencyKeySchema, + ) -> tuple[MessageSchema, bool]: """ - Блокировка чата и вставка — одна транзакция; lock снимается на COMMIT. + Блокировка чата, проверка ключа, вставка сообщения и ключа — одна + транзакция; lock снимается на COMMIT. Запись в один чат сериализуется, записи в разные чаты не мешают друг другу. + Ключ идемпотентности привязан к чату, поэтому та же блокировка + сериализует и одновременные повторы с одним ключом с разных реплик: + второй ждёт COMMIT первого и находит его ключ + created_at строго больше последнего в чате, даже если часы ушли назад: порядок коммитов совпадает с порядком (created_at, id), и читатель с cursor не пропустит сообщение. MAX берётся из индекса @@ -28,6 +40,22 @@ class PostgresMessageRepository(MessageRepository): lock_query = """ SELECT pg_advisory_xact_lock($1, hashtext($2::uuid::text)); """ + find_key_query = """ + SELECT + idem.request_hash, + message.id, + message.chat_id, + message.sender_id, + message.text, + message.created_at + FROM message_idempotency_keys idem + JOIN messages message ON message.id = idem.message_id + WHERE + idem.sender_id = $1 + AND idem.chat_id = $2 + AND idem.idempotency_key = $3 + AND idem.expires_at > clock_timestamp(); + """ insert_query = """ INSERT INTO messages ( @@ -45,11 +73,56 @@ class PostgresMessageRepository(MessageRepository): WHERE chat_id = $1 RETURNING id, chat_id, sender_id, text, created_at; """ + + save_key_query = """ + INSERT INTO message_idempotency_keys + ( + sender_id, + chat_id, + idempotency_key, + request_hash, + message_id, + expires_at + ) + VALUES + ( + $1, + $2, + $3, + $4, + $5, + clock_timestamp() + make_interval(secs => $6::integer) + ) + ON CONFLICT (sender_id, chat_id, idempotency_key) DO UPDATE + SET + request_hash = EXCLUDED.request_hash, + message_id = EXCLUDED.message_id, + created_at = EXCLUDED.created_at, + expires_at = EXCLUDED.expires_at; + """ async with transaction(self._pool) as conn: await conn.execute(lock_query, CHAT_LOCK_NAMESPACE, chat_id) + stored = await conn.fetchrow( + find_key_query, sender_id, chat_id, idempotency.key + ) + if stored is not None: + if stored["request_hash"] != idempotency.request_hash: + raise ConflictError( + "Idempotency-Key уже использован с другим телом запроса" + ) + return self._row_to_message(stored), False row = await conn.fetchrow(insert_query, chat_id, sender_id, text) - assert row is not None - return self._row_to_message(row) + assert row is not None + await conn.execute( + save_key_query, + sender_id, + chat_id, + idempotency.key, + idempotency.request_hash, + row["id"], + idempotency.ttl_seconds, + ) + return self._row_to_message(row), True @repository_logs async def list_for_chat( diff --git a/src/infrastructure/services/message_service/message_service.py b/src/infrastructure/services/message_service/message_service.py index 3b2061a3..96659108 100644 --- a/src/infrastructure/services/message_service/message_service.py +++ b/src/infrastructure/services/message_service/message_service.py @@ -1,7 +1,14 @@ +import hashlib from uuid import UUID from config import LOGGER -from domain import CreateMessageSchema, MessagePageSchema, MessageSchema, UserSchema +from domain import ( + CreateMessageSchema, + IdempotencyKeySchema, + MessagePageSchema, + MessageSchema, + UserSchema, +) from domain.errors import NotFoundError from infrastructure.utils import build_page, decode_cursor from interfaces import ChatRepository, MessageRepository @@ -9,21 +16,47 @@ from interfaces import ChatRepository, MessageRepository class MessageService: def __init__( - self, chat_repository: ChatRepository, message_repository: MessageRepository + self, + chat_repository: ChatRepository, + message_repository: MessageRepository, + idempotency_ttl_seconds: int, ) -> None: self._chats = chat_repository self._messages = message_repository + self._idempotency_ttl = idempotency_ttl_seconds async def send_message( - self, current_user: UserSchema, chat_id: UUID, data: CreateMessageSchema + self, + current_user: UserSchema, + chat_id: UUID, + data: CreateMessageSchema, + idempotency_key: str, ) -> MessageSchema: - """Отправитель берётся из текущей идентификации; текст в логи не пишется""" + """ + Отправитель берётся из текущей идентификации; текст в логи не пишется. + + Повтор с тем же Idempotency-Key заново проверяет доступ к чату + и возвращает исходное сообщение. + """ await self._ensure_member(current_user, chat_id) - message = await self._messages.create(chat_id, current_user.id, data.text) - LOGGER.info( - f"Сообщение {message.id} отправлено в чат {chat_id} " - f"пользователем {current_user.id}" + idempotency = IdempotencyKeySchema( + key=idempotency_key, + request_hash=self._request_hash(data), + ttl_seconds=self._idempotency_ttl, ) + message, created = await self._messages.create( + chat_id, current_user.id, data.text, idempotency + ) + if created: + LOGGER.info( + f"Сообщение {message.id} отправлено в чат {chat_id} " + f"пользователем {current_user.id}" + ) + else: + LOGGER.info( + f"Повтор по Idempotency-Key: возвращено сообщение {message.id} " + f"чата {chat_id}" + ) return message async def list_messages( @@ -40,3 +73,11 @@ class MessageService: """Посторонний получает 404: существование чужого чата не раскрывается""" if await self._chats.get_for_member(chat_id, current_user.id) is None: raise NotFoundError() + + @staticmethod + def _request_hash(data: CreateMessageSchema) -> bytes: + """ + SHA-256 канонического тела. Порядок полей задаёт схема, а не клиент, + поэтому JSON с другим порядком ключей считается тем же запросом. + """ + return hashlib.sha256(data.model_dump_json().encode()).digest() diff --git a/src/interfaces/message_repository.py b/src/interfaces/message_repository.py index 1b878c83..3bb4057f 100644 --- a/src/interfaces/message_repository.py +++ b/src/interfaces/message_repository.py @@ -1,18 +1,28 @@ from abc import ABC, abstractmethod from uuid import UUID -from domain import MessageSchema, PageKey +from domain import IdempotencyKeySchema, MessageSchema, PageKey class MessageRepository(ABC): """Слой абстракции хранилища сообщений""" @abstractmethod - async def create(self, chat_id: UUID, sender_id: UUID, text: str) -> MessageSchema: + async def create( + self, + chat_id: UUID, + sender_id: UUID, + text: str, + idempotency: IdempotencyKeySchema, + ) -> tuple[MessageSchema, bool]: """ Добавление сообщения в конец истории чата. id и created_at назначает хранилище; created_at строго возрастает внутри чата, поэтому сообщение не может появиться перед уже выданным cursor. + + Сообщение и запись ключа идемпотентности сохраняются атомарно. + Одновременные запросы с одним ключом через любые реплики создают + ровно одно сообщение; повтор возвращает его же. """ pass diff --git a/src/migrate.py b/src/migrate.py new file mode 100644 index 00000000..0ec58ace --- /dev/null +++ b/src/migrate.py @@ -0,0 +1,24 @@ +""" +Однократное применение миграций до запуска API-реплик. + +Запускается сервисом migrate (compose) и завершается; реплики стартуют +после его успешного завершения и только проверяют версию схемы. +""" + +import asyncio + +from config import DB_STARTUP_TIMEOUT_SEC, LOGGER +from infrastructure import database_pool_manager + + +async def main() -> None: + await database_pool_manager.create_pool(DB_STARTUP_TIMEOUT_SEC) + try: + await database_pool_manager.apply_migrations() + finally: + await database_pool_manager.close() + LOGGER.info("Миграции применены, migrate завершается") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/student-tests/conftest.py b/student-tests/conftest.py index b22e64e0..13c81718 100644 --- a/student-tests/conftest.py +++ b/student-tests/conftest.py @@ -5,7 +5,7 @@ from pathlib import Path import httpx import pytest -from helpers import auth, basic, unique_password, unique_username +from helpers import auth, basic, message_headers, unique_password, unique_username ROOT = Path(__file__).resolve().parents[1] DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url"] @@ -68,7 +68,7 @@ def send_message(api: httpx.Client) -> Callable[..., dict]: response = api.post( f"/api/v1/chats/{chat['id']}/messages", json={"text": text}, - headers=auth(user), + headers=message_headers(user), ) assert response.status_code == 201, response.text return response.json() diff --git a/student-tests/helpers.py b/student-tests/helpers.py index fc4ecda4..bb3adaaf 100644 --- a/student-tests/helpers.py +++ b/student-tests/helpers.py @@ -30,6 +30,11 @@ def auth(user: dict) -> dict[str, str]: return {"Authorization": f"Bearer {user['token']}"} +def message_headers(user: dict, key: str | None = None) -> dict[str, str]: + """Bearer и Idempotency-Key (новый, если не задан)""" + return {**auth(user), "Idempotency-Key": key or uuid.uuid4().hex} + + def public_user(user: dict) -> dict: """User из ответа API без тестовых полей token/password""" return {field: user[field] for field in USER_FIELDS} diff --git a/student-tests/test_balancing.py b/student-tests/test_balancing.py new file mode 100644 index 00000000..5b89e1cb --- /dev/null +++ b/student-tests/test_balancing.py @@ -0,0 +1,74 @@ +""" +Балансировка без sticky sessions: запросы одной сессии обслуживают разные +реплики, а состояние (сессии, сообщения, logout) общее для всех. +""" + +import uuid +from collections import Counter + +import httpx +from helpers import assert_unauthorized, auth, message_headers, public_user + +REQUESTS = 20 + + +def test_one_session_is_served_by_several_replicas( + api: httpx.Client, make_user +) -> None: + """Одно keep-alive соединение клиента к LB, без повторного входа""" + alice = make_user("alice") + instances: Counter[str] = Counter() + for _ in range(REQUESTS): + response = api.get("/api/v1/users/me", headers=auth(alice)) + assert response.status_code == 200, response.text + assert response.json() == public_user(alice) + instances[response.headers["X-Instance-Id"]] += 1 + print("\nINSTANCES FOR ONE SESSION:", dict(instances)) + assert len(instances) >= 2 + + +def test_write_is_visible_through_every_replica( + api: httpx.Client, make_user, make_chat +) -> None: + alice = make_user("alice") + bob = make_user("bob") + chat = make_chat(alice, [alice, bob]) + sent = api.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": f"видно везде {uuid.uuid4().hex}"}, + headers=message_headers(alice), + ) + assert sent.status_code == 201, sent.text + readers: set[str] = set() + for _ in range(REQUESTS): + page = api.get(f"/api/v1/chats/{chat['id']}/messages", headers=auth(bob)) + assert page.status_code == 200, page.text + assert page.json()["items"] == [sent.json()] + readers.add(page.headers["X-Instance-Id"]) + assert len(readers) >= 2 + + +def test_logout_applies_to_all_replicas(api: httpx.Client, make_user) -> None: + """Сессия попала в общий кэш через обе реплики, затем отозвана на одной""" + alice = make_user("alice") + for _ in range(4): + assert api.get("/api/v1/users/me", headers=auth(alice)).status_code == 200 + logout = api.delete("/api/v1/auth/sessions/current", headers=auth(alice)) + assert logout.status_code == 204 + rejected_by: set[str] = set() + for _ in range(REQUESTS): + response = api.get("/api/v1/users/me", headers=auth(alice)) + assert_unauthorized(response) + rejected_by.add(response.headers["X-Instance-Id"]) + assert len(rejected_by) >= 2 + + +def test_every_api_response_has_instance_id(api: httpx.Client) -> None: + for response in ( + api.get("/health/live"), + api.get("/health/ready"), + api.get("/api/v1/unknown"), + api.get("/api/v1/users/me"), + ): + assert response.headers["X-Instance-Id"] + assert response.headers["X-Request-Id"] diff --git a/student-tests/test_concurrency.py b/student-tests/test_concurrency.py index 19524415..ab8c495d 100644 --- a/student-tests/test_concurrency.py +++ b/student-tests/test_concurrency.py @@ -5,7 +5,7 @@ import json import time import httpx -from helpers import auth, read_all_messages +from helpers import auth, message_headers, read_all_messages CLIENTS = 10 MESSAGES_PER_CLIENT = 10 @@ -29,6 +29,7 @@ async def _send_concurrently( client.post( f"/api/v1/chats/{chat_id}/messages", json={"text": f"{user['username']}#{n}"}, + headers=message_headers(user), ) for n in range(MESSAGES_PER_CLIENT) ) diff --git a/student-tests/test_foreign_chat.py b/student-tests/test_foreign_chat.py index e720cf9e..658995fa 100644 --- a/student-tests/test_foreign_chat.py +++ b/student-tests/test_foreign_chat.py @@ -3,7 +3,7 @@ import uuid import httpx -from helpers import assert_error, auth +from helpers import assert_error, auth, message_headers def test_outsider_cannot_access_chat( @@ -18,7 +18,9 @@ def test_outsider_cannot_access_chat( assert_error(api.get(path, headers=auth(eve)), 404, "not_found") assert_error(api.get(f"{path}/messages", headers=auth(eve)), 404, "not_found") - response = api.post(f"{path}/messages", json={"text": "чужое"}, headers=auth(eve)) + response = api.post( + f"{path}/messages", json={"text": "чужое"}, headers=message_headers(eve) + ) assert_error(response, 404, "not_found") history = api.get(f"{path}/messages", headers=auth(bob)).json() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..3c9c3b22 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,249 @@ +"""Idempotency-Key отправки сообщения: повтор, конфликт, область ключа, гонки""" + +import asyncio +import json +import uuid +from collections import Counter + +import httpx +import pytest +from helpers import assert_error, auth, message_headers, psql, read_all_messages + +PARALLEL_REPLAYS = 20 + + +def post_message( + api: httpx.Client, user: dict, chat: dict, text: str, key: str +) -> httpx.Response: + return api.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers=message_headers(user, key), + ) + + +def history(api: httpx.Client, user: dict, chat: dict) -> list[dict]: + items, _ = read_all_messages(api, user, chat["id"], 100) + return items + + +def test_replay_returns_same_message(api: httpx.Client, make_user, make_chat) -> None: + alice = make_user("alice") + chat = make_chat(alice, [alice]) + key = uuid.uuid4().hex + + first = post_message(api, alice, chat, "один раз", key) + second = post_message(api, alice, chat, "один раз", key) + + assert first.status_code == second.status_code == 201, second.text + assert first.json() == second.json() + assert history(api, alice, chat) == [first.json()] + + +def test_replay_ignores_json_formatting( + api: httpx.Client, make_user, make_chat +) -> None: + """Пробелы и форма записи JSON не меняют смысл тела: это тот же запрос""" + alice = make_user("alice") + chat = make_chat(alice, [alice]) + headers = {**message_headers(alice), "Content-Type": "application/json"} + path = f"/api/v1/chats/{chat['id']}/messages" + + escaped = b'{"text":"\\u041f\\u0440\\u0438"}' + compact = api.post(path, content=escaped, headers=headers) + spaced = api.post(path, content='{ "text" : "При" }'.encode(), headers=headers) + + assert compact.status_code == spaced.status_code == 201, spaced.text + assert compact.json() == spaced.json() + assert len(history(api, alice, chat)) == 1 + + +def test_same_key_with_other_body_is_conflict( + api: httpx.Client, make_user, make_chat +) -> None: + alice = make_user("alice") + chat = make_chat(alice, [alice]) + key = uuid.uuid4().hex + original = post_message(api, alice, chat, "оригинал", key).json() + + assert_error(post_message(api, alice, chat, "другой", key), 409, "conflict") + assert history(api, alice, chat) == [original] + + +@pytest.mark.parametrize( + "headers", + [ + {}, + {"Idempotency-Key": ""}, + {"Idempotency-Key": "with space"}, + {"Idempotency-Key": "ключ".encode()}, + {"Idempotency-Key": "a/b"}, + {"Idempotency-Key": "x" * 129}, + ], +) +def test_missing_or_invalid_key_is_rejected( + api: httpx.Client, make_user, make_chat, headers: dict +) -> None: + alice = make_user("alice") + chat = make_chat(alice, [alice]) + response = api.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": "без ключа"}, + headers={**auth(alice), **headers}, + ) + assert_error(response, 400, "invalid_request") + assert history(api, alice, chat) == [] + + +@pytest.mark.parametrize("key", ["x" * 128, "Az09._:-"]) +def test_key_boundaries_are_accepted( + api: httpx.Client, make_user, make_chat, key: str +) -> None: + alice = make_user("alice") + chat = make_chat(alice, [alice]) + assert post_message(api, alice, chat, "граница", key).status_code == 201 + + +def test_key_scope_is_user_and_chat(api: httpx.Client, make_user, make_chat) -> None: + """Один ключ у разных пользователей и в разных чатах — независимые сообщения""" + alice = make_user("alice") + bob = make_user("bob") + chat = make_chat(alice, [alice, bob]) + other_chat = make_chat(alice, [alice]) + key = uuid.uuid4().hex + + from_alice = post_message(api, alice, chat, "текст", key).json() + from_bob = post_message(api, bob, chat, "текст", key).json() + in_other_chat = post_message(api, alice, other_chat, "текст", key).json() + + assert len({from_alice["id"], from_bob["id"], in_other_chat["id"]}) == 3 + assert from_bob["sender_id"] == bob["id"] + assert len(history(api, alice, chat)) == 2 + assert len(history(api, alice, other_chat)) == 1 + + +def test_replay_rechecks_session_and_access( + api: httpx.Client, make_user, make_chat +) -> None: + alice = make_user("alice") + eve = make_user("eve") + chat = make_chat(alice, [alice]) + key = uuid.uuid4().hex + post_message(api, alice, chat, "секрет", key) + + # Чужой с тем же ключом не получает сохранённое сообщение + assert_error(post_message(api, eve, chat, "секрет", key), 404, "not_found") + # Отозванная сессия не даёт повтора + logout = api.delete("/api/v1/auth/sessions/current", headers=auth(alice)) + assert logout.status_code == 204 + replay = post_message(api, alice, chat, "секрет", key) + assert_error(replay, 401, "unauthorized") + + +def test_expired_key_can_be_reused(api: httpx.Client, make_user, make_chat) -> None: + """После срока хранения (24 ч) ключ снова создаёт сообщение; срок сдвигается в БД""" + alice = make_user("alice") + chat = make_chat(alice, [alice]) + key = uuid.uuid4().hex + first = post_message(api, alice, chat, "вчера", key).json() + psql( + "UPDATE message_idempotency_keys " + "SET created_at = now() - interval '2 days', " + "expires_at = now() - interval '1 day' " + f"WHERE chat_id = '{chat['id']}' AND idempotency_key = '{key}';" + ) + + second = post_message(api, alice, chat, "сегодня", key) + + assert second.status_code == 201, second.text + assert second.json()["id"] != first["id"] + assert [item["text"] for item in history(api, alice, chat)] == ["вчера", "сегодня"] + stored = psql( + "SELECT count(*) FROM message_idempotency_keys " + f"WHERE chat_id = '{chat['id']}' AND idempotency_key = '{key}';" + ) + assert stored == "1" + + +async def _post_concurrently( + base_url: str, user: dict, chat: dict, key: str, texts: list[str] +) -> list[httpx.Response]: + """Все запросы стартуют по одному сигналу; отдельные соединения к LB""" + start = asyncio.Event() + limits = httpx.Limits(max_connections=len(texts)) + + async with httpx.AsyncClient( + base_url=base_url, timeout=30, limits=limits + ) as client: + + async def post(text: str) -> httpx.Response: + await start.wait() + return await client.post( + f"/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers=message_headers(user, key), + ) + + tasks = [asyncio.create_task(post(text)) for text in texts] + await asyncio.sleep(0) + start.set() + return list(await asyncio.gather(*tasks)) + + +def test_parallel_replays_through_replicas_create_one_message( + api: httpx.Client, base_url: str, make_user, make_chat +) -> None: + alice = make_user("alice") + chat = make_chat(alice, [alice]) + key = uuid.uuid4().hex + + responses = asyncio.run( + _post_concurrently( + base_url, alice, chat, key, ["одно сообщение"] * PARALLEL_REPLAYS + ) + ) + + statuses = Counter(response.status_code for response in responses) + instances = Counter(response.headers["X-Instance-Id"] for response in responses) + bodies = {json.dumps(response.json(), sort_keys=True) for response in responses} + items = history(api, alice, chat) + rows = psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}';") + print( + "\nPARALLEL SAME KEY:", + json.dumps( + { + "requests": len(responses), + "statuses": dict(statuses), + "instances": dict(instances), + "distinct_bodies": len(bodies), + "messages_in_history": len(items), + "rows_in_db": int(rows), + } + ), + ) + assert statuses == {201: PARALLEL_REPLAYS} + assert len(instances) >= 2, "запросы не распределились по репликам" + assert len(bodies) == 1 + assert len(items) == 1 and rows == "1" + + +def test_parallel_same_key_with_different_bodies( + api: httpx.Client, base_url: str, make_user, make_chat +) -> None: + """Победитель гонки один; совпавшие с ним тела — 201, остальные — 409""" + alice = make_user("alice") + chat = make_chat(alice, [alice]) + key = uuid.uuid4().hex + texts = ["вариант A", "вариант B"] * (PARALLEL_REPLAYS // 2) + + responses = asyncio.run(_post_concurrently(base_url, alice, chat, key, texts)) + + items = history(api, alice, chat) + assert len(items) == 1 + winner = items[0] + for text, response in zip(texts, responses): + if text == winner["text"]: + assert response.status_code == 201, response.text + assert response.json() == winner + else: + assert_error(response, 409, "conflict") diff --git a/student-tests/test_invalid_json.py b/student-tests/test_invalid_json.py index df887973..65106047 100644 --- a/student-tests/test_invalid_json.py +++ b/student-tests/test_invalid_json.py @@ -2,7 +2,7 @@ import httpx import pytest -from helpers import assert_error, auth +from helpers import assert_error, auth, message_headers PASSWORD = "Valid-password-123" @@ -32,8 +32,16 @@ def test_malformed_json_on_create_chat_and_message( assert_error( api.post("/api/v1/chats", content=raw, headers=headers), 400, "invalid_request" ) + message_post_headers = { + **message_headers(alice), + "Content-Type": "application/json", + } assert_error( - api.post(f"/api/v1/chats/{chat['id']}/messages", content=raw, headers=headers), + api.post( + f"/api/v1/chats/{chat['id']}/messages", + content=raw, + headers=message_post_headers, + ), 400, "invalid_request", ) @@ -78,7 +86,9 @@ def test_invalid_message_payload( alice = make_user("alice") chat = make_chat(alice, [alice]) response = api.post( - f"/api/v1/chats/{chat['id']}/messages", json=body, headers=auth(alice) + f"/api/v1/chats/{chat['id']}/messages", + json=body, + headers=message_headers(alice), ) assert_error(response, 400, "invalid_request") diff --git a/student-tests/test_messages.py b/student-tests/test_messages.py index 0319354f..786cb2f9 100644 --- a/student-tests/test_messages.py +++ b/student-tests/test_messages.py @@ -2,7 +2,7 @@ import httpx import pytest -from helpers import assert_error, auth +from helpers import assert_error, auth, message_headers @pytest.mark.parametrize("text", ["", " ", " \n\t ", " "]) @@ -12,7 +12,9 @@ def test_empty_or_blank_message_is_rejected( alice = make_user("alice") chat = make_chat(alice, [alice]) response = api.post( - f"/api/v1/chats/{chat['id']}/messages", json={"text": text}, headers=auth(alice) + f"/api/v1/chats/{chat['id']}/messages", + json={"text": text}, + headers=message_headers(alice), ) assert_error(response, 400, "invalid_request") history = api.get(f"/api/v1/chats/{chat['id']}/messages", headers=auth(alice)) @@ -29,7 +31,7 @@ def test_message_length_is_counted_in_code_points( response = api.post( f"/api/v1/chats/{chat['id']}/messages", json={"text": "😀" * 4001}, - headers=auth(alice), + headers=message_headers(alice), ) assert_error(response, 400, "invalid_request") @@ -56,7 +58,7 @@ def test_nul_character_is_rejected(api: httpx.Client, make_user, make_chat) -> N response = api.post( f"/api/v1/chats/{chat['id']}/messages", json={"text": "до\u0000после"}, - headers=auth(alice), + headers=message_headers(alice), ) assert_error(response, 400, "invalid_request") response = api.post(