feat: конф нгинкс, тесты, ключ идмпт etc

This commit is contained in:
Enterxx2
2026-10-20 10:41:56 +03:00
parent aa971fbd27
commit b53e5d1077
63 changed files with 5013 additions and 153 deletions
+2
View File
@@ -33,3 +33,5 @@ requirements-dev.txt
lab.json
.mypy_cache
.ruff_cache
infra
.venv-validate
+5
View File
@@ -6,6 +6,9 @@ APP_ENV=lab
LOG_LEVEL=INFO
SHUTDOWN_TIMEOUT_SEC=5
SHUTDOWN_DRAIN_SEC=1
APP_KEEPALIVE_TIMEOUT_SEC=75
DB_USER=test
DB_PASS=test
DB_NAME=test
@@ -30,4 +33,6 @@ PASSWORD_HASH_WORKERS=4
# lab3: число API-реплик за балансировщиком и срок хранения Idempotency-Key
API_REPLICAS=2
# Публичный порт балансировщика на 127.0.0.1 (порт для make test-expiry — LB_EXPIRY_PORT=8081)
LB_PORT=8080
IDEMPOTENCY_TTL_SECONDS=86400
+1
View File
@@ -33,3 +33,4 @@ coverage/
.pytest_cache/
.mypy_cache/
.ruff_cache/
.venv-validate/
+4
View File
@@ -15,6 +15,10 @@ COPY src ./src
RUN useradd --system --uid 10001 --no-create-home app
USER app
ARG APP_REVISION=dev
ENV APP_REVISION=${APP_REVISION}
LABEL org.opencontainers.image.revision=${APP_REVISION}
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
CMD ["python", "src/healthcheck.py"]
+21 -3
View File
@@ -4,11 +4,15 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover install up down restart ps logs wait-ready destroy-data student-test concurrency-test test-expiry test-failures test-migrations explain sigterm-check
export BUILDX_NO_DEFAULT_ATTESTATIONS := 1
.PHONY: help check validate test load test-persistence test-failover install up down restart restore instances rolling-update ps logs wait-ready destroy-data student-test concurrency-test test-expiry test-failures test-migrations explain sigterm-check experiment-failover experiment-rolling
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "up/down/restart/ps/logs: compose stack; destroy-data: DELETE volumes intentionally"
@echo "instances: API containers <-> X-Instance-Id; restore: start replicas stopped by stop-one.sh; rolling-update: replace replicas one by one"
@echo "experiment-failover / experiment-rolling: load + stop each replica / rolling update, metrics to evidence/"
@echo "student-test: own tests; test-migrations: from empty volume; test-expiry: short SESSION_TTL; test-failures: stop Valkey/PostgreSQL; explain: EXPLAIN history"
check:
@@ -44,11 +48,20 @@ restart:
./scripts/restart.sh
@$(MAKE) --no-print-directory wait-ready
restore:
./scripts/restore.sh
instances:
./scripts/instances.sh
rolling-update:
./scripts/rolling-update.sh
ps:
docker compose ps
logs:
docker compose logs -f api
docker compose logs -f api nginx
wait-ready:
@for i in $$(seq 1 60); do \
@@ -65,7 +78,6 @@ student-test:
concurrency-test:
BASE_URL="$(BASE_URL)" poetry run pytest student-tests/test_concurrency.py -s
test-expiry:
docker compose --profile expiry up -d --build --wait api-short-ttl
EXPIRY_BASE_URL=http://localhost:8081 poetry run pytest student-tests/test_expiry.py -s; \
status=$$?; docker compose --profile expiry stop api-short-ttl; exit $$status
@@ -83,3 +95,9 @@ explain:
sigterm-check:
$(PYTHON) scripts/sigterm_check.py --base-url "$(BASE_URL)"
experiment-failover:
$(PYTHON) scripts/experiment.py failover --base-url "$(BASE_URL)" --output evidence/experiment-failover.json
experiment-rolling:
$(PYTHON) scripts/experiment.py rolling --base-url "$(BASE_URL)" --output evidence/experiment-rolling.json
+140 -17
View File
@@ -1,37 +1,160 @@
# Отчёт — лабораторная 3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Саватеев Артём, БВТ2302
- Целевая оценка: **4** (весь уровень 3 + весь уровень 4);
- Источники: документация [nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) (`resolve`, `max_fails`, `keepalive_timeout`) и [proxy](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) (`proxy_next_upstream`, повтор неидемпотентных запросов), [Compose services/deploy](https://docs.docker.com/reference/compose-file/services/), исходники uvicorn 0.54 (`Server.shutdown`, h11 keep-alive). Библиотеки — из lab2 (FastAPI, asyncpg, redis-py, argon2-cffi), новых зависимостей нет. Помощь ии
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
```sh
cp .env.example .env # затем задать свои DB_PASS и VALKEY_PASS (синтетические)
docker compose --env-file .env config --quiet
make up # = docker compose up -d --build --wait: build → postgres/valkey →
# migrate (однократно) → api ×API_REPLICAS → nginx; ждёт healthy
make check test # стенд доступен только через LB: http://localhost:8080
python3 -m venv .venv-validate && .venv-validate/bin/python -m pip install -r requirements-dev.txt
make validate PYTHON=.venv-validate/bin/python # полная OpenAPI-валидация; не в .venv —
# его Poetry принял бы за окружение проекта (Python 3.9)
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
make instances # какой контейнер остановлен: контейнер ↔ X-Instance-Id ↔ PID ↔ ревизия
make restore # вернуть остановленную реплику (работающие не пересоздаются)
make student-test # собственные тесты (poetry install --no-root один раз)
make rolling-update # выкладка новой сборки по одной реплике без простоя
make down # безопасная остановка, volume с данными сохраняется
make destroy-data # намеренное удаление учебных данных (down -v)
```
Действия оператора: создать `.env` с секретами (в Git не попадает), после `stop-one.sh` вручную выполнить `make restore`. Миграции отдельно запускать не нужно — их выполняет сервис `migrate` внутри `make up`/`rolling-update`. Сборку через `make`/скрипты лучше, чем голый `docker compose up --build`: Makefile экспортирует `BUILDX_NO_DEFAULT_ATTESTATIONS=1`, иначе Docker Desktop добавляет к образу provenance-attestation, ID образа меняется на каждой сборке и Compose пересоздаёт все реплики разом. Обновлять работающий стенд — только `make rolling-update`.
Воспроизводимость проверена копией ровно тех файлов, что попадут в Git (без `.env`), отдельным compose-проектом на пустом volume: build, migrate с нуля, smoke, failover, persistence — [evidence/15-clean-clone.txt](evidence/15-clean-clone.txt).
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
клиент ──► nginx :8080 (127.0.0.1, единственный опубликованный порт)
│ round-robin, без sticky; Docker DNS «api» перечитывается каждые 2 с
├──► api-1 ─┐ сеть edge: только LB и API
└──► api-2 ─┤
├──► postgres (источник истины: пользователи, чаты, сообщения,
│ сессии, ключи идемпотентности; volume postgres_data)
migrate ────┤
└──► valkey (read-through кэш сессий, не источник истины)
сеть backend: internal, LB туда не подключён
```
**Перенесено из lab2:** API, PostgreSQL как источник истины, сессии в PostgreSQL с кэшем в Valkey (logout инвалидирует общий кэш), Argon2id, keyset-пагинация, монотонный `created_at` под advisory lock чата, обработка отказов Valkey/PostgreSQL. Локального состояния, от которого зависит корректность, в API нет: circuit breaker Valkey и пул соединений — только оптимизации процесса.
**Изменено в lab3:**
| Что | Решение | Где |
| --- | --- | --- |
| Реплики и LB | `api` с `deploy.replicas: ${API_REPLICAS:-2}`, без `container_name` и портов; nginx 1.28 `upstream api { server api:8080 resolve; }` — новые/остановленные контейнеры подхватываются без reload | [compose.yaml](compose.yaml), [infra/nginx/nginx.conf](infra/nginx/nginx.conf) |
| X-Instance-Id | случайный alias процесса (`secrets.token_hex(6)`), без hostname/IP; в каждом ответе API; процесс пишет его в `/tmp/instance-id` для оператора | [request_context.py](src/application/api/middlewares/request_context.py), [settings.py](src/config/settings.py) |
| X-Request-Id | nginx передаёт `$request_id`, API принимает его (только 32 hex) — одна строка access-лога LB и журнал реплики связаны одним ID | там же, [evidence/04](evidence/04-balancing-idempotency.txt) |
| Idempotency-Key | обязательный заголовок `[A-Za-z0-9._:-]{1,128}` (нет/неверный — 400); таблица `message_idempotency_keys`, PK `(sender_id, chat_id, key)`, SHA-256 канонического тела, TTL `IDEMPOTENCY_TTL_SECONDS` = 24 ч | [0002_message_idempotency.sql](migrations/0002_message_idempotency.sql), [postgres.py](src/infrastructure/repositories/message_repository/postgres.py), [message_service.py](src/infrastructure/services/message_service/message_service.py) |
| Миграции | one-shot сервис `migrate` (`src/migrate.py`) до реплик (`service_completed_successfully`); реплика миграции не применяет, а проверяет версию схемы и не стартует, если схема старее кода | [database.py](src/infrastructure/dependencies/database.py), [migrate.py](src/migrate.py) |
| Graceful shutdown | SIGTERM → drain 1 с (порт закрыт, readiness 503, ответы по открытым keep-alive с `Connection: close`) → graceful uvicorn до 5 с → закрытие пулов | [gateway.py](src/gateway.py) |
| Ошибки LB | 502/503/504, созданные nginx, отдаются как `503 {"error":{"code":"unavailable",…}}` с `X-Request-Id`; ответы реплик проходят без изменений | [infra/nginx/errors.conf](infra/nginx/errors.conf) |
**Транзакционная граница идемпотентности.** В одной транзакции: `pg_advisory_xact_lock(чат)` → поиск действующего ключа → (повтор: вернуть сохранённое сообщение или 409 при другом теле) → `INSERT` сообщения → `INSERT` ключа. Ключ привязан к чату, поэтому блокировка чата, уже нужная для монотонного `created_at`, сериализует и одновременные повторы с разных реплик: второй ждёт COMMIT первого и находит ключ (READ COMMITTED). Ошибка до COMMIT откатывает и сообщение, и ключ — транзиентный 5xx не фиксируется как результат. Повтор заново проходит проверку сессии и членства. Область «пользователь + метод + путь + ключ» сведена к `(sender_id, chat_id, key)`: метод и путь фиксированы. Порядок ключей и пробелы JSON не влияют (хеш от тела, сериализованного по схеме). Истёкший ключ переиспользуется (`ON CONFLICT … DO UPDATE`) и создаёт новое сообщение.
**Почему nginx не повторяет POST сам.** `proxy_next_upstream error timeout` без `non_idempotent`: запрос, не дошедший до реплики (connect refused/timeout), nginx отправляет следующей реплике — это безопасно для любого метода. POST, уже отправленный реплике, не повторяется: прозрачный повтор `POST /users`/`/chats` создал бы дубль (контракт это запрещает). Неопределённый POST сообщения повторяет клиент с тем же ключом. 503 приложения тоже не повторяется и не исключает реплику: он означает отказ общей зависимости, другая реплика зависит от той же БД.
### Health, readiness, таймауты и время обнаружения
| Механизм | Значение | Назначение |
| --- | --- | --- |
| `/health/live` | 200 пока жив процесс | liveness, проверки через LB |
| `/health/ready` | 200 если не завершается и PostgreSQL отвечает (Valkey не критичен — сессии читаются из БД) | Docker healthcheck API: `interval 5s, timeout 3s, retries 2, start_period 20s, start_interval 1s`; используется `up --wait`, `depends_on` nginx, `restore.sh`, `rolling-update.sh` |
| Порт реплики | открывается только после startup (БД + версия схемы), закрывается первым при SIGTERM | LB не шлёт трафик в неготовую/завершающуюся реплику |
| nginx `proxy_connect_timeout` | 0,5 с | исчезнувший IP (SIGKILL/крах контейнера) |
| nginx `proxy_read_timeout` / `send` | 10 с | зависшая реплика; дольше самого долгого запроса API (ожидание пула 3 с + statement_timeout 5 с) |
| `max_fails=1 fail_timeout=5s` | | после ошибки соединения реплика исключена на 5 с, затем пробный запрос |
| `resolver 127.0.0.11 valid=2s` | | остановленный контейнер пропадает из upstream ≤ 2 с, новый появляется ≤ 2 с |
| keep-alive | nginx → API `keepalive_timeout 60s` < uvicorn `timeout_keep_alive 75 s` | idle-соединение закрывает LB, а не реплика: нет гонки «запрос в закрытое соединение» |
| SIGTERM | drain 1 с + graceful ≤ 5 с < `stop_grace_period 10s` | SIGKILL не прерывает завершение |
У nginx open source нет активных health checks — проверка пассивная, по реальным запросам. **Максимальное время обнаружения:** остановка процесса/SIGTERM — 0 с (порт закрыт → `connection refused` → тот же запрос сразу уходит на другую реплику, клиент ошибки не видит); крах контейнера — ≤ 0,5 с на запрос (connect timeout), затем исключение на 5 с и удаление из DNS ≤ 2 с; зависание без закрытия порта — ≤ 10 с (read timeout; GET повторяется, POST получает 503 и повторяется клиентом с тем же ключом). Docker healthcheck обнаруживает неготовность за ≤ 5 с × 2 = 10 с, но в Compose он не исключает контейнер из DNS — поэтому балансировку обеспечивает nginx, а не `depends_on`/healthcheck.
### Ресурсы, сети, секреты
| Роль | Service | Образ | CPU / RAM | Сети | Порт наружу | Volume | Credentials |
| --- | --- | --- | --- | --- | --- | --- | --- |
| LB | `nginx` | nginx:1.28.0-alpine3.21 | 0,5 / 64M | edge | 127.0.0.1:8080 (+8081 только для `test-expiry`) | — | нет |
| API ×2 | `api` | mtusi-messenger:lab3 (из исходников) | 1,0 / 512M каждая | edge, backend | нет | — | DB_PASS, VALKEY_PASS из `.env` |
| Миграции | `migrate` | тот же | 0,5 / 256M | backend | нет | — | DB_PASS |
| БД | `postgres` | postgres:18.2-alpine3.22 | 1,5 / 512M | backend | нет | `postgres_data` | POSTGRES_PASSWORD = DB_PASS |
| Кэш сессий | `valkey` | valkey/valkey:8.1.4-alpine3.22 | 0,25 / 96M | backend | нет | нет (не источник истины) | VALKEY_PASS |
Сеть `backend` — `internal`: nginx не видит БД и Valkey (даже не резолвит имя), хранилища не имеют выхода наружу. Порт PostgreSQL больше не публикуется на хост (в lab2 был 127.0.0.1:5432); тесты обращаются к БД через `docker compose exec`. Подтверждение: [evidence/11-limits-ports-networks.txt](evidence/11-limits-ports-networks.txt). 512M на реплику: Argon2id RFC 9106 low-memory — 64 MiB × `PASSWORD_HASH_WORKERS=4`. Пул БД: 2 реплики × `DB_POOL_MAX_SIZE=10` + migrate + реплика при rolling update = 40 ≤ `max_connections=100`.
### Соответствие .env.example
| Переменная | Используется | Примечание |
| --- | --- | --- |
| `APP_HOST`, `APP_PORT` | uvicorn | 0.0.0.0:8080 внутри контейнера |
| `APP_ENV`, `LOG_LEVEL` | журнал | |
| `APP_KEEPALIVE_TIMEOUT_SEC=75` | uvicorn `timeout_keep_alive` | новое; > keepalive_timeout nginx |
| `SHUTDOWN_TIMEOUT_SEC=5` | uvicorn `timeout_graceful_shutdown` | |
| `SHUTDOWN_DRAIN_SEC=1` | `DrainingServer` | новое |
| `DB_*` | asyncpg pool | `DB_HOST/DB_PORT` переопределены в compose (`postgres:5432`); `DB_PORT` больше не публикуется |
| `VALKEY_*` | redis-py | `VALKEY_HOST/PORT` переопределены в compose |
| `SESSION_TTL_SECONDS`, `SESSION_CACHE_TTL_SECONDS`, `PASSWORD_HASH_WORKERS` | как в lab2 | `api-short-ttl` переопределяет TTL = 2 |
| `API_REPLICAS=2` | `deploy.replicas` | |
| `IDEMPOTENCY_TTL_SECONDS=86400` | срок ключа | |
| `LB_PORT=8080` (+ `LB_EXPIRY_PORT`, по умолчанию 8081) | публикация nginx | новое |
| `APP_REVISION` | build arg → метка образа и журнал старта | не в `.env`; задаёт `rolling-update.sh` |
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Формат файлов — транскрипт терминала: `$ команда`, затем сырой вывод.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Публичный smoke | `make check test` | PASS lab3 public API scenario | [02](evidence/02-check-smoke.txt) |
| 3: стенд одной командой, ≥2 API, один вход, общие БД и сессии | `make up`, `docker compose ps`, `make instances` | 2 реплики healthy, migrate Exited (0), наружу только nginx | [01](evidence/01-up.txt) |
| 3: одна сессия → разные реплики без повторного входа | `pytest test_balancing.py -s`, access-лог nginx | 20 запросов одним токеном: 10/10 по двум `X-Instance-Id`; logout на одной реплике действует на всех | [04](evidence/04-balancing-idempotency.txt) |
| 3: остановка любой реплики, чтение/запись ≤ 10 с | `make test-failover` для `api-1`, затем для `api-2` | 129 / 128 / 129 успешных записей, **0 неудачных попыток**, первая запись после возврата hook 1,6–2,7 с; `instances` и `docker inspect` показывают реально остановленный контейнер (exit 0, журнал drain/shutdown) | [05a](evidence/05-failover-replica-a.txt), [05b](evidence/05-failover-replica-b.txt) |
| 3: Idempotency-Key без дублей, включая конкурентный повтор и смену реплики | `pytest test_idempotency.py -s` | 20 одновременных POST с одним ключом через обе реплики (10/10): все 201, одно тело, 1 строка в БД; разные тела под одним ключом — один победитель, остальные 409; 400 для нет/неверного ключа; область user+chat; повтор после logout — 401, чужой — 404; истёкший ключ создаёт новое | [04](evidence/04-balancing-idempotency.txt) |
| 3: persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; `StartedAt` всех 5 контейнеров изменился, volume тот же (created 16:10:27Z) | [06](evidence/06-persistence.txt) |
| Регрессия lab2 | `make student-test`, `test-expiry`, `test-failures` | 107 passed; TTL 2 с через LB :8081; отказ Valkey/PostgreSQL как в lab2 | [03](evidence/03-student-tests.txt), [07](evidence/07-expiry.txt), [08](evidence/08-dependency-failures.txt) |
| 4: health/readiness/таймауты/SIGTERM согласованы; LB исключает backend | `make sigterm-check` | exit 0 за 1,53 с; 354 запроса через LB после SIGTERM без ошибок; после выхода реплика не отвечает | [10](evidence/10-sigterm.txt) |
| 4: поочерёдная остановка каждой реплики, RPS/ошибки/восстановление | `make experiment-failover` | см. «Эксперименты»: SIGTERM — 0 ошибок; SIGKILL — 1 и 3 ошибки 503, повтор тем же ключом; потерь/дублей 0 | [13](evidence/13-experiment-failover.txt), [json](evidence/experiment-failover.json) |
| 4: идемпотентность параллельными запросами | `test_parallel_*` | см. строку уровня 3 | [04](evidence/04-balancing-idempotency.txt) |
| 4: лимиты CPU/RAM, API без портов, сети/секреты, нет `container_name` | `docker inspect`, `docker network inspect` | лимиты у всех контейнеров; порты только у nginx; backend internal | [11](evidence/11-limits-ports-networks.txt) |
| 4: rolling update под нагрузкой | `make experiment-rolling` | обе реплики заменены новой ревизией, 0 ошибок, наибольший разрыв между подтверждениями записи 0,098 с | [14](evidence/14-experiment-rolling.txt), [json](evidence/experiment-rolling.json) |
| 4: миграции однократно и совместимы с работающей версией | `make test-migrations`, `scripts/check_previous_version.sh` | с пустого volume, 3 повтора идемпотентны; API до migrate отказывается стартовать; **lab2-код (`aa971fb`) работает на схеме v2**: вход, чат, запись, чтение | [09](evidence/09-migrations.txt) |
| Чистый clone | копия файлов Git, отдельный проект | build → smoke → failover → persistence PASS, затем `down -v` | [15](evidence/15-clean-clone.txt) |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Гипотеза.** При остановке одной из двух реплик (SIGTERM) клиент не видит ошибок: порт закрывается раньше, чем реплика перестаёт отвечать, и nginx переотправляет ещё не доставленные запросы; при SIGKILL теряются только запросы, бывшие в обработке у убитой реплики, а повтор с тем же ключом не создаёт дублей. Пропускная способность при одной реплике падает, так как API ограничена 1 CPU.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
**Методика** ([scripts/experiment.py](scripts/experiment.py)). Стенд из таблицы ресурсов на Apple M4 / Docker Desktop VM 10 CPU, 8 GiB. 4 потока closed loop через LB: `POST` сообщения (неподтверждённый POST повторяется с тем же ключом и телом каждые 50 мс до 201) + `GET ?limit=1`; timeout клиента 5 с; новый чат на прогон (≈ 10–12 тыс. сообщений за прогон). Фазы: 8 с до отказа → отказ (`docker stop` = SIGTERM, затем `docker kill -s SIGKILL`) → 10 с с одной репликой → `scripts/restore.sh` → 8 с. Порядок: `api-1`, `api-2` для каждого сигнала, с восстановлением предыдущей. После прогона история читается целиком: каждое подтверждённое сообщение на месте и не изменено, нет дублей id/текстов. «Время восстановления записи» — от первой ошибки после отказа до первой успешной записи после последней ошибки; «разрыв» — наибольший интервал между подтверждениями записи. Один прогон на сценарий; CPU/RAM — `docker stats` в середине фазы «до».
| Воздействие | Что ожидаем | Что наблюдали (запись ok RPS до / во время / после) | Ошибки, восстановление, данные |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| SIGTERM `api-1` | 0 ошибок | 382 / 277 / 399 | 0 ошибок; разрыв 0,063 с; реплика в ротации через 0,97 с после restore; потерь/дублей 0 |
| SIGTERM `api-2` | 0 ошибок | 401 / 282 / 362 | 0 ошибок; разрыв 0,071 с; в ротации через 3,7 с; 0 / 0 |
| SIGKILL `api-1` | ошибки только у запросов в обработке | 427 / 281 / 408 | 1 × 503 (POST, оборван убитой репликой), восстановление < 0,01 с, разрыв 0,070 с; повтор ключом создал сообщение один раз; 0 / 0 |
| SIGKILL `api-2` | то же | 387 / 277 / 373 | 3 × 503, восстановление 0,006 с, разрыв 0,061 с; 0 / 0 |
| Rolling update 2 → 3 → 2 → 3 → 2 | 0 ошибок | 420 / 308 / 345 | 0 ошибок; разрыв 0,098 с; старые alias исчезли через 8,1 и 14,9 с от начала, новые вошли на 4,9 и 13,9 с; обе реплики на новой ревизии; 0 / 0 |
| Valkey / PostgreSQL недоступны | обход кэша / ограниченный 503 | как в lab2, через LB | [08](evidence/08-dependency-failures.txt) |
| Рестарт всего стенда | данные и токен сохраняются | PASS | [06](evidence/06-persistence.txt) |
Нагрузка до отказа: API ≈ 84 % от лимита 1 CPU каждая, PostgreSQL ≈ 28 %, nginx ≈ 9 %, память API ≈ 48 MiB. Во время отказа запись падает на ≈ 30 % (не вдвое: при двух репликах узким местом частично являются клиент и БД). Базовый публичный генератор (`tests/load.py`, 30 с, concurrency 4, только POST): 713 успешных RPS, 0 ошибок, p50/p95/p99 = 4,7 / 8,4 / 22,5 мс — [evidence/12](evidence/12-load-baseline.txt), [load.json](evidence/load.json).
**Подтверждённое улучшение — drain при SIGTERM.** Без drain uvicorn по SIGTERM сразу закрывает idle keep-alive соединения, и nginx изредка отправлял запрос в уже закрываемое соединение: в пробном rolling update был `upstream prematurely closed connection` (GET nginx повторил; POST в той же ситуации получил бы 503). С drain в итоговых прогонах SIGTERM и rolling update таких событий 0 — все 4 случая «prematurely closed» в журнале nginx пришлись на прогоны SIGKILL (сопоставлено по времени, где это ожидаемо).
**Ограничения измерения.** Closed loop с 4 потоками страдает coordinated omission и ограничен клиентом на той же машине; по одному прогону на сценарий, без доверительных интервалов; RPS — подтверждения HTTP, а не COMMIT. Числа описывают этот ноутбук, не универсальную ёмкость.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **Выдерживается:** отказ/остановка/крах одного API-процесса или контейнера, поочерёдная замена реплик, перезапуск всего стенда с volumes, отказ Valkey. **Не выдерживается:** отказ nginx, PostgreSQL (одна реплика БД — SPOF; API честно отвечает 503) и Docker-host — все контейнеры на одном узле. Отказ процесса ≠ отказ узла: здесь проверен только первый.
- nginx open source — только пассивные проверки: реплика, которая принимает соединения, но не отвечает, обнаруживается за `proxy_read_timeout` (10 с) и снова получает пробный запрос каждые 5 с; healthcheck Docker в Compose её не исключает. Активные проверки дал бы HAProxy/Traefik или nginx Plus.
- `docker stop`/`docker kill` не перезапускаются политикой `unless-stopped` (так и задумано для `stop-one.sh`); автоматического восстановления упавшей реплики Compose не даёт — `make restore` выполняет оператор.
- Истёкшие ключи идемпотентности удаляются только при повторном использовании ключа; фоновой очистки нет (строки маленькие, но таблица растёт).
- Совместимость миграций обеспечивается правилом expand-only и проверена для пары lab2 → lab3 на уровне схемы; поведение API при смешанных версиях (старая реплика не требует ключа и не отдаёт `X-Instance-Id`) во время такой выкладки различается — это переход контракта, а не схемы.
- `api-short-ttl` для `make test-expiry` тоже за LB (порт 8081), но 8081 публикуется nginx постоянно; без профиля `expiry` он отвечает 503.
+93 -16
View File
@@ -1,27 +1,100 @@
name: messenger-lab3
x-api: &api
build: .
x-app: &app
build:
context: .
args:
APP_REVISION: ${APP_REVISION:-dev}
image: mtusi-messenger:lab3
restart: unless-stopped
env_file:
- .env
environment: &api-network
environment: &app-network
DB_HOST: postgres
DB_PORT: "5432"
VALKEY_HOST: valkey
VALKEY_PORT: "6379"
x-api-resources: &api-resources
resources:
limits:
cpus: "1.0"
memory: 512M
reservations:
cpus: "0.25"
memory: 128M
x-api: &api
<<: *app
restart: unless-stopped
depends_on:
migrate:
condition: service_completed_successfully
postgres:
condition: service_healthy
valkey:
condition: service_started
networks:
- edge
- backend
- default
healthcheck:
test: ["CMD", "python", "src/healthcheck.py"]
interval: 5s
timeout: 3s
retries: 2
start_period: 20s
start_interval: 1s
stop_grace_period: 10s
services:
nginx:
image: nginx:1.28.0-alpine3.21
restart: unless-stopped
depends_on:
api:
condition: service_healthy
ports:
- "127.0.0.1:${LB_PORT:-8080}:8080"
- "127.0.0.1:${LB_EXPIRY_PORT:-8081}:8081"
volumes:
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
- ./infra/nginx/errors.conf:/etc/nginx/errors.conf:ro
networks:
- edge
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8090/healthz"]
interval: 5s
timeout: 2s
retries: 3
stop_grace_period: 10s
deploy:
resources:
limits:
cpus: "0.5"
memory: 64M
api:
<<: *api
deploy:
<<: *api-resources
replicas: ${API_REPLICAS:-2}
migrate:
<<: *app
command: ["python", "src/migrate.py"]
restart: "no"
depends_on:
postgres:
condition: service_healthy
networks:
- backend
healthcheck:
disable: true
deploy:
resources:
limits:
cpus: "0.5"
memory: 256M
postgres:
image: postgres:18.2-alpine3.22
restart: unless-stopped
@@ -37,12 +110,14 @@ services:
timeout: 3s
retries: 15
start_period: 10s
ports:
- "127.0.0.1:${DB_PORT}:5432"
networks:
- backend
- default
stop_grace_period: 30s
deploy:
resources:
limits:
cpus: "1.5"
memory: 512M
valkey:
image: valkey/valkey:8.1.4-alpine3.22
@@ -68,23 +143,25 @@ services:
retries: 15
networks:
- backend
api:
<<: *api
ports:
- "127.0.0.1:${APP_PORT}:${APP_PORT}"
deploy:
resources:
limits:
cpus: "0.25"
memory: 96M
api-short-ttl:
<<: *api
profiles:
- expiry
environment:
<<: *api-network
<<: *app-network
SESSION_TTL_SECONDS: "2"
ports:
- "127.0.0.1:8081:${APP_PORT}"
deploy:
<<: *api-resources
replicas: 1
networks:
edge: {}
backend:
internal: true
+21
View File
@@ -0,0 +1,21 @@
$ date -u +%Y-%m-%dT%H:%M:%SZ
2026-10-19T16:59:47Z
$ git log -1 --format="%h %s" && git status --short | wc -l | xargs echo "uncommitted files:"
aa971fb feat: перенос core-логики 2 лабы
uncommitted files: 42
$ sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize
Apple M4
10
17179869184
$ docker version --format "Docker Engine {{.Server.Version}}, client {{.Client.Version}}"
Docker Engine 29.0.1, client 29.0.1
$ docker compose version
Docker Compose version v2.40.3-desktop.1
$ docker info --format "Docker VM: {{.NCPU}} CPU, {{.MemTotal}} bytes RAM, {{.OperatingSystem}}, kernel {{.KernelVersion}}"
Docker VM: 10 CPU, 8217448448 bytes RAM, Docker Desktop, kernel 6.12.54-linuxkit
$ python3 --version
Python 3.9.6
$ poetry run python --version
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
warnings.warn(
Python 3.12.12
+150
View File
@@ -0,0 +1,150 @@
$ docker compose --env-file .env config --quiet && echo "compose config: OK"
compose config: OK
$ make up
docker compose up -d --build --wait
#1 [internal] load local bake definitions
#1 reading from stdin 1.17kB done
#1 DONE 0.0s
#2 [api internal] load build definition from Dockerfile
#2 transferring dockerfile: 914B done
#2 DONE 0.0s
#3 [api internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm
#3 DONE 0.3s
#4 [migrate internal] load .dockerignore
#4 transferring context: 358B done
#4 DONE 0.0s
#5 [migrate 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done
#5 DONE 0.0s
#6 [api internal] load build context
#6 transferring context: 16.17kB done
#6 DONE 0.0s
#7 [api 3/7] COPY poetry.lock pyproject.toml ./
#7 CACHED
#8 [api 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction
#8 CACHED
#9 [api 2/7] WORKDIR /app
#9 CACHED
#10 [api 5/7] COPY migrations ./migrations
#10 CACHED
#11 [migrate 6/7] COPY src ./src
#11 DONE 0.0s
#12 [migrate 7/7] RUN useradd --system --uid 10001 --no-create-home app
#12 0.110 useradd warning: app's uid 10001 is greater than SYS_UID_MAX 999
#12 DONE 0.1s
#13 [api] exporting to image
#13 exporting layers 0.0s done
#13 exporting manifest sha256:8d0545b5703e6c70c35f1136bdfd37c4ceb4a70d08b5bad21b70d6755df05b5d done
#13 exporting config sha256:28c6ec56ed30844b944958f2749a370cea79cfd768ca3f491807eae51ad534a0 done
#13 naming to docker.io/library/mtusi-messenger:lab3 done
#13 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done
#13 DONE 0.1s
#14 [migrate] exporting to image
#14 exporting layers 0.0s done
#14 exporting manifest sha256:9802819ef7b235f44ad03fed838cf83fcd4573a342405936538c7c969adfd573 done
#14 exporting config sha256:e2b0c133c10bc42d53160db701c32694ed2cbac54f0522999289f4cf7cef21e1 done
#14 naming to docker.io/library/mtusi-messenger:lab3 done
#14 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done
#14 DONE 0.1s
#15 [api] resolving provenance for metadata file
#15 DONE 0.0s
#16 [migrate] resolving provenance for metadata file
#16 DONE 0.0s
mtusi-messenger:lab3 Built
mtusi-messenger:lab3 Built
Network messenger-lab3_edge Creating
Network messenger-lab3_edge Created
Network messenger-lab3_backend Creating
Network messenger-lab3_backend Created
Container messenger-lab3-postgres-1 Creating
Container messenger-lab3-valkey-1 Creating
Container messenger-lab3-valkey-1 Created
Container messenger-lab3-postgres-1 Created
Container messenger-lab3-migrate-1 Creating
Container messenger-lab3-migrate-1 Created
Container messenger-lab3-api-2 Creating
Container messenger-lab3-api-1 Creating
Container messenger-lab3-api-1 Created
Container messenger-lab3-api-2 Created
Container messenger-lab3-nginx-1 Creating
Container messenger-lab3-nginx-1 Created
Container messenger-lab3-postgres-1 Starting
Container messenger-lab3-valkey-1 Starting
Container messenger-lab3-valkey-1 Started
Container messenger-lab3-postgres-1 Started
Container messenger-lab3-postgres-1 Waiting
Container messenger-lab3-postgres-1 Healthy
Container messenger-lab3-migrate-1 Starting
Container messenger-lab3-migrate-1 Started
Container messenger-lab3-migrate-1 Waiting
Container messenger-lab3-postgres-1 Waiting
Container messenger-lab3-postgres-1 Healthy
Container messenger-lab3-migrate-1 Exited
Container messenger-lab3-api-1 Starting
Container messenger-lab3-api-1 Started
Container messenger-lab3-api-2 Starting
Container messenger-lab3-api-2 Started
Container messenger-lab3-api-1 Waiting
Container messenger-lab3-api-2 Waiting
Container messenger-lab3-api-1 Healthy
Container messenger-lab3-api-2 Healthy
Container messenger-lab3-nginx-1 Starting
Container messenger-lab3-nginx-1 Started
Container messenger-lab3-valkey-1 Waiting
Container messenger-lab3-nginx-1 Waiting
Container messenger-lab3-api-1 Waiting
Container messenger-lab3-api-2 Waiting
Container messenger-lab3-migrate-1 Waiting
Container messenger-lab3-postgres-1 Waiting
Container messenger-lab3-migrate-1 Exited
Container messenger-lab3-postgres-1 Healthy
Container messenger-lab3-valkey-1 Healthy
Container messenger-lab3-api-1 Healthy
Container messenger-lab3-api-2 Healthy
Container messenger-lab3-nginx-1 Healthy
$ docker compose ps -a
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
messenger-lab3-api-1 mtusi-messenger:lab3 "python src/gateway.…" api 15 seconds ago Up 7 seconds (healthy) 8080/tcp
messenger-lab3-api-2 mtusi-messenger:lab3 "python src/gateway.…" api 15 seconds ago Up 7 seconds (healthy) 8080/tcp
messenger-lab3-migrate-1 mtusi-messenger:lab3 "python src/migrate.…" migrate 15 seconds ago Exited (0) 7 seconds ago
messenger-lab3-nginx-1 nginx:1.28.0-alpine3.21 "/docker-entrypoint.…" nginx 15 seconds ago Up 5 seconds (healthy) 127.0.0.1:8080-8081->8080-8081/tcp
messenger-lab3-postgres-1 postgres:18.2-alpine3.22 "docker-entrypoint.s…" postgres 15 seconds ago Up 14 seconds (healthy) 5432/tcp
messenger-lab3-valkey-1 valkey/valkey:8.1.4-alpine3.22 "docker-entrypoint.s…" valkey 15 seconds ago Up 14 seconds (healthy) 6379/tcp
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy 978f2777b379 95632 dev
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
$ docker compose logs migrate
migrate-1 | 2026-10-19 17:00:03.868 | INFO | infrastructure.dependencies.database:create_pool:69 - Установка соединения с базой данных psql
migrate-1 | 2026-10-19 17:00:03.937 | INFO | infrastructure.dependencies.database:create_pool:99 - Соединение с базой данных psql установлено. DB_NAME: test, pool: 2..10
migrate-1 | 2026-10-19 17:00:03.939 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии
migrate-1 | 2026-10-19 17:00:03.940 | INFO | __main__:main:20 - Миграции применены, migrate завершается
$ docker compose logs api | grep -E "Схема БД|Application started"
api-2 | 2026-10-19 17:00:05.191 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2
api-2 | 2026-10-19 17:00:05.226 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=389b271f592b, session ttl=3600 s
api-1 | 2026-10-19 17:00:05.106 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2
api-1 | 2026-10-19 17:00:05.173 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=978f2777b379, session ttl=3600 s
$ curl -s -o /dev/null -w "%{http_code}\n" --max-time 2 http://localhost:8080/health/ready
200
$ docker compose exec -T nginx sh -c "wget -q -O - --timeout 2 http://api:8080/health/live; echo; nslookup api 127.0.0.11 | tail -4"
{"status":"ok"}
Address: 192.168.48.3
Name: api
Address: 192.168.48.2
+6
View File
@@ -0,0 +1,6 @@
$ make check
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
$ make test
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
+128
View File
@@ -0,0 +1,128 @@
$ make student-test
BASE_URL="http://localhost:8080" poetry run pytest
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
warnings.warn(
============================= test session starts ==============================
platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python
cachedir: .pytest_cache
rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3
configfile: pyproject.toml
testpaths: student-tests
plugins: anyio-4.15.1
collecting ... collected 113 items
student-tests/test_auth.py::test_login_returns_opaque_bearer_session PASSED [ 0%]
student-tests/test_auth.py::test_wrong_password_and_unknown_user_are_indistinguishable PASSED [ 1%]
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[None] PASSED [ 2%]
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[] PASSED [ 3%]
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic] PASSED [ 4%]
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic !!!not-base64!!!] PASSED [ 5%]
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic bm8tY29sb24taGVyZQ==] PASSED [ 6%]
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic //46cGFzc3dvcmQ=] PASSED [ 7%]
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Bearer some-token] PASSED [ 7%]
student-tests/test_auth.py::test_utf8_password_with_colon PASSED [ 8%]
student-tests/test_auth.py::test_basic_is_rejected_on_business_endpoints PASSED [ 9%]
student-tests/test_auth.py::test_logout_revokes_only_current_session PASSED [ 10%]
student-tests/test_auth.py::test_logout_without_token PASSED [ 11%]
student-tests/test_auth.py::test_relogin_after_logout PASSED [ 12%]
student-tests/test_auth.py::test_password_is_not_stored_in_plaintext PASSED [ 13%]
student-tests/test_balancing.py::test_one_session_is_served_by_several_replicas PASSED [ 14%]
student-tests/test_balancing.py::test_write_is_visible_through_every_replica PASSED [ 15%]
student-tests/test_balancing.py::test_logout_applies_to_all_replicas PASSED [ 15%]
student-tests/test_balancing.py::test_every_api_response_has_instance_id PASSED [ 16%]
student-tests/test_concurrency.py::test_parallel_messages_are_not_lost_or_duplicated PASSED [ 17%]
student-tests/test_dependency_failures.py::test_valkey_down_cache_is_bypassed SKIPPED [ 18%]
student-tests/test_dependency_failures.py::test_logout_during_valkey_outage_is_not_resurrected SKIPPED [ 19%]
student-tests/test_dependency_failures.py::test_postgres_down_returns_bounded_503_and_no_anonymous_access SKIPPED [ 20%]
student-tests/test_dependency_failures.py::test_session_survives_postgres_restart SKIPPED [ 21%]
student-tests/test_expiry.py::test_session_expires_after_short_ttl SKIPPED [ 22%]
student-tests/test_expiry.py::test_expires_at_matches_short_ttl SKIPPED [ 23%]
student-tests/test_foreign_chat.py::test_outsider_cannot_access_chat PASSED [ 23%]
student-tests/test_foreign_chat.py::test_foreign_and_missing_chat_are_indistinguishable PASSED [ 24%]
student-tests/test_foreign_chat.py::test_chat_list_contains_only_own_chats PASSED [ 25%]
student-tests/test_health.py::test_health_endpoints_have_request_id PASSED [ 26%]
student-tests/test_health.py::test_unknown_route_uses_error_format PASSED [ 27%]
student-tests/test_idempotency.py::test_replay_returns_same_message PASSED [ 28%]
student-tests/test_idempotency.py::test_replay_ignores_json_formatting PASSED [ 29%]
student-tests/test_idempotency.py::test_same_key_with_other_body_is_conflict PASSED [ 30%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers0] PASSED [ 30%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers1] PASSED [ 31%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers2] PASSED [ 32%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers3] PASSED [ 33%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers4] PASSED [ 34%]
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers5] PASSED [ 35%]
student-tests/test_idempotency.py::test_key_boundaries_are_accepted[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 36%]
student-tests/test_idempotency.py::test_key_boundaries_are_accepted[Az09._:-] PASSED [ 37%]
student-tests/test_idempotency.py::test_key_scope_is_user_and_chat PASSED [ 38%]
student-tests/test_idempotency.py::test_replay_rechecks_session_and_access PASSED [ 38%]
student-tests/test_idempotency.py::test_expired_key_can_be_reused PASSED [ 39%]
student-tests/test_idempotency.py::test_parallel_replays_through_replicas_create_one_message PASSED [ 40%]
student-tests/test_idempotency.py::test_parallel_same_key_with_different_bodies PASSED [ 41%]
student-tests/test_invalid_json.py::test_malformed_json_on_register[{"username": "abc",] PASSED [ 42%]
student-tests/test_invalid_json.py::test_malformed_json_on_register[not json at all] PASSED [ 43%]
student-tests/test_invalid_json.py::test_malformed_json_on_register[{"text": "\xff\xfe"}] PASSED [ 44%]
student-tests/test_invalid_json.py::test_malformed_json_on_register[] PASSED [ 45%]
student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[{"username": "abc",] PASSED [ 46%]
student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[not json at all] PASSED [ 46%]
student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[{"text": "\xff\xfe"}] PASSED [ 47%]
student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[] PASSED [ 48%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body0] PASSED [ 49%]
student-tests/test_invalid_json.py::test_invalid_user_payload[alice] PASSED [ 50%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body2] PASSED [ 51%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body3] PASSED [ 52%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body4] PASSED [ 53%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body5] PASSED [ 53%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body6] PASSED [ 54%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body7] PASSED [ 55%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body8] PASSED [ 56%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body9] PASSED [ 57%]
student-tests/test_invalid_json.py::test_invalid_user_payload[body10] PASSED [ 58%]
student-tests/test_invalid_json.py::test_invalid_message_payload[body0] PASSED [ 59%]
student-tests/test_invalid_json.py::test_invalid_message_payload[body1] PASSED [ 60%]
student-tests/test_invalid_json.py::test_invalid_message_payload[body2] PASSED [ 61%]
student-tests/test_invalid_json.py::test_invalid_chat_payload PASSED [ 61%]
student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[] PASSED [ 62%]
student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[ ] PASSED [ 63%]
student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[ \n\t ] PASSED [ 64%]
student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[\u3000] PASSED [ 65%]
student-tests/test_messages.py::test_message_length_is_counted_in_code_points PASSED [ 66%]
student-tests/test_messages.py::test_unicode_is_preserved PASSED [ 67%]
student-tests/test_messages.py::test_nul_character_is_rejected PASSED [ 68%]
student-tests/test_messages.py::test_sender_is_taken_from_session PASSED [ 69%]
student-tests/test_pagination.py::test_walk_message_pages[7-3-3] PASSED [ 69%]
student-tests/test_pagination.py::test_walk_message_pages[6-3-2] PASSED [ 70%]
student-tests/test_pagination.py::test_walk_message_pages[5-1-5] PASSED [ 71%]
student-tests/test_pagination.py::test_walk_message_pages[3-100-1] PASSED [ 72%]
student-tests/test_pagination.py::test_limit_can_change_between_pages PASSED [ 73%]
student-tests/test_pagination.py::test_new_message_appears_after_cursor PASSED [ 74%]
student-tests/test_pagination.py::test_cursor_is_bound_to_chat PASSED [ 75%]
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params0] PASSED [ 76%]
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params1] PASSED [ 76%]
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params2] PASSED [ 77%]
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params3] PASSED [ 78%]
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params4] PASSED [ 79%]
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params5] PASSED [ 80%]
student-tests/test_pagination.py::test_walk_chat_list_pages PASSED [ 81%]
student-tests/test_pagination.py::test_chat_cursor_is_bound_to_user PASSED [ 82%]
student-tests/test_transactions.py::test_same_username_race_creates_exactly_one_user PASSED [ 83%]
student-tests/test_transactions.py::test_failed_chat_creation_is_rolled_back PASSED [ 84%]
student-tests/test_transactions.py::test_successful_chat_creation_commits_all_members PASSED [ 84%]
student-tests/test_users.py::test_duplicate_username_is_conflict PASSED [ 85%]
student-tests/test_users.py::test_registration_never_returns_password PASSED [ 86%]
student-tests/test_users.py::test_invalid_password_is_rejected[None] PASSED [ 87%]
student-tests/test_users.py::test_invalid_password_is_rejected[] PASSED [ 88%]
student-tests/test_users.py::test_invalid_password_is_rejected[\u043a\u043e\u0440\u043e\u0442\u043a\u0438\u0439] PASSED [ 89%]
student-tests/test_users.py::test_invalid_password_is_rejected[xxxxxxxxxxx] PASSED [ 90%]
student-tests/test_users.py::test_invalid_password_is_rejected[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 91%]
student-tests/test_users.py::test_invalid_password_is_rejected[123456789012] PASSED [ 92%]
student-tests/test_users.py::test_password_length_is_counted_in_code_points PASSED [ 92%]
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers0] PASSED [ 93%]
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers1] PASSED [ 94%]
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers2] PASSED [ 95%]
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers3] PASSED [ 96%]
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers4] PASSED [ 97%]
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers5] PASSED [ 98%]
student-tests/test_users.py::test_x_user_id_gives_no_access PASSED [ 99%]
student-tests/test_users.py::test_current_user PASSED [100%]
======================= 107 passed, 6 skipped in 14.98s ========================
+50
View File
@@ -0,0 +1,50 @@
$ BASE_URL=http://localhost:8080 poetry run pytest student-tests/test_balancing.py student-tests/test_idempotency.py student-tests/test_concurrency.py -s
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
warnings.warn(
============================= test session starts ==============================
platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python
cachedir: .pytest_cache
rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3
configfile: pyproject.toml
plugins: anyio-4.15.1
collecting ... collected 21 items
student-tests/test_balancing.py::test_one_session_is_served_by_several_replicas
INSTANCES FOR ONE SESSION: {'389b271f592b': 10, '978f2777b379': 10}
PASSED
student-tests/test_balancing.py::test_write_is_visible_through_every_replica PASSED
student-tests/test_balancing.py::test_logout_applies_to_all_replicas PASSED
student-tests/test_balancing.py::test_every_api_response_has_instance_id PASSED
student-tests/test_idempotency.py::test_replay_returns_same_message PASSED
student-tests/test_idempotency.py::test_replay_ignores_json_formatting PASSED
student-tests/test_idempotency.py::test_same_key_with_other_body_is_conflict PASSED
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers0] PASSED
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers1] PASSED
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers2] PASSED
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers3] PASSED
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers4] PASSED
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers5] PASSED
student-tests/test_idempotency.py::test_key_boundaries_are_accepted[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED
student-tests/test_idempotency.py::test_key_boundaries_are_accepted[Az09._:-] PASSED
student-tests/test_idempotency.py::test_key_scope_is_user_and_chat PASSED
student-tests/test_idempotency.py::test_replay_rechecks_session_and_access PASSED
student-tests/test_idempotency.py::test_expired_key_can_be_reused PASSED
student-tests/test_idempotency.py::test_parallel_replays_through_replicas_create_one_message
PARALLEL SAME KEY: {"requests": 20, "statuses": {"201": 20}, "instances": {"389b271f592b": 10, "978f2777b379": 10}, "distinct_bodies": 1, "messages_in_history": 1, "rows_in_db": 1}
PASSED
student-tests/test_idempotency.py::test_parallel_same_key_with_different_bodies PASSED
student-tests/test_concurrency.py::test_parallel_messages_are_not_lost_or_duplicated
CONCURRENCY RESULT: {"clients": 10, "messages_sent": 100, "status_201": 100, "messages_read": 100, "unique_ids": 100, "unique_texts": 100, "pages_read": 4, "send_seconds": 0.227}
PASSED
============================== 21 passed in 3.79s ==============================
$ docker compose logs --no-log-prefix nginx | grep "\"uri\":\"/api/v1/users/me\",\"status\":200" | tail -6
{"time":"2026-10-19T17:01:21+00:00","request_id":"d754e7abc1b3b0e09ab4bf0d9fd50270","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.002","instance":"389b271f592b"}
{"time":"2026-10-19T17:01:21+00:00","request_id":"aeaf60de5c114ed306169370ede170b6","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.002,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"}
{"time":"2026-10-19T17:01:21+00:00","request_id":"7caafb063b53d8985d17356cd62d812d","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.002,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"389b271f592b"}
{"time":"2026-10-19T17:01:21+00:00","request_id":"a8e4fb898cd5ff5e56bdd1669067a60f","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"}
{"time":"2026-10-19T17:01:21+00:00","request_id":"fb8e6ba34f06d739933b9872b9aadd89","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.000","instance":"389b271f592b"}
{"time":"2026-10-19T17:01:21+00:00","request_id":"e166ba8766667300b8bcc65996ab6827","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"}
$ rid=$(docker compose logs --no-log-prefix nginx | grep "\"uri\":\"/api/v1/users/me\",\"status\":200" | tail -1 | sed -E "s/.*\"request_id\":\"([0-9a-f]+)\".*/\1/"); echo "request_id=$rid"; docker compose logs api | grep "$rid"
request_id=e166ba8766667300b8bcc65996ab6827
api-1 | 2026-10-19 17:01:21.285 | INFO | application.api.middlewares.request_context:_log_request:70 - GET /api/v1/users/me -> 200 | 0.6 ms | request_id=e166ba8766667300b8bcc65996ab6827
+41
View File
@@ -0,0 +1,41 @@
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy 978f2777b379 95632 dev
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 2.693}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 exited/unhealthy - 0 dev
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api)
/messenger-lab3-api-1 status=exited exit=0 finished=2026-10-19T17:01:38.389851544Z
/messenger-lab3-api-2 status=running exit=0 finished=0001-01-01T00:00:00Z
$ docker compose logs api | grep -E "SIGTERM|Application stopped|Finished server process"
api-1 | 2026-10-19 17:01:37.194 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=978f2777b379: порт закрыт, keep-alive соединения закрываются после ответа
api-1 | 2026-10-19 17:01:38.301 | INFO | __main__:lifespan:54 - Application stopped, instance=978f2777b379, connections closed
api-1 | INFO: Finished server process [1]
$ docker compose logs --no-log-prefix nginx | grep -E "\[(warn|error)\]" | tail -4
2026/10/19 17:00:46 [warn] 30#30: *43 a client request body is buffered to a temporary file /var/cache/nginx/client_temp/0000000001, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551b-c8fe-7b27-bef6-22c34e5e5367/messages HTTP/1.1", host: "localhost:8080"
2026/10/19 17:00:46 [warn] 30#30: *43 a client request body is buffered to a temporary file /var/cache/nginx/client_temp/0000000002, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551b-c8fe-7b27-bef6-22c34e5e5367/messages HTTP/1.1", host: "localhost:8080"
2026/10/19 17:01:38 [error] 30#30: *803 connect() failed (111: Connection refused) while connecting to upstream, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages HTTP/1.1", upstream: "http://192.168.48.2:8080/api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages", host: "localhost:8080"
2026/10/19 17:01:38 [warn] 30#30: *803 upstream server temporarily disabled while connecting to upstream, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages HTTP/1.1", upstream: "http://192.168.48.2:8080/api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages", host: "localhost:8080"
$ make restore
./scripts/restore.sh
Container messenger-lab3-api-2 Running
Container messenger-lab3-api-1 Starting
Container messenger-lab3-api-1 Started
Container messenger-lab3-api-1 Waiting
Container messenger-lab3-api-2 Waiting
Container messenger-lab3-api-1 Healthy
Container messenger-lab3-api-2 Healthy
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy b0b074af6d9c 18674 dev
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
+69
View File
@@ -0,0 +1,69 @@
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy b0b074af6d9c 18674 dev
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.616}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 exited/unhealthy - 0 dev
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api)
/messenger-lab3-api-1 status=exited exit=0 finished=2026-10-19T17:02:10.511955087Z
/messenger-lab3-api-2 status=running exit=0 finished=0001-01-01T00:00:00Z
$ make restore
./scripts/restore.sh
Container messenger-lab3-api-2 Running
Container messenger-lab3-api-1 Starting
Container messenger-lab3-api-1 Started
Container messenger-lab3-api-1 Waiting
Container messenger-lab3-api-2 Waiting
Container messenger-lab3-api-1 Healthy
Container messenger-lab3-api-2 Healthy
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
$ curl -s -o /dev/null -w "shift round-robin: %{http_code}\n" http://localhost:8080/health/live
shift round-robin: 200
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 1.618}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev
messenger-lab3-api-2 exited/unhealthy - 0 dev
$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api)
/messenger-lab3-api-1 status=running exit=0 finished=2026-10-19T17:02:10.511955087Z
/messenger-lab3-api-2 status=exited exit=0 finished=2026-10-19T17:02:43.134886046Z
$ docker compose logs api-2 2>/dev/null | grep -E "SIGTERM|Application stopped|Finished server process" || docker compose logs api | grep -E "SIGTERM|Application stopped" | tail -2
api-1 | 2026-10-19 17:02:09.336 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=b0b074af6d9c: порт закрыт, keep-alive соединения закрываются после ответа
api-1 | 2026-10-19 17:02:10.443 | INFO | __main__:lifespan:54 - Application stopped, instance=b0b074af6d9c, connections closed
$ make restore
./scripts/restore.sh
Container messenger-lab3-api-1 Running
Container messenger-lab3-api-2 Starting
Container messenger-lab3-api-2 Started
Container messenger-lab3-api-1 Waiting
Container messenger-lab3-api-2 Waiting
Container messenger-lab3-api-1 Healthy
Container messenger-lab3-api-2 Healthy
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev
messenger-lab3-api-2 running/healthy 774df521af21 22387 dev
$ docker logs messenger-lab3-api-2 2>&1 | grep -E "instance=389b271f592b|Finished server process"
2026-10-19 17:00:05.226 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=389b271f592b, session ttl=3600 s
2026-10-19 17:02:41.948 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=389b271f592b: порт закрыт, keep-alive соединения закрываются после ответа
2026-10-19 17:02:43.068 | INFO | __main__:lifespan:54 - Application stopped, instance=389b271f592b, connections closed
INFO: Finished server process [1]
+37
View File
@@ -0,0 +1,37 @@
$ cat scripts/restart.sh
#!/bin/sh
set -eu
# Перезапуск всего стенда с сохранением volumes: хранилища (PostgreSQL, Valkey),
# затем все API-реплики и балансировщик. Никаких down -v, flush или повторного
# seed; migrate не запускается — схема уже в volume. Готовность ждёт тест.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
docker compose --env-file "$ENV_FILE" restart --no-deps postgres valkey
docker compose --env-file "$ENV_FILE" restart --no-deps api
docker compose --env-file "$ENV_FILE" restart --no-deps nginx
$ docker inspect -f "{{.Name}} started={{.State.StartedAt}}" $(docker compose ps -q)
/messenger-lab3-api-1 started=2026-10-19T17:02:39.217903586Z
/messenger-lab3-api-2 started=2026-10-19T17:02:57.444596886Z
/messenger-lab3-nginx-1 started=2026-10-19T17:00:06.004414293Z
/messenger-lab3-postgres-1 started=2026-10-19T16:59:57.066060636Z
/messenger-lab3-valkey-1 started=2026-10-19T16:59:57.064188719Z
$ docker volume inspect -f "{{.Name}} created={{.CreatedAt}}" messenger-lab3_postgres_data
messenger-lab3_postgres_data created=2026-10-19T16:10:27Z
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
$ docker inspect -f "{{.Name}} started={{.State.StartedAt}}" $(docker compose ps -q)
/messenger-lab3-api-1 started=2026-10-19T17:03:17.125014673Z
/messenger-lab3-api-2 started=2026-10-19T17:03:17.151381881Z
/messenger-lab3-nginx-1 started=2026-10-19T17:03:17.57774484Z
/messenger-lab3-postgres-1 started=2026-10-19T17:03:15.504400631Z
/messenger-lab3-valkey-1 started=2026-10-19T17:03:15.535674256Z
$ docker volume inspect -f "{{.Name}} created={{.CreatedAt}}" messenger-lab3_postgres_data
messenger-lab3_postgres_data created=2026-10-19T16:10:27Z
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy bfb3f7bbb85d 23878 dev
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
+108
View File
@@ -0,0 +1,108 @@
$ make test-expiry
docker compose --profile expiry up -d --build --wait api-short-ttl
#1 [internal] load local bake definitions
#1 reading from stdin 1.20kB done
#1 DONE 0.0s
#2 [api-short-ttl internal] load build definition from Dockerfile
#2 transferring dockerfile: 914B done
#2 DONE 0.0s
#3 [api-short-ttl internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm
#3 DONE 0.1s
#4 [api-short-ttl internal] load .dockerignore
#4 transferring context: 358B done
#4 DONE 0.0s
#5 [api-short-ttl 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done
#5 DONE 0.0s
#6 [api-short-ttl internal] load build context
#6 transferring context: 7.39kB done
#6 DONE 0.0s
#7 [migrate 5/7] COPY migrations ./migrations
#7 CACHED
#8 [migrate 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction
#8 CACHED
#9 [migrate 2/7] WORKDIR /app
#9 CACHED
#10 [migrate 6/7] COPY src ./src
#10 CACHED
#11 [migrate 3/7] COPY poetry.lock pyproject.toml ./
#11 CACHED
#12 [api-short-ttl 7/7] RUN useradd --system --uid 10001 --no-create-home app
#12 CACHED
#13 [migrate] exporting to image
#13 exporting layers done
#13 exporting manifest sha256:9802819ef7b235f44ad03fed838cf83fcd4573a342405936538c7c969adfd573 done
#13 exporting config sha256:e2b0c133c10bc42d53160db701c32694ed2cbac54f0522999289f4cf7cef21e1 done
#13 naming to docker.io/library/mtusi-messenger:lab3 done
#13 unpacking to docker.io/library/mtusi-messenger:lab3 done
#13 DONE 0.0s
#14 [api-short-ttl] exporting to image
#14 exporting layers done
#14 exporting manifest sha256:036b7dde37a28a3eeec751ce6ada2034e1a9a1d76cd9310dbd3cd157824ea087 done
#14 exporting config sha256:26fb210928fa7201fedcce1e64d506079d863ddc42532dc4d25f1f77d673e3b3 done
#14 naming to docker.io/library/mtusi-messenger:lab3 done
#14 unpacking to docker.io/library/mtusi-messenger:lab3 done
#14 DONE 0.0s
#15 [migrate] resolving provenance for metadata file
#15 DONE 0.0s
#16 [api-short-ttl] resolving provenance for metadata file
#16 DONE 0.0s
mtusi-messenger:lab3 Built
mtusi-messenger:lab3 Built
Container messenger-lab3-valkey-1 Running
Container messenger-lab3-postgres-1 Running
Container messenger-lab3-migrate-1 Recreate
Container messenger-lab3-migrate-1 Recreated
Container messenger-lab3-api-short-ttl-1 Creating
Container messenger-lab3-api-short-ttl-1 Created
Container messenger-lab3-postgres-1 Waiting
Container messenger-lab3-postgres-1 Healthy
Container messenger-lab3-migrate-1 Starting
Container messenger-lab3-migrate-1 Started
Container messenger-lab3-migrate-1 Waiting
Container messenger-lab3-postgres-1 Waiting
Container messenger-lab3-postgres-1 Healthy
Container messenger-lab3-migrate-1 Exited
Container messenger-lab3-api-short-ttl-1 Starting
Container messenger-lab3-api-short-ttl-1 Started
Container messenger-lab3-api-short-ttl-1 Waiting
Container messenger-lab3-migrate-1 Waiting
Container messenger-lab3-postgres-1 Waiting
Container messenger-lab3-valkey-1 Waiting
Container messenger-lab3-migrate-1 Exited
Container messenger-lab3-valkey-1 Healthy
Container messenger-lab3-postgres-1 Healthy
Container messenger-lab3-api-short-ttl-1 Healthy
EXPIRY_BASE_URL=http://localhost:8081 poetry run pytest student-tests/test_expiry.py -s; \
status=$?; docker compose --profile expiry stop api-short-ttl; exit $status
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
warnings.warn(
============================= test session starts ==============================
platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python
cachedir: .pytest_cache
rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3
configfile: pyproject.toml
plugins: anyio-4.15.1
collecting ... collected 2 items
student-tests/test_expiry.py::test_session_expires_after_short_ttl PASSED
student-tests/test_expiry.py::test_expires_at_matches_short_ttl PASSED
============================== 2 passed in 3.54s ===============================
Container messenger-lab3-api-short-ttl-1 Stopping
Container messenger-lab3-api-short-ttl-1 Stopped
+24
View File
@@ -0,0 +1,24 @@
$ make test-failures
BASE_URL="http://localhost:8080" RUN_FAILURE_TESTS=1 poetry run pytest student-tests/test_dependency_failures.py -s
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
warnings.warn(
============================= test session starts ==============================
platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python
cachedir: .pytest_cache
rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3
configfile: pyproject.toml
plugins: anyio-4.15.1
collecting ... collected 4 items
student-tests/test_dependency_failures.py::test_valkey_down_cache_is_bypassed PASSED
student-tests/test_dependency_failures.py::test_logout_during_valkey_outage_is_not_resurrected PASSED
student-tests/test_dependency_failures.py::test_postgres_down_returns_bounded_503_and_no_anonymous_access PASSED
student-tests/test_dependency_failures.py::test_session_survives_postgres_restart PASSED
============================== 4 passed in 11.79s ==============================
$ docker compose ps --format "{{.Name}}\t{{.Status}}"
messenger-lab3-api-1 Up 12 hours (healthy)
messenger-lab3-api-2 Up 12 hours (healthy)
messenger-lab3-nginx-1 Up 12 hours (healthy)
messenger-lab3-postgres-1 Up Less than a second (health: starting)
messenger-lab3-valkey-1 Up 8 seconds (healthy)
+22
View File
@@ -0,0 +1,22 @@
$ make test-migrations
./scripts/check_migrations.sh
2026-10-20 04:55:31.041 | INFO | infrastructure.dependencies.database:create_pool:69 - Установка соединения с базой данных psql
2026-10-20 04:55:31.118 | INFO | infrastructure.dependencies.database:create_pool:99 - Соединение с базой данных psql установлено. DB_NAME: test, pool: 2..10
пустой volume: таблиц в public = 0
API до migrate: В БД не применены миграции [1, 2]: сначала запустите migrate
2026-10-20 04:55:31.132 | INFO | infrastructure.dependencies.database:apply_migrations:152 - Применена миграция 0001_init.sql
2026-10-20 04:55:31.134 | INFO | infrastructure.dependencies.database:apply_migrations:152 - Применена миграция 0002_message_idempotency.sql
2026-10-20 04:55:31.134 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии
запуск 1: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7
2026-10-20 04:55:31.135 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии
запуск 2: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7
2026-10-20 04:55:31.136 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии
запуск 3: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7
2026-10-20 04:55:31.136 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2
API после migrate: схема совместима
$ BUILDX_NO_DEFAULT_ATTESTATIONS=1 ./scripts/check_previous_version.sh
previous version image: mtusi-messenger:compat-aa971fb (aa971fb feat: перенос core-логики 2 лабы)
schema on stand: 0001_init, 0002_message_idempotency
{"ready": true, "register": 201, "login": 201, "create_chat": 201, "send_message_without_key": 201, "read_history": 200, "history_matches": true}
startup log: ['Схема БД в актуальной версии']
PASS: previous API version works on the current schema (expand-only migration)
+54
View File
@@ -0,0 +1,54 @@
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy bfb3f7bbb85d 23878 dev
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
$ make sigterm-check
python3 scripts/sigterm_check.py --base-url "http://localhost:8080"
Журнал реплики после SIGTERM:
2026-10-20 04:55:46.580 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=bfb3f7bbb85d: порт закрыт, keep-alive соединения закрываются после ответа
INFO: Shutting down
INFO: Waiting for application shutdown.
2026-10-20 04:55:47.701 | INFO | __main__:lifespan:54 - Application stopped, instance=bfb3f7bbb85d, connections closed
INFO: Application shutdown complete.
INFO: Finished server process [1]
{
"container": "messenger-lab3-api-1",
"instance": "bfb3f7bbb85d",
"exit_code": 0,
"shutdown_seconds": 1.53,
"limit_seconds": 10,
"instances_before": [
"6d2f6cd2b04a",
"bfb3f7bbb85d"
],
"probes_ok_after_sigterm": 354,
"instances_after_sigterm": [
"6d2f6cd2b04a",
"bfb3f7bbb85d"
],
"last_target_response_after_sigterm_seconds": 0.133,
"target_responses_after_exit": 0,
"errors_after_sigterm": 0,
"error_kinds": []
}
PASS: SIGTERM завершил messenger-lab3-api-1 за 1.53 с, LB обслужил 354 запросов без ошибок
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 exited/unhealthy - 0 dev
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
$ make restore
./scripts/restore.sh
Container messenger-lab3-api-2 Running
Container messenger-lab3-api-1 Starting
Container messenger-lab3-api-1 Started
Container messenger-lab3-api-1 Waiting
Container messenger-lab3-api-2 Waiting
Container messenger-lab3-api-1 Healthy
Container messenger-lab3-api-2 Healthy
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy 7535b2ed4434 69183 dev
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
+33
View File
@@ -0,0 +1,33 @@
$ docker inspect -f "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} restart={{.HostConfig.RestartPolicy.Name}} ports={{json .HostConfig.PortBindings}}" $(docker compose ps -aq)
/messenger-lab3-api-1 cpus=1000000000 memory=536870912 restart=unless-stopped ports={}
/messenger-lab3-api-2 cpus=1000000000 memory=536870912 restart=unless-stopped ports={}
/messenger-lab3-api-short-ttl-1 cpus=1000000000 memory=536870912 restart=unless-stopped ports={}
/messenger-lab3-migrate-1 cpus=500000000 memory=268435456 restart=no ports={}
/messenger-lab3-nginx-1 cpus=500000000 memory=67108864 restart=unless-stopped ports={"8080/tcp":[{"HostIp":"127.0.0.1","HostPort":"8080"}],"8081/tcp":[{"HostIp":"127.0.0.1","HostPort":"8081"}]}
/messenger-lab3-postgres-1 cpus=1500000000 memory=536870912 restart=unless-stopped ports={}
/messenger-lab3-valkey-1 cpus=250000000 memory=100663296 restart=unless-stopped ports={}
$ for n in messenger-lab3_edge messenger-lab3_backend; do docker network inspect -f "{{.Name}} internal={{.Internal}} containers={{range .Containers}}{{.Name}} {{end}}" $n; done
messenger-lab3_edge internal=false containers=messenger-lab3-api-2 messenger-lab3-api-1 messenger-lab3-nginx-1
messenger-lab3_backend internal=true containers=messenger-lab3-api-2 messenger-lab3-valkey-1 messenger-lab3-postgres-1 messenger-lab3-api-1
$ grep -n "container_name" compose.yaml || echo "container_name: not used"
container_name: not used
$ docker compose config | grep -E "^\s+(published|target):" | sort | uniq -c
1 published: "8080"
1 published: "8081"
1 target: /etc/nginx/errors.conf
1 target: /etc/nginx/nginx.conf
1 target: /var/lib/postgresql
1 target: 8080
1 target: 8081
$ docker compose exec -T nginx sh -c "wget -q -O - --timeout 2 http://postgres:5432 2>&1 | head -1; nslookup postgres 127.0.0.11 2>&1 | tail -2"
wget: bad address 'postgres:5432'
** server can't find postgres: NXDOMAIN
$ curl -s --max-time 2 http://localhost:5432 >/dev/null 2>&1 && echo "5432 open" || echo "host 127.0.0.1:5432: closed"
host 127.0.0.1:5432: closed
$ docker stats --no-stream --format "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" $(docker compose ps -q)
messenger-lab3-api-1 0.25% 45.73MiB / 512MiB
messenger-lab3-api-2 0.21% 45.42MiB / 512MiB
messenger-lab3-nginx-1 0.03% 2.59MiB / 64MiB
messenger-lab3-postgres-1 1.15% 30.5MiB / 512MiB
messenger-lab3-valkey-1 0.22% 11.04MiB / 96MiB
+18
View File
@@ -0,0 +1,18 @@
$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.024,
"concurrency": 4,
"successful_requests": 21418,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 713.369,
"successful_latency_ms": {
"p50": 4.748,
"p95": 8.395,
"p99": 22.531
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+22
View File
@@ -0,0 +1,22 @@
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy 7535b2ed4434 69183 dev
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
$ make experiment-failover
python3 scripts/experiment.py failover --base-url "http://localhost:8080" --output evidence/experiment-failover.json
== SIGTERM messenger-lab3-api-1 (instance 7535b2ed4434)
{"scenario": "SIGTERM messenger-lab3-api-1", "rps_ok_write": {"before": 381.6, "during": 276.9, "after": 398.9}, "rps_ok_read": {"before": 381.6, "during": 276.8, "after": 399.0}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.063}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.971, "instances_seen_from_action": {"6d2f6cd2b04a": [-10.128, 21.291], "7535b2ed4434": [-10.128, 0.032], "eb1dace786c6": [14.257, 21.289]}, "integrity": {"messages_in_history": 10740, "confirmed_unique": 10740, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
== SIGTERM messenger-lab3-api-2 (instance 6d2f6cd2b04a)
{"scenario": "SIGTERM messenger-lab3-api-2", "rps_ok_write": {"before": 401.3, "during": 282.3, "after": 362.4}, "rps_ok_read": {"before": 401.1, "during": 282.4, "after": 362.5}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.071}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 3.736, "instances_seen_from_action": {"6d2f6cd2b04a": [-10.122, 0.079], "eb1dace786c6": [-10.121, 21.19], "577316c523cb": [16.921, 21.191]}, "integrity": {"messages_in_history": 10687, "confirmed_unique": 10687, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
== SIGKILL messenger-lab3-api-1 (instance eb1dace786c6)
{"scenario": "SIGKILL messenger-lab3-api-1", "rps_ok_write": {"before": 427.4, "during": 281.4, "after": 408.2}, "rps_ok_read": {"before": 427.3, "during": 281.3, "after": 408.5}, "errors": {"before": 0, "during": 1, "after": 0}, "recovery": {"write_errors": 1, "error_kinds": {"HTTP 503": 1}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.07}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.79, "instances_seen_from_action": {"eb1dace786c6": [-9.73, 0.024], "577316c523cb": [-9.73, 20.11], "506b547ba744": [12.893, 20.111]}, "integrity": {"messages_in_history": 10837, "confirmed_unique": 10837, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
== SIGKILL messenger-lab3-api-2 (instance 577316c523cb)
{"scenario": "SIGKILL messenger-lab3-api-2", "rps_ok_write": {"before": 387.4, "during": 276.7, "after": 373.2}, "rps_ok_read": {"before": 387.1, "during": 276.8, "after": 373.3}, "errors": {"before": 0, "during": 3, "after": 0}, "recovery": {"write_errors": 3, "error_kinds": {"HTTP 503": 3}, "write_recovery_seconds": 0.006, "max_write_gap_seconds": 0.061}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.0, "instances_seen_from_action": {"506b547ba744": [-9.94, 20.189], "577316c523cb": [-9.939, 0.028], "1c20e7f6dbde": [11.911, 20.19]}, "integrity": {"messages_in_history": 10212, "confirmed_unique": 10212, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
saved: evidence/experiment-failover.json
PASS: прогонов 4, подтверждённые сообщения целы, дублей нет
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy 506b547ba744 76140 dev
messenger-lab3-api-2 running/healthy 1c20e7f6dbde 77892 dev
+18
View File
@@ -0,0 +1,18 @@
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-1 running/healthy 506b547ba744 76140 dev
messenger-lab3-api-2 running/healthy 1c20e7f6dbde 77892 dev
$ make experiment-rolling
python3 scripts/experiment.py rolling --base-url "http://localhost:8080" --output evidence/experiment-rolling.json
== rolling update to revision exp-1792472327
{"scenario": "rolling update", "rps_ok_write": {"before": 420.1, "during": 308.0, "after": 345.4}, "rps_ok_read": {"before": 419.9, "during": 307.9, "after": 345.5}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.098}, "target_responses_while_stopped": null, "restored_replica_in_rotation_seconds": null, "instances_seen_from_action": {"506b547ba744": [-9.228, 8.14], "1c20e7f6dbde": [-9.227, 14.911], "27601bb40a21": [4.875, 24.355], "e7e986ee1b58": [13.885, 24.355]}, "integrity": {"messages_in_history": 11682, "confirmed_unique": 11682, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
saved: evidence/experiment-rolling.json
PASS: прогонов 1, подтверждённые сообщения целы, дублей нет
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
messenger-lab3-api-3 running/healthy 27601bb40a21 79887 exp-1792472327
messenger-lab3-api-4 running/healthy e7e986ee1b58 80346 exp-1792472327
$ docker compose logs --no-log-prefix nginx --since 3m | grep -cE "prematurely closed" || true
4
+192
View File
@@ -0,0 +1,192 @@
$ echo "COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME (отдельный проект: рабочий стенд не затрагивается)"
COMPOSE_PROJECT_NAME=lab3-clean (отдельный проект: рабочий стенд не затрагивается)
$ cp .env.example .env && sed -i "" "s/^DB_PASS=.*/DB_PASS=$(openssl rand -hex 16)/; s/^VALKEY_PASS=.*/VALKEY_PASS=$(openssl rand -hex 16)/; s/^LB_PORT=.*/LB_PORT=18080/" .env && echo "LB_EXPIRY_PORT=18081" >> .env && echo ".env created from .env.example with random synthetic secrets"
.env created from .env.example with random synthetic secrets
$ docker compose --env-file .env config --quiet && echo "compose config: OK"
compose config: OK
$ make up
docker compose up -d --build --wait
#1 [internal] load local bake definitions
#1 reading from stdin 1.47kB done
#1 DONE 0.0s
#2 [api internal] load build definition from Dockerfile
#2 transferring dockerfile: 914B done
#2 DONE 0.0s
#3 [api internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm
#3 DONE 0.0s
#4 [api internal] load .dockerignore
#4 transferring context: 358B done
#4 DONE 0.0s
#5 [api 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done
#5 DONE 0.0s
#6 [api internal] load build context
#6 transferring context: 184.06kB 0.0s done
#6 DONE 0.0s
#7 [api 2/7] WORKDIR /app
#7 CACHED
#8 [api 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction
#8 CACHED
#9 [api 3/7] COPY poetry.lock pyproject.toml ./
#9 CACHED
#10 [migrate 5/7] COPY migrations ./migrations
#10 CACHED
#11 [migrate 6/7] COPY src ./src
#11 DONE 0.0s
#12 [api 7/7] RUN useradd --system --uid 10001 --no-create-home app
#12 0.118 useradd warning: app's uid 10001 is greater than SYS_UID_MAX 999
#12 DONE 0.1s
#13 [migrate] exporting to image
#13 exporting layers 0.0s done
#13 exporting manifest sha256:a99a66062da552c44bf2543747fc3ac215605d7487b93882f0248f46ae71dbf5 done
#13 exporting config sha256:53e2ac5d5830524705f0737d78b69d4f9c1ca9670b8a24788c8717a7d772898f done
#13 naming to docker.io/library/mtusi-messenger:lab3 done
#13 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done
#13 DONE 0.1s
#14 [api] exporting to image
#14 exporting layers 0.0s done
#14 exporting manifest sha256:47e4d0732ca8d624529f0f775abf4cd8457445931de74053dec5bbab49de873b done
#14 exporting config sha256:d6557e51b469086507b892558ee5960f31b77943d8c8fddc381e955e4eb3718a done
#14 naming to docker.io/library/mtusi-messenger:lab3 done
#14 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done
#14 DONE 0.1s
#15 [migrate] resolving provenance for metadata file
#15 DONE 0.0s
#16 [api] resolving provenance for metadata file
#16 DONE 0.0s
mtusi-messenger:lab3 Built
mtusi-messenger:lab3 Built
Network lab3-clean_backend Creating
Network lab3-clean_backend Created
Network lab3-clean_edge Creating
Network lab3-clean_edge Created
Volume lab3-clean_postgres_data Creating
Volume lab3-clean_postgres_data Created
Container lab3-clean-valkey-1 Creating
Container lab3-clean-postgres-1 Creating
Container lab3-clean-valkey-1 Created
Container lab3-clean-postgres-1 Created
Container lab3-clean-migrate-1 Creating
Container lab3-clean-migrate-1 Created
Container lab3-clean-api-2 Creating
Container lab3-clean-api-1 Creating
Container lab3-clean-api-2 Created
Container lab3-clean-api-1 Created
Container lab3-clean-nginx-1 Creating
Container lab3-clean-nginx-1 Created
Container lab3-clean-valkey-1 Starting
Container lab3-clean-postgres-1 Starting
Container lab3-clean-postgres-1 Started
Container lab3-clean-postgres-1 Waiting
Container lab3-clean-valkey-1 Started
Container lab3-clean-postgres-1 Healthy
Container lab3-clean-migrate-1 Starting
Container lab3-clean-migrate-1 Started
Container lab3-clean-migrate-1 Waiting
Container lab3-clean-postgres-1 Waiting
Container lab3-clean-postgres-1 Healthy
Container lab3-clean-migrate-1 Exited
Container lab3-clean-api-1 Starting
Container lab3-clean-api-1 Started
Container lab3-clean-api-2 Starting
Container lab3-clean-api-2 Started
Container lab3-clean-api-1 Waiting
Container lab3-clean-api-2 Waiting
Container lab3-clean-api-1 Healthy
Container lab3-clean-api-2 Healthy
Container lab3-clean-nginx-1 Starting
Container lab3-clean-nginx-1 Started
Container lab3-clean-postgres-1 Waiting
Container lab3-clean-valkey-1 Waiting
Container lab3-clean-nginx-1 Waiting
Container lab3-clean-api-1 Waiting
Container lab3-clean-api-2 Waiting
Container lab3-clean-migrate-1 Waiting
Container lab3-clean-valkey-1 Healthy
Container lab3-clean-postgres-1 Healthy
Container lab3-clean-migrate-1 Exited
Container lab3-clean-api-1 Healthy
Container lab3-clean-api-2 Healthy
Container lab3-clean-nginx-1 Healthy
$ make check test BASE_URL=http://localhost:18080
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
python3 tests/smoke.py --base-url "http://localhost:18080"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:18080
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 2.118}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
$ make instances
./scripts/instances.sh
CONTAINER STATE INSTANCE PID REVISION
lab3-clean-api-1 exited/unhealthy - 0 dev
lab3-clean-api-2 running/healthy 4a226e142873 83992 dev
$ make restore
./scripts/restore.sh
Container lab3-clean-api-2 Running
Container lab3-clean-api-1 Starting
Container lab3-clean-api-1 Started
Container lab3-clean-api-1 Waiting
Container lab3-clean-api-2 Waiting
Container lab3-clean-api-1 Healthy
Container lab3-clean-api-2 Healthy
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18080
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
$ docker compose ps -a
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
lab3-clean-api-1 mtusi-messenger:lab3 "python src/gateway.…" api 43 seconds ago Up 6 seconds (healthy) 8080/tcp
lab3-clean-api-2 mtusi-messenger:lab3 "python src/gateway.…" api 43 seconds ago Up 6 seconds (healthy) 8080/tcp
lab3-clean-migrate-1 mtusi-messenger:lab3 "python src/migrate.…" migrate 43 seconds ago Exited (0) 35 seconds ago
lab3-clean-nginx-1 nginx:1.28.0-alpine3.21 "/docker-entrypoint.…" nginx 43 seconds ago Up 5 seconds (healthy) 127.0.0.1:18080->8080/tcp, 127.0.0.1:18081->8081/tcp
lab3-clean-postgres-1 postgres:18.2-alpine3.22 "docker-entrypoint.s…" postgres 44 seconds ago Up 8 seconds (healthy) 5432/tcp
lab3-clean-valkey-1 valkey/valkey:8.1.4-alpine3.22 "docker-entrypoint.s…" valkey 44 seconds ago Up 8 seconds (healthy) 6379/tcp
$ docker compose --profile expiry down -v
Container lab3-clean-nginx-1 Stopping
Container lab3-clean-nginx-1 Stopped
Container lab3-clean-nginx-1 Removing
Container lab3-clean-nginx-1 Removed
Container lab3-clean-api-2 Stopping
Container lab3-clean-api-1 Stopping
Container lab3-clean-api-1 Stopped
Container lab3-clean-api-1 Removing
Container lab3-clean-api-1 Removed
Container lab3-clean-api-2 Stopped
Container lab3-clean-api-2 Removing
Container lab3-clean-api-2 Removed
Container lab3-clean-valkey-1 Stopping
Container lab3-clean-migrate-1 Stopping
Container lab3-clean-migrate-1 Stopped
Container lab3-clean-migrate-1 Removing
Container lab3-clean-migrate-1 Removed
Container lab3-clean-postgres-1 Stopping
Container lab3-clean-postgres-1 Stopped
Container lab3-clean-postgres-1 Removing
Container lab3-clean-postgres-1 Removed
Container lab3-clean-valkey-1 Stopped
Container lab3-clean-valkey-1 Removing
Container lab3-clean-valkey-1 Removed
Volume lab3-clean_postgres_data Removing
Network lab3-clean_edge Removing
Network lab3-clean_backend Removing
Volume lab3-clean_postgres_data Removed
Network lab3-clean_edge Removed
Network lab3-clean_backend Removed
File diff suppressed because it is too large Load Diff
+482
View File
@@ -0,0 +1,482 @@
{
"mode": "rolling",
"started": "2026-10-20T09:50:22+0300",
"profile": {
"workers": 4,
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
"client_timeout_seconds": 5,
"phases_seconds": {
"before": 8,
"outage": 10,
"after": 8
}
},
"runs": [
{
"scenario": "rolling update",
"target_instance": null,
"workers": 4,
"events_seconds": {
"action_start": 10.323,
"action_end": 26.786,
"end": 34.791
},
"instances_before": [
"109172df7747",
"f6e244c650a2"
],
"instances_after": [
"500528cb388e",
"de52b5578b6b"
],
"target_responses_while_stopped": null,
"restored_replica_in_rotation_seconds": null,
"instances_seen_from_action": {
"109172df7747": [
-10.3,
14.813
],
"f6e244c650a2": [
-10.296,
8.226
],
"500528cb388e": [
6.241,
24.466
],
"de52b5578b6b": [
13.791,
24.468
]
},
"phases": {
"before": {
"seconds": 10.323,
"write": {
"ok": 3687,
"errors": 0,
"ok_rps": 357.2,
"latency_ms": {
"p50": 5.4,
"p95": 10.3,
"p99": 28.9
}
},
"read": {
"ok": 3684,
"errors": 0,
"ok_rps": 356.9,
"latency_ms": {
"p50": 4.2,
"p95": 7.8,
"p99": 19.9
}
}
},
"during": {
"seconds": 16.463,
"write": {
"ok": 5071,
"errors": 0,
"ok_rps": 308.0,
"latency_ms": {
"p50": 5.4,
"p95": 16.9,
"p99": 38.6
}
},
"read": {
"ok": 5073,
"errors": 0,
"ok_rps": 308.1,
"latency_ms": {
"p50": 4.2,
"p95": 12.3,
"p99": 31.4
}
}
},
"after": {
"seconds": 8.005,
"write": {
"ok": 2800,
"errors": 0,
"ok_rps": 349.8,
"latency_ms": {
"p50": 4.8,
"p95": 10.9,
"p99": 48.3
}
},
"read": {
"ok": 2801,
"errors": 0,
"ok_rps": 349.9,
"latency_ms": {
"p50": 3.7,
"p95": 8.0,
"p99": 47.3
}
}
}
},
"recovery": {
"write_errors": 0,
"error_kinds": {},
"write_recovery_seconds": 0.0,
"max_write_gap_seconds": 0.299
},
"integrity": {
"messages_in_history": 11558,
"confirmed_unique": 11558,
"confirmed_missing_or_changed": 0,
"duplicate_ids": 0,
"duplicate_texts": 0,
"retried_writes_confirmed_as_replay": 0
},
"container_stats_before_action": [
"messenger-lab3-nginx-1\t10.92%\t3.227MiB / 64MiB",
"messenger-lab3-postgres-1\t30.04%\t54.82MiB / 512MiB",
"messenger-lab3-api-1\t85.53%\t52.1MiB / 512MiB",
"messenger-lab3-api-2\t84.90%\t54.01MiB / 512MiB"
],
"action_output": [
"[09:50:33] build revision exp-1792479022",
"[09:50:34] migrate",
"[09:50:36] scale api to 3, wait healthy",
"[09:50:41] graceful stop messenger-lab3-api-1",
"[09:50:42] scale api to 3, wait healthy",
"[09:50:47] graceful stop messenger-lab3-api-2",
"[09:50:49] done: all API replicas run revision exp-1792479022"
],
"restore_output_tail": [],
"timeline": [
{
"t": 0,
"ok": 705,
"errors": 0,
"instances": {
"109172df7747": 354,
"f6e244c650a2": 351
}
},
{
"t": 1,
"ok": 522,
"errors": 0,
"instances": {
"f6e244c650a2": 263,
"109172df7747": 259
}
},
{
"t": 2,
"ok": 733,
"errors": 0,
"instances": {
"109172df7747": 367,
"f6e244c650a2": 366
}
},
{
"t": 3,
"ok": 771,
"errors": 0,
"instances": {
"109172df7747": 385,
"f6e244c650a2": 386
}
},
{
"t": 4,
"ok": 725,
"errors": 0,
"instances": {
"109172df7747": 363,
"f6e244c650a2": 362
}
},
{
"t": 5,
"ok": 803,
"errors": 0,
"instances": {
"109172df7747": 401,
"f6e244c650a2": 402
}
},
{
"t": 6,
"ok": 449,
"errors": 0,
"instances": {
"109172df7747": 224,
"f6e244c650a2": 225
}
},
{
"t": 7,
"ok": 777,
"errors": 0,
"instances": {
"109172df7747": 390,
"f6e244c650a2": 387
}
},
{
"t": 8,
"ok": 803,
"errors": 0,
"instances": {
"f6e244c650a2": 402,
"109172df7747": 401
}
},
{
"t": 9,
"ok": 811,
"errors": 0,
"instances": {
"109172df7747": 405,
"f6e244c650a2": 406
}
},
{
"t": 10,
"ok": 677,
"errors": 0,
"instances": {
"109172df7747": 339,
"f6e244c650a2": 338
}
},
{
"t": 11,
"ok": 271,
"errors": 0,
"instances": {
"f6e244c650a2": 135,
"109172df7747": 136
}
},
{
"t": 12,
"ok": 432,
"errors": 0,
"instances": {
"f6e244c650a2": 217,
"109172df7747": 215
}
},
{
"t": 13,
"ok": 422,
"errors": 0,
"instances": {
"109172df7747": 211,
"f6e244c650a2": 211
}
},
{
"t": 14,
"ok": 533,
"errors": 0,
"instances": {
"109172df7747": 267,
"f6e244c650a2": 266
}
},
{
"t": 15,
"ok": 737,
"errors": 0,
"instances": {
"f6e244c650a2": 369,
"109172df7747": 368
}
},
{
"t": 16,
"ok": 683,
"errors": 0,
"instances": {
"109172df7747": 276,
"f6e244c650a2": 277,
"500528cb388e": 130
}
},
{
"t": 17,
"ok": 635,
"errors": 0,
"instances": {
"500528cb388e": 212,
"109172df7747": 212,
"f6e244c650a2": 211
}
},
{
"t": 18,
"ok": 841,
"errors": 0,
"instances": {
"f6e244c650a2": 153,
"500528cb388e": 345,
"109172df7747": 343
}
},
{
"t": 19,
"ok": 782,
"errors": 0,
"instances": {
"109172df7747": 391,
"500528cb388e": 391
}
},
{
"t": 20,
"ok": 254,
"errors": 0,
"instances": {
"109172df7747": 128,
"500528cb388e": 126
}
},
{
"t": 21,
"ok": 767,
"errors": 0,
"instances": {
"500528cb388e": 384,
"109172df7747": 383
}
},
{
"t": 22,
"ok": 761,
"errors": 0,
"instances": {
"109172df7747": 380,
"500528cb388e": 381
}
},
{
"t": 23,
"ok": 795,
"errors": 0,
"instances": {
"500528cb388e": 397,
"109172df7747": 398
}
},
{
"t": 24,
"ok": 924,
"errors": 0,
"instances": {
"109172df7747": 322,
"500528cb388e": 321,
"de52b5578b6b": 281
}
},
{
"t": 25,
"ok": 681,
"errors": 0,
"instances": {
"500528cb388e": 320,
"109172df7747": 41,
"de52b5578b6b": 320
}
},
{
"t": 26,
"ok": 232,
"errors": 0,
"instances": {
"de52b5578b6b": 115,
"500528cb388e": 117
}
},
{
"t": 27,
"ok": 252,
"errors": 0,
"instances": {
"de52b5578b6b": 126,
"500528cb388e": 126
}
},
{
"t": 28,
"ok": 633,
"errors": 0,
"instances": {
"de52b5578b6b": 317,
"500528cb388e": 316
}
},
{
"t": 29,
"ok": 940,
"errors": 0,
"instances": {
"500528cb388e": 470,
"de52b5578b6b": 470
}
},
{
"t": 30,
"ok": 814,
"errors": 0,
"instances": {
"de52b5578b6b": 407,
"500528cb388e": 407
}
},
{
"t": 31,
"ok": 626,
"errors": 0,
"instances": {
"de52b5578b6b": 313,
"500528cb388e": 313
}
},
{
"t": 32,
"ok": 878,
"errors": 0,
"instances": {
"de52b5578b6b": 439,
"500528cb388e": 439
}
},
{
"t": 33,
"ok": 740,
"errors": 0,
"instances": {
"500528cb388e": 370,
"de52b5578b6b": 370
}
},
{
"t": 34,
"ok": 707,
"errors": 0,
"instances": {
"500528cb388e": 353,
"de52b5578b6b": 354
}
}
],
"revisions_after": [
"CONTAINER STATE INSTANCE PID REVISION",
"messenger-lab3-api-3 running/healthy 500528cb388e 60335 exp-1792479022",
"messenger-lab3-api-4 running/healthy de52b5578b6b 60779 exp-1792479022"
]
}
]
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.024,
"concurrency": 4,
"successful_requests": 21418,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 713.369,
"successful_latency_ms": {
"p50": 4.748,
"p95": 8.395,
"p99": 22.531
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+15
View File
@@ -0,0 +1,15 @@
# Ошибки, созданные самим балансировщиком, — в формате ошибок контракта.
# Ответы реплик (включая их 503) проходят без изменений: proxy_intercept_errors off.
# X-Instance-Id здесь нет: ответ создан LB, а не API-репликой.
location @unavailable {
default_type application/json;
add_header X-Request-Id $request_id always;
return 503 '{"error":{"code":"unavailable","message":"Нет доступной API-реплики","request_id":"$request_id"}}';
}
location @too_large {
default_type application/json;
add_header X-Request-Id $request_id always;
return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}';
}
+101
View File
@@ -0,0 +1,101 @@
worker_processes 1;
pid /tmp/nginx.pid;
error_log /dev/stderr warn;
events {
worker_connections 1024;
}
http {
resolver 127.0.0.11 valid=2s ipv6=off;
resolver_timeout 1s;
log_format upstream_json escape=json
'{"time":"$time_iso8601","request_id":"$request_id",'
'"method":"$request_method","uri":"$uri","status":$status,'
'"request_time":$request_time,"upstream_addr":"$upstream_addr",'
'"upstream_status":"$upstream_status",'
'"upstream_response_time":"$upstream_response_time",'
'"instance":"$upstream_http_x_instance_id"}';
access_log /dev/stdout upstream_json;
server_tokens off;
client_max_body_size 1m;
client_body_buffer_size 64k;
client_header_timeout 10s;
client_body_timeout 10s;
send_timeout 10s;
keepalive_timeout 65s;
upstream api {
zone api 64k;
server api:8080 resolve max_fails=1 fail_timeout=5s;
keepalive 32;
# Меньше APP_KEEPALIVE_TIMEOUT_SEC (75 с): idle-соединение первым
# закрывает nginx, а не реплика, — нет гонки «запрос в закрытое соединение»
keepalive_timeout 60s;
}
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
# Один ID на LB и реплику: API принимает $request_id и возвращает его
proxy_set_header X-Request-Id $request_id;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
# connect внутри bridge-сети завершает ядро за доли мс даже у занятой реплики.
# Самый долгий запрос API: ожидание пула 3 с + statement_timeout 5 с
proxy_connect_timeout 500ms;
proxy_send_timeout 10s;
proxy_read_timeout 10s;
# Повтор на другой реплике только при ошибке соединения/таймауте.
# POST, уже отправленный реплике, nginx не повторяет (non_idempotent выключен):
# повтор сообщения делает клиент с тем же Idempotency-Key.
# 503 приложения (общая зависимость недоступна) не повторяется и не исключает
# реплику: другая реплика зависит от той же БД.
proxy_next_upstream error timeout;
proxy_next_upstream_tries 3;
proxy_next_upstream_timeout 5s;
server {
listen 8080;
location / {
proxy_pass http://api;
}
error_page 502 503 504 = @unavailable;
error_page 413 = @too_large;
include /etc/nginx/errors.conf;
}
server {
listen 8081;
# Реплика api-short-ttl только для make test-expiry (профиль expiry):
# адрес резолвится на каждый запрос, без профиля — 503 от LB
location / {
set $api_short_ttl http://api-short-ttl:8080;
proxy_pass $api_short_ttl;
}
error_page 502 503 504 = @unavailable;
error_page 413 = @too_large;
include /etc/nginx/errors.conf;
}
# Liveness самого балансировщика для healthcheck контейнера; наружу не публикуется
server {
listen 127.0.0.1:8090;
access_log off;
location = /healthz {
default_type text/plain;
return 200 "ok\n";
}
}
}
+18
View File
@@ -0,0 +1,18 @@
-- Idempotency-Key отправки сообщения (lab3). Область по контракту:
-- пользователь + метод + путь + ключ. Метод и путь фиксированы
-- (POST /api/v1/chats/{chat_id}/messages), поэтому ключ — (sender_id, chat_id, key).
-- Только добавление таблицы (expand): предыдущая версия API работает на новой схеме.
CREATE TABLE message_idempotency_keys (
sender_id uuid NOT NULL,
chat_id uuid NOT NULL,
idempotency_key text NOT NULL,
request_hash bytea NOT NULL,
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
PRIMARY KEY (sender_id, chat_id, idempotency_key),
CONSTRAINT message_idempotency_keys_key_format
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT message_idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32),
CONSTRAINT message_idempotency_keys_expiry_order CHECK (expires_at > created_at)
);
+17 -6
View File
@@ -1,17 +1,19 @@
#!/bin/sh
set -eu
# Миграции с нуля и повторно: отдельный compose-проект с собственным пустым
# volume. Рабочий стенд и его данные не затрагиваются; временный volume
# этого проекта удаляется в конце.
# Миграции с нуля и повторно, отказ API на непримененной схеме: отдельный
# compose-проект с собственным пустым volume. Рабочий стенд и его данные
# не затрагиваются; временный volume этого проекта удаляется в конце.
cd "$(dirname "$0")/.."
PROJECT=messenger-lab3-migrations-check
# Свой порт на хосте: основной стенд уже занимает DB_PORT
export DB_PORT="${MIGRATIONS_CHECK_DB_PORT:-55432}"
# Публикуется только порт LB: свои порты, чтобы не конфликтовать с основным стендом
export LB_PORT="${MIGRATIONS_CHECK_LB_PORT:-58080}"
export LB_EXPIRY_PORT="${MIGRATIONS_CHECK_LB_EXPIRY_PORT:-58081}"
COMPOSE="docker compose --env-file ${ENV_FILE:-.env} -p $PROJECT"
trap '$COMPOSE down -v --remove-orphans >/dev/null 2>&1' EXIT
$COMPOSE up -d --wait postgres >/dev/null 2>&1
$COMPOSE run --rm --no-deps -T api python - <<'EOF'
# Тот же образ и сеть, что у сервиса migrate; apply_migrations трижды подряд
$COMPOSE run --rm --no-deps -T migrate python - <<'EOF'
import asyncio
import sys
@@ -27,6 +29,13 @@ async def main() -> None:
"SELECT count(*) FROM pg_tables WHERE schemaname = 'public';"
)
print(f"пустой volume: таблиц в public = {tables}")
# Так стартует API-реплика: без migrate она отказывается работать
try:
await db.check_migrations()
except RuntimeError as error:
print(f"API до migrate: {error}")
else:
raise SystemExit("FAIL: API стартовал бы на пустой схеме")
for attempt in (1, 2, 3):
await db.apply_migrations()
async with db.acquire() as conn:
@@ -40,6 +49,8 @@ async def main() -> None:
f"запуск {attempt}: schema_migrations = "
f"{[(r['version'], r['name']) for r in versions]}, таблиц = {tables}"
)
await db.check_migrations()
print("API после migrate: схема совместима")
await db.close()
+89
View File
@@ -0,0 +1,89 @@
#!/bin/sh
set -eu
# Совместимость текущей схемы с предыдущей версией API — условие rolling update:
# миграции применяются до замены реплик, и старые реплики продолжают работать.
# Образ собирается из коммита PREVIOUS_REVISION (по умолчанию lab2-код aa971fb),
# запускается одним процессом в сети backend рабочего стенда (наружу порт
# не публикуется) и выполняет вход, создание чата, запись и чтение сообщения.
# Создаются только синтетические данные; volumes не затрагиваются.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
REVISION="${PREVIOUS_REVISION:-aa971fb}"
IMAGE="mtusi-messenger:compat-$REVISION"
NETWORK=messenger-lab3_backend
git archive "$REVISION" | docker build --quiet --tag "$IMAGE" - >/dev/null
echo "previous version image: $IMAGE ($(git log -1 --format='%h %s' "$REVISION"))"
echo "schema on stand: $(docker compose --env-file "$ENV_FILE" exec -T postgres sh -c \
'psql -X -q -tA -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT string_agg(name, \$\$, \$\$ ORDER BY version) FROM schema_migrations;"')"
docker run --rm -i --network "$NETWORK" --env-file "$ENV_FILE" \
-e DB_HOST=postgres -e DB_PORT=5432 -e VALKEY_HOST=valkey -e VALKEY_PORT=6379 \
"$IMAGE" sh -c 'python src/gateway.py >/tmp/api.log 2>&1 & exec python -' <<'EOF'
import base64
import json
import time
import urllib.error
import urllib.request
import uuid
BASE = "http://127.0.0.1:8080"
def call(method, path, body=None, headers=None):
data = None if body is None else json.dumps(body).encode()
request = urllib.request.Request(BASE + path, data=data, method=method)
request.add_header("Content-Type", "application/json")
for name, value in (headers or {}).items():
request.add_header(name, value)
try:
with urllib.request.urlopen(request, timeout=5) as response:
return response.status, json.loads(response.read() or b"null")
except urllib.error.HTTPError as error:
return error.code, None
for _ in range(60):
try:
if call("GET", "/health/ready")[0] == 200:
break
except OSError:
pass
time.sleep(0.5)
else:
print(open("/tmp/api.log").read()[-2000:])
raise SystemExit("FAIL: previous version did not become ready on current schema")
username = "compat_" + uuid.uuid4().hex[:12]
password = "Compat-" + uuid.uuid4().hex
status_user, user = call(
"POST", "/api/v1/users",
{"username": username, "display_name": "Compat", "password": password},
)
credentials = base64.b64encode(f"{username}:{password}".encode()).decode()
status_login, session = call(
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + credentials}
)
bearer = {"Authorization": "Bearer " + session["token"]}
status_chat, chat = call(
"POST", "/api/v1/chats", {"title": "compat", "member_ids": [user["id"]]}, bearer
)
path = f"/api/v1/chats/{chat['id']}/messages"
status_send, message = call("POST", path, {"text": "старая версия пишет"}, bearer)
status_read, page = call("GET", path, headers=bearer)
result = {
"ready": True,
"register": status_user,
"login": status_login,
"create_chat": status_chat,
"send_message_without_key": status_send,
"read_history": status_read,
"history_matches": page["items"] == [message],
}
print(json.dumps(result, ensure_ascii=False))
log = open("/tmp/api.log").read()
print("startup log:", [line.split(" - ", 1)[-1] for line in log.splitlines() if "миграц" in line.lower() or "Схема" in line])
if any(code >= 400 for code in result.values() if isinstance(code, int)) or not result["history_matches"]:
raise SystemExit("FAIL: previous version is not compatible with current schema")
print("PASS: previous API version works on the current schema (expand-only migration)")
EOF
+480
View File
@@ -0,0 +1,480 @@
#!/usr/bin/env python3
"""Нагрузка через LB во время отказа/замены API-реплик; метрики и проверка данных.
failover — для каждой API-реплики по очереди: нагрузка → остановка этой реплики
(docker stop = SIGTERM, затем docker kill = SIGKILL) → восстановление
(scripts/restore.sh) → нагрузка после восстановления.
rolling — нагрузка во время scripts/rolling-update.sh (поочерёдная замена реплик).
Клиент ведёт себя как настоящий: неподтверждённый POST повторяет с тем же
Idempotency-Key и телом. После прогона вся история читается заново: каждое
подтверждённое сообщение на месте и не изменено, дублей нет.
Python 3.9+, стандартная библиотека; tokens не печатаются и не сохраняются.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import subprocess
import sys
import threading
import time
import urllib.error
import urllib.request
import uuid
from dataclasses import dataclass
from pathlib import Path
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client # noqa: E402
REQUEST_TIMEOUT_SEC = 5
RETRY_PAUSE_SEC = 0.05
STATS_FORMAT = "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
@dataclass
class Attempt:
"""Одна HTTP-попытка клиента; время — секунды от начала прогона"""
op: str
started: float
finished: float
status: int | None
error: str | None
instance: str | None
retry: bool
@property
def ok(self) -> bool:
return self.status in (200, 201)
class Recorder:
def __init__(self) -> None:
self.started = time.monotonic()
self.lock = threading.Lock()
self.attempts: list[Attempt] = []
self.confirmed: list[dict] = []
self.events: dict[str, float] = {}
def now(self) -> float:
return time.monotonic() - self.started
def mark(self, name: str) -> None:
self.events[name] = round(self.now(), 3)
def add(self, attempt: Attempt, message: dict | None = None) -> None:
with self.lock:
self.attempts.append(attempt)
if message is not None:
self.confirmed.append(message)
def request(
base: str, method: str, path: str, token: str, body: dict | None, key: str | None
) -> tuple[int | None, str | None, str | None, dict | None]:
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
data = None
if body is not None:
data = json.dumps(body, ensure_ascii=False).encode()
headers["Content-Type"] = "application/json"
if key is not None:
headers["Idempotency-Key"] = key
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
try:
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as response:
payload = json.loads(response.read())
return response.status, None, response.headers.get("X-Instance-Id"), payload
except urllib.error.HTTPError as error:
error.read()
return error.code, None, error.headers.get("X-Instance-Id"), None
except OSError as error:
return None, type(error).__name__, None, None
def worker(
base: str,
token: str,
chat_id: str,
worker_id: int,
recorder: Recorder,
stop: threading.Event,
) -> None:
"""Запись с повтором по тому же ключу до подтверждения, затем чтение"""
path = f"/api/v1/chats/{chat_id}/messages"
index = 0
while not stop.is_set():
key = uuid.uuid4().hex
body = {"text": f"exp-{worker_id}-{index}-{key[:8]}"}
retry = False
while not stop.is_set():
started = recorder.now()
status, error, instance, payload = request(
base, "POST", path, token, body, key
)
attempt = Attempt(
"write", started, recorder.now(), status, error, instance, retry
)
recorder.add(attempt, payload if status == 201 else None)
if status == 201:
break
retry = True
time.sleep(RETRY_PAUSE_SEC)
started = recorder.now()
status, error, instance, _ = request(
base, "GET", path + "?limit=1", token, None, None
)
recorder.add(
Attempt("read", started, recorder.now(), status, error, instance, False)
)
index += 1
def percentile(values: list[float], p: float) -> float | None:
if not values:
return None
ordered = sorted(values)
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1)
def phase_stats(attempts: list[Attempt], start: float, end: float) -> dict:
inside = [a for a in attempts if start <= a.finished < end]
duration = max(end - start, 1e-9)
result: dict = {"seconds": round(duration, 3)}
for op in ("write", "read"):
ops = [a for a in inside if a.op == op]
ok = [a for a in ops if a.ok]
result[op] = {
"ok": len(ok),
"errors": len(ops) - len(ok),
"ok_rps": round(len(ok) / duration, 1),
"latency_ms": {
"p50": percentile([a.finished - a.started for a in ok], 0.50),
"p95": percentile([a.finished - a.started for a in ok], 0.95),
"p99": percentile([a.finished - a.started for a in ok], 0.99),
},
}
return result
def recovery_stats(attempts: list[Attempt], action_start: float, end: float) -> dict:
"""
Время восстановления записи: от первой ошибки после воздействия до первой
успешной записи после последней ошибки; без ошибок — 0. Отдельно —
наибольший разрыв между подтверждениями записи (задержка без ошибок).
"""
window = sorted(
(a for a in attempts if a.op == "write" and a.finished >= action_start - 1),
key=lambda a: a.finished,
)
errors = [a for a in window if not a.ok and a.finished >= action_start]
kinds: dict[str, int] = {}
for a in errors:
kind = a.error or f"HTTP {a.status}"
kinds[kind] = kinds.get(kind, 0) + 1
if errors:
last_error = errors[-1].finished
first_ok_after = next(
(a.finished for a in window if a.ok and a.finished > last_error), end
)
recovery = first_ok_after - errors[0].finished
else:
recovery = 0.0
oks = [a.finished for a in window if a.ok and a.finished <= end]
gap = max((b - a for a, b in zip(oks, oks[1:])), default=None)
return {
"write_errors": len(errors),
"error_kinds": kinds,
"write_recovery_seconds": round(recovery, 3),
"max_write_gap_seconds": None if gap is None else round(gap, 3),
}
def timeline(attempts: list[Attempt], end: float) -> list[dict]:
rows = []
for second in range(int(end) + 1):
inside = [a for a in attempts if second <= a.finished < second + 1]
instances: dict[str, int] = {}
for a in inside:
if a.ok and a.instance:
instances[a.instance] = instances.get(a.instance, 0) + 1
rows.append(
{
"t": second,
"ok": sum(a.ok for a in inside),
"errors": sum(not a.ok for a in inside),
"instances": instances,
}
)
return rows
def read_history(client: Client, actor: dict, chat_id: str) -> list[dict]:
path = f"/api/v1/chats/{chat_id}/messages?limit=100"
items: list[dict] = []
while True:
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
items.extend(page["items"])
if page["next_cursor"] is None:
return items
path = f"/api/v1/chats/{chat_id}/messages?limit=100&cursor=" + quote(
page["next_cursor"], safe=""
)
def integrity(client: Client, actor: dict, chat_id: str, recorder: Recorder) -> dict:
history = read_history(client, actor, chat_id)
by_id = {m["id"]: m for m in history}
texts = [m["text"] for m in history]
confirmed = {m["id"]: m for m in recorder.confirmed}
return {
"messages_in_history": len(history),
"confirmed_unique": len(confirmed),
"confirmed_missing_or_changed": sum(
by_id.get(i) != m for i, m in confirmed.items()
),
"duplicate_ids": len(history) - len(by_id),
"duplicate_texts": len(texts) - len(set(texts)),
"retried_writes_confirmed_as_replay": len(recorder.confirmed) - len(confirmed),
}
def run(command: list[str], env: dict | None = None) -> str:
result = subprocess.run(
command, cwd=ROOT, env=env, check=True, capture_output=True, text=True
)
return result.stdout.strip()
def api_containers() -> list[str]:
ids = run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).split()
names = [run(["docker", "inspect", "-f", "{{.Name}}", i]).lstrip("/") for i in ids]
return sorted(names)
def instance_of(container: str) -> str:
return run(["docker", "exec", container, "cat", "/tmp/instance-id"])
def container_stats() -> list[str]:
names = ["messenger-lab3-nginx-1", "messenger-lab3-postgres-1", *api_containers()]
output = run(["docker", "stats", "--no-stream", "--format", STATS_FORMAT, *names])
return output.splitlines()
def scenario(
args: argparse.Namespace,
label: str,
action: list[str],
restore: list[str] | None,
target: str | None = None,
env: dict | None = None,
) -> dict:
client = Client(args.base_url, timeout=REQUEST_TIMEOUT_SEC)
alice, _, _, chat = client.bootstrap()
base = client.base
recorder = Recorder()
stop = threading.Event()
threads = [
threading.Thread(
target=worker,
args=(base, alice["token"], chat["id"], n, recorder, stop),
daemon=True,
)
for n in range(args.workers)
]
for thread in threads:
thread.start()
time.sleep(args.before / 2)
stats = container_stats()
time.sleep(args.before / 2)
instances_before = sorted(
{a.instance for a in recorder.attempts if a.ok and a.instance}
)
recorder.mark("action_start")
action_output = run(action, env)
recorder.mark("action_end")
restore_output = ""
if restore is not None:
time.sleep(args.outage)
recorder.mark("restore_start")
restore_output = run(restore)
recorder.mark("restore_end")
time.sleep(args.after)
stop.set()
for thread in threads:
thread.join(timeout=REQUEST_TIMEOUT_SEC + 1)
recorder.mark("end")
events = recorder.events
end = events["end"]
after_start = events.get("restore_end", events["action_end"])
attempts = recorder.attempts
seen_after = [
a for a in attempts if a.ok and a.finished >= after_start and a.instance
]
back_in_rotation = next(
(
round(a.finished - after_start, 3)
for a in sorted(seen_after, key=lambda a: a.finished)
if a.instance not in instances_before
),
None,
)
served = sorted(
(a for a in attempts if a.ok and a.instance), key=lambda a: a.finished
)
instances_seen: dict[str, list[float]] = {}
for a in served:
offset = round(a.finished - events["action_start"], 3)
instances_seen.setdefault(a.instance, [offset, offset])[1] = offset
phases = {
"before": phase_stats(attempts, 0, events["action_start"]),
"during": phase_stats(attempts, events["action_start"], after_start),
"after": phase_stats(attempts, after_start, end),
}
target_after_stop = None
if target is not None:
target_after_stop = sum(
1
for a in attempts
if a.ok
and a.instance == target
and a.finished > events["action_end"]
and a.finished < events.get("restore_start", end)
)
return {
"scenario": label,
"target_instance": target,
"workers": args.workers,
"events_seconds": events,
"instances_before": instances_before,
"instances_after": sorted({a.instance for a in seen_after}),
"target_responses_while_stopped": target_after_stop,
"restored_replica_in_rotation_seconds": back_in_rotation
if restore is not None
else None,
# alias → [первый, последний] успешный ответ, секунды от начала воздействия
"instances_seen_from_action": instances_seen,
"phases": phases,
"recovery": recovery_stats(attempts, events["action_start"], end),
"integrity": integrity(client, alice, chat["id"], recorder),
"container_stats_before_action": stats,
"action_output": action_output.splitlines(),
"restore_output_tail": restore_output.splitlines()[-2:],
"timeline": timeline(attempts, end),
}
def failover(args: argparse.Namespace) -> list[dict]:
results = []
for signal in args.signals:
for container in api_containers():
target = instance_of(container)
if signal == "SIGTERM":
action = ["docker", "stop", "--timeout", "10", container]
else:
action = ["docker", "kill", "--signal", "SIGKILL", container]
label = f"{signal} {container}"
print(f"== {label} (instance {target})", flush=True)
result = scenario(
args, label, action, ["./scripts/restore.sh"], target=target
)
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
results.append(result)
return results
def rolling(args: argparse.Namespace) -> list[dict]:
env = dict(os.environ)
env.setdefault("APP_REVISION", f"exp-{int(time.time())}")
print(f"== rolling update to revision {env['APP_REVISION']}", flush=True)
result = scenario(
args, "rolling update", ["./scripts/rolling-update.sh"], None, env=env
)
result["revisions_after"] = run(["./scripts/instances.sh"]).splitlines()
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
return [result]
def brief(result: dict) -> dict:
"""Краткая строка прогона для терминала; полная версия — в --output"""
return {
"scenario": result["scenario"],
"rps_ok_write": {k: v["write"]["ok_rps"] for k, v in result["phases"].items()},
"rps_ok_read": {k: v["read"]["ok_rps"] for k, v in result["phases"].items()},
"errors": {
k: v["write"]["errors"] + v["read"]["errors"]
for k, v in result["phases"].items()
},
"recovery": result["recovery"],
"target_responses_while_stopped": result["target_responses_while_stopped"],
"restored_replica_in_rotation_seconds": result[
"restored_replica_in_rotation_seconds"
],
"instances_seen_from_action": result["instances_seen_from_action"],
"integrity": result["integrity"],
}
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("mode", choices=["failover", "rolling"])
parser.add_argument("--base-url", default=None)
parser.add_argument("--workers", type=int, default=4)
parser.add_argument("--before", type=float, default=8, help="нагрузка до отказа")
parser.add_argument("--outage", type=float, default=10, help="реплика остановлена")
parser.add_argument("--after", type=float, default=8, help="после восстановления")
parser.add_argument(
"--signals",
nargs="+",
default=["SIGTERM", "SIGKILL"],
choices=["SIGTERM", "SIGKILL"],
)
parser.add_argument("--output", default=None, help="JSON с полными результатами")
args = parser.parse_args()
started = time.strftime("%Y-%m-%dT%H:%M:%S%z")
results = failover(args) if args.mode == "failover" else rolling(args)
report = {
"mode": args.mode,
"started": started,
"profile": {
"workers": args.workers,
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
"client_timeout_seconds": REQUEST_TIMEOUT_SEC,
"phases_seconds": {
"before": args.before,
"outage": args.outage,
"after": args.after,
},
},
"runs": results,
}
if args.output:
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
print(f"saved: {args.output}")
failed = [
r["scenario"]
for r in results
if r["integrity"]["confirmed_missing_or_changed"]
or r["integrity"]["duplicate_ids"]
or r["integrity"]["duplicate_texts"]
or r["target_responses_while_stopped"]
]
if failed:
raise SystemExit(f"FAIL: потеря/дубль или отказ не изолирован: {failed}")
print(f"PASS: прогонов {len(results)}, подтверждённые сообщения целы, дублей нет")
if __name__ == "__main__":
main()
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
# Операторский вид API-реплик: контейнер ↔ alias X-Instance-Id ↔ ревизия образа.
# Нужен, чтобы показать, какой реальный контейнер обслужил запрос или остановлен:
# заголовок сам по себе не доказывает разные процессы.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
printf '%-24s %-28s %-14s %-8s %s\n' CONTAINER STATE INSTANCE PID REVISION
for container in $(docker compose --env-file "$ENV_FILE" ps -a -q api); do
name=$(docker inspect -f '{{.Name}}' "$container")
state=$(docker inspect -f '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$container")
pid=$(docker inspect -f '{{.State.Pid}}' "$container")
revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container")
alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || echo -)
printf '%-24s %-28s %-14s %-8s %s\n' "${name#/}" "$state" "$alias" "$pid" "$revision"
done
+6 -4
View File
@@ -1,9 +1,11 @@
#!/bin/sh
set -eu
# Перезапуск API и всех используемых хранилищ (PostgreSQL, Valkey) с сохранением
# volumes: никаких down -v, flush или повторного seed. Готовность ждёт тест.
# Перезапуск всего стенда с сохранением volumes: хранилища (PostgreSQL, Valkey),
# затем все API-реплики и балансировщик. Никаких down -v, flush или повторного
# seed; migrate не запускается — схема уже в volume. Готовность ждёт тест.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
docker compose --env-file "$ENV_FILE" restart postgres valkey
docker compose --env-file "$ENV_FILE" restart api
docker compose --env-file "$ENV_FILE" restart --no-deps postgres valkey
docker compose --env-file "$ENV_FILE" restart --no-deps api
docker compose --env-file "$ENV_FILE" restart --no-deps nginx
+9
View File
@@ -0,0 +1,9 @@
#!/bin/sh
set -eu
# Возврат API-реплик к числу API_REPLICAS после stop-one.sh: запуск остановленных
# контейнеров и ожидание healthy. Работающие реплики не пересоздаются
# (--no-recreate), хранилища, migrate и LB не перезапускаются.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
docker compose --env-file "$ENV_FILE" up -d --no-deps --no-recreate --wait --wait-timeout 60 api
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
set -eu
# Поочерёдная замена API-реплик новой сборкой без остановки обслуживания.
# 1. build нового образа (ревизия видна в метке образа и журнале старта);
# 2. migrate однократно: миграции только расширяют схему, старые реплики
# продолжают работать на новой схеме;
# 3. для каждой старой реплики: +1 новая → ждать healthy → пауза на обновление
# DNS в nginx → graceful stop старой (SIGTERM, LB уводит трафик) → удалить.
# Во время замены реплик на одну больше, чем API_REPLICAS: ресурс заложен заранее.
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
COMPOSE="docker compose --env-file $ENV_FILE"
APP_REVISION="${APP_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date +%s)}"
export APP_REVISION
# ID образа зависит только от содержимого (без provenance-attestation)
export BUILDX_NO_DEFAULT_ATTESTATIONS=1
# Больше resolver valid=2s в nginx: новая реплика уже в upstream до остановки старой
LB_DNS_REFRESH_SEC=3
STOP_TIMEOUT_SEC=10
log() {
echo "[$(date +%H:%M:%S)] $*"
}
log "build revision $APP_REVISION"
$COMPOSE build --quiet api
log "migrate"
$COMPOSE run --rm --no-deps migrate >/dev/null
old_containers=$($COMPOSE ps -q api)
for container in $old_containers; do
name=$(docker inspect -f '{{.Name}}' "$container")
running=$($COMPOSE ps -q api | wc -l | tr -d ' ')
log "scale api to $((running + 1)), wait healthy"
$COMPOSE up -d --no-deps --no-recreate --wait --wait-timeout 60 \
--scale "api=$((running + 1))" api
sleep "$LB_DNS_REFRESH_SEC"
log "graceful stop ${name#/}"
docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null
docker rm "$container" >/dev/null
done
log "done: all API replicas run revision $APP_REVISION"
+78 -57
View File
@@ -1,8 +1,11 @@
#!/usr/bin/env python3
"""SIGTERM: процесс перестаёт принимать запросы и завершается не дольше чем за 10 секунд.
"""SIGTERM одной API-реплики за балансировщиком.
Во время проверки фоновый клиент непрерывно шлёт GET /health/live: до сигнала
ответы 200, после — отказ в соединении; ответов 5xx быть не должно.
Реплика перестаёт принимать соединения, дожидается текущих запросов и
завершается не дольше чем за 10 секунд; LB уводит трафик на другие реплики.
Во время проверки фоновые клиенты непрерывно шлют GET /health/live через LB:
ошибок (5xx/обрыв) быть не должно, после выхода реплики её X-Instance-Id
в ответах не появляется.
"""
import argparse
@@ -20,106 +23,124 @@ DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url
SHUTDOWN_LIMIT_SEC = 10
PROBE_INTERVAL_SEC = 0.02
PROBERS = 4
class Prober(threading.Thread):
"""Фоновый клиент: каждое обращение — новое соединение"""
"""Фоновый клиент через LB: время ответа и обслужившая реплика"""
def __init__(self, url: str) -> None:
super().__init__(daemon=True)
self.url = url
self.stop = threading.Event()
self.ok: list[float] = []
self.refused: list[float] = []
self.server_errors = 0
self.ok: list[tuple[float, str]] = []
self.errors: list[tuple[float, str]] = []
def run(self) -> None:
while not self.stop.is_set():
try:
with urllib.request.urlopen(self.url, timeout=2) as response:
if response.status == 200:
self.ok.append(time.monotonic())
self.ok.append(
(time.monotonic(), response.headers.get("X-Instance-Id", ""))
)
except urllib.error.HTTPError as error:
if error.code >= 500:
self.server_errors += 1
except OSError:
self.refused.append(time.monotonic())
self.errors.append((time.monotonic(), f"HTTP {error.code}"))
except OSError as error:
self.errors.append((time.monotonic(), type(error).__name__))
time.sleep(PROBE_INTERVAL_SEC)
def docker(*args: str) -> str:
result = subprocess.run(
["docker", *args], check=True, capture_output=True, text=True
)
return result.stdout.strip()
def first_api_container() -> str:
ids = docker("compose", "--env-file", ".env", "ps", "-q", "api").split()
if not ids:
raise SystemExit("FAIL: нет запущенных API-реплик")
return docker("inspect", "-f", "{{.Name}}", ids[0]).lstrip("/")
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--base-url", default=os.environ.get("BASE_URL") or DEFAULT_BASE_URL
)
parser.add_argument(
"--container", default=os.environ.get("CONTAINER", "messenger-lab3-api-1")
)
parser.add_argument("--container", default=os.environ.get("CONTAINER"))
args = parser.parse_args()
os.chdir(ROOT)
container = args.container or first_api_container()
instance = docker("exec", container, "cat", "/tmp/instance-id")
logs = subprocess.Popen(
["docker", "logs", "--follow", "--since", "0s", args.container],
["docker", "logs", "--follow", "--since", "0s", container],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
text=True,
)
waiter = subprocess.Popen(
["docker", "wait", args.container], stdout=subprocess.PIPE, text=True
["docker", "wait", container], stdout=subprocess.PIPE, text=True
)
prober = Prober(args.base_url.rstrip("/") + "/health/live")
prober.start()
probers = [
Prober(args.base_url.rstrip("/") + "/health/live") for _ in range(PROBERS)
]
for prober in probers:
prober.start()
time.sleep(1)
if not prober.ok:
raise SystemExit("FAIL: API не отвечает до отправки SIGTERM")
before = {alias for prober in probers for _, alias in prober.ok}
if instance not in before:
raise SystemExit("FAIL: целевая реплика не обслуживает запросы до SIGTERM")
sent_at = time.monotonic()
subprocess.run(
["docker", "kill", "--signal", "SIGTERM", args.container],
check=True,
capture_output=True,
)
docker("kill", "--signal", "SIGTERM", container)
exit_code = int(waiter.communicate(timeout=SHUTDOWN_LIMIT_SEC + 5)[0].strip())
exited_at = time.monotonic()
time.sleep(0.5)
prober.stop.set()
prober.join()
time.sleep(1)
for prober in probers:
prober.stop.set()
prober.join()
logs.terminate()
shutdown_log = logs.communicate(timeout=5)[0]
shutdown_sec = exited_at - sent_at
ok_after = [t for t in prober.ok if t > sent_at]
first_refused = min((t for t in prober.refused if t > sent_at), default=None)
print("Логи контейнера после SIGTERM:")
ok_after = [(t, a) for p in probers for t, a in p.ok if t > sent_at]
errors_after = [(t, e) for p in probers for t, e in p.errors if t > sent_at]
target_after_exit = [t for t, a in ok_after if a == instance and t > exited_at]
last_target = max((t for t, a in ok_after if a == instance), default=None)
print("Журнал реплики после SIGTERM:")
print(
"\n".join(line for line in shutdown_log.splitlines() if "/health/" not in line)
)
print(
json.dumps(
{
"container": args.container,
"exit_code": exit_code,
"shutdown_seconds": round(shutdown_sec, 3),
"limit_seconds": SHUTDOWN_LIMIT_SEC,
"probes_ok_before_sigterm": len(prober.ok) - len(ok_after),
"probes_ok_after_sigterm": len(ok_after),
"first_refused_after_sigterm_seconds": None
if first_refused is None
else round(first_refused - sent_at, 3),
"probes_refused_after_sigterm": len(prober.refused),
"server_errors_5xx": prober.server_errors,
},
indent=2,
)
)
summary = {
"container": container,
"instance": instance,
"exit_code": exit_code,
"shutdown_seconds": round(exited_at - sent_at, 3),
"limit_seconds": SHUTDOWN_LIMIT_SEC,
"instances_before": sorted(before),
"probes_ok_after_sigterm": len(ok_after),
"instances_after_sigterm": sorted({a for _, a in ok_after}),
"last_target_response_after_sigterm_seconds": None
if last_target is None
else round(last_target - sent_at, 3),
"target_responses_after_exit": len(target_after_exit),
"errors_after_sigterm": len(errors_after),
"error_kinds": sorted({e for _, e in errors_after}),
}
print(json.dumps(summary, indent=2, ensure_ascii=False))
if (
exit_code != 0
or shutdown_sec > SHUTDOWN_LIMIT_SEC
or prober.server_errors
or first_refused is None
or exited_at - sent_at > SHUTDOWN_LIMIT_SEC
or errors_after
or target_after_exit
or not ok_after
):
raise SystemExit("FAIL: завершение по SIGTERM не соответствует требованиям")
print(
"PASS: SIGTERM завершил процесс за %.2f с, новые соединения отклоняются"
% shutdown_sec
"PASS: SIGTERM завершил %s за %.2f с, LB обслужил %d запросов без ошибок"
% (container, exited_at - sent_at, len(ok_after))
)
+24
View File
@@ -0,0 +1,24 @@
#!/bin/sh
set -eu
# Остановка ОДНОЙ API-реплики, обслужившей тест: TARGET_INSTANCE_ID → контейнер.
# Каждый процесс API пишет свой alias (X-Instance-Id) в /tmp/instance-id.
# docker stop: SIGTERM → graceful shutdown; restart-политика unless-stopped
# не поднимает вручную остановленный контейнер, он остаётся остановленным.
# БД, LB и volumes не затрагиваются. Восстановление — scripts/restore.sh.
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
cd "$(dirname "$0")/.."
ENV_FILE="${ENV_FILE:-.env}"
# Меньше 10 с лимита hook; больше drain + graceful shutdown (1 + 5 с)
STOP_TIMEOUT_SEC=8
for container in $(docker compose --env-file "$ENV_FILE" ps -q api); do
alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || true)
if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then
name=$(docker inspect -f '{{.Name}}' "$container")
docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null
echo "stopped ${name#/} (instance $alias)" >&2
exit 0
fi
done
echo "no running API replica with instance $TARGET_INSTANCE_ID" >&2
exit 1
@@ -1,20 +1,26 @@
import re
import time
import uuid
from fastapi import Request, Response
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
from application.api.utils import REQUEST_ID_HEADER, error_response
from config import LOGGER
from application.api.utils import INSTANCE_ID_HEADER, REQUEST_ID_HEADER, error_response
from config import INSTANCE_ID, LOGGER
from domain.errors import AppError
HEALTH_PATH_PREFIX = "/health/"
INCOMING_REQUEST_ID = re.compile(r"^[0-9a-f]{32}$")
class RequestContextMiddleware(BaseHTTPMiddleware):
"""
X-Request-Id в каждом ответе, журнал запросов и защита от необработанных ошибок.
X-Request-Id и X-Instance-Id в каждом ответе, журнал запросов и защита
от необработанных ошибок.
X-Request-Id берётся у балансировщика (nginx выставляет $request_id),
поэтому одна строка access-лога LB и журнал реплики связаны одним ID.
В лог попадают метод, путь, статус и длительность — без тел запросов,
query-параметров и текста сообщений.
"""
@@ -22,7 +28,7 @@ class RequestContextMiddleware(BaseHTTPMiddleware):
async def dispatch(
self, request: Request, call_next: RequestResponseEndpoint
) -> Response:
request_id = uuid.uuid4().hex
request_id = self._request_id(request)
request.state.request_id = request_id
started = time.perf_counter()
try:
@@ -34,9 +40,19 @@ class RequestContextMiddleware(BaseHTTPMiddleware):
request_id, error.status_code, error.code, error.message
)
response.headers[REQUEST_ID_HEADER] = request_id
response.headers[INSTANCE_ID_HEADER] = INSTANCE_ID
if getattr(request.app.state, "is_draining", False):
response.headers["Connection"] = "close"
self._log_request(request, response, request_id, started)
return response
@staticmethod
def _request_id(request: Request) -> str:
incoming = request.headers.get(REQUEST_ID_HEADER, "")
if INCOMING_REQUEST_ID.fullmatch(incoming):
return incoming
return uuid.uuid4().hex
@staticmethod
def _log_request(
request: Request, response: Response, request_id: str, started: float
@@ -1,10 +1,12 @@
from typing import Annotated
from uuid import UUID
from fastapi import APIRouter, Body, Depends, Query, status
from fastapi import APIRouter, Body, Depends, Header, Query, status
from application.api.dependencies import get_current_user
from domain import (
IDEMPOTENCY_KEY_MAX_LENGTH,
IDEMPOTENCY_KEY_PATTERN,
CreateMessageSchema,
MessagePageSchema,
MessageSchema,
@@ -26,10 +28,23 @@ messages_router = APIRouter(
async def send_message(
chat_id: UUID,
body: Annotated[CreateMessageSchema, Body(...)],
idempotency_key: Annotated[
str,
Header(
alias="Idempotency-Key",
min_length=1,
max_length=IDEMPOTENCY_KEY_MAX_LENGTH,
pattern=IDEMPOTENCY_KEY_PATTERN,
),
],
current_user: UserSchema = Depends(get_current_user),
service: MessageService = Depends(ServiceCreator.get_message_service),
) -> MessageSchema:
return await service.send_message(current_user, chat_id, body)
"""
Idempotency-Key обязателен: повтор того же тела возвращает тот же 201
и то же сообщение, другое тело с тем же ключом — 409. Нет/неверный ключ — 400.
"""
return await service.send_message(current_user, chat_id, body, idempotency_key)
@messages_router.get(
+12 -2
View File
@@ -1,3 +1,13 @@
from .errors import REQUEST_ID_HEADER, error_response, get_request_id
from .errors import (
INSTANCE_ID_HEADER,
REQUEST_ID_HEADER,
error_response,
get_request_id,
)
__all__ = ["REQUEST_ID_HEADER", "error_response", "get_request_id"]
__all__ = [
"REQUEST_ID_HEADER",
"INSTANCE_ID_HEADER",
"error_response",
"get_request_id",
]
+1
View File
@@ -6,6 +6,7 @@ from fastapi.responses import JSONResponse
from domain.errors import ErrorBodySchema, ErrorCode, ErrorSchema
REQUEST_ID_HEADER = "X-Request-Id"
INSTANCE_ID_HEADER = "X-Instance-Id"
def get_request_id(request: Request) -> str:
+12
View File
@@ -2,7 +2,9 @@ from .log_config import LOGGER
from .settings import (
APP_ENV,
APP_HOST,
APP_KEEPALIVE_TIMEOUT_SEC,
APP_PORT,
APP_REVISION,
DB_COMMAND_TIMEOUT_SEC,
DB_CONNECT_TIMEOUT_SEC,
DB_HOST,
@@ -13,10 +15,14 @@ from .settings import (
DB_PORT,
DB_STARTUP_TIMEOUT_SEC,
DB_USER,
IDEMPOTENCY_TTL_SECONDS,
INSTANCE_ID,
INSTANCE_ID_FILE,
LOG_LEVEL,
PASSWORD_HASH_WORKERS,
SESSION_CACHE_TTL_SECONDS,
SESSION_TTL_SECONDS,
SHUTDOWN_DRAIN_SEC,
SHUTDOWN_TIMEOUT_SEC,
VALKEY_HOST,
VALKEY_PASS,
@@ -31,8 +37,11 @@ __all__ = [
"APP_HOST",
"APP_PORT",
"APP_ENV",
"APP_REVISION",
"APP_KEEPALIVE_TIMEOUT_SEC",
"LOG_LEVEL",
"SHUTDOWN_TIMEOUT_SEC",
"SHUTDOWN_DRAIN_SEC",
"DB_USER",
"DB_PASS",
"DB_NAME",
@@ -52,4 +61,7 @@ __all__ = [
"SESSION_TTL_SECONDS",
"SESSION_CACHE_TTL_SECONDS",
"PASSWORD_HASH_WORKERS",
"IDEMPOTENCY_TTL_SECONDS",
"INSTANCE_ID",
"INSTANCE_ID_FILE",
]
+10
View File
@@ -1,4 +1,6 @@
import os
import secrets
import tempfile
from pathlib import Path
from envparse import Env
@@ -12,9 +14,13 @@ if ENV_FILE.is_file():
APP_HOST = env.str("APP_HOST")
APP_PORT = env.int("APP_PORT")
APP_ENV = env.str("APP_ENV")
APP_REVISION = env.str("APP_REVISION", default="dev")
# Больше keepalive_timeout балансировщика: idle-соединение закрывает LB, а не API
APP_KEEPALIVE_TIMEOUT_SEC = env.int("APP_KEEPALIVE_TIMEOUT_SEC")
LOG_LEVEL = env.str("LOG_LEVEL")
SHUTDOWN_TIMEOUT_SEC = env.int("SHUTDOWN_TIMEOUT_SEC")
SHUTDOWN_DRAIN_SEC = env.float("SHUTDOWN_DRAIN_SEC")
DB_USER = env.str("DB_USER")
DB_PASS = env.str("DB_PASS")
@@ -37,3 +43,7 @@ VALKEY_RETRY_AFTER_SEC = env.float("VALKEY_RETRY_AFTER_SEC")
SESSION_TTL_SECONDS = env.int("SESSION_TTL_SECONDS")
SESSION_CACHE_TTL_SECONDS = env.int("SESSION_CACHE_TTL_SECONDS")
PASSWORD_HASH_WORKERS = env.int("PASSWORD_HASH_WORKERS")
IDEMPOTENCY_TTL_SECONDS = env.int("IDEMPOTENCY_TTL_SECONDS")
INSTANCE_ID = secrets.token_hex(6)
INSTANCE_ID_FILE = Path(tempfile.gettempdir()) / "instance-id"
+11 -1
View File
@@ -1,7 +1,14 @@
from .auth import ActiveSessionSchema, BasicCredentialsSchema, SessionSchema
from .chats import ChatPageSchema, ChatSchema, CreateChatSchema
from .health import HealthSchema
from .messages import CreateMessageSchema, MessagePageSchema, MessageSchema
from .messages import (
IDEMPOTENCY_KEY_MAX_LENGTH,
IDEMPOTENCY_KEY_PATTERN,
CreateMessageSchema,
IdempotencyKeySchema,
MessagePageSchema,
MessageSchema,
)
from .pagination import PageCursorSchema, PageKey
from .users import CreateUserSchema, UserCredentialsSchema, UserSchema
@@ -16,6 +23,9 @@ __all__ = [
"ChatSchema",
"ChatPageSchema",
"CreateMessageSchema",
"IdempotencyKeySchema",
"IDEMPOTENCY_KEY_PATTERN",
"IDEMPOTENCY_KEY_MAX_LENGTH",
"MessageSchema",
"MessagePageSchema",
"HealthSchema",
+11 -1
View File
@@ -1,7 +1,17 @@
from .schemas import CreateMessageSchema, MessagePageSchema, MessageSchema
from .schemas import (
IDEMPOTENCY_KEY_MAX_LENGTH,
IDEMPOTENCY_KEY_PATTERN,
CreateMessageSchema,
IdempotencyKeySchema,
MessagePageSchema,
MessageSchema,
)
__all__ = [
"IDEMPOTENCY_KEY_PATTERN",
"IDEMPOTENCY_KEY_MAX_LENGTH",
"CreateMessageSchema",
"IdempotencyKeySchema",
"MessageSchema",
"MessagePageSchema",
]
+18
View File
@@ -7,6 +7,9 @@ from domain.utils import NotBlankStr, UtcDateTime
MessageText = Annotated[NotBlankStr, Field(min_length=1, max_length=4000)]
IDEMPOTENCY_KEY_PATTERN = r"^[A-Za-z0-9._:-]{1,128}$"
IDEMPOTENCY_KEY_MAX_LENGTH = 128
class CreateMessageSchema(BaseModel):
"""Запрос отправки сообщения: отправитель берётся из текущей сессии"""
@@ -33,3 +36,18 @@ class MessagePageSchema(BaseModel):
items: list[MessageSchema]
next_cursor: str | None
class IdempotencyKeySchema(BaseModel):
"""
Ключ повторной отправки сообщения.
request_hash — SHA-256 канонического тела: повтор с тем же ключом и тем же
телом возвращает исходное сообщение, с другим телом — конфликт.
"""
model_config = ConfigDict(frozen=True)
key: str
request_hash: bytes
ttl_seconds: int
+63 -8
View File
@@ -1,3 +1,5 @@
import asyncio
import socket
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
@@ -10,11 +12,16 @@ from application.api.middlewares import RequestContextMiddleware
from config import (
APP_ENV,
APP_HOST,
APP_KEEPALIVE_TIMEOUT_SEC,
APP_PORT,
APP_REVISION,
DB_STARTUP_TIMEOUT_SEC,
INSTANCE_ID,
INSTANCE_ID_FILE,
LOGGER,
PASSWORD_HASH_WORKERS,
SESSION_TTL_SECONDS,
SHUTDOWN_DRAIN_SEC,
SHUTDOWN_TIMEOUT_SEC,
)
from infrastructure import PasswordHasher, cache_manager, database_pool_manager
@@ -22,29 +29,35 @@ from infrastructure import PasswordHasher, cache_manager, database_pool_manager
@asynccontextmanager
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
"""
uvicorn открывает порт только после startup: реплика без БД или со старой
схемой не принимает соединений, и LB отправляет запросы другим репликам.
"""
await database_pool_manager.create_pool(DB_STARTUP_TIMEOUT_SEC)
await database_pool_manager.apply_migrations()
await database_pool_manager.check_migrations()
cache_manager.create_client()
password_hasher = PasswordHasher(PASSWORD_HASH_WORKERS)
app.state.database = database_pool_manager
app.state.cache = cache_manager
app.state.password_hasher = password_hasher
INSTANCE_ID_FILE.write_text(INSTANCE_ID)
app.state.is_ready = True
LOGGER.info(
f"Application started, env={APP_ENV}, session ttl={SESSION_TTL_SECONDS} s"
f"Application started, env={APP_ENV}, revision={APP_REVISION}, "
f"instance={INSTANCE_ID}, session ttl={SESSION_TTL_SECONDS} s"
)
yield
app.state.is_ready = False
await cache_manager.close()
await database_pool_manager.close()
password_hasher.close()
LOGGER.info("Application stopped, connections closed")
LOGGER.info(f"Application stopped, instance={INSTANCE_ID}, connections closed")
app = FastAPI(
title="mtuci-labs-messanger",
description="мессенджер-конкурент-максу",
version="1.2.0",
version="1.3.0",
lifespan=lifespan,
)
@@ -55,17 +68,59 @@ app.include_router(health_endpoints)
app.include_router(api_endpoints)
def main(host: str, port: int, shutdown_timeout: int) -> None:
"""Функция для запуска api"""
uvicorn.run(
class DrainingServer(uvicorn.Server):
"""
SIGTERM: сначала drain, затем штатный graceful shutdown uvicorn.
"""
def __init__(self, config: uvicorn.Config, drain_seconds: float) -> None:
super().__init__(config)
self._drain_seconds = drain_seconds
async def shutdown(self, sockets: list[socket.socket] | None = None) -> None:
for server in getattr(self, "servers", []):
server.close()
app.state.is_ready = False
app.state.is_draining = True
LOGGER.info(
f"SIGTERM: drain {self._drain_seconds} с, instance={INSTANCE_ID}: "
"порт закрыт, keep-alive соединения закрываются после ответа"
)
await asyncio.sleep(self._drain_seconds)
await super().shutdown(sockets)
def main(
host: str,
port: int,
shutdown_timeout: int,
keepalive_timeout: int,
drain_seconds: float,
) -> None:
"""
Функция для запуска api.
SIGTERM: drain_seconds без приёма новых соединений, затем uvicorn ждёт
текущих запросов не дольше shutdown_timeout, и lifespan закрывает пулы.
Сумма меньше stop_grace_period контейнера (10 с).
"""
config = uvicorn.Config(
app,
host=host,
port=port,
timeout_graceful_shutdown=shutdown_timeout,
timeout_keep_alive=keepalive_timeout,
access_log=False,
server_header=False,
)
DrainingServer(config, drain_seconds).run()
if __name__ == "__main__":
main(host=APP_HOST, port=APP_PORT, shutdown_timeout=SHUTDOWN_TIMEOUT_SEC)
main(
host=APP_HOST,
port=APP_PORT,
shutdown_timeout=SHUTDOWN_TIMEOUT_SEC,
keepalive_timeout=APP_KEEPALIVE_TIMEOUT_SEC,
drain_seconds=SHUTDOWN_DRAIN_SEC,
)
+33 -3
View File
@@ -106,8 +106,10 @@ class DatabasePoolManager:
"""
Применение миграций NNNN_name.sql по порядку версий.
Вызывается однократно отдельным процессом (src/migrate.py, сервис migrate)
до запуска API-реплик, а не каждой репликой при старте.
Каждая миграция и запись о ней в schema_migrations — одна транзакция.
Advisory lock не даёт двум процессам API применять миграции одновременно;
Advisory lock — страховка от двух одновременных запусков migrate;
уже применённые версии пропускаются, поэтому повторный запуск безопасен.
"""
async with self.acquire() as conn:
@@ -133,8 +135,7 @@ class DatabasePoolManager:
"SELECT version FROM schema_migrations;"
)
}
for path in sorted(directory.glob("*.sql")):
version = int(path.name.split("_", 1)[0])
for version, path in self._migration_files(directory):
if version in applied:
continue
async with conn.transaction():
@@ -155,6 +156,35 @@ class DatabasePoolManager:
)
LOGGER.info("Схема БД в актуальной версии")
async def check_migrations(self, directory: Path = MIGRATIONS_DIR) -> None:
"""
Схема БД не старше кода: все миграции из образа уже применены.
API миграции не применяет. Если migrate ещё не отработал, реплика не
стартует и не начинает слушать порт, балансировщик её не использует
"""
expected = {version for version, _ in self._migration_files(directory)}
async with self.acquire() as conn:
try:
records = await conn.fetch("SELECT version FROM schema_migrations;")
except asyncpg.UndefinedTableError:
records = []
applied = {record["version"] for record in records}
missing = sorted(expected - applied)
if missing:
raise RuntimeError(
f"В БД не применены миграции {missing}: сначала запустите migrate"
)
LOGGER.info(f"Схема БД совместима: версия {max(applied, default=0)}")
@staticmethod
def _migration_files(directory: Path) -> list[tuple[int, Path]]:
"""Файлы NNNN_name.sql по возрастанию версии"""
return sorted(
(int(path.name.split("_", 1)[0]), path) for path in directory.glob("*.sql")
)
@asynccontextmanager
async def acquire(self) -> AsyncIterator[asyncpg.Connection]:
"""
@@ -1,6 +1,10 @@
from fastapi import Depends
from config import SESSION_CACHE_TTL_SECONDS, SESSION_TTL_SECONDS
from config import (
IDEMPOTENCY_TTL_SECONDS,
SESSION_CACHE_TTL_SECONDS,
SESSION_TTL_SECONDS,
)
from infrastructure.dependencies.cache import CacheManager, get_cache
from infrastructure.dependencies.database import DatabasePoolManager, get_database
from infrastructure.repositories import (
@@ -90,4 +94,6 @@ class ServiceCreator:
chat_repository: ChatRepository = Depends(_get_chat_repository),
message_repository: MessageRepository = Depends(_get_message_repository),
) -> MessageService:
return MessageService(chat_repository, message_repository)
return MessageService(
chat_repository, message_repository, IDEMPOTENCY_TTL_SECONDS
)
@@ -2,7 +2,8 @@ from uuid import UUID
import asyncpg
from domain import MessageSchema, PageKey
from domain import IdempotencyKeySchema, MessageSchema, PageKey
from domain.errors import ConflictError
from infrastructure.utils import Pool, repository_logs, transaction
from interfaces import MessageRepository
@@ -15,11 +16,22 @@ class PostgresMessageRepository(MessageRepository):
self._pool = pool
@repository_logs
async def create(self, chat_id: UUID, sender_id: UUID, text: str) -> MessageSchema:
async def create(
self,
chat_id: UUID,
sender_id: UUID,
text: str,
idempotency: IdempotencyKeySchema,
) -> tuple[MessageSchema, bool]:
"""
Блокировка чата и вставка — одна транзакция; lock снимается на COMMIT.
Блокировка чата, проверка ключа, вставка сообщения и ключа — одна
транзакция; lock снимается на COMMIT.
Запись в один чат сериализуется, записи в разные чаты не мешают друг другу.
Ключ идемпотентности привязан к чату, поэтому та же блокировка
сериализует и одновременные повторы с одним ключом с разных реплик:
второй ждёт COMMIT первого и находит его ключ
created_at строго больше последнего в чате, даже если часы ушли назад:
порядок коммитов совпадает с порядком (created_at, id), и читатель
с cursor не пропустит сообщение. MAX берётся из индекса
@@ -28,6 +40,22 @@ class PostgresMessageRepository(MessageRepository):
lock_query = """
SELECT pg_advisory_xact_lock($1, hashtext($2::uuid::text));
"""
find_key_query = """
SELECT
idem.request_hash,
message.id,
message.chat_id,
message.sender_id,
message.text,
message.created_at
FROM message_idempotency_keys idem
JOIN messages message ON message.id = idem.message_id
WHERE
idem.sender_id = $1
AND idem.chat_id = $2
AND idem.idempotency_key = $3
AND idem.expires_at > clock_timestamp();
"""
insert_query = """
INSERT INTO messages
(
@@ -45,11 +73,56 @@ class PostgresMessageRepository(MessageRepository):
WHERE chat_id = $1
RETURNING id, chat_id, sender_id, text, created_at;
"""
save_key_query = """
INSERT INTO message_idempotency_keys
(
sender_id,
chat_id,
idempotency_key,
request_hash,
message_id,
expires_at
)
VALUES
(
$1,
$2,
$3,
$4,
$5,
clock_timestamp() + make_interval(secs => $6::integer)
)
ON CONFLICT (sender_id, chat_id, idempotency_key) DO UPDATE
SET
request_hash = EXCLUDED.request_hash,
message_id = EXCLUDED.message_id,
created_at = EXCLUDED.created_at,
expires_at = EXCLUDED.expires_at;
"""
async with transaction(self._pool) as conn:
await conn.execute(lock_query, CHAT_LOCK_NAMESPACE, chat_id)
stored = await conn.fetchrow(
find_key_query, sender_id, chat_id, idempotency.key
)
if stored is not None:
if stored["request_hash"] != idempotency.request_hash:
raise ConflictError(
"Idempotency-Key уже использован с другим телом запроса"
)
return self._row_to_message(stored), False
row = await conn.fetchrow(insert_query, chat_id, sender_id, text)
assert row is not None
return self._row_to_message(row)
assert row is not None
await conn.execute(
save_key_query,
sender_id,
chat_id,
idempotency.key,
idempotency.request_hash,
row["id"],
idempotency.ttl_seconds,
)
return self._row_to_message(row), True
@repository_logs
async def list_for_chat(
@@ -1,7 +1,14 @@
import hashlib
from uuid import UUID
from config import LOGGER
from domain import CreateMessageSchema, MessagePageSchema, MessageSchema, UserSchema
from domain import (
CreateMessageSchema,
IdempotencyKeySchema,
MessagePageSchema,
MessageSchema,
UserSchema,
)
from domain.errors import NotFoundError
from infrastructure.utils import build_page, decode_cursor
from interfaces import ChatRepository, MessageRepository
@@ -9,21 +16,47 @@ from interfaces import ChatRepository, MessageRepository
class MessageService:
def __init__(
self, chat_repository: ChatRepository, message_repository: MessageRepository
self,
chat_repository: ChatRepository,
message_repository: MessageRepository,
idempotency_ttl_seconds: int,
) -> None:
self._chats = chat_repository
self._messages = message_repository
self._idempotency_ttl = idempotency_ttl_seconds
async def send_message(
self, current_user: UserSchema, chat_id: UUID, data: CreateMessageSchema
self,
current_user: UserSchema,
chat_id: UUID,
data: CreateMessageSchema,
idempotency_key: str,
) -> MessageSchema:
"""Отправитель берётся из текущей идентификации; текст в логи не пишется"""
"""
Отправитель берётся из текущей идентификации; текст в логи не пишется.
Повтор с тем же Idempotency-Key заново проверяет доступ к чату
и возвращает исходное сообщение.
"""
await self._ensure_member(current_user, chat_id)
message = await self._messages.create(chat_id, current_user.id, data.text)
LOGGER.info(
f"Сообщение {message.id} отправлено в чат {chat_id} "
f"пользователем {current_user.id}"
idempotency = IdempotencyKeySchema(
key=idempotency_key,
request_hash=self._request_hash(data),
ttl_seconds=self._idempotency_ttl,
)
message, created = await self._messages.create(
chat_id, current_user.id, data.text, idempotency
)
if created:
LOGGER.info(
f"Сообщение {message.id} отправлено в чат {chat_id} "
f"пользователем {current_user.id}"
)
else:
LOGGER.info(
f"Повтор по Idempotency-Key: возвращено сообщение {message.id} "
f"чата {chat_id}"
)
return message
async def list_messages(
@@ -40,3 +73,11 @@ class MessageService:
"""Посторонний получает 404: существование чужого чата не раскрывается"""
if await self._chats.get_for_member(chat_id, current_user.id) is None:
raise NotFoundError()
@staticmethod
def _request_hash(data: CreateMessageSchema) -> bytes:
"""
SHA-256 канонического тела. Порядок полей задаёт схема, а не клиент,
поэтому JSON с другим порядком ключей считается тем же запросом.
"""
return hashlib.sha256(data.model_dump_json().encode()).digest()
+12 -2
View File
@@ -1,18 +1,28 @@
from abc import ABC, abstractmethod
from uuid import UUID
from domain import MessageSchema, PageKey
from domain import IdempotencyKeySchema, MessageSchema, PageKey
class MessageRepository(ABC):
"""Слой абстракции хранилища сообщений"""
@abstractmethod
async def create(self, chat_id: UUID, sender_id: UUID, text: str) -> MessageSchema:
async def create(
self,
chat_id: UUID,
sender_id: UUID,
text: str,
idempotency: IdempotencyKeySchema,
) -> tuple[MessageSchema, bool]:
"""
Добавление сообщения в конец истории чата. id и created_at назначает
хранилище; created_at строго возрастает внутри чата, поэтому сообщение
не может появиться перед уже выданным cursor.
Сообщение и запись ключа идемпотентности сохраняются атомарно.
Одновременные запросы с одним ключом через любые реплики создают
ровно одно сообщение; повтор возвращает его же.
"""
pass
+24
View File
@@ -0,0 +1,24 @@
"""
Однократное применение миграций до запуска API-реплик.
Запускается сервисом migrate (compose) и завершается; реплики стартуют
после его успешного завершения и только проверяют версию схемы.
"""
import asyncio
from config import DB_STARTUP_TIMEOUT_SEC, LOGGER
from infrastructure import database_pool_manager
async def main() -> None:
await database_pool_manager.create_pool(DB_STARTUP_TIMEOUT_SEC)
try:
await database_pool_manager.apply_migrations()
finally:
await database_pool_manager.close()
LOGGER.info("Миграции применены, migrate завершается")
if __name__ == "__main__":
asyncio.run(main())
+2 -2
View File
@@ -5,7 +5,7 @@ from pathlib import Path
import httpx
import pytest
from helpers import auth, basic, unique_password, unique_username
from helpers import auth, basic, message_headers, unique_password, unique_username
ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url"]
@@ -68,7 +68,7 @@ def send_message(api: httpx.Client) -> Callable[..., dict]:
response = api.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": text},
headers=auth(user),
headers=message_headers(user),
)
assert response.status_code == 201, response.text
return response.json()
+5
View File
@@ -30,6 +30,11 @@ def auth(user: dict) -> dict[str, str]:
return {"Authorization": f"Bearer {user['token']}"}
def message_headers(user: dict, key: str | None = None) -> dict[str, str]:
"""Bearer и Idempotency-Key (новый, если не задан)"""
return {**auth(user), "Idempotency-Key": key or uuid.uuid4().hex}
def public_user(user: dict) -> dict:
"""User из ответа API без тестовых полей token/password"""
return {field: user[field] for field in USER_FIELDS}
+74
View File
@@ -0,0 +1,74 @@
"""
Балансировка без sticky sessions: запросы одной сессии обслуживают разные
реплики, а состояние (сессии, сообщения, logout) общее для всех.
"""
import uuid
from collections import Counter
import httpx
from helpers import assert_unauthorized, auth, message_headers, public_user
REQUESTS = 20
def test_one_session_is_served_by_several_replicas(
api: httpx.Client, make_user
) -> None:
"""Одно keep-alive соединение клиента к LB, без повторного входа"""
alice = make_user("alice")
instances: Counter[str] = Counter()
for _ in range(REQUESTS):
response = api.get("/api/v1/users/me", headers=auth(alice))
assert response.status_code == 200, response.text
assert response.json() == public_user(alice)
instances[response.headers["X-Instance-Id"]] += 1
print("\nINSTANCES FOR ONE SESSION:", dict(instances))
assert len(instances) >= 2
def test_write_is_visible_through_every_replica(
api: httpx.Client, make_user, make_chat
) -> None:
alice = make_user("alice")
bob = make_user("bob")
chat = make_chat(alice, [alice, bob])
sent = api.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": f"видно везде {uuid.uuid4().hex}"},
headers=message_headers(alice),
)
assert sent.status_code == 201, sent.text
readers: set[str] = set()
for _ in range(REQUESTS):
page = api.get(f"/api/v1/chats/{chat['id']}/messages", headers=auth(bob))
assert page.status_code == 200, page.text
assert page.json()["items"] == [sent.json()]
readers.add(page.headers["X-Instance-Id"])
assert len(readers) >= 2
def test_logout_applies_to_all_replicas(api: httpx.Client, make_user) -> None:
"""Сессия попала в общий кэш через обе реплики, затем отозвана на одной"""
alice = make_user("alice")
for _ in range(4):
assert api.get("/api/v1/users/me", headers=auth(alice)).status_code == 200
logout = api.delete("/api/v1/auth/sessions/current", headers=auth(alice))
assert logout.status_code == 204
rejected_by: set[str] = set()
for _ in range(REQUESTS):
response = api.get("/api/v1/users/me", headers=auth(alice))
assert_unauthorized(response)
rejected_by.add(response.headers["X-Instance-Id"])
assert len(rejected_by) >= 2
def test_every_api_response_has_instance_id(api: httpx.Client) -> None:
for response in (
api.get("/health/live"),
api.get("/health/ready"),
api.get("/api/v1/unknown"),
api.get("/api/v1/users/me"),
):
assert response.headers["X-Instance-Id"]
assert response.headers["X-Request-Id"]
+2 -1
View File
@@ -5,7 +5,7 @@ import json
import time
import httpx
from helpers import auth, read_all_messages
from helpers import auth, message_headers, read_all_messages
CLIENTS = 10
MESSAGES_PER_CLIENT = 10
@@ -29,6 +29,7 @@ async def _send_concurrently(
client.post(
f"/api/v1/chats/{chat_id}/messages",
json={"text": f"{user['username']}#{n}"},
headers=message_headers(user),
)
for n in range(MESSAGES_PER_CLIENT)
)
+4 -2
View File
@@ -3,7 +3,7 @@
import uuid
import httpx
from helpers import assert_error, auth
from helpers import assert_error, auth, message_headers
def test_outsider_cannot_access_chat(
@@ -18,7 +18,9 @@ def test_outsider_cannot_access_chat(
assert_error(api.get(path, headers=auth(eve)), 404, "not_found")
assert_error(api.get(f"{path}/messages", headers=auth(eve)), 404, "not_found")
response = api.post(f"{path}/messages", json={"text": "чужое"}, headers=auth(eve))
response = api.post(
f"{path}/messages", json={"text": "чужое"}, headers=message_headers(eve)
)
assert_error(response, 404, "not_found")
history = api.get(f"{path}/messages", headers=auth(bob)).json()
+249
View File
@@ -0,0 +1,249 @@
"""Idempotency-Key отправки сообщения: повтор, конфликт, область ключа, гонки"""
import asyncio
import json
import uuid
from collections import Counter
import httpx
import pytest
from helpers import assert_error, auth, message_headers, psql, read_all_messages
PARALLEL_REPLAYS = 20
def post_message(
api: httpx.Client, user: dict, chat: dict, text: str, key: str
) -> httpx.Response:
return api.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": text},
headers=message_headers(user, key),
)
def history(api: httpx.Client, user: dict, chat: dict) -> list[dict]:
items, _ = read_all_messages(api, user, chat["id"], 100)
return items
def test_replay_returns_same_message(api: httpx.Client, make_user, make_chat) -> None:
alice = make_user("alice")
chat = make_chat(alice, [alice])
key = uuid.uuid4().hex
first = post_message(api, alice, chat, "один раз", key)
second = post_message(api, alice, chat, "один раз", key)
assert first.status_code == second.status_code == 201, second.text
assert first.json() == second.json()
assert history(api, alice, chat) == [first.json()]
def test_replay_ignores_json_formatting(
api: httpx.Client, make_user, make_chat
) -> None:
"""Пробелы и форма записи JSON не меняют смысл тела: это тот же запрос"""
alice = make_user("alice")
chat = make_chat(alice, [alice])
headers = {**message_headers(alice), "Content-Type": "application/json"}
path = f"/api/v1/chats/{chat['id']}/messages"
escaped = b'{"text":"\\u041f\\u0440\\u0438"}'
compact = api.post(path, content=escaped, headers=headers)
spaced = api.post(path, content='{ "text" : "При" }'.encode(), headers=headers)
assert compact.status_code == spaced.status_code == 201, spaced.text
assert compact.json() == spaced.json()
assert len(history(api, alice, chat)) == 1
def test_same_key_with_other_body_is_conflict(
api: httpx.Client, make_user, make_chat
) -> None:
alice = make_user("alice")
chat = make_chat(alice, [alice])
key = uuid.uuid4().hex
original = post_message(api, alice, chat, "оригинал", key).json()
assert_error(post_message(api, alice, chat, "другой", key), 409, "conflict")
assert history(api, alice, chat) == [original]
@pytest.mark.parametrize(
"headers",
[
{},
{"Idempotency-Key": ""},
{"Idempotency-Key": "with space"},
{"Idempotency-Key": "ключ".encode()},
{"Idempotency-Key": "a/b"},
{"Idempotency-Key": "x" * 129},
],
)
def test_missing_or_invalid_key_is_rejected(
api: httpx.Client, make_user, make_chat, headers: dict
) -> None:
alice = make_user("alice")
chat = make_chat(alice, [alice])
response = api.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": "без ключа"},
headers={**auth(alice), **headers},
)
assert_error(response, 400, "invalid_request")
assert history(api, alice, chat) == []
@pytest.mark.parametrize("key", ["x" * 128, "Az09._:-"])
def test_key_boundaries_are_accepted(
api: httpx.Client, make_user, make_chat, key: str
) -> None:
alice = make_user("alice")
chat = make_chat(alice, [alice])
assert post_message(api, alice, chat, "граница", key).status_code == 201
def test_key_scope_is_user_and_chat(api: httpx.Client, make_user, make_chat) -> None:
"""Один ключ у разных пользователей и в разных чатах — независимые сообщения"""
alice = make_user("alice")
bob = make_user("bob")
chat = make_chat(alice, [alice, bob])
other_chat = make_chat(alice, [alice])
key = uuid.uuid4().hex
from_alice = post_message(api, alice, chat, "текст", key).json()
from_bob = post_message(api, bob, chat, "текст", key).json()
in_other_chat = post_message(api, alice, other_chat, "текст", key).json()
assert len({from_alice["id"], from_bob["id"], in_other_chat["id"]}) == 3
assert from_bob["sender_id"] == bob["id"]
assert len(history(api, alice, chat)) == 2
assert len(history(api, alice, other_chat)) == 1
def test_replay_rechecks_session_and_access(
api: httpx.Client, make_user, make_chat
) -> None:
alice = make_user("alice")
eve = make_user("eve")
chat = make_chat(alice, [alice])
key = uuid.uuid4().hex
post_message(api, alice, chat, "секрет", key)
# Чужой с тем же ключом не получает сохранённое сообщение
assert_error(post_message(api, eve, chat, "секрет", key), 404, "not_found")
# Отозванная сессия не даёт повтора
logout = api.delete("/api/v1/auth/sessions/current", headers=auth(alice))
assert logout.status_code == 204
replay = post_message(api, alice, chat, "секрет", key)
assert_error(replay, 401, "unauthorized")
def test_expired_key_can_be_reused(api: httpx.Client, make_user, make_chat) -> None:
"""После срока хранения (24 ч) ключ снова создаёт сообщение; срок сдвигается в БД"""
alice = make_user("alice")
chat = make_chat(alice, [alice])
key = uuid.uuid4().hex
first = post_message(api, alice, chat, "вчера", key).json()
psql(
"UPDATE message_idempotency_keys "
"SET created_at = now() - interval '2 days', "
"expires_at = now() - interval '1 day' "
f"WHERE chat_id = '{chat['id']}' AND idempotency_key = '{key}';"
)
second = post_message(api, alice, chat, "сегодня", key)
assert second.status_code == 201, second.text
assert second.json()["id"] != first["id"]
assert [item["text"] for item in history(api, alice, chat)] == ["вчера", "сегодня"]
stored = psql(
"SELECT count(*) FROM message_idempotency_keys "
f"WHERE chat_id = '{chat['id']}' AND idempotency_key = '{key}';"
)
assert stored == "1"
async def _post_concurrently(
base_url: str, user: dict, chat: dict, key: str, texts: list[str]
) -> list[httpx.Response]:
"""Все запросы стартуют по одному сигналу; отдельные соединения к LB"""
start = asyncio.Event()
limits = httpx.Limits(max_connections=len(texts))
async with httpx.AsyncClient(
base_url=base_url, timeout=30, limits=limits
) as client:
async def post(text: str) -> httpx.Response:
await start.wait()
return await client.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": text},
headers=message_headers(user, key),
)
tasks = [asyncio.create_task(post(text)) for text in texts]
await asyncio.sleep(0)
start.set()
return list(await asyncio.gather(*tasks))
def test_parallel_replays_through_replicas_create_one_message(
api: httpx.Client, base_url: str, make_user, make_chat
) -> None:
alice = make_user("alice")
chat = make_chat(alice, [alice])
key = uuid.uuid4().hex
responses = asyncio.run(
_post_concurrently(
base_url, alice, chat, key, ["одно сообщение"] * PARALLEL_REPLAYS
)
)
statuses = Counter(response.status_code for response in responses)
instances = Counter(response.headers["X-Instance-Id"] for response in responses)
bodies = {json.dumps(response.json(), sort_keys=True) for response in responses}
items = history(api, alice, chat)
rows = psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}';")
print(
"\nPARALLEL SAME KEY:",
json.dumps(
{
"requests": len(responses),
"statuses": dict(statuses),
"instances": dict(instances),
"distinct_bodies": len(bodies),
"messages_in_history": len(items),
"rows_in_db": int(rows),
}
),
)
assert statuses == {201: PARALLEL_REPLAYS}
assert len(instances) >= 2, "запросы не распределились по репликам"
assert len(bodies) == 1
assert len(items) == 1 and rows == "1"
def test_parallel_same_key_with_different_bodies(
api: httpx.Client, base_url: str, make_user, make_chat
) -> None:
"""Победитель гонки один; совпавшие с ним тела — 201, остальные — 409"""
alice = make_user("alice")
chat = make_chat(alice, [alice])
key = uuid.uuid4().hex
texts = ["вариант A", "вариант B"] * (PARALLEL_REPLAYS // 2)
responses = asyncio.run(_post_concurrently(base_url, alice, chat, key, texts))
items = history(api, alice, chat)
assert len(items) == 1
winner = items[0]
for text, response in zip(texts, responses):
if text == winner["text"]:
assert response.status_code == 201, response.text
assert response.json() == winner
else:
assert_error(response, 409, "conflict")
+13 -3
View File
@@ -2,7 +2,7 @@
import httpx
import pytest
from helpers import assert_error, auth
from helpers import assert_error, auth, message_headers
PASSWORD = "Valid-password-123"
@@ -32,8 +32,16 @@ def test_malformed_json_on_create_chat_and_message(
assert_error(
api.post("/api/v1/chats", content=raw, headers=headers), 400, "invalid_request"
)
message_post_headers = {
**message_headers(alice),
"Content-Type": "application/json",
}
assert_error(
api.post(f"/api/v1/chats/{chat['id']}/messages", content=raw, headers=headers),
api.post(
f"/api/v1/chats/{chat['id']}/messages",
content=raw,
headers=message_post_headers,
),
400,
"invalid_request",
)
@@ -78,7 +86,9 @@ def test_invalid_message_payload(
alice = make_user("alice")
chat = make_chat(alice, [alice])
response = api.post(
f"/api/v1/chats/{chat['id']}/messages", json=body, headers=auth(alice)
f"/api/v1/chats/{chat['id']}/messages",
json=body,
headers=message_headers(alice),
)
assert_error(response, 400, "invalid_request")
+6 -4
View File
@@ -2,7 +2,7 @@
import httpx
import pytest
from helpers import assert_error, auth
from helpers import assert_error, auth, message_headers
@pytest.mark.parametrize("text", ["", " ", " \n\t ", " "])
@@ -12,7 +12,9 @@ def test_empty_or_blank_message_is_rejected(
alice = make_user("alice")
chat = make_chat(alice, [alice])
response = api.post(
f"/api/v1/chats/{chat['id']}/messages", json={"text": text}, headers=auth(alice)
f"/api/v1/chats/{chat['id']}/messages",
json={"text": text},
headers=message_headers(alice),
)
assert_error(response, 400, "invalid_request")
history = api.get(f"/api/v1/chats/{chat['id']}/messages", headers=auth(alice))
@@ -29,7 +31,7 @@ def test_message_length_is_counted_in_code_points(
response = api.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": "😀" * 4001},
headers=auth(alice),
headers=message_headers(alice),
)
assert_error(response, 400, "invalid_request")
@@ -56,7 +58,7 @@ def test_nul_character_is_rejected(api: httpx.Client, make_user, make_chat) -> N
response = api.post(
f"/api/v1/chats/{chat['id']}/messages",
json={"text": "до\u0000после"},
headers=auth(alice),
headers=message_headers(alice),
)
assert_error(response, 400, "invalid_request")
response = api.post(