feat: конф нгинкс, тесты, ключ идмпт etc
This commit is contained in:
@@ -33,3 +33,5 @@ requirements-dev.txt
|
||||
lab.json
|
||||
.mypy_cache
|
||||
.ruff_cache
|
||||
infra
|
||||
.venv-validate
|
||||
|
||||
@@ -6,6 +6,9 @@ APP_ENV=lab
|
||||
LOG_LEVEL=INFO
|
||||
SHUTDOWN_TIMEOUT_SEC=5
|
||||
|
||||
SHUTDOWN_DRAIN_SEC=1
|
||||
APP_KEEPALIVE_TIMEOUT_SEC=75
|
||||
|
||||
DB_USER=test
|
||||
DB_PASS=test
|
||||
DB_NAME=test
|
||||
@@ -30,4 +33,6 @@ PASSWORD_HASH_WORKERS=4
|
||||
|
||||
# lab3: число API-реплик за балансировщиком и срок хранения Idempotency-Key
|
||||
API_REPLICAS=2
|
||||
# Публичный порт балансировщика на 127.0.0.1 (порт для make test-expiry — LB_EXPIRY_PORT=8081)
|
||||
LB_PORT=8080
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
@@ -33,3 +33,4 @@ coverage/
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
.venv-validate/
|
||||
|
||||
@@ -15,6 +15,10 @@ COPY src ./src
|
||||
RUN useradd --system --uid 10001 --no-create-home app
|
||||
USER app
|
||||
|
||||
ARG APP_REVISION=dev
|
||||
ENV APP_REVISION=${APP_REVISION}
|
||||
LABEL org.opencontainers.image.revision=${APP_REVISION}
|
||||
|
||||
EXPOSE 8080
|
||||
HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \
|
||||
CMD ["python", "src/healthcheck.py"]
|
||||
|
||||
@@ -4,11 +4,15 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover install up down restart ps logs wait-ready destroy-data student-test concurrency-test test-expiry test-failures test-migrations explain sigterm-check
|
||||
export BUILDX_NO_DEFAULT_ATTESTATIONS := 1
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover install up down restart restore instances rolling-update ps logs wait-ready destroy-data student-test concurrency-test test-expiry test-failures test-migrations explain sigterm-check experiment-failover experiment-rolling
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "up/down/restart/ps/logs: compose stack; destroy-data: DELETE volumes intentionally"
|
||||
@echo "instances: API containers <-> X-Instance-Id; restore: start replicas stopped by stop-one.sh; rolling-update: replace replicas one by one"
|
||||
@echo "experiment-failover / experiment-rolling: load + stop each replica / rolling update, metrics to evidence/"
|
||||
@echo "student-test: own tests; test-migrations: from empty volume; test-expiry: short SESSION_TTL; test-failures: stop Valkey/PostgreSQL; explain: EXPLAIN history"
|
||||
|
||||
check:
|
||||
@@ -44,11 +48,20 @@ restart:
|
||||
./scripts/restart.sh
|
||||
@$(MAKE) --no-print-directory wait-ready
|
||||
|
||||
restore:
|
||||
./scripts/restore.sh
|
||||
|
||||
instances:
|
||||
./scripts/instances.sh
|
||||
|
||||
rolling-update:
|
||||
./scripts/rolling-update.sh
|
||||
|
||||
ps:
|
||||
docker compose ps
|
||||
|
||||
logs:
|
||||
docker compose logs -f api
|
||||
docker compose logs -f api nginx
|
||||
|
||||
wait-ready:
|
||||
@for i in $$(seq 1 60); do \
|
||||
@@ -65,7 +78,6 @@ student-test:
|
||||
concurrency-test:
|
||||
BASE_URL="$(BASE_URL)" poetry run pytest student-tests/test_concurrency.py -s
|
||||
|
||||
test-expiry:
|
||||
docker compose --profile expiry up -d --build --wait api-short-ttl
|
||||
EXPIRY_BASE_URL=http://localhost:8081 poetry run pytest student-tests/test_expiry.py -s; \
|
||||
status=$$?; docker compose --profile expiry stop api-short-ttl; exit $$status
|
||||
@@ -83,3 +95,9 @@ explain:
|
||||
|
||||
sigterm-check:
|
||||
$(PYTHON) scripts/sigterm_check.py --base-url "$(BASE_URL)"
|
||||
|
||||
experiment-failover:
|
||||
$(PYTHON) scripts/experiment.py failover --base-url "$(BASE_URL)" --output evidence/experiment-failover.json
|
||||
|
||||
experiment-rolling:
|
||||
$(PYTHON) scripts/experiment.py rolling --base-url "$(BASE_URL)" --output evidence/experiment-rolling.json
|
||||
|
||||
@@ -1,37 +1,160 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Саватеев Артём, БВТ2302
|
||||
- Целевая оценка: **4** (весь уровень 3 + весь уровень 4);
|
||||
- Источники: документация [nginx upstream](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) (`resolve`, `max_fails`, `keepalive_timeout`) и [proxy](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream) (`proxy_next_upstream`, повтор неидемпотентных запросов), [Compose services/deploy](https://docs.docker.com/reference/compose-file/services/), исходники uvicorn 0.54 (`Server.shutdown`, h11 keep-alive). Библиотеки — из lab2 (FastAPI, asyncpg, redis-py, argon2-cffi), новых зависимостей нет. Помощь ии
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
|
||||
```sh
|
||||
cp .env.example .env # затем задать свои DB_PASS и VALKEY_PASS (синтетические)
|
||||
docker compose --env-file .env config --quiet
|
||||
make up # = docker compose up -d --build --wait: build → postgres/valkey →
|
||||
# migrate (однократно) → api ×API_REPLICAS → nginx; ждёт healthy
|
||||
make check test # стенд доступен только через LB: http://localhost:8080
|
||||
python3 -m venv .venv-validate && .venv-validate/bin/python -m pip install -r requirements-dev.txt
|
||||
make validate PYTHON=.venv-validate/bin/python # полная OpenAPI-валидация; не в .venv —
|
||||
# его Poetry принял бы за окружение проекта (Python 3.9)
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
make instances # какой контейнер остановлен: контейнер ↔ X-Instance-Id ↔ PID ↔ ревизия
|
||||
make restore # вернуть остановленную реплику (работающие не пересоздаются)
|
||||
make student-test # собственные тесты (poetry install --no-root один раз)
|
||||
make rolling-update # выкладка новой сборки по одной реплике без простоя
|
||||
make down # безопасная остановка, volume с данными сохраняется
|
||||
make destroy-data # намеренное удаление учебных данных (down -v)
|
||||
```
|
||||
|
||||
Действия оператора: создать `.env` с секретами (в Git не попадает), после `stop-one.sh` вручную выполнить `make restore`. Миграции отдельно запускать не нужно — их выполняет сервис `migrate` внутри `make up`/`rolling-update`. Сборку через `make`/скрипты лучше, чем голый `docker compose up --build`: Makefile экспортирует `BUILDX_NO_DEFAULT_ATTESTATIONS=1`, иначе Docker Desktop добавляет к образу provenance-attestation, ID образа меняется на каждой сборке и Compose пересоздаёт все реплики разом. Обновлять работающий стенд — только `make rolling-update`.
|
||||
|
||||
Воспроизводимость проверена копией ровно тех файлов, что попадут в Git (без `.env`), отдельным compose-проектом на пустом volume: build, migrate с нуля, smoke, failover, persistence — [evidence/15-clean-clone.txt](evidence/15-clean-clone.txt).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент ──► nginx :8080 (127.0.0.1, единственный опубликованный порт)
|
||||
│ round-robin, без sticky; Docker DNS «api» перечитывается каждые 2 с
|
||||
├──► api-1 ─┐ сеть edge: только LB и API
|
||||
└──► api-2 ─┤
|
||||
├──► postgres (источник истины: пользователи, чаты, сообщения,
|
||||
│ сессии, ключи идемпотентности; volume postgres_data)
|
||||
migrate ────┤
|
||||
└──► valkey (read-through кэш сессий, не источник истины)
|
||||
сеть backend: internal, LB туда не подключён
|
||||
```
|
||||
|
||||
**Перенесено из lab2:** API, PostgreSQL как источник истины, сессии в PostgreSQL с кэшем в Valkey (logout инвалидирует общий кэш), Argon2id, keyset-пагинация, монотонный `created_at` под advisory lock чата, обработка отказов Valkey/PostgreSQL. Локального состояния, от которого зависит корректность, в API нет: circuit breaker Valkey и пул соединений — только оптимизации процесса.
|
||||
|
||||
**Изменено в lab3:**
|
||||
|
||||
| Что | Решение | Где |
|
||||
| --- | --- | --- |
|
||||
| Реплики и LB | `api` с `deploy.replicas: ${API_REPLICAS:-2}`, без `container_name` и портов; nginx 1.28 `upstream api { server api:8080 resolve; }` — новые/остановленные контейнеры подхватываются без reload | [compose.yaml](compose.yaml), [infra/nginx/nginx.conf](infra/nginx/nginx.conf) |
|
||||
| X-Instance-Id | случайный alias процесса (`secrets.token_hex(6)`), без hostname/IP; в каждом ответе API; процесс пишет его в `/tmp/instance-id` для оператора | [request_context.py](src/application/api/middlewares/request_context.py), [settings.py](src/config/settings.py) |
|
||||
| X-Request-Id | nginx передаёт `$request_id`, API принимает его (только 32 hex) — одна строка access-лога LB и журнал реплики связаны одним ID | там же, [evidence/04](evidence/04-balancing-idempotency.txt) |
|
||||
| Idempotency-Key | обязательный заголовок `[A-Za-z0-9._:-]{1,128}` (нет/неверный — 400); таблица `message_idempotency_keys`, PK `(sender_id, chat_id, key)`, SHA-256 канонического тела, TTL `IDEMPOTENCY_TTL_SECONDS` = 24 ч | [0002_message_idempotency.sql](migrations/0002_message_idempotency.sql), [postgres.py](src/infrastructure/repositories/message_repository/postgres.py), [message_service.py](src/infrastructure/services/message_service/message_service.py) |
|
||||
| Миграции | one-shot сервис `migrate` (`src/migrate.py`) до реплик (`service_completed_successfully`); реплика миграции не применяет, а проверяет версию схемы и не стартует, если схема старее кода | [database.py](src/infrastructure/dependencies/database.py), [migrate.py](src/migrate.py) |
|
||||
| Graceful shutdown | SIGTERM → drain 1 с (порт закрыт, readiness 503, ответы по открытым keep-alive с `Connection: close`) → graceful uvicorn до 5 с → закрытие пулов | [gateway.py](src/gateway.py) |
|
||||
| Ошибки LB | 502/503/504, созданные nginx, отдаются как `503 {"error":{"code":"unavailable",…}}` с `X-Request-Id`; ответы реплик проходят без изменений | [infra/nginx/errors.conf](infra/nginx/errors.conf) |
|
||||
|
||||
**Транзакционная граница идемпотентности.** В одной транзакции: `pg_advisory_xact_lock(чат)` → поиск действующего ключа → (повтор: вернуть сохранённое сообщение или 409 при другом теле) → `INSERT` сообщения → `INSERT` ключа. Ключ привязан к чату, поэтому блокировка чата, уже нужная для монотонного `created_at`, сериализует и одновременные повторы с разных реплик: второй ждёт COMMIT первого и находит ключ (READ COMMITTED). Ошибка до COMMIT откатывает и сообщение, и ключ — транзиентный 5xx не фиксируется как результат. Повтор заново проходит проверку сессии и членства. Область «пользователь + метод + путь + ключ» сведена к `(sender_id, chat_id, key)`: метод и путь фиксированы. Порядок ключей и пробелы JSON не влияют (хеш от тела, сериализованного по схеме). Истёкший ключ переиспользуется (`ON CONFLICT … DO UPDATE`) и создаёт новое сообщение.
|
||||
|
||||
**Почему nginx не повторяет POST сам.** `proxy_next_upstream error timeout` без `non_idempotent`: запрос, не дошедший до реплики (connect refused/timeout), nginx отправляет следующей реплике — это безопасно для любого метода. POST, уже отправленный реплике, не повторяется: прозрачный повтор `POST /users`/`/chats` создал бы дубль (контракт это запрещает). Неопределённый POST сообщения повторяет клиент с тем же ключом. 503 приложения тоже не повторяется и не исключает реплику: он означает отказ общей зависимости, другая реплика зависит от той же БД.
|
||||
|
||||
### Health, readiness, таймауты и время обнаружения
|
||||
|
||||
| Механизм | Значение | Назначение |
|
||||
| --- | --- | --- |
|
||||
| `/health/live` | 200 пока жив процесс | liveness, проверки через LB |
|
||||
| `/health/ready` | 200 если не завершается и PostgreSQL отвечает (Valkey не критичен — сессии читаются из БД) | Docker healthcheck API: `interval 5s, timeout 3s, retries 2, start_period 20s, start_interval 1s`; используется `up --wait`, `depends_on` nginx, `restore.sh`, `rolling-update.sh` |
|
||||
| Порт реплики | открывается только после startup (БД + версия схемы), закрывается первым при SIGTERM | LB не шлёт трафик в неготовую/завершающуюся реплику |
|
||||
| nginx `proxy_connect_timeout` | 0,5 с | исчезнувший IP (SIGKILL/крах контейнера) |
|
||||
| nginx `proxy_read_timeout` / `send` | 10 с | зависшая реплика; дольше самого долгого запроса API (ожидание пула 3 с + statement_timeout 5 с) |
|
||||
| `max_fails=1 fail_timeout=5s` | | после ошибки соединения реплика исключена на 5 с, затем пробный запрос |
|
||||
| `resolver 127.0.0.11 valid=2s` | | остановленный контейнер пропадает из upstream ≤ 2 с, новый появляется ≤ 2 с |
|
||||
| keep-alive | nginx → API `keepalive_timeout 60s` < uvicorn `timeout_keep_alive 75 s` | idle-соединение закрывает LB, а не реплика: нет гонки «запрос в закрытое соединение» |
|
||||
| SIGTERM | drain 1 с + graceful ≤ 5 с < `stop_grace_period 10s` | SIGKILL не прерывает завершение |
|
||||
|
||||
У nginx open source нет активных health checks — проверка пассивная, по реальным запросам. **Максимальное время обнаружения:** остановка процесса/SIGTERM — 0 с (порт закрыт → `connection refused` → тот же запрос сразу уходит на другую реплику, клиент ошибки не видит); крах контейнера — ≤ 0,5 с на запрос (connect timeout), затем исключение на 5 с и удаление из DNS ≤ 2 с; зависание без закрытия порта — ≤ 10 с (read timeout; GET повторяется, POST получает 503 и повторяется клиентом с тем же ключом). Docker healthcheck обнаруживает неготовность за ≤ 5 с × 2 = 10 с, но в Compose он не исключает контейнер из DNS — поэтому балансировку обеспечивает nginx, а не `depends_on`/healthcheck.
|
||||
|
||||
### Ресурсы, сети, секреты
|
||||
|
||||
| Роль | Service | Образ | CPU / RAM | Сети | Порт наружу | Volume | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| LB | `nginx` | nginx:1.28.0-alpine3.21 | 0,5 / 64M | edge | 127.0.0.1:8080 (+8081 только для `test-expiry`) | — | нет |
|
||||
| API ×2 | `api` | mtusi-messenger:lab3 (из исходников) | 1,0 / 512M каждая | edge, backend | нет | — | DB_PASS, VALKEY_PASS из `.env` |
|
||||
| Миграции | `migrate` | тот же | 0,5 / 256M | backend | нет | — | DB_PASS |
|
||||
| БД | `postgres` | postgres:18.2-alpine3.22 | 1,5 / 512M | backend | нет | `postgres_data` | POSTGRES_PASSWORD = DB_PASS |
|
||||
| Кэш сессий | `valkey` | valkey/valkey:8.1.4-alpine3.22 | 0,25 / 96M | backend | нет | нет (не источник истины) | VALKEY_PASS |
|
||||
|
||||
Сеть `backend` — `internal`: nginx не видит БД и Valkey (даже не резолвит имя), хранилища не имеют выхода наружу. Порт PostgreSQL больше не публикуется на хост (в lab2 был 127.0.0.1:5432); тесты обращаются к БД через `docker compose exec`. Подтверждение: [evidence/11-limits-ports-networks.txt](evidence/11-limits-ports-networks.txt). 512M на реплику: Argon2id RFC 9106 low-memory — 64 MiB × `PASSWORD_HASH_WORKERS=4`. Пул БД: 2 реплики × `DB_POOL_MAX_SIZE=10` + migrate + реплика при rolling update = 40 ≤ `max_connections=100`.
|
||||
|
||||
### Соответствие .env.example
|
||||
|
||||
| Переменная | Используется | Примечание |
|
||||
| --- | --- | --- |
|
||||
| `APP_HOST`, `APP_PORT` | uvicorn | 0.0.0.0:8080 внутри контейнера |
|
||||
| `APP_ENV`, `LOG_LEVEL` | журнал | |
|
||||
| `APP_KEEPALIVE_TIMEOUT_SEC=75` | uvicorn `timeout_keep_alive` | новое; > keepalive_timeout nginx |
|
||||
| `SHUTDOWN_TIMEOUT_SEC=5` | uvicorn `timeout_graceful_shutdown` | |
|
||||
| `SHUTDOWN_DRAIN_SEC=1` | `DrainingServer` | новое |
|
||||
| `DB_*` | asyncpg pool | `DB_HOST/DB_PORT` переопределены в compose (`postgres:5432`); `DB_PORT` больше не публикуется |
|
||||
| `VALKEY_*` | redis-py | `VALKEY_HOST/PORT` переопределены в compose |
|
||||
| `SESSION_TTL_SECONDS`, `SESSION_CACHE_TTL_SECONDS`, `PASSWORD_HASH_WORKERS` | как в lab2 | `api-short-ttl` переопределяет TTL = 2 |
|
||||
| `API_REPLICAS=2` | `deploy.replicas` | |
|
||||
| `IDEMPOTENCY_TTL_SECONDS=86400` | срок ключа | |
|
||||
| `LB_PORT=8080` (+ `LB_EXPIRY_PORT`, по умолчанию 8081) | публикация nginx | новое |
|
||||
| `APP_REVISION` | build arg → метка образа и журнал старта | не в `.env`; задаёт `rolling-update.sh` |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
Формат файлов — транскрипт терминала: `$ команда`, затем сырой вывод.
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | `make check test` | PASS lab3 public API scenario | [02](evidence/02-check-smoke.txt) |
|
||||
| 3: стенд одной командой, ≥2 API, один вход, общие БД и сессии | `make up`, `docker compose ps`, `make instances` | 2 реплики healthy, migrate Exited (0), наружу только nginx | [01](evidence/01-up.txt) |
|
||||
| 3: одна сессия → разные реплики без повторного входа | `pytest test_balancing.py -s`, access-лог nginx | 20 запросов одним токеном: 10/10 по двум `X-Instance-Id`; logout на одной реплике действует на всех | [04](evidence/04-balancing-idempotency.txt) |
|
||||
| 3: остановка любой реплики, чтение/запись ≤ 10 с | `make test-failover` для `api-1`, затем для `api-2` | 129 / 128 / 129 успешных записей, **0 неудачных попыток**, первая запись после возврата hook 1,6–2,7 с; `instances` и `docker inspect` показывают реально остановленный контейнер (exit 0, журнал drain/shutdown) | [05a](evidence/05-failover-replica-a.txt), [05b](evidence/05-failover-replica-b.txt) |
|
||||
| 3: Idempotency-Key без дублей, включая конкурентный повтор и смену реплики | `pytest test_idempotency.py -s` | 20 одновременных POST с одним ключом через обе реплики (10/10): все 201, одно тело, 1 строка в БД; разные тела под одним ключом — один победитель, остальные 409; 400 для нет/неверного ключа; область user+chat; повтор после logout — 401, чужой — 404; истёкший ключ создаёт новое | [04](evidence/04-balancing-idempotency.txt) |
|
||||
| 3: persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; `StartedAt` всех 5 контейнеров изменился, volume тот же (created 16:10:27Z) | [06](evidence/06-persistence.txt) |
|
||||
| Регрессия lab2 | `make student-test`, `test-expiry`, `test-failures` | 107 passed; TTL 2 с через LB :8081; отказ Valkey/PostgreSQL как в lab2 | [03](evidence/03-student-tests.txt), [07](evidence/07-expiry.txt), [08](evidence/08-dependency-failures.txt) |
|
||||
| 4: health/readiness/таймауты/SIGTERM согласованы; LB исключает backend | `make sigterm-check` | exit 0 за 1,53 с; 354 запроса через LB после SIGTERM без ошибок; после выхода реплика не отвечает | [10](evidence/10-sigterm.txt) |
|
||||
| 4: поочерёдная остановка каждой реплики, RPS/ошибки/восстановление | `make experiment-failover` | см. «Эксперименты»: SIGTERM — 0 ошибок; SIGKILL — 1 и 3 ошибки 503, повтор тем же ключом; потерь/дублей 0 | [13](evidence/13-experiment-failover.txt), [json](evidence/experiment-failover.json) |
|
||||
| 4: идемпотентность параллельными запросами | `test_parallel_*` | см. строку уровня 3 | [04](evidence/04-balancing-idempotency.txt) |
|
||||
| 4: лимиты CPU/RAM, API без портов, сети/секреты, нет `container_name` | `docker inspect`, `docker network inspect` | лимиты у всех контейнеров; порты только у nginx; backend internal | [11](evidence/11-limits-ports-networks.txt) |
|
||||
| 4: rolling update под нагрузкой | `make experiment-rolling` | обе реплики заменены новой ревизией, 0 ошибок, наибольший разрыв между подтверждениями записи 0,098 с | [14](evidence/14-experiment-rolling.txt), [json](evidence/experiment-rolling.json) |
|
||||
| 4: миграции однократно и совместимы с работающей версией | `make test-migrations`, `scripts/check_previous_version.sh` | с пустого volume, 3 повтора идемпотентны; API до migrate отказывается стартовать; **lab2-код (`aa971fb`) работает на схеме v2**: вход, чат, запись, чтение | [09](evidence/09-migrations.txt) |
|
||||
| Чистый clone | копия файлов Git, отдельный проект | build → smoke → failover → persistence PASS, затем `down -v` | [15](evidence/15-clean-clone.txt) |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Гипотеза.** При остановке одной из двух реплик (SIGTERM) клиент не видит ошибок: порт закрывается раньше, чем реплика перестаёт отвечать, и nginx переотправляет ещё не доставленные запросы; при SIGKILL теряются только запросы, бывшие в обработке у убитой реплики, а повтор с тем же ключом не создаёт дублей. Пропускная способность при одной реплике падает, так как API ограничена 1 CPU.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
**Методика** ([scripts/experiment.py](scripts/experiment.py)). Стенд из таблицы ресурсов на Apple M4 / Docker Desktop VM 10 CPU, 8 GiB. 4 потока closed loop через LB: `POST` сообщения (неподтверждённый POST повторяется с тем же ключом и телом каждые 50 мс до 201) + `GET ?limit=1`; timeout клиента 5 с; новый чат на прогон (≈ 10–12 тыс. сообщений за прогон). Фазы: 8 с до отказа → отказ (`docker stop` = SIGTERM, затем `docker kill -s SIGKILL`) → 10 с с одной репликой → `scripts/restore.sh` → 8 с. Порядок: `api-1`, `api-2` для каждого сигнала, с восстановлением предыдущей. После прогона история читается целиком: каждое подтверждённое сообщение на месте и не изменено, нет дублей id/текстов. «Время восстановления записи» — от первой ошибки после отказа до первой успешной записи после последней ошибки; «разрыв» — наибольший интервал между подтверждениями записи. Один прогон на сценарий; CPU/RAM — `docker stats` в середине фазы «до».
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали (запись ok RPS до / во время / после) | Ошибки, восстановление, данные |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| SIGTERM `api-1` | 0 ошибок | 382 / 277 / 399 | 0 ошибок; разрыв 0,063 с; реплика в ротации через 0,97 с после restore; потерь/дублей 0 |
|
||||
| SIGTERM `api-2` | 0 ошибок | 401 / 282 / 362 | 0 ошибок; разрыв 0,071 с; в ротации через 3,7 с; 0 / 0 |
|
||||
| SIGKILL `api-1` | ошибки только у запросов в обработке | 427 / 281 / 408 | 1 × 503 (POST, оборван убитой репликой), восстановление < 0,01 с, разрыв 0,070 с; повтор ключом создал сообщение один раз; 0 / 0 |
|
||||
| SIGKILL `api-2` | то же | 387 / 277 / 373 | 3 × 503, восстановление 0,006 с, разрыв 0,061 с; 0 / 0 |
|
||||
| Rolling update 2 → 3 → 2 → 3 → 2 | 0 ошибок | 420 / 308 / 345 | 0 ошибок; разрыв 0,098 с; старые alias исчезли через 8,1 и 14,9 с от начала, новые вошли на 4,9 и 13,9 с; обе реплики на новой ревизии; 0 / 0 |
|
||||
| Valkey / PostgreSQL недоступны | обход кэша / ограниченный 503 | как в lab2, через LB | [08](evidence/08-dependency-failures.txt) |
|
||||
| Рестарт всего стенда | данные и токен сохраняются | PASS | [06](evidence/06-persistence.txt) |
|
||||
|
||||
Нагрузка до отказа: API ≈ 84 % от лимита 1 CPU каждая, PostgreSQL ≈ 28 %, nginx ≈ 9 %, память API ≈ 48 MiB. Во время отказа запись падает на ≈ 30 % (не вдвое: при двух репликах узким местом частично являются клиент и БД). Базовый публичный генератор (`tests/load.py`, 30 с, concurrency 4, только POST): 713 успешных RPS, 0 ошибок, p50/p95/p99 = 4,7 / 8,4 / 22,5 мс — [evidence/12](evidence/12-load-baseline.txt), [load.json](evidence/load.json).
|
||||
|
||||
**Подтверждённое улучшение — drain при SIGTERM.** Без drain uvicorn по SIGTERM сразу закрывает idle keep-alive соединения, и nginx изредка отправлял запрос в уже закрываемое соединение: в пробном rolling update был `upstream prematurely closed connection` (GET nginx повторил; POST в той же ситуации получил бы 503). С drain в итоговых прогонах SIGTERM и rolling update таких событий 0 — все 4 случая «prematurely closed» в журнале nginx пришлись на прогоны SIGKILL (сопоставлено по времени, где это ожидаемо).
|
||||
|
||||
**Ограничения измерения.** Closed loop с 4 потоками страдает coordinated omission и ограничен клиентом на той же машине; по одному прогону на сценарий, без доверительных интервалов; RPS — подтверждения HTTP, а не COMMIT. Числа описывают этот ноутбук, не универсальную ёмкость.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **Выдерживается:** отказ/остановка/крах одного API-процесса или контейнера, поочерёдная замена реплик, перезапуск всего стенда с volumes, отказ Valkey. **Не выдерживается:** отказ nginx, PostgreSQL (одна реплика БД — SPOF; API честно отвечает 503) и Docker-host — все контейнеры на одном узле. Отказ процесса ≠ отказ узла: здесь проверен только первый.
|
||||
- nginx open source — только пассивные проверки: реплика, которая принимает соединения, но не отвечает, обнаруживается за `proxy_read_timeout` (10 с) и снова получает пробный запрос каждые 5 с; healthcheck Docker в Compose её не исключает. Активные проверки дал бы HAProxy/Traefik или nginx Plus.
|
||||
- `docker stop`/`docker kill` не перезапускаются политикой `unless-stopped` (так и задумано для `stop-one.sh`); автоматического восстановления упавшей реплики Compose не даёт — `make restore` выполняет оператор.
|
||||
- Истёкшие ключи идемпотентности удаляются только при повторном использовании ключа; фоновой очистки нет (строки маленькие, но таблица растёт).
|
||||
- Совместимость миграций обеспечивается правилом expand-only и проверена для пары lab2 → lab3 на уровне схемы; поведение API при смешанных версиях (старая реплика не требует ключа и не отдаёт `X-Instance-Id`) во время такой выкладки различается — это переход контракта, а не схемы.
|
||||
- `api-short-ttl` для `make test-expiry` тоже за LB (порт 8081), но 8081 публикуется nginx постоянно; без профиля `expiry` он отвечает 503.
|
||||
+93
-16
@@ -1,27 +1,100 @@
|
||||
name: messenger-lab3
|
||||
|
||||
x-api: &api
|
||||
build: .
|
||||
x-app: &app
|
||||
build:
|
||||
context: .
|
||||
args:
|
||||
APP_REVISION: ${APP_REVISION:-dev}
|
||||
image: mtusi-messenger:lab3
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env
|
||||
environment: &api-network
|
||||
environment: &app-network
|
||||
DB_HOST: postgres
|
||||
DB_PORT: "5432"
|
||||
VALKEY_HOST: valkey
|
||||
VALKEY_PORT: "6379"
|
||||
|
||||
x-api-resources: &api-resources
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
x-api: &api
|
||||
<<: *app
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_started
|
||||
networks:
|
||||
- edge
|
||||
- backend
|
||||
- default
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "src/healthcheck.py"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 2
|
||||
start_period: 20s
|
||||
start_interval: 1s
|
||||
stop_grace_period: 10s
|
||||
|
||||
services:
|
||||
nginx:
|
||||
image: nginx:1.28.0-alpine3.21
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||||
- "127.0.0.1:${LB_EXPIRY_PORT:-8081}:8081"
|
||||
volumes:
|
||||
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/nginx/errors.conf:/etc/nginx/errors.conf:ro
|
||||
networks:
|
||||
- edge
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8090/healthz"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
stop_grace_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 64M
|
||||
|
||||
api:
|
||||
<<: *api
|
||||
deploy:
|
||||
<<: *api-resources
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
|
||||
migrate:
|
||||
<<: *app
|
||||
command: ["python", "src/migrate.py"]
|
||||
restart: "no"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
networks:
|
||||
- backend
|
||||
healthcheck:
|
||||
disable: true
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
|
||||
postgres:
|
||||
image: postgres:18.2-alpine3.22
|
||||
restart: unless-stopped
|
||||
@@ -37,12 +110,14 @@ services:
|
||||
timeout: 3s
|
||||
retries: 15
|
||||
start_period: 10s
|
||||
ports:
|
||||
- "127.0.0.1:${DB_PORT}:5432"
|
||||
networks:
|
||||
- backend
|
||||
- default
|
||||
stop_grace_period: 30s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.5"
|
||||
memory: 512M
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1.4-alpine3.22
|
||||
@@ -68,23 +143,25 @@ services:
|
||||
retries: 15
|
||||
networks:
|
||||
- backend
|
||||
|
||||
api:
|
||||
<<: *api
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT}:${APP_PORT}"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 96M
|
||||
|
||||
api-short-ttl:
|
||||
<<: *api
|
||||
profiles:
|
||||
- expiry
|
||||
environment:
|
||||
<<: *api-network
|
||||
<<: *app-network
|
||||
SESSION_TTL_SECONDS: "2"
|
||||
ports:
|
||||
- "127.0.0.1:8081:${APP_PORT}"
|
||||
deploy:
|
||||
<<: *api-resources
|
||||
replicas: 1
|
||||
|
||||
networks:
|
||||
edge: {}
|
||||
backend:
|
||||
internal: true
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
$ date -u +%Y-%m-%dT%H:%M:%SZ
|
||||
2026-10-19T16:59:47Z
|
||||
$ git log -1 --format="%h %s" && git status --short | wc -l | xargs echo "uncommitted files:"
|
||||
aa971fb feat: перенос core-логики 2 лабы
|
||||
uncommitted files: 42
|
||||
$ sysctl -n machdep.cpu.brand_string hw.ncpu hw.memsize
|
||||
Apple M4
|
||||
10
|
||||
17179869184
|
||||
$ docker version --format "Docker Engine {{.Server.Version}}, client {{.Client.Version}}"
|
||||
Docker Engine 29.0.1, client 29.0.1
|
||||
$ docker compose version
|
||||
Docker Compose version v2.40.3-desktop.1
|
||||
$ docker info --format "Docker VM: {{.NCPU}} CPU, {{.MemTotal}} bytes RAM, {{.OperatingSystem}}, kernel {{.KernelVersion}}"
|
||||
Docker VM: 10 CPU, 8217448448 bytes RAM, Docker Desktop, kernel 6.12.54-linuxkit
|
||||
$ python3 --version
|
||||
Python 3.9.6
|
||||
$ poetry run python --version
|
||||
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
|
||||
warnings.warn(
|
||||
Python 3.12.12
|
||||
@@ -0,0 +1,150 @@
|
||||
$ docker compose --env-file .env config --quiet && echo "compose config: OK"
|
||||
compose config: OK
|
||||
$ make up
|
||||
docker compose up -d --build --wait
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 1.17kB done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [api internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 914B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [api internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm
|
||||
#3 DONE 0.3s
|
||||
|
||||
#4 [migrate internal] load .dockerignore
|
||||
#4 transferring context: 358B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [migrate 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
|
||||
#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [api internal] load build context
|
||||
#6 transferring context: 16.17kB done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [api 3/7] COPY poetry.lock pyproject.toml ./
|
||||
#7 CACHED
|
||||
|
||||
#8 [api 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction
|
||||
#8 CACHED
|
||||
|
||||
#9 [api 2/7] WORKDIR /app
|
||||
#9 CACHED
|
||||
|
||||
#10 [api 5/7] COPY migrations ./migrations
|
||||
#10 CACHED
|
||||
|
||||
#11 [migrate 6/7] COPY src ./src
|
||||
#11 DONE 0.0s
|
||||
|
||||
#12 [migrate 7/7] RUN useradd --system --uid 10001 --no-create-home app
|
||||
#12 0.110 useradd warning: app's uid 10001 is greater than SYS_UID_MAX 999
|
||||
#12 DONE 0.1s
|
||||
|
||||
#13 [api] exporting to image
|
||||
#13 exporting layers 0.0s done
|
||||
#13 exporting manifest sha256:8d0545b5703e6c70c35f1136bdfd37c4ceb4a70d08b5bad21b70d6755df05b5d done
|
||||
#13 exporting config sha256:28c6ec56ed30844b944958f2749a370cea79cfd768ca3f491807eae51ad534a0 done
|
||||
#13 naming to docker.io/library/mtusi-messenger:lab3 done
|
||||
#13 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done
|
||||
#13 DONE 0.1s
|
||||
|
||||
#14 [migrate] exporting to image
|
||||
#14 exporting layers 0.0s done
|
||||
#14 exporting manifest sha256:9802819ef7b235f44ad03fed838cf83fcd4573a342405936538c7c969adfd573 done
|
||||
#14 exporting config sha256:e2b0c133c10bc42d53160db701c32694ed2cbac54f0522999289f4cf7cef21e1 done
|
||||
#14 naming to docker.io/library/mtusi-messenger:lab3 done
|
||||
#14 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done
|
||||
#14 DONE 0.1s
|
||||
|
||||
#15 [api] resolving provenance for metadata file
|
||||
#15 DONE 0.0s
|
||||
|
||||
#16 [migrate] resolving provenance for metadata file
|
||||
#16 DONE 0.0s
|
||||
mtusi-messenger:lab3 Built
|
||||
mtusi-messenger:lab3 Built
|
||||
Network messenger-lab3_edge Creating
|
||||
Network messenger-lab3_edge Created
|
||||
Network messenger-lab3_backend Creating
|
||||
Network messenger-lab3_backend Created
|
||||
Container messenger-lab3-postgres-1 Creating
|
||||
Container messenger-lab3-valkey-1 Creating
|
||||
Container messenger-lab3-valkey-1 Created
|
||||
Container messenger-lab3-postgres-1 Created
|
||||
Container messenger-lab3-migrate-1 Creating
|
||||
Container messenger-lab3-migrate-1 Created
|
||||
Container messenger-lab3-api-2 Creating
|
||||
Container messenger-lab3-api-1 Creating
|
||||
Container messenger-lab3-api-1 Created
|
||||
Container messenger-lab3-api-2 Created
|
||||
Container messenger-lab3-nginx-1 Creating
|
||||
Container messenger-lab3-nginx-1 Created
|
||||
Container messenger-lab3-postgres-1 Starting
|
||||
Container messenger-lab3-valkey-1 Starting
|
||||
Container messenger-lab3-valkey-1 Started
|
||||
Container messenger-lab3-postgres-1 Started
|
||||
Container messenger-lab3-postgres-1 Waiting
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-migrate-1 Starting
|
||||
Container messenger-lab3-migrate-1 Started
|
||||
Container messenger-lab3-migrate-1 Waiting
|
||||
Container messenger-lab3-postgres-1 Waiting
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-migrate-1 Exited
|
||||
Container messenger-lab3-api-1 Starting
|
||||
Container messenger-lab3-api-1 Started
|
||||
Container messenger-lab3-api-2 Starting
|
||||
Container messenger-lab3-api-2 Started
|
||||
Container messenger-lab3-api-1 Waiting
|
||||
Container messenger-lab3-api-2 Waiting
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
Container messenger-lab3-nginx-1 Starting
|
||||
Container messenger-lab3-nginx-1 Started
|
||||
Container messenger-lab3-valkey-1 Waiting
|
||||
Container messenger-lab3-nginx-1 Waiting
|
||||
Container messenger-lab3-api-1 Waiting
|
||||
Container messenger-lab3-api-2 Waiting
|
||||
Container messenger-lab3-migrate-1 Waiting
|
||||
Container messenger-lab3-postgres-1 Waiting
|
||||
Container messenger-lab3-migrate-1 Exited
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-valkey-1 Healthy
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
Container messenger-lab3-nginx-1 Healthy
|
||||
$ docker compose ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
messenger-lab3-api-1 mtusi-messenger:lab3 "python src/gateway.…" api 15 seconds ago Up 7 seconds (healthy) 8080/tcp
|
||||
messenger-lab3-api-2 mtusi-messenger:lab3 "python src/gateway.…" api 15 seconds ago Up 7 seconds (healthy) 8080/tcp
|
||||
messenger-lab3-migrate-1 mtusi-messenger:lab3 "python src/migrate.…" migrate 15 seconds ago Exited (0) 7 seconds ago
|
||||
messenger-lab3-nginx-1 nginx:1.28.0-alpine3.21 "/docker-entrypoint.…" nginx 15 seconds ago Up 5 seconds (healthy) 127.0.0.1:8080-8081->8080-8081/tcp
|
||||
messenger-lab3-postgres-1 postgres:18.2-alpine3.22 "docker-entrypoint.s…" postgres 15 seconds ago Up 14 seconds (healthy) 5432/tcp
|
||||
messenger-lab3-valkey-1 valkey/valkey:8.1.4-alpine3.22 "docker-entrypoint.s…" valkey 15 seconds ago Up 14 seconds (healthy) 6379/tcp
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy 978f2777b379 95632 dev
|
||||
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
|
||||
$ docker compose logs migrate
|
||||
migrate-1 | 2026-10-19 17:00:03.868 | INFO | infrastructure.dependencies.database:create_pool:69 - Установка соединения с базой данных psql
|
||||
migrate-1 | 2026-10-19 17:00:03.937 | INFO | infrastructure.dependencies.database:create_pool:99 - Соединение с базой данных psql установлено. DB_NAME: test, pool: 2..10
|
||||
migrate-1 | 2026-10-19 17:00:03.939 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии
|
||||
migrate-1 | 2026-10-19 17:00:03.940 | INFO | __main__:main:20 - Миграции применены, migrate завершается
|
||||
$ docker compose logs api | grep -E "Схема БД|Application started"
|
||||
api-2 | 2026-10-19 17:00:05.191 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2
|
||||
api-2 | 2026-10-19 17:00:05.226 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=389b271f592b, session ttl=3600 s
|
||||
api-1 | 2026-10-19 17:00:05.106 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2
|
||||
api-1 | 2026-10-19 17:00:05.173 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=978f2777b379, session ttl=3600 s
|
||||
$ curl -s -o /dev/null -w "%{http_code}\n" --max-time 2 http://localhost:8080/health/ready
|
||||
200
|
||||
$ docker compose exec -T nginx sh -c "wget -q -O - --timeout 2 http://api:8080/health/live; echo; nslookup api 127.0.0.11 | tail -4"
|
||||
{"status":"ok"}
|
||||
Address: 192.168.48.3
|
||||
Name: api
|
||||
Address: 192.168.48.2
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
$ make check
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
$ make test
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,128 @@
|
||||
$ make student-test
|
||||
BASE_URL="http://localhost:8080" poetry run pytest
|
||||
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
|
||||
warnings.warn(
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python
|
||||
cachedir: .pytest_cache
|
||||
rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3
|
||||
configfile: pyproject.toml
|
||||
testpaths: student-tests
|
||||
plugins: anyio-4.15.1
|
||||
collecting ... collected 113 items
|
||||
|
||||
student-tests/test_auth.py::test_login_returns_opaque_bearer_session PASSED [ 0%]
|
||||
student-tests/test_auth.py::test_wrong_password_and_unknown_user_are_indistinguishable PASSED [ 1%]
|
||||
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[None] PASSED [ 2%]
|
||||
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[] PASSED [ 3%]
|
||||
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic] PASSED [ 4%]
|
||||
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic !!!not-base64!!!] PASSED [ 5%]
|
||||
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic bm8tY29sb24taGVyZQ==] PASSED [ 6%]
|
||||
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Basic //46cGFzc3dvcmQ=] PASSED [ 7%]
|
||||
student-tests/test_auth.py::test_malformed_basic_is_unauthorized[Bearer some-token] PASSED [ 7%]
|
||||
student-tests/test_auth.py::test_utf8_password_with_colon PASSED [ 8%]
|
||||
student-tests/test_auth.py::test_basic_is_rejected_on_business_endpoints PASSED [ 9%]
|
||||
student-tests/test_auth.py::test_logout_revokes_only_current_session PASSED [ 10%]
|
||||
student-tests/test_auth.py::test_logout_without_token PASSED [ 11%]
|
||||
student-tests/test_auth.py::test_relogin_after_logout PASSED [ 12%]
|
||||
student-tests/test_auth.py::test_password_is_not_stored_in_plaintext PASSED [ 13%]
|
||||
student-tests/test_balancing.py::test_one_session_is_served_by_several_replicas PASSED [ 14%]
|
||||
student-tests/test_balancing.py::test_write_is_visible_through_every_replica PASSED [ 15%]
|
||||
student-tests/test_balancing.py::test_logout_applies_to_all_replicas PASSED [ 15%]
|
||||
student-tests/test_balancing.py::test_every_api_response_has_instance_id PASSED [ 16%]
|
||||
student-tests/test_concurrency.py::test_parallel_messages_are_not_lost_or_duplicated PASSED [ 17%]
|
||||
student-tests/test_dependency_failures.py::test_valkey_down_cache_is_bypassed SKIPPED [ 18%]
|
||||
student-tests/test_dependency_failures.py::test_logout_during_valkey_outage_is_not_resurrected SKIPPED [ 19%]
|
||||
student-tests/test_dependency_failures.py::test_postgres_down_returns_bounded_503_and_no_anonymous_access SKIPPED [ 20%]
|
||||
student-tests/test_dependency_failures.py::test_session_survives_postgres_restart SKIPPED [ 21%]
|
||||
student-tests/test_expiry.py::test_session_expires_after_short_ttl SKIPPED [ 22%]
|
||||
student-tests/test_expiry.py::test_expires_at_matches_short_ttl SKIPPED [ 23%]
|
||||
student-tests/test_foreign_chat.py::test_outsider_cannot_access_chat PASSED [ 23%]
|
||||
student-tests/test_foreign_chat.py::test_foreign_and_missing_chat_are_indistinguishable PASSED [ 24%]
|
||||
student-tests/test_foreign_chat.py::test_chat_list_contains_only_own_chats PASSED [ 25%]
|
||||
student-tests/test_health.py::test_health_endpoints_have_request_id PASSED [ 26%]
|
||||
student-tests/test_health.py::test_unknown_route_uses_error_format PASSED [ 27%]
|
||||
student-tests/test_idempotency.py::test_replay_returns_same_message PASSED [ 28%]
|
||||
student-tests/test_idempotency.py::test_replay_ignores_json_formatting PASSED [ 29%]
|
||||
student-tests/test_idempotency.py::test_same_key_with_other_body_is_conflict PASSED [ 30%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers0] PASSED [ 30%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers1] PASSED [ 31%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers2] PASSED [ 32%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers3] PASSED [ 33%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers4] PASSED [ 34%]
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers5] PASSED [ 35%]
|
||||
student-tests/test_idempotency.py::test_key_boundaries_are_accepted[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 36%]
|
||||
student-tests/test_idempotency.py::test_key_boundaries_are_accepted[Az09._:-] PASSED [ 37%]
|
||||
student-tests/test_idempotency.py::test_key_scope_is_user_and_chat PASSED [ 38%]
|
||||
student-tests/test_idempotency.py::test_replay_rechecks_session_and_access PASSED [ 38%]
|
||||
student-tests/test_idempotency.py::test_expired_key_can_be_reused PASSED [ 39%]
|
||||
student-tests/test_idempotency.py::test_parallel_replays_through_replicas_create_one_message PASSED [ 40%]
|
||||
student-tests/test_idempotency.py::test_parallel_same_key_with_different_bodies PASSED [ 41%]
|
||||
student-tests/test_invalid_json.py::test_malformed_json_on_register[{"username": "abc",] PASSED [ 42%]
|
||||
student-tests/test_invalid_json.py::test_malformed_json_on_register[not json at all] PASSED [ 43%]
|
||||
student-tests/test_invalid_json.py::test_malformed_json_on_register[{"text": "\xff\xfe"}] PASSED [ 44%]
|
||||
student-tests/test_invalid_json.py::test_malformed_json_on_register[] PASSED [ 45%]
|
||||
student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[{"username": "abc",] PASSED [ 46%]
|
||||
student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[not json at all] PASSED [ 46%]
|
||||
student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[{"text": "\xff\xfe"}] PASSED [ 47%]
|
||||
student-tests/test_invalid_json.py::test_malformed_json_on_create_chat_and_message[] PASSED [ 48%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body0] PASSED [ 49%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[alice] PASSED [ 50%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body2] PASSED [ 51%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body3] PASSED [ 52%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body4] PASSED [ 53%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body5] PASSED [ 53%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body6] PASSED [ 54%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body7] PASSED [ 55%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body8] PASSED [ 56%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body9] PASSED [ 57%]
|
||||
student-tests/test_invalid_json.py::test_invalid_user_payload[body10] PASSED [ 58%]
|
||||
student-tests/test_invalid_json.py::test_invalid_message_payload[body0] PASSED [ 59%]
|
||||
student-tests/test_invalid_json.py::test_invalid_message_payload[body1] PASSED [ 60%]
|
||||
student-tests/test_invalid_json.py::test_invalid_message_payload[body2] PASSED [ 61%]
|
||||
student-tests/test_invalid_json.py::test_invalid_chat_payload PASSED [ 61%]
|
||||
student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[] PASSED [ 62%]
|
||||
student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[ ] PASSED [ 63%]
|
||||
student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[ \n\t ] PASSED [ 64%]
|
||||
student-tests/test_messages.py::test_empty_or_blank_message_is_rejected[\u3000] PASSED [ 65%]
|
||||
student-tests/test_messages.py::test_message_length_is_counted_in_code_points PASSED [ 66%]
|
||||
student-tests/test_messages.py::test_unicode_is_preserved PASSED [ 67%]
|
||||
student-tests/test_messages.py::test_nul_character_is_rejected PASSED [ 68%]
|
||||
student-tests/test_messages.py::test_sender_is_taken_from_session PASSED [ 69%]
|
||||
student-tests/test_pagination.py::test_walk_message_pages[7-3-3] PASSED [ 69%]
|
||||
student-tests/test_pagination.py::test_walk_message_pages[6-3-2] PASSED [ 70%]
|
||||
student-tests/test_pagination.py::test_walk_message_pages[5-1-5] PASSED [ 71%]
|
||||
student-tests/test_pagination.py::test_walk_message_pages[3-100-1] PASSED [ 72%]
|
||||
student-tests/test_pagination.py::test_limit_can_change_between_pages PASSED [ 73%]
|
||||
student-tests/test_pagination.py::test_new_message_appears_after_cursor PASSED [ 74%]
|
||||
student-tests/test_pagination.py::test_cursor_is_bound_to_chat PASSED [ 75%]
|
||||
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params0] PASSED [ 76%]
|
||||
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params1] PASSED [ 76%]
|
||||
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params2] PASSED [ 77%]
|
||||
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params3] PASSED [ 78%]
|
||||
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params4] PASSED [ 79%]
|
||||
student-tests/test_pagination.py::test_invalid_limit_or_cursor[params5] PASSED [ 80%]
|
||||
student-tests/test_pagination.py::test_walk_chat_list_pages PASSED [ 81%]
|
||||
student-tests/test_pagination.py::test_chat_cursor_is_bound_to_user PASSED [ 82%]
|
||||
student-tests/test_transactions.py::test_same_username_race_creates_exactly_one_user PASSED [ 83%]
|
||||
student-tests/test_transactions.py::test_failed_chat_creation_is_rolled_back PASSED [ 84%]
|
||||
student-tests/test_transactions.py::test_successful_chat_creation_commits_all_members PASSED [ 84%]
|
||||
student-tests/test_users.py::test_duplicate_username_is_conflict PASSED [ 85%]
|
||||
student-tests/test_users.py::test_registration_never_returns_password PASSED [ 86%]
|
||||
student-tests/test_users.py::test_invalid_password_is_rejected[None] PASSED [ 87%]
|
||||
student-tests/test_users.py::test_invalid_password_is_rejected[] PASSED [ 88%]
|
||||
student-tests/test_users.py::test_invalid_password_is_rejected[\u043a\u043e\u0440\u043e\u0442\u043a\u0438\u0439] PASSED [ 89%]
|
||||
student-tests/test_users.py::test_invalid_password_is_rejected[xxxxxxxxxxx] PASSED [ 90%]
|
||||
student-tests/test_users.py::test_invalid_password_is_rejected[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED [ 91%]
|
||||
student-tests/test_users.py::test_invalid_password_is_rejected[123456789012] PASSED [ 92%]
|
||||
student-tests/test_users.py::test_password_length_is_counted_in_code_points PASSED [ 92%]
|
||||
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers0] PASSED [ 93%]
|
||||
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers1] PASSED [ 94%]
|
||||
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers2] PASSED [ 95%]
|
||||
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers3] PASSED [ 96%]
|
||||
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers4] PASSED [ 97%]
|
||||
student-tests/test_users.py::test_missing_or_unknown_token_is_unauthorized[headers5] PASSED [ 98%]
|
||||
student-tests/test_users.py::test_x_user_id_gives_no_access PASSED [ 99%]
|
||||
student-tests/test_users.py::test_current_user PASSED [100%]
|
||||
|
||||
======================= 107 passed, 6 skipped in 14.98s ========================
|
||||
@@ -0,0 +1,50 @@
|
||||
$ BASE_URL=http://localhost:8080 poetry run pytest student-tests/test_balancing.py student-tests/test_idempotency.py student-tests/test_concurrency.py -s
|
||||
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
|
||||
warnings.warn(
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python
|
||||
cachedir: .pytest_cache
|
||||
rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3
|
||||
configfile: pyproject.toml
|
||||
plugins: anyio-4.15.1
|
||||
collecting ... collected 21 items
|
||||
|
||||
student-tests/test_balancing.py::test_one_session_is_served_by_several_replicas
|
||||
INSTANCES FOR ONE SESSION: {'389b271f592b': 10, '978f2777b379': 10}
|
||||
PASSED
|
||||
student-tests/test_balancing.py::test_write_is_visible_through_every_replica PASSED
|
||||
student-tests/test_balancing.py::test_logout_applies_to_all_replicas PASSED
|
||||
student-tests/test_balancing.py::test_every_api_response_has_instance_id PASSED
|
||||
student-tests/test_idempotency.py::test_replay_returns_same_message PASSED
|
||||
student-tests/test_idempotency.py::test_replay_ignores_json_formatting PASSED
|
||||
student-tests/test_idempotency.py::test_same_key_with_other_body_is_conflict PASSED
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers0] PASSED
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers1] PASSED
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers2] PASSED
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers3] PASSED
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers4] PASSED
|
||||
student-tests/test_idempotency.py::test_missing_or_invalid_key_is_rejected[headers5] PASSED
|
||||
student-tests/test_idempotency.py::test_key_boundaries_are_accepted[xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx] PASSED
|
||||
student-tests/test_idempotency.py::test_key_boundaries_are_accepted[Az09._:-] PASSED
|
||||
student-tests/test_idempotency.py::test_key_scope_is_user_and_chat PASSED
|
||||
student-tests/test_idempotency.py::test_replay_rechecks_session_and_access PASSED
|
||||
student-tests/test_idempotency.py::test_expired_key_can_be_reused PASSED
|
||||
student-tests/test_idempotency.py::test_parallel_replays_through_replicas_create_one_message
|
||||
PARALLEL SAME KEY: {"requests": 20, "statuses": {"201": 20}, "instances": {"389b271f592b": 10, "978f2777b379": 10}, "distinct_bodies": 1, "messages_in_history": 1, "rows_in_db": 1}
|
||||
PASSED
|
||||
student-tests/test_idempotency.py::test_parallel_same_key_with_different_bodies PASSED
|
||||
student-tests/test_concurrency.py::test_parallel_messages_are_not_lost_or_duplicated
|
||||
CONCURRENCY RESULT: {"clients": 10, "messages_sent": 100, "status_201": 100, "messages_read": 100, "unique_ids": 100, "unique_texts": 100, "pages_read": 4, "send_seconds": 0.227}
|
||||
PASSED
|
||||
|
||||
============================== 21 passed in 3.79s ==============================
|
||||
$ docker compose logs --no-log-prefix nginx | grep "\"uri\":\"/api/v1/users/me\",\"status\":200" | tail -6
|
||||
{"time":"2026-10-19T17:01:21+00:00","request_id":"d754e7abc1b3b0e09ab4bf0d9fd50270","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.002","instance":"389b271f592b"}
|
||||
{"time":"2026-10-19T17:01:21+00:00","request_id":"aeaf60de5c114ed306169370ede170b6","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.002,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"}
|
||||
{"time":"2026-10-19T17:01:21+00:00","request_id":"7caafb063b53d8985d17356cd62d812d","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.002,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"389b271f592b"}
|
||||
{"time":"2026-10-19T17:01:21+00:00","request_id":"a8e4fb898cd5ff5e56bdd1669067a60f","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"}
|
||||
{"time":"2026-10-19T17:01:21+00:00","request_id":"fb8e6ba34f06d739933b9872b9aadd89","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.3:8080","upstream_status":"200","upstream_response_time":"0.000","instance":"389b271f592b"}
|
||||
{"time":"2026-10-19T17:01:21+00:00","request_id":"e166ba8766667300b8bcc65996ab6827","method":"GET","uri":"/api/v1/users/me","status":200,"request_time":0.001,"upstream_addr":"192.168.48.2:8080","upstream_status":"200","upstream_response_time":"0.001","instance":"978f2777b379"}
|
||||
$ rid=$(docker compose logs --no-log-prefix nginx | grep "\"uri\":\"/api/v1/users/me\",\"status\":200" | tail -1 | sed -E "s/.*\"request_id\":\"([0-9a-f]+)\".*/\1/"); echo "request_id=$rid"; docker compose logs api | grep "$rid"
|
||||
request_id=e166ba8766667300b8bcc65996ab6827
|
||||
api-1 | 2026-10-19 17:01:21.285 | INFO | application.api.middlewares.request_context:_log_request:70 - GET /api/v1/users/me -> 200 | 0.6 ms | request_id=e166ba8766667300b8bcc65996ab6827
|
||||
@@ -0,0 +1,41 @@
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy 978f2777b379 95632 dev
|
||||
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 2.693}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 exited/unhealthy - 0 dev
|
||||
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
|
||||
$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api)
|
||||
/messenger-lab3-api-1 status=exited exit=0 finished=2026-10-19T17:01:38.389851544Z
|
||||
/messenger-lab3-api-2 status=running exit=0 finished=0001-01-01T00:00:00Z
|
||||
$ docker compose logs api | grep -E "SIGTERM|Application stopped|Finished server process"
|
||||
api-1 | 2026-10-19 17:01:37.194 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=978f2777b379: порт закрыт, keep-alive соединения закрываются после ответа
|
||||
api-1 | 2026-10-19 17:01:38.301 | INFO | __main__:lifespan:54 - Application stopped, instance=978f2777b379, connections closed
|
||||
api-1 | INFO: Finished server process [1]
|
||||
$ docker compose logs --no-log-prefix nginx | grep -E "\[(warn|error)\]" | tail -4
|
||||
2026/10/19 17:00:46 [warn] 30#30: *43 a client request body is buffered to a temporary file /var/cache/nginx/client_temp/0000000001, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551b-c8fe-7b27-bef6-22c34e5e5367/messages HTTP/1.1", host: "localhost:8080"
|
||||
2026/10/19 17:00:46 [warn] 30#30: *43 a client request body is buffered to a temporary file /var/cache/nginx/client_temp/0000000002, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551b-c8fe-7b27-bef6-22c34e5e5367/messages HTTP/1.1", host: "localhost:8080"
|
||||
2026/10/19 17:01:38 [error] 30#30: *803 connect() failed (111: Connection refused) while connecting to upstream, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages HTTP/1.1", upstream: "http://192.168.48.2:8080/api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages", host: "localhost:8080"
|
||||
2026/10/19 17:01:38 [warn] 30#30: *803 upstream server temporarily disabled while connecting to upstream, client: 192.168.48.1, server: , request: "POST /api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages HTTP/1.1", upstream: "http://192.168.48.2:8080/api/v1/chats/01a1551c-8845-7e45-aa74-7d88537eac05/messages", host: "localhost:8080"
|
||||
$ make restore
|
||||
./scripts/restore.sh
|
||||
Container messenger-lab3-api-2 Running
|
||||
Container messenger-lab3-api-1 Starting
|
||||
Container messenger-lab3-api-1 Started
|
||||
Container messenger-lab3-api-1 Waiting
|
||||
Container messenger-lab3-api-2 Waiting
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy b0b074af6d9c 18674 dev
|
||||
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
|
||||
@@ -0,0 +1,69 @@
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy b0b074af6d9c 18674 dev
|
||||
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.616}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 exited/unhealthy - 0 dev
|
||||
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
|
||||
$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api)
|
||||
/messenger-lab3-api-1 status=exited exit=0 finished=2026-10-19T17:02:10.511955087Z
|
||||
/messenger-lab3-api-2 status=running exit=0 finished=0001-01-01T00:00:00Z
|
||||
$ make restore
|
||||
./scripts/restore.sh
|
||||
Container messenger-lab3-api-2 Running
|
||||
Container messenger-lab3-api-1 Starting
|
||||
Container messenger-lab3-api-1 Started
|
||||
Container messenger-lab3-api-1 Waiting
|
||||
Container messenger-lab3-api-2 Waiting
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev
|
||||
messenger-lab3-api-2 running/healthy 389b271f592b 95685 dev
|
||||
$ curl -s -o /dev/null -w "shift round-robin: %{http_code}\n" http://localhost:8080/health/live
|
||||
shift round-robin: 200
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 129, "failed_attempts": 0, "first_recovery_seconds": 1.618}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev
|
||||
messenger-lab3-api-2 exited/unhealthy - 0 dev
|
||||
$ docker inspect -f "{{.Name}} status={{.State.Status}} exit={{.State.ExitCode}} finished={{.State.FinishedAt}}" $(docker compose ps -aq api)
|
||||
/messenger-lab3-api-1 status=running exit=0 finished=2026-10-19T17:02:10.511955087Z
|
||||
/messenger-lab3-api-2 status=exited exit=0 finished=2026-10-19T17:02:43.134886046Z
|
||||
$ docker compose logs api-2 2>/dev/null | grep -E "SIGTERM|Application stopped|Finished server process" || docker compose logs api | grep -E "SIGTERM|Application stopped" | tail -2
|
||||
api-1 | 2026-10-19 17:02:09.336 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=b0b074af6d9c: порт закрыт, keep-alive соединения закрываются после ответа
|
||||
api-1 | 2026-10-19 17:02:10.443 | INFO | __main__:lifespan:54 - Application stopped, instance=b0b074af6d9c, connections closed
|
||||
$ make restore
|
||||
./scripts/restore.sh
|
||||
Container messenger-lab3-api-1 Running
|
||||
Container messenger-lab3-api-2 Starting
|
||||
Container messenger-lab3-api-2 Started
|
||||
Container messenger-lab3-api-1 Waiting
|
||||
Container messenger-lab3-api-2 Waiting
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy e06c3c21d163 21196 dev
|
||||
messenger-lab3-api-2 running/healthy 774df521af21 22387 dev
|
||||
$ docker logs messenger-lab3-api-2 2>&1 | grep -E "instance=389b271f592b|Finished server process"
|
||||
2026-10-19 17:00:05.226 | INFO | __main__:lifespan:45 - Application started, env=lab, revision=dev, instance=389b271f592b, session ttl=3600 s
|
||||
2026-10-19 17:02:41.948 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=389b271f592b: порт закрыт, keep-alive соединения закрываются после ответа
|
||||
2026-10-19 17:02:43.068 | INFO | __main__:lifespan:54 - Application stopped, instance=389b271f592b, connections closed
|
||||
INFO: Finished server process [1]
|
||||
@@ -0,0 +1,37 @@
|
||||
$ cat scripts/restart.sh
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Перезапуск всего стенда с сохранением volumes: хранилища (PostgreSQL, Valkey),
|
||||
# затем все API-реплики и балансировщик. Никаких down -v, flush или повторного
|
||||
# seed; migrate не запускается — схема уже в volume. Готовность ждёт тест.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps postgres valkey
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps api
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps nginx
|
||||
$ docker inspect -f "{{.Name}} started={{.State.StartedAt}}" $(docker compose ps -q)
|
||||
/messenger-lab3-api-1 started=2026-10-19T17:02:39.217903586Z
|
||||
/messenger-lab3-api-2 started=2026-10-19T17:02:57.444596886Z
|
||||
/messenger-lab3-nginx-1 started=2026-10-19T17:00:06.004414293Z
|
||||
/messenger-lab3-postgres-1 started=2026-10-19T16:59:57.066060636Z
|
||||
/messenger-lab3-valkey-1 started=2026-10-19T16:59:57.064188719Z
|
||||
$ docker volume inspect -f "{{.Name}} created={{.CreatedAt}}" messenger-lab3_postgres_data
|
||||
messenger-lab3_postgres_data created=2026-10-19T16:10:27Z
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ docker inspect -f "{{.Name}} started={{.State.StartedAt}}" $(docker compose ps -q)
|
||||
/messenger-lab3-api-1 started=2026-10-19T17:03:17.125014673Z
|
||||
/messenger-lab3-api-2 started=2026-10-19T17:03:17.151381881Z
|
||||
/messenger-lab3-nginx-1 started=2026-10-19T17:03:17.57774484Z
|
||||
/messenger-lab3-postgres-1 started=2026-10-19T17:03:15.504400631Z
|
||||
/messenger-lab3-valkey-1 started=2026-10-19T17:03:15.535674256Z
|
||||
$ docker volume inspect -f "{{.Name}} created={{.CreatedAt}}" messenger-lab3_postgres_data
|
||||
messenger-lab3_postgres_data created=2026-10-19T16:10:27Z
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy bfb3f7bbb85d 23878 dev
|
||||
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
|
||||
@@ -0,0 +1,108 @@
|
||||
$ make test-expiry
|
||||
docker compose --profile expiry up -d --build --wait api-short-ttl
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 1.20kB done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [api-short-ttl internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 914B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [api-short-ttl internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm
|
||||
#3 DONE 0.1s
|
||||
|
||||
#4 [api-short-ttl internal] load .dockerignore
|
||||
#4 transferring context: 358B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [api-short-ttl 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
|
||||
#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [api-short-ttl internal] load build context
|
||||
#6 transferring context: 7.39kB done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [migrate 5/7] COPY migrations ./migrations
|
||||
#7 CACHED
|
||||
|
||||
#8 [migrate 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction
|
||||
#8 CACHED
|
||||
|
||||
#9 [migrate 2/7] WORKDIR /app
|
||||
#9 CACHED
|
||||
|
||||
#10 [migrate 6/7] COPY src ./src
|
||||
#10 CACHED
|
||||
|
||||
#11 [migrate 3/7] COPY poetry.lock pyproject.toml ./
|
||||
#11 CACHED
|
||||
|
||||
#12 [api-short-ttl 7/7] RUN useradd --system --uid 10001 --no-create-home app
|
||||
#12 CACHED
|
||||
|
||||
#13 [migrate] exporting to image
|
||||
#13 exporting layers done
|
||||
#13 exporting manifest sha256:9802819ef7b235f44ad03fed838cf83fcd4573a342405936538c7c969adfd573 done
|
||||
#13 exporting config sha256:e2b0c133c10bc42d53160db701c32694ed2cbac54f0522999289f4cf7cef21e1 done
|
||||
#13 naming to docker.io/library/mtusi-messenger:lab3 done
|
||||
#13 unpacking to docker.io/library/mtusi-messenger:lab3 done
|
||||
#13 DONE 0.0s
|
||||
|
||||
#14 [api-short-ttl] exporting to image
|
||||
#14 exporting layers done
|
||||
#14 exporting manifest sha256:036b7dde37a28a3eeec751ce6ada2034e1a9a1d76cd9310dbd3cd157824ea087 done
|
||||
#14 exporting config sha256:26fb210928fa7201fedcce1e64d506079d863ddc42532dc4d25f1f77d673e3b3 done
|
||||
#14 naming to docker.io/library/mtusi-messenger:lab3 done
|
||||
#14 unpacking to docker.io/library/mtusi-messenger:lab3 done
|
||||
#14 DONE 0.0s
|
||||
|
||||
#15 [migrate] resolving provenance for metadata file
|
||||
#15 DONE 0.0s
|
||||
|
||||
#16 [api-short-ttl] resolving provenance for metadata file
|
||||
#16 DONE 0.0s
|
||||
mtusi-messenger:lab3 Built
|
||||
mtusi-messenger:lab3 Built
|
||||
Container messenger-lab3-valkey-1 Running
|
||||
Container messenger-lab3-postgres-1 Running
|
||||
Container messenger-lab3-migrate-1 Recreate
|
||||
Container messenger-lab3-migrate-1 Recreated
|
||||
Container messenger-lab3-api-short-ttl-1 Creating
|
||||
Container messenger-lab3-api-short-ttl-1 Created
|
||||
Container messenger-lab3-postgres-1 Waiting
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-migrate-1 Starting
|
||||
Container messenger-lab3-migrate-1 Started
|
||||
Container messenger-lab3-migrate-1 Waiting
|
||||
Container messenger-lab3-postgres-1 Waiting
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-migrate-1 Exited
|
||||
Container messenger-lab3-api-short-ttl-1 Starting
|
||||
Container messenger-lab3-api-short-ttl-1 Started
|
||||
Container messenger-lab3-api-short-ttl-1 Waiting
|
||||
Container messenger-lab3-migrate-1 Waiting
|
||||
Container messenger-lab3-postgres-1 Waiting
|
||||
Container messenger-lab3-valkey-1 Waiting
|
||||
Container messenger-lab3-migrate-1 Exited
|
||||
Container messenger-lab3-valkey-1 Healthy
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-api-short-ttl-1 Healthy
|
||||
EXPIRY_BASE_URL=http://localhost:8081 poetry run pytest student-tests/test_expiry.py -s; \
|
||||
status=$?; docker compose --profile expiry stop api-short-ttl; exit $status
|
||||
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
|
||||
warnings.warn(
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python
|
||||
cachedir: .pytest_cache
|
||||
rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3
|
||||
configfile: pyproject.toml
|
||||
plugins: anyio-4.15.1
|
||||
collecting ... collected 2 items
|
||||
|
||||
student-tests/test_expiry.py::test_session_expires_after_short_ttl PASSED
|
||||
student-tests/test_expiry.py::test_expires_at_matches_short_ttl PASSED
|
||||
|
||||
============================== 2 passed in 3.54s ===============================
|
||||
Container messenger-lab3-api-short-ttl-1 Stopping
|
||||
Container messenger-lab3-api-short-ttl-1 Stopped
|
||||
@@ -0,0 +1,24 @@
|
||||
$ make test-failures
|
||||
BASE_URL="http://localhost:8080" RUN_FAILURE_TESTS=1 poetry run pytest student-tests/test_dependency_failures.py -s
|
||||
/Users/enterxx/Library/Python/3.9/lib/python/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020
|
||||
warnings.warn(
|
||||
============================= test session starts ==============================
|
||||
platform darwin -- Python 3.12.12, pytest-9.1.1, pluggy-1.6.0 -- /Users/enterxx/Library/Caches/pypoetry/virtualenvs/max2-jSniokBX-py3.12/bin/python
|
||||
cachedir: .pytest_cache
|
||||
rootdir: /Users/enterxx/Desktop/university/4course/mtusi2026lab3
|
||||
configfile: pyproject.toml
|
||||
plugins: anyio-4.15.1
|
||||
collecting ... collected 4 items
|
||||
|
||||
student-tests/test_dependency_failures.py::test_valkey_down_cache_is_bypassed PASSED
|
||||
student-tests/test_dependency_failures.py::test_logout_during_valkey_outage_is_not_resurrected PASSED
|
||||
student-tests/test_dependency_failures.py::test_postgres_down_returns_bounded_503_and_no_anonymous_access PASSED
|
||||
student-tests/test_dependency_failures.py::test_session_survives_postgres_restart PASSED
|
||||
|
||||
============================== 4 passed in 11.79s ==============================
|
||||
$ docker compose ps --format "{{.Name}}\t{{.Status}}"
|
||||
messenger-lab3-api-1 Up 12 hours (healthy)
|
||||
messenger-lab3-api-2 Up 12 hours (healthy)
|
||||
messenger-lab3-nginx-1 Up 12 hours (healthy)
|
||||
messenger-lab3-postgres-1 Up Less than a second (health: starting)
|
||||
messenger-lab3-valkey-1 Up 8 seconds (healthy)
|
||||
@@ -0,0 +1,22 @@
|
||||
$ make test-migrations
|
||||
./scripts/check_migrations.sh
|
||||
2026-10-20 04:55:31.041 | INFO | infrastructure.dependencies.database:create_pool:69 - Установка соединения с базой данных psql
|
||||
2026-10-20 04:55:31.118 | INFO | infrastructure.dependencies.database:create_pool:99 - Соединение с базой данных psql установлено. DB_NAME: test, pool: 2..10
|
||||
пустой volume: таблиц в public = 0
|
||||
API до migrate: В БД не применены миграции [1, 2]: сначала запустите migrate
|
||||
2026-10-20 04:55:31.132 | INFO | infrastructure.dependencies.database:apply_migrations:152 - Применена миграция 0001_init.sql
|
||||
2026-10-20 04:55:31.134 | INFO | infrastructure.dependencies.database:apply_migrations:152 - Применена миграция 0002_message_idempotency.sql
|
||||
2026-10-20 04:55:31.134 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии
|
||||
запуск 1: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7
|
||||
2026-10-20 04:55:31.135 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии
|
||||
запуск 2: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7
|
||||
2026-10-20 04:55:31.136 | INFO | infrastructure.dependencies.database:apply_migrations:157 - Схема БД в актуальной версии
|
||||
запуск 3: schema_migrations = [(1, '0001_init'), (2, '0002_message_idempotency')], таблиц = 7
|
||||
2026-10-20 04:55:31.136 | INFO | infrastructure.dependencies.database:check_migrations:183 - Схема БД совместима: версия 2
|
||||
API после migrate: схема совместима
|
||||
$ BUILDX_NO_DEFAULT_ATTESTATIONS=1 ./scripts/check_previous_version.sh
|
||||
previous version image: mtusi-messenger:compat-aa971fb (aa971fb feat: перенос core-логики 2 лабы)
|
||||
schema on stand: 0001_init, 0002_message_idempotency
|
||||
{"ready": true, "register": 201, "login": 201, "create_chat": 201, "send_message_without_key": 201, "read_history": 200, "history_matches": true}
|
||||
startup log: ['Схема БД в актуальной версии']
|
||||
PASS: previous API version works on the current schema (expand-only migration)
|
||||
@@ -0,0 +1,54 @@
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy bfb3f7bbb85d 23878 dev
|
||||
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
|
||||
$ make sigterm-check
|
||||
python3 scripts/sigterm_check.py --base-url "http://localhost:8080"
|
||||
Журнал реплики после SIGTERM:
|
||||
2026-10-20 04:55:46.580 | INFO | __main__:shutdown:91 - SIGTERM: drain 1.0 с, instance=bfb3f7bbb85d: порт закрыт, keep-alive соединения закрываются после ответа
|
||||
INFO: Shutting down
|
||||
INFO: Waiting for application shutdown.
|
||||
2026-10-20 04:55:47.701 | INFO | __main__:lifespan:54 - Application stopped, instance=bfb3f7bbb85d, connections closed
|
||||
INFO: Application shutdown complete.
|
||||
INFO: Finished server process [1]
|
||||
{
|
||||
"container": "messenger-lab3-api-1",
|
||||
"instance": "bfb3f7bbb85d",
|
||||
"exit_code": 0,
|
||||
"shutdown_seconds": 1.53,
|
||||
"limit_seconds": 10,
|
||||
"instances_before": [
|
||||
"6d2f6cd2b04a",
|
||||
"bfb3f7bbb85d"
|
||||
],
|
||||
"probes_ok_after_sigterm": 354,
|
||||
"instances_after_sigterm": [
|
||||
"6d2f6cd2b04a",
|
||||
"bfb3f7bbb85d"
|
||||
],
|
||||
"last_target_response_after_sigterm_seconds": 0.133,
|
||||
"target_responses_after_exit": 0,
|
||||
"errors_after_sigterm": 0,
|
||||
"error_kinds": []
|
||||
}
|
||||
PASS: SIGTERM завершил messenger-lab3-api-1 за 1.53 с, LB обслужил 354 запросов без ошибок
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 exited/unhealthy - 0 dev
|
||||
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
|
||||
$ make restore
|
||||
./scripts/restore.sh
|
||||
Container messenger-lab3-api-2 Running
|
||||
Container messenger-lab3-api-1 Starting
|
||||
Container messenger-lab3-api-1 Started
|
||||
Container messenger-lab3-api-1 Waiting
|
||||
Container messenger-lab3-api-2 Waiting
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy 7535b2ed4434 69183 dev
|
||||
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
|
||||
@@ -0,0 +1,33 @@
|
||||
$ docker inspect -f "{{.Name}} cpus={{.HostConfig.NanoCpus}} memory={{.HostConfig.Memory}} restart={{.HostConfig.RestartPolicy.Name}} ports={{json .HostConfig.PortBindings}}" $(docker compose ps -aq)
|
||||
/messenger-lab3-api-1 cpus=1000000000 memory=536870912 restart=unless-stopped ports={}
|
||||
/messenger-lab3-api-2 cpus=1000000000 memory=536870912 restart=unless-stopped ports={}
|
||||
/messenger-lab3-api-short-ttl-1 cpus=1000000000 memory=536870912 restart=unless-stopped ports={}
|
||||
/messenger-lab3-migrate-1 cpus=500000000 memory=268435456 restart=no ports={}
|
||||
/messenger-lab3-nginx-1 cpus=500000000 memory=67108864 restart=unless-stopped ports={"8080/tcp":[{"HostIp":"127.0.0.1","HostPort":"8080"}],"8081/tcp":[{"HostIp":"127.0.0.1","HostPort":"8081"}]}
|
||||
/messenger-lab3-postgres-1 cpus=1500000000 memory=536870912 restart=unless-stopped ports={}
|
||||
/messenger-lab3-valkey-1 cpus=250000000 memory=100663296 restart=unless-stopped ports={}
|
||||
$ for n in messenger-lab3_edge messenger-lab3_backend; do docker network inspect -f "{{.Name}} internal={{.Internal}} containers={{range .Containers}}{{.Name}} {{end}}" $n; done
|
||||
messenger-lab3_edge internal=false containers=messenger-lab3-api-2 messenger-lab3-api-1 messenger-lab3-nginx-1
|
||||
messenger-lab3_backend internal=true containers=messenger-lab3-api-2 messenger-lab3-valkey-1 messenger-lab3-postgres-1 messenger-lab3-api-1
|
||||
$ grep -n "container_name" compose.yaml || echo "container_name: not used"
|
||||
container_name: not used
|
||||
$ docker compose config | grep -E "^\s+(published|target):" | sort | uniq -c
|
||||
1 published: "8080"
|
||||
1 published: "8081"
|
||||
1 target: /etc/nginx/errors.conf
|
||||
1 target: /etc/nginx/nginx.conf
|
||||
1 target: /var/lib/postgresql
|
||||
1 target: 8080
|
||||
1 target: 8081
|
||||
$ docker compose exec -T nginx sh -c "wget -q -O - --timeout 2 http://postgres:5432 2>&1 | head -1; nslookup postgres 127.0.0.11 2>&1 | tail -2"
|
||||
wget: bad address 'postgres:5432'
|
||||
** server can't find postgres: NXDOMAIN
|
||||
|
||||
$ curl -s --max-time 2 http://localhost:5432 >/dev/null 2>&1 && echo "5432 open" || echo "host 127.0.0.1:5432: closed"
|
||||
host 127.0.0.1:5432: closed
|
||||
$ docker stats --no-stream --format "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}" $(docker compose ps -q)
|
||||
messenger-lab3-api-1 0.25% 45.73MiB / 512MiB
|
||||
messenger-lab3-api-2 0.21% 45.42MiB / 512MiB
|
||||
messenger-lab3-nginx-1 0.03% 2.59MiB / 64MiB
|
||||
messenger-lab3-postgres-1 1.15% 30.5MiB / 512MiB
|
||||
messenger-lab3-valkey-1 0.22% 11.04MiB / 96MiB
|
||||
@@ -0,0 +1,18 @@
|
||||
$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/load.json
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.024,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 21418,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 713.369,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.748,
|
||||
"p95": 8.395,
|
||||
"p99": 22.531
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy 7535b2ed4434 69183 dev
|
||||
messenger-lab3-api-2 running/healthy 6d2f6cd2b04a 23926 dev
|
||||
$ make experiment-failover
|
||||
python3 scripts/experiment.py failover --base-url "http://localhost:8080" --output evidence/experiment-failover.json
|
||||
== SIGTERM messenger-lab3-api-1 (instance 7535b2ed4434)
|
||||
{"scenario": "SIGTERM messenger-lab3-api-1", "rps_ok_write": {"before": 381.6, "during": 276.9, "after": 398.9}, "rps_ok_read": {"before": 381.6, "during": 276.8, "after": 399.0}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.063}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.971, "instances_seen_from_action": {"6d2f6cd2b04a": [-10.128, 21.291], "7535b2ed4434": [-10.128, 0.032], "eb1dace786c6": [14.257, 21.289]}, "integrity": {"messages_in_history": 10740, "confirmed_unique": 10740, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
|
||||
== SIGTERM messenger-lab3-api-2 (instance 6d2f6cd2b04a)
|
||||
{"scenario": "SIGTERM messenger-lab3-api-2", "rps_ok_write": {"before": 401.3, "during": 282.3, "after": 362.4}, "rps_ok_read": {"before": 401.1, "during": 282.4, "after": 362.5}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.071}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 3.736, "instances_seen_from_action": {"6d2f6cd2b04a": [-10.122, 0.079], "eb1dace786c6": [-10.121, 21.19], "577316c523cb": [16.921, 21.191]}, "integrity": {"messages_in_history": 10687, "confirmed_unique": 10687, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
|
||||
== SIGKILL messenger-lab3-api-1 (instance eb1dace786c6)
|
||||
{"scenario": "SIGKILL messenger-lab3-api-1", "rps_ok_write": {"before": 427.4, "during": 281.4, "after": 408.2}, "rps_ok_read": {"before": 427.3, "during": 281.3, "after": 408.5}, "errors": {"before": 0, "during": 1, "after": 0}, "recovery": {"write_errors": 1, "error_kinds": {"HTTP 503": 1}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.07}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.79, "instances_seen_from_action": {"eb1dace786c6": [-9.73, 0.024], "577316c523cb": [-9.73, 20.11], "506b547ba744": [12.893, 20.111]}, "integrity": {"messages_in_history": 10837, "confirmed_unique": 10837, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
|
||||
== SIGKILL messenger-lab3-api-2 (instance 577316c523cb)
|
||||
{"scenario": "SIGKILL messenger-lab3-api-2", "rps_ok_write": {"before": 387.4, "during": 276.7, "after": 373.2}, "rps_ok_read": {"before": 387.1, "during": 276.8, "after": 373.3}, "errors": {"before": 0, "during": 3, "after": 0}, "recovery": {"write_errors": 3, "error_kinds": {"HTTP 503": 3}, "write_recovery_seconds": 0.006, "max_write_gap_seconds": 0.061}, "target_responses_while_stopped": 0, "restored_replica_in_rotation_seconds": 0.0, "instances_seen_from_action": {"506b547ba744": [-9.94, 20.189], "577316c523cb": [-9.939, 0.028], "1c20e7f6dbde": [11.911, 20.19]}, "integrity": {"messages_in_history": 10212, "confirmed_unique": 10212, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
|
||||
saved: evidence/experiment-failover.json
|
||||
PASS: прогонов 4, подтверждённые сообщения целы, дублей нет
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy 506b547ba744 76140 dev
|
||||
messenger-lab3-api-2 running/healthy 1c20e7f6dbde 77892 dev
|
||||
@@ -0,0 +1,18 @@
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-1 running/healthy 506b547ba744 76140 dev
|
||||
messenger-lab3-api-2 running/healthy 1c20e7f6dbde 77892 dev
|
||||
$ make experiment-rolling
|
||||
python3 scripts/experiment.py rolling --base-url "http://localhost:8080" --output evidence/experiment-rolling.json
|
||||
== rolling update to revision exp-1792472327
|
||||
{"scenario": "rolling update", "rps_ok_write": {"before": 420.1, "during": 308.0, "after": 345.4}, "rps_ok_read": {"before": 419.9, "during": 307.9, "after": 345.5}, "errors": {"before": 0, "during": 0, "after": 0}, "recovery": {"write_errors": 0, "error_kinds": {}, "write_recovery_seconds": 0.0, "max_write_gap_seconds": 0.098}, "target_responses_while_stopped": null, "restored_replica_in_rotation_seconds": null, "instances_seen_from_action": {"506b547ba744": [-9.228, 8.14], "1c20e7f6dbde": [-9.227, 14.911], "27601bb40a21": [4.875, 24.355], "e7e986ee1b58": [13.885, 24.355]}, "integrity": {"messages_in_history": 11682, "confirmed_unique": 11682, "confirmed_missing_or_changed": 0, "duplicate_ids": 0, "duplicate_texts": 0, "retried_writes_confirmed_as_replay": 0}}
|
||||
saved: evidence/experiment-rolling.json
|
||||
PASS: прогонов 1, подтверждённые сообщения целы, дублей нет
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
messenger-lab3-api-3 running/healthy 27601bb40a21 79887 exp-1792472327
|
||||
messenger-lab3-api-4 running/healthy e7e986ee1b58 80346 exp-1792472327
|
||||
$ docker compose logs --no-log-prefix nginx --since 3m | grep -cE "prematurely closed" || true
|
||||
4
|
||||
@@ -0,0 +1,192 @@
|
||||
$ echo "COMPOSE_PROJECT_NAME=$COMPOSE_PROJECT_NAME (отдельный проект: рабочий стенд не затрагивается)"
|
||||
COMPOSE_PROJECT_NAME=lab3-clean (отдельный проект: рабочий стенд не затрагивается)
|
||||
$ cp .env.example .env && sed -i "" "s/^DB_PASS=.*/DB_PASS=$(openssl rand -hex 16)/; s/^VALKEY_PASS=.*/VALKEY_PASS=$(openssl rand -hex 16)/; s/^LB_PORT=.*/LB_PORT=18080/" .env && echo "LB_EXPIRY_PORT=18081" >> .env && echo ".env created from .env.example with random synthetic secrets"
|
||||
.env created from .env.example with random synthetic secrets
|
||||
$ docker compose --env-file .env config --quiet && echo "compose config: OK"
|
||||
compose config: OK
|
||||
$ make up
|
||||
docker compose up -d --build --wait
|
||||
#1 [internal] load local bake definitions
|
||||
#1 reading from stdin 1.47kB done
|
||||
#1 DONE 0.0s
|
||||
|
||||
#2 [api internal] load build definition from Dockerfile
|
||||
#2 transferring dockerfile: 914B done
|
||||
#2 DONE 0.0s
|
||||
|
||||
#3 [api internal] load metadata for docker.io/library/python:3.12.12-slim-bookworm
|
||||
#3 DONE 0.0s
|
||||
|
||||
#4 [api internal] load .dockerignore
|
||||
#4 transferring context: 358B done
|
||||
#4 DONE 0.0s
|
||||
|
||||
#5 [api 1/7] FROM docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c
|
||||
#5 resolve docker.io/library/python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c done
|
||||
#5 DONE 0.0s
|
||||
|
||||
#6 [api internal] load build context
|
||||
#6 transferring context: 184.06kB 0.0s done
|
||||
#6 DONE 0.0s
|
||||
|
||||
#7 [api 2/7] WORKDIR /app
|
||||
#7 CACHED
|
||||
|
||||
#8 [api 4/7] RUN pip install --no-cache-dir --timeout 60 --retries 10 "poetry==2.2.1" && poetry install --only main --no-root --no-interaction
|
||||
#8 CACHED
|
||||
|
||||
#9 [api 3/7] COPY poetry.lock pyproject.toml ./
|
||||
#9 CACHED
|
||||
|
||||
#10 [migrate 5/7] COPY migrations ./migrations
|
||||
#10 CACHED
|
||||
|
||||
#11 [migrate 6/7] COPY src ./src
|
||||
#11 DONE 0.0s
|
||||
|
||||
#12 [api 7/7] RUN useradd --system --uid 10001 --no-create-home app
|
||||
#12 0.118 useradd warning: app's uid 10001 is greater than SYS_UID_MAX 999
|
||||
#12 DONE 0.1s
|
||||
|
||||
#13 [migrate] exporting to image
|
||||
#13 exporting layers 0.0s done
|
||||
#13 exporting manifest sha256:a99a66062da552c44bf2543747fc3ac215605d7487b93882f0248f46ae71dbf5 done
|
||||
#13 exporting config sha256:53e2ac5d5830524705f0737d78b69d4f9c1ca9670b8a24788c8717a7d772898f done
|
||||
#13 naming to docker.io/library/mtusi-messenger:lab3 done
|
||||
#13 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done
|
||||
#13 DONE 0.1s
|
||||
|
||||
#14 [api] exporting to image
|
||||
#14 exporting layers 0.0s done
|
||||
#14 exporting manifest sha256:47e4d0732ca8d624529f0f775abf4cd8457445931de74053dec5bbab49de873b done
|
||||
#14 exporting config sha256:d6557e51b469086507b892558ee5960f31b77943d8c8fddc381e955e4eb3718a done
|
||||
#14 naming to docker.io/library/mtusi-messenger:lab3 done
|
||||
#14 unpacking to docker.io/library/mtusi-messenger:lab3 0.0s done
|
||||
#14 DONE 0.1s
|
||||
|
||||
#15 [migrate] resolving provenance for metadata file
|
||||
#15 DONE 0.0s
|
||||
|
||||
#16 [api] resolving provenance for metadata file
|
||||
#16 DONE 0.0s
|
||||
mtusi-messenger:lab3 Built
|
||||
mtusi-messenger:lab3 Built
|
||||
Network lab3-clean_backend Creating
|
||||
Network lab3-clean_backend Created
|
||||
Network lab3-clean_edge Creating
|
||||
Network lab3-clean_edge Created
|
||||
Volume lab3-clean_postgres_data Creating
|
||||
Volume lab3-clean_postgres_data Created
|
||||
Container lab3-clean-valkey-1 Creating
|
||||
Container lab3-clean-postgres-1 Creating
|
||||
Container lab3-clean-valkey-1 Created
|
||||
Container lab3-clean-postgres-1 Created
|
||||
Container lab3-clean-migrate-1 Creating
|
||||
Container lab3-clean-migrate-1 Created
|
||||
Container lab3-clean-api-2 Creating
|
||||
Container lab3-clean-api-1 Creating
|
||||
Container lab3-clean-api-2 Created
|
||||
Container lab3-clean-api-1 Created
|
||||
Container lab3-clean-nginx-1 Creating
|
||||
Container lab3-clean-nginx-1 Created
|
||||
Container lab3-clean-valkey-1 Starting
|
||||
Container lab3-clean-postgres-1 Starting
|
||||
Container lab3-clean-postgres-1 Started
|
||||
Container lab3-clean-postgres-1 Waiting
|
||||
Container lab3-clean-valkey-1 Started
|
||||
Container lab3-clean-postgres-1 Healthy
|
||||
Container lab3-clean-migrate-1 Starting
|
||||
Container lab3-clean-migrate-1 Started
|
||||
Container lab3-clean-migrate-1 Waiting
|
||||
Container lab3-clean-postgres-1 Waiting
|
||||
Container lab3-clean-postgres-1 Healthy
|
||||
Container lab3-clean-migrate-1 Exited
|
||||
Container lab3-clean-api-1 Starting
|
||||
Container lab3-clean-api-1 Started
|
||||
Container lab3-clean-api-2 Starting
|
||||
Container lab3-clean-api-2 Started
|
||||
Container lab3-clean-api-1 Waiting
|
||||
Container lab3-clean-api-2 Waiting
|
||||
Container lab3-clean-api-1 Healthy
|
||||
Container lab3-clean-api-2 Healthy
|
||||
Container lab3-clean-nginx-1 Starting
|
||||
Container lab3-clean-nginx-1 Started
|
||||
Container lab3-clean-postgres-1 Waiting
|
||||
Container lab3-clean-valkey-1 Waiting
|
||||
Container lab3-clean-nginx-1 Waiting
|
||||
Container lab3-clean-api-1 Waiting
|
||||
Container lab3-clean-api-2 Waiting
|
||||
Container lab3-clean-migrate-1 Waiting
|
||||
Container lab3-clean-valkey-1 Healthy
|
||||
Container lab3-clean-postgres-1 Healthy
|
||||
Container lab3-clean-migrate-1 Exited
|
||||
Container lab3-clean-api-1 Healthy
|
||||
Container lab3-clean-api-2 Healthy
|
||||
Container lab3-clean-nginx-1 Healthy
|
||||
$ make check test BASE_URL=http://localhost:18080
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
python3 tests/smoke.py --base-url "http://localhost:18080"
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://localhost:18080
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:18080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 2.118}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ make instances
|
||||
./scripts/instances.sh
|
||||
CONTAINER STATE INSTANCE PID REVISION
|
||||
lab3-clean-api-1 exited/unhealthy - 0 dev
|
||||
lab3-clean-api-2 running/healthy 4a226e142873 83992 dev
|
||||
$ make restore
|
||||
./scripts/restore.sh
|
||||
Container lab3-clean-api-2 Running
|
||||
Container lab3-clean-api-1 Starting
|
||||
Container lab3-clean-api-1 Started
|
||||
Container lab3-clean-api-1 Waiting
|
||||
Container lab3-clean-api-2 Waiting
|
||||
Container lab3-clean-api-1 Healthy
|
||||
Container lab3-clean-api-2 Healthy
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://localhost:18080
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:18080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
$ docker compose ps -a
|
||||
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
|
||||
lab3-clean-api-1 mtusi-messenger:lab3 "python src/gateway.…" api 43 seconds ago Up 6 seconds (healthy) 8080/tcp
|
||||
lab3-clean-api-2 mtusi-messenger:lab3 "python src/gateway.…" api 43 seconds ago Up 6 seconds (healthy) 8080/tcp
|
||||
lab3-clean-migrate-1 mtusi-messenger:lab3 "python src/migrate.…" migrate 43 seconds ago Exited (0) 35 seconds ago
|
||||
lab3-clean-nginx-1 nginx:1.28.0-alpine3.21 "/docker-entrypoint.…" nginx 43 seconds ago Up 5 seconds (healthy) 127.0.0.1:18080->8080/tcp, 127.0.0.1:18081->8081/tcp
|
||||
lab3-clean-postgres-1 postgres:18.2-alpine3.22 "docker-entrypoint.s…" postgres 44 seconds ago Up 8 seconds (healthy) 5432/tcp
|
||||
lab3-clean-valkey-1 valkey/valkey:8.1.4-alpine3.22 "docker-entrypoint.s…" valkey 44 seconds ago Up 8 seconds (healthy) 6379/tcp
|
||||
$ docker compose --profile expiry down -v
|
||||
Container lab3-clean-nginx-1 Stopping
|
||||
Container lab3-clean-nginx-1 Stopped
|
||||
Container lab3-clean-nginx-1 Removing
|
||||
Container lab3-clean-nginx-1 Removed
|
||||
Container lab3-clean-api-2 Stopping
|
||||
Container lab3-clean-api-1 Stopping
|
||||
Container lab3-clean-api-1 Stopped
|
||||
Container lab3-clean-api-1 Removing
|
||||
Container lab3-clean-api-1 Removed
|
||||
Container lab3-clean-api-2 Stopped
|
||||
Container lab3-clean-api-2 Removing
|
||||
Container lab3-clean-api-2 Removed
|
||||
Container lab3-clean-valkey-1 Stopping
|
||||
Container lab3-clean-migrate-1 Stopping
|
||||
Container lab3-clean-migrate-1 Stopped
|
||||
Container lab3-clean-migrate-1 Removing
|
||||
Container lab3-clean-migrate-1 Removed
|
||||
Container lab3-clean-postgres-1 Stopping
|
||||
Container lab3-clean-postgres-1 Stopped
|
||||
Container lab3-clean-postgres-1 Removing
|
||||
Container lab3-clean-postgres-1 Removed
|
||||
Container lab3-clean-valkey-1 Stopped
|
||||
Container lab3-clean-valkey-1 Removing
|
||||
Container lab3-clean-valkey-1 Removed
|
||||
Volume lab3-clean_postgres_data Removing
|
||||
Network lab3-clean_edge Removing
|
||||
Network lab3-clean_backend Removing
|
||||
Volume lab3-clean_postgres_data Removed
|
||||
Network lab3-clean_edge Removed
|
||||
Network lab3-clean_backend Removed
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,482 @@
|
||||
{
|
||||
"mode": "rolling",
|
||||
"started": "2026-10-20T09:50:22+0300",
|
||||
"profile": {
|
||||
"workers": 4,
|
||||
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
|
||||
"client_timeout_seconds": 5,
|
||||
"phases_seconds": {
|
||||
"before": 8,
|
||||
"outage": 10,
|
||||
"after": 8
|
||||
}
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"scenario": "rolling update",
|
||||
"target_instance": null,
|
||||
"workers": 4,
|
||||
"events_seconds": {
|
||||
"action_start": 10.323,
|
||||
"action_end": 26.786,
|
||||
"end": 34.791
|
||||
},
|
||||
"instances_before": [
|
||||
"109172df7747",
|
||||
"f6e244c650a2"
|
||||
],
|
||||
"instances_after": [
|
||||
"500528cb388e",
|
||||
"de52b5578b6b"
|
||||
],
|
||||
"target_responses_while_stopped": null,
|
||||
"restored_replica_in_rotation_seconds": null,
|
||||
"instances_seen_from_action": {
|
||||
"109172df7747": [
|
||||
-10.3,
|
||||
14.813
|
||||
],
|
||||
"f6e244c650a2": [
|
||||
-10.296,
|
||||
8.226
|
||||
],
|
||||
"500528cb388e": [
|
||||
6.241,
|
||||
24.466
|
||||
],
|
||||
"de52b5578b6b": [
|
||||
13.791,
|
||||
24.468
|
||||
]
|
||||
},
|
||||
"phases": {
|
||||
"before": {
|
||||
"seconds": 10.323,
|
||||
"write": {
|
||||
"ok": 3687,
|
||||
"errors": 0,
|
||||
"ok_rps": 357.2,
|
||||
"latency_ms": {
|
||||
"p50": 5.4,
|
||||
"p95": 10.3,
|
||||
"p99": 28.9
|
||||
}
|
||||
},
|
||||
"read": {
|
||||
"ok": 3684,
|
||||
"errors": 0,
|
||||
"ok_rps": 356.9,
|
||||
"latency_ms": {
|
||||
"p50": 4.2,
|
||||
"p95": 7.8,
|
||||
"p99": 19.9
|
||||
}
|
||||
}
|
||||
},
|
||||
"during": {
|
||||
"seconds": 16.463,
|
||||
"write": {
|
||||
"ok": 5071,
|
||||
"errors": 0,
|
||||
"ok_rps": 308.0,
|
||||
"latency_ms": {
|
||||
"p50": 5.4,
|
||||
"p95": 16.9,
|
||||
"p99": 38.6
|
||||
}
|
||||
},
|
||||
"read": {
|
||||
"ok": 5073,
|
||||
"errors": 0,
|
||||
"ok_rps": 308.1,
|
||||
"latency_ms": {
|
||||
"p50": 4.2,
|
||||
"p95": 12.3,
|
||||
"p99": 31.4
|
||||
}
|
||||
}
|
||||
},
|
||||
"after": {
|
||||
"seconds": 8.005,
|
||||
"write": {
|
||||
"ok": 2800,
|
||||
"errors": 0,
|
||||
"ok_rps": 349.8,
|
||||
"latency_ms": {
|
||||
"p50": 4.8,
|
||||
"p95": 10.9,
|
||||
"p99": 48.3
|
||||
}
|
||||
},
|
||||
"read": {
|
||||
"ok": 2801,
|
||||
"errors": 0,
|
||||
"ok_rps": 349.9,
|
||||
"latency_ms": {
|
||||
"p50": 3.7,
|
||||
"p95": 8.0,
|
||||
"p99": 47.3
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"recovery": {
|
||||
"write_errors": 0,
|
||||
"error_kinds": {},
|
||||
"write_recovery_seconds": 0.0,
|
||||
"max_write_gap_seconds": 0.299
|
||||
},
|
||||
"integrity": {
|
||||
"messages_in_history": 11558,
|
||||
"confirmed_unique": 11558,
|
||||
"confirmed_missing_or_changed": 0,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0,
|
||||
"retried_writes_confirmed_as_replay": 0
|
||||
},
|
||||
"container_stats_before_action": [
|
||||
"messenger-lab3-nginx-1\t10.92%\t3.227MiB / 64MiB",
|
||||
"messenger-lab3-postgres-1\t30.04%\t54.82MiB / 512MiB",
|
||||
"messenger-lab3-api-1\t85.53%\t52.1MiB / 512MiB",
|
||||
"messenger-lab3-api-2\t84.90%\t54.01MiB / 512MiB"
|
||||
],
|
||||
"action_output": [
|
||||
"[09:50:33] build revision exp-1792479022",
|
||||
"[09:50:34] migrate",
|
||||
"[09:50:36] scale api to 3, wait healthy",
|
||||
"[09:50:41] graceful stop messenger-lab3-api-1",
|
||||
"[09:50:42] scale api to 3, wait healthy",
|
||||
"[09:50:47] graceful stop messenger-lab3-api-2",
|
||||
"[09:50:49] done: all API replicas run revision exp-1792479022"
|
||||
],
|
||||
"restore_output_tail": [],
|
||||
"timeline": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 705,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 354,
|
||||
"f6e244c650a2": 351
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 522,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"f6e244c650a2": 263,
|
||||
"109172df7747": 259
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 733,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 367,
|
||||
"f6e244c650a2": 366
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 771,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 385,
|
||||
"f6e244c650a2": 386
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 725,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 363,
|
||||
"f6e244c650a2": 362
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 803,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 401,
|
||||
"f6e244c650a2": 402
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 449,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 224,
|
||||
"f6e244c650a2": 225
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 777,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 390,
|
||||
"f6e244c650a2": 387
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 803,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"f6e244c650a2": 402,
|
||||
"109172df7747": 401
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 811,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 405,
|
||||
"f6e244c650a2": 406
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 677,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 339,
|
||||
"f6e244c650a2": 338
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 271,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"f6e244c650a2": 135,
|
||||
"109172df7747": 136
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 432,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"f6e244c650a2": 217,
|
||||
"109172df7747": 215
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 422,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 211,
|
||||
"f6e244c650a2": 211
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 533,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 267,
|
||||
"f6e244c650a2": 266
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 737,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"f6e244c650a2": 369,
|
||||
"109172df7747": 368
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 683,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 276,
|
||||
"f6e244c650a2": 277,
|
||||
"500528cb388e": 130
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 635,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"500528cb388e": 212,
|
||||
"109172df7747": 212,
|
||||
"f6e244c650a2": 211
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 841,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"f6e244c650a2": 153,
|
||||
"500528cb388e": 345,
|
||||
"109172df7747": 343
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 782,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 391,
|
||||
"500528cb388e": 391
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 254,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 128,
|
||||
"500528cb388e": 126
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 767,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"500528cb388e": 384,
|
||||
"109172df7747": 383
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 761,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 380,
|
||||
"500528cb388e": 381
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 795,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"500528cb388e": 397,
|
||||
"109172df7747": 398
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 924,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"109172df7747": 322,
|
||||
"500528cb388e": 321,
|
||||
"de52b5578b6b": 281
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 681,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"500528cb388e": 320,
|
||||
"109172df7747": 41,
|
||||
"de52b5578b6b": 320
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 232,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"de52b5578b6b": 115,
|
||||
"500528cb388e": 117
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 252,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"de52b5578b6b": 126,
|
||||
"500528cb388e": 126
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 633,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"de52b5578b6b": 317,
|
||||
"500528cb388e": 316
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 940,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"500528cb388e": 470,
|
||||
"de52b5578b6b": 470
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 814,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"de52b5578b6b": 407,
|
||||
"500528cb388e": 407
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 626,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"de52b5578b6b": 313,
|
||||
"500528cb388e": 313
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 878,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"de52b5578b6b": 439,
|
||||
"500528cb388e": 439
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 740,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"500528cb388e": 370,
|
||||
"de52b5578b6b": 370
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 707,
|
||||
"errors": 0,
|
||||
"instances": {
|
||||
"500528cb388e": 353,
|
||||
"de52b5578b6b": 354
|
||||
}
|
||||
}
|
||||
],
|
||||
"revisions_after": [
|
||||
"CONTAINER STATE INSTANCE PID REVISION",
|
||||
"messenger-lab3-api-3 running/healthy 500528cb388e 60335 exp-1792479022",
|
||||
"messenger-lab3-api-4 running/healthy de52b5578b6b 60779 exp-1792479022"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.024,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 21418,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 713.369,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.748,
|
||||
"p95": 8.395,
|
||||
"p99": 22.531
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# Ошибки, созданные самим балансировщиком, — в формате ошибок контракта.
|
||||
# Ответы реплик (включая их 503) проходят без изменений: proxy_intercept_errors off.
|
||||
# X-Instance-Id здесь нет: ответ создан LB, а не API-репликой.
|
||||
|
||||
location @unavailable {
|
||||
default_type application/json;
|
||||
add_header X-Request-Id $request_id always;
|
||||
return 503 '{"error":{"code":"unavailable","message":"Нет доступной API-реплики","request_id":"$request_id"}}';
|
||||
}
|
||||
|
||||
location @too_large {
|
||||
default_type application/json;
|
||||
add_header X-Request-Id $request_id always;
|
||||
return 413 '{"error":{"code":"payload_too_large","message":"Тело запроса слишком большое","request_id":"$request_id"}}';
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
worker_processes 1;
|
||||
pid /tmp/nginx.pid;
|
||||
error_log /dev/stderr warn;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
|
||||
log_format upstream_json escape=json
|
||||
'{"time":"$time_iso8601","request_id":"$request_id",'
|
||||
'"method":"$request_method","uri":"$uri","status":$status,'
|
||||
'"request_time":$request_time,"upstream_addr":"$upstream_addr",'
|
||||
'"upstream_status":"$upstream_status",'
|
||||
'"upstream_response_time":"$upstream_response_time",'
|
||||
'"instance":"$upstream_http_x_instance_id"}';
|
||||
access_log /dev/stdout upstream_json;
|
||||
|
||||
server_tokens off;
|
||||
client_max_body_size 1m;
|
||||
client_body_buffer_size 64k;
|
||||
client_header_timeout 10s;
|
||||
client_body_timeout 10s;
|
||||
send_timeout 10s;
|
||||
keepalive_timeout 65s;
|
||||
|
||||
upstream api {
|
||||
zone api 64k;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=5s;
|
||||
keepalive 32;
|
||||
# Меньше APP_KEEPALIVE_TIMEOUT_SEC (75 с): idle-соединение первым
|
||||
# закрывает nginx, а не реплика, — нет гонки «запрос в закрытое соединение»
|
||||
keepalive_timeout 60s;
|
||||
}
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
# Один ID на LB и реплику: API принимает $request_id и возвращает его
|
||||
proxy_set_header X-Request-Id $request_id;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
# connect внутри bridge-сети завершает ядро за доли мс даже у занятой реплики.
|
||||
# Самый долгий запрос API: ожидание пула 3 с + statement_timeout 5 с
|
||||
proxy_connect_timeout 500ms;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_read_timeout 10s;
|
||||
|
||||
# Повтор на другой реплике только при ошибке соединения/таймауте.
|
||||
# POST, уже отправленный реплике, nginx не повторяет (non_idempotent выключен):
|
||||
# повтор сообщения делает клиент с тем же Idempotency-Key.
|
||||
# 503 приложения (общая зависимость недоступна) не повторяется и не исключает
|
||||
# реплику: другая реплика зависит от той же БД.
|
||||
proxy_next_upstream error timeout;
|
||||
proxy_next_upstream_tries 3;
|
||||
proxy_next_upstream_timeout 5s;
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
}
|
||||
|
||||
error_page 502 503 504 = @unavailable;
|
||||
error_page 413 = @too_large;
|
||||
|
||||
include /etc/nginx/errors.conf;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8081;
|
||||
|
||||
# Реплика api-short-ttl только для make test-expiry (профиль expiry):
|
||||
# адрес резолвится на каждый запрос, без профиля — 503 от LB
|
||||
location / {
|
||||
set $api_short_ttl http://api-short-ttl:8080;
|
||||
proxy_pass $api_short_ttl;
|
||||
}
|
||||
|
||||
error_page 502 503 504 = @unavailable;
|
||||
error_page 413 = @too_large;
|
||||
|
||||
include /etc/nginx/errors.conf;
|
||||
}
|
||||
|
||||
# Liveness самого балансировщика для healthcheck контейнера; наружу не публикуется
|
||||
server {
|
||||
listen 127.0.0.1:8090;
|
||||
access_log off;
|
||||
|
||||
location = /healthz {
|
||||
default_type text/plain;
|
||||
return 200 "ok\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
-- Idempotency-Key отправки сообщения (lab3). Область по контракту:
|
||||
-- пользователь + метод + путь + ключ. Метод и путь фиксированы
|
||||
-- (POST /api/v1/chats/{chat_id}/messages), поэтому ключ — (sender_id, chat_id, key).
|
||||
-- Только добавление таблицы (expand): предыдущая версия API работает на новой схеме.
|
||||
CREATE TABLE message_idempotency_keys (
|
||||
sender_id uuid NOT NULL,
|
||||
chat_id uuid NOT NULL,
|
||||
idempotency_key text NOT NULL,
|
||||
request_hash bytea NOT NULL,
|
||||
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (sender_id, chat_id, idempotency_key),
|
||||
CONSTRAINT message_idempotency_keys_key_format
|
||||
CHECK (idempotency_key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
CONSTRAINT message_idempotency_keys_hash_len CHECK (octet_length(request_hash) = 32),
|
||||
CONSTRAINT message_idempotency_keys_expiry_order CHECK (expires_at > created_at)
|
||||
);
|
||||
@@ -1,17 +1,19 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Миграции с нуля и повторно: отдельный compose-проект с собственным пустым
|
||||
# volume. Рабочий стенд и его данные не затрагиваются; временный volume
|
||||
# этого проекта удаляется в конце.
|
||||
# Миграции с нуля и повторно, отказ API на непримененной схеме: отдельный
|
||||
# compose-проект с собственным пустым volume. Рабочий стенд и его данные
|
||||
# не затрагиваются; временный volume этого проекта удаляется в конце.
|
||||
cd "$(dirname "$0")/.."
|
||||
PROJECT=messenger-lab3-migrations-check
|
||||
# Свой порт на хосте: основной стенд уже занимает DB_PORT
|
||||
export DB_PORT="${MIGRATIONS_CHECK_DB_PORT:-55432}"
|
||||
# Публикуется только порт LB: свои порты, чтобы не конфликтовать с основным стендом
|
||||
export LB_PORT="${MIGRATIONS_CHECK_LB_PORT:-58080}"
|
||||
export LB_EXPIRY_PORT="${MIGRATIONS_CHECK_LB_EXPIRY_PORT:-58081}"
|
||||
COMPOSE="docker compose --env-file ${ENV_FILE:-.env} -p $PROJECT"
|
||||
trap '$COMPOSE down -v --remove-orphans >/dev/null 2>&1' EXIT
|
||||
|
||||
$COMPOSE up -d --wait postgres >/dev/null 2>&1
|
||||
$COMPOSE run --rm --no-deps -T api python - <<'EOF'
|
||||
# Тот же образ и сеть, что у сервиса migrate; apply_migrations трижды подряд
|
||||
$COMPOSE run --rm --no-deps -T migrate python - <<'EOF'
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
@@ -27,6 +29,13 @@ async def main() -> None:
|
||||
"SELECT count(*) FROM pg_tables WHERE schemaname = 'public';"
|
||||
)
|
||||
print(f"пустой volume: таблиц в public = {tables}")
|
||||
# Так стартует API-реплика: без migrate она отказывается работать
|
||||
try:
|
||||
await db.check_migrations()
|
||||
except RuntimeError as error:
|
||||
print(f"API до migrate: {error}")
|
||||
else:
|
||||
raise SystemExit("FAIL: API стартовал бы на пустой схеме")
|
||||
for attempt in (1, 2, 3):
|
||||
await db.apply_migrations()
|
||||
async with db.acquire() as conn:
|
||||
@@ -40,6 +49,8 @@ async def main() -> None:
|
||||
f"запуск {attempt}: schema_migrations = "
|
||||
f"{[(r['version'], r['name']) for r in versions]}, таблиц = {tables}"
|
||||
)
|
||||
await db.check_migrations()
|
||||
print("API после migrate: схема совместима")
|
||||
await db.close()
|
||||
|
||||
|
||||
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Совместимость текущей схемы с предыдущей версией API — условие rolling update:
|
||||
# миграции применяются до замены реплик, и старые реплики продолжают работать.
|
||||
# Образ собирается из коммита PREVIOUS_REVISION (по умолчанию lab2-код aa971fb),
|
||||
# запускается одним процессом в сети backend рабочего стенда (наружу порт
|
||||
# не публикуется) и выполняет вход, создание чата, запись и чтение сообщения.
|
||||
# Создаются только синтетические данные; volumes не затрагиваются.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
REVISION="${PREVIOUS_REVISION:-aa971fb}"
|
||||
IMAGE="mtusi-messenger:compat-$REVISION"
|
||||
NETWORK=messenger-lab3_backend
|
||||
|
||||
git archive "$REVISION" | docker build --quiet --tag "$IMAGE" - >/dev/null
|
||||
echo "previous version image: $IMAGE ($(git log -1 --format='%h %s' "$REVISION"))"
|
||||
echo "schema on stand: $(docker compose --env-file "$ENV_FILE" exec -T postgres sh -c \
|
||||
'psql -X -q -tA -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c "SELECT string_agg(name, \$\$, \$\$ ORDER BY version) FROM schema_migrations;"')"
|
||||
|
||||
docker run --rm -i --network "$NETWORK" --env-file "$ENV_FILE" \
|
||||
-e DB_HOST=postgres -e DB_PORT=5432 -e VALKEY_HOST=valkey -e VALKEY_PORT=6379 \
|
||||
"$IMAGE" sh -c 'python src/gateway.py >/tmp/api.log 2>&1 & exec python -' <<'EOF'
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
BASE = "http://127.0.0.1:8080"
|
||||
|
||||
|
||||
def call(method, path, body=None, headers=None):
|
||||
data = None if body is None else json.dumps(body).encode()
|
||||
request = urllib.request.Request(BASE + path, data=data, method=method)
|
||||
request.add_header("Content-Type", "application/json")
|
||||
for name, value in (headers or {}).items():
|
||||
request.add_header(name, value)
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=5) as response:
|
||||
return response.status, json.loads(response.read() or b"null")
|
||||
except urllib.error.HTTPError as error:
|
||||
return error.code, None
|
||||
|
||||
|
||||
for _ in range(60):
|
||||
try:
|
||||
if call("GET", "/health/ready")[0] == 200:
|
||||
break
|
||||
except OSError:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
print(open("/tmp/api.log").read()[-2000:])
|
||||
raise SystemExit("FAIL: previous version did not become ready on current schema")
|
||||
|
||||
username = "compat_" + uuid.uuid4().hex[:12]
|
||||
password = "Compat-" + uuid.uuid4().hex
|
||||
status_user, user = call(
|
||||
"POST", "/api/v1/users",
|
||||
{"username": username, "display_name": "Compat", "password": password},
|
||||
)
|
||||
credentials = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
status_login, session = call(
|
||||
"POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + credentials}
|
||||
)
|
||||
bearer = {"Authorization": "Bearer " + session["token"]}
|
||||
status_chat, chat = call(
|
||||
"POST", "/api/v1/chats", {"title": "compat", "member_ids": [user["id"]]}, bearer
|
||||
)
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
status_send, message = call("POST", path, {"text": "старая версия пишет"}, bearer)
|
||||
status_read, page = call("GET", path, headers=bearer)
|
||||
result = {
|
||||
"ready": True,
|
||||
"register": status_user,
|
||||
"login": status_login,
|
||||
"create_chat": status_chat,
|
||||
"send_message_without_key": status_send,
|
||||
"read_history": status_read,
|
||||
"history_matches": page["items"] == [message],
|
||||
}
|
||||
print(json.dumps(result, ensure_ascii=False))
|
||||
log = open("/tmp/api.log").read()
|
||||
print("startup log:", [line.split(" - ", 1)[-1] for line in log.splitlines() if "миграц" in line.lower() or "Схема" in line])
|
||||
if any(code >= 400 for code in result.values() if isinstance(code, int)) or not result["history_matches"]:
|
||||
raise SystemExit("FAIL: previous version is not compatible with current schema")
|
||||
print("PASS: previous API version works on the current schema (expand-only migration)")
|
||||
EOF
|
||||
@@ -0,0 +1,480 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Нагрузка через LB во время отказа/замены API-реплик; метрики и проверка данных.
|
||||
|
||||
failover — для каждой API-реплики по очереди: нагрузка → остановка этой реплики
|
||||
(docker stop = SIGTERM, затем docker kill = SIGKILL) → восстановление
|
||||
(scripts/restore.sh) → нагрузка после восстановления.
|
||||
rolling — нагрузка во время scripts/rolling-update.sh (поочерёдная замена реплик).
|
||||
|
||||
Клиент ведёт себя как настоящий: неподтверждённый POST повторяет с тем же
|
||||
Idempotency-Key и телом. После прогона вся история читается заново: каждое
|
||||
подтверждённое сообщение на месте и не изменено, дублей нет.
|
||||
Python 3.9+, стандартная библиотека; tokens не печатаются и не сохраняются.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client # noqa: E402
|
||||
|
||||
REQUEST_TIMEOUT_SEC = 5
|
||||
RETRY_PAUSE_SEC = 0.05
|
||||
STATS_FORMAT = "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"
|
||||
|
||||
|
||||
@dataclass
|
||||
class Attempt:
|
||||
"""Одна HTTP-попытка клиента; время — секунды от начала прогона"""
|
||||
|
||||
op: str
|
||||
started: float
|
||||
finished: float
|
||||
status: int | None
|
||||
error: str | None
|
||||
instance: str | None
|
||||
retry: bool
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.status in (200, 201)
|
||||
|
||||
|
||||
class Recorder:
|
||||
def __init__(self) -> None:
|
||||
self.started = time.monotonic()
|
||||
self.lock = threading.Lock()
|
||||
self.attempts: list[Attempt] = []
|
||||
self.confirmed: list[dict] = []
|
||||
self.events: dict[str, float] = {}
|
||||
|
||||
def now(self) -> float:
|
||||
return time.monotonic() - self.started
|
||||
|
||||
def mark(self, name: str) -> None:
|
||||
self.events[name] = round(self.now(), 3)
|
||||
|
||||
def add(self, attempt: Attempt, message: dict | None = None) -> None:
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
if message is not None:
|
||||
self.confirmed.append(message)
|
||||
|
||||
|
||||
def request(
|
||||
base: str, method: str, path: str, token: str, body: dict | None, key: str | None
|
||||
) -> tuple[int | None, str | None, str | None, dict | None]:
|
||||
headers = {"Authorization": "Bearer " + token, "Accept": "application/json"}
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body, ensure_ascii=False).encode()
|
||||
headers["Content-Type"] = "application/json"
|
||||
if key is not None:
|
||||
headers["Idempotency-Key"] = key
|
||||
req = urllib.request.Request(base + path, data=data, headers=headers, method=method)
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_SEC) as response:
|
||||
payload = json.loads(response.read())
|
||||
return response.status, None, response.headers.get("X-Instance-Id"), payload
|
||||
except urllib.error.HTTPError as error:
|
||||
error.read()
|
||||
return error.code, None, error.headers.get("X-Instance-Id"), None
|
||||
except OSError as error:
|
||||
return None, type(error).__name__, None, None
|
||||
|
||||
|
||||
def worker(
|
||||
base: str,
|
||||
token: str,
|
||||
chat_id: str,
|
||||
worker_id: int,
|
||||
recorder: Recorder,
|
||||
stop: threading.Event,
|
||||
) -> None:
|
||||
"""Запись с повтором по тому же ключу до подтверждения, затем чтение"""
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
index = 0
|
||||
while not stop.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
body = {"text": f"exp-{worker_id}-{index}-{key[:8]}"}
|
||||
retry = False
|
||||
while not stop.is_set():
|
||||
started = recorder.now()
|
||||
status, error, instance, payload = request(
|
||||
base, "POST", path, token, body, key
|
||||
)
|
||||
attempt = Attempt(
|
||||
"write", started, recorder.now(), status, error, instance, retry
|
||||
)
|
||||
recorder.add(attempt, payload if status == 201 else None)
|
||||
if status == 201:
|
||||
break
|
||||
retry = True
|
||||
time.sleep(RETRY_PAUSE_SEC)
|
||||
started = recorder.now()
|
||||
status, error, instance, _ = request(
|
||||
base, "GET", path + "?limit=1", token, None, None
|
||||
)
|
||||
recorder.add(
|
||||
Attempt("read", started, recorder.now(), status, error, instance, False)
|
||||
)
|
||||
index += 1
|
||||
|
||||
|
||||
def percentile(values: list[float], p: float) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
return round(1000 * ordered[max(0, math.ceil(p * len(ordered)) - 1)], 1)
|
||||
|
||||
|
||||
def phase_stats(attempts: list[Attempt], start: float, end: float) -> dict:
|
||||
inside = [a for a in attempts if start <= a.finished < end]
|
||||
duration = max(end - start, 1e-9)
|
||||
result: dict = {"seconds": round(duration, 3)}
|
||||
for op in ("write", "read"):
|
||||
ops = [a for a in inside if a.op == op]
|
||||
ok = [a for a in ops if a.ok]
|
||||
result[op] = {
|
||||
"ok": len(ok),
|
||||
"errors": len(ops) - len(ok),
|
||||
"ok_rps": round(len(ok) / duration, 1),
|
||||
"latency_ms": {
|
||||
"p50": percentile([a.finished - a.started for a in ok], 0.50),
|
||||
"p95": percentile([a.finished - a.started for a in ok], 0.95),
|
||||
"p99": percentile([a.finished - a.started for a in ok], 0.99),
|
||||
},
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def recovery_stats(attempts: list[Attempt], action_start: float, end: float) -> dict:
|
||||
"""
|
||||
Время восстановления записи: от первой ошибки после воздействия до первой
|
||||
успешной записи после последней ошибки; без ошибок — 0. Отдельно —
|
||||
наибольший разрыв между подтверждениями записи (задержка без ошибок).
|
||||
"""
|
||||
window = sorted(
|
||||
(a for a in attempts if a.op == "write" and a.finished >= action_start - 1),
|
||||
key=lambda a: a.finished,
|
||||
)
|
||||
errors = [a for a in window if not a.ok and a.finished >= action_start]
|
||||
kinds: dict[str, int] = {}
|
||||
for a in errors:
|
||||
kind = a.error or f"HTTP {a.status}"
|
||||
kinds[kind] = kinds.get(kind, 0) + 1
|
||||
if errors:
|
||||
last_error = errors[-1].finished
|
||||
first_ok_after = next(
|
||||
(a.finished for a in window if a.ok and a.finished > last_error), end
|
||||
)
|
||||
recovery = first_ok_after - errors[0].finished
|
||||
else:
|
||||
recovery = 0.0
|
||||
oks = [a.finished for a in window if a.ok and a.finished <= end]
|
||||
gap = max((b - a for a, b in zip(oks, oks[1:])), default=None)
|
||||
return {
|
||||
"write_errors": len(errors),
|
||||
"error_kinds": kinds,
|
||||
"write_recovery_seconds": round(recovery, 3),
|
||||
"max_write_gap_seconds": None if gap is None else round(gap, 3),
|
||||
}
|
||||
|
||||
|
||||
def timeline(attempts: list[Attempt], end: float) -> list[dict]:
|
||||
rows = []
|
||||
for second in range(int(end) + 1):
|
||||
inside = [a for a in attempts if second <= a.finished < second + 1]
|
||||
instances: dict[str, int] = {}
|
||||
for a in inside:
|
||||
if a.ok and a.instance:
|
||||
instances[a.instance] = instances.get(a.instance, 0) + 1
|
||||
rows.append(
|
||||
{
|
||||
"t": second,
|
||||
"ok": sum(a.ok for a in inside),
|
||||
"errors": sum(not a.ok for a in inside),
|
||||
"instances": instances,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def read_history(client: Client, actor: dict, chat_id: str) -> list[dict]:
|
||||
path = f"/api/v1/chats/{chat_id}/messages?limit=100"
|
||||
items: list[dict] = []
|
||||
while True:
|
||||
page = client.call("GET", path, actor=actor, schema="MessagePage")["json"]
|
||||
items.extend(page["items"])
|
||||
if page["next_cursor"] is None:
|
||||
return items
|
||||
path = f"/api/v1/chats/{chat_id}/messages?limit=100&cursor=" + quote(
|
||||
page["next_cursor"], safe=""
|
||||
)
|
||||
|
||||
|
||||
def integrity(client: Client, actor: dict, chat_id: str, recorder: Recorder) -> dict:
|
||||
history = read_history(client, actor, chat_id)
|
||||
by_id = {m["id"]: m for m in history}
|
||||
texts = [m["text"] for m in history]
|
||||
confirmed = {m["id"]: m for m in recorder.confirmed}
|
||||
return {
|
||||
"messages_in_history": len(history),
|
||||
"confirmed_unique": len(confirmed),
|
||||
"confirmed_missing_or_changed": sum(
|
||||
by_id.get(i) != m for i, m in confirmed.items()
|
||||
),
|
||||
"duplicate_ids": len(history) - len(by_id),
|
||||
"duplicate_texts": len(texts) - len(set(texts)),
|
||||
"retried_writes_confirmed_as_replay": len(recorder.confirmed) - len(confirmed),
|
||||
}
|
||||
|
||||
|
||||
def run(command: list[str], env: dict | None = None) -> str:
|
||||
result = subprocess.run(
|
||||
command, cwd=ROOT, env=env, check=True, capture_output=True, text=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def api_containers() -> list[str]:
|
||||
ids = run(["docker", "compose", "--env-file", ".env", "ps", "-q", "api"]).split()
|
||||
names = [run(["docker", "inspect", "-f", "{{.Name}}", i]).lstrip("/") for i in ids]
|
||||
return sorted(names)
|
||||
|
||||
|
||||
def instance_of(container: str) -> str:
|
||||
return run(["docker", "exec", container, "cat", "/tmp/instance-id"])
|
||||
|
||||
|
||||
def container_stats() -> list[str]:
|
||||
names = ["messenger-lab3-nginx-1", "messenger-lab3-postgres-1", *api_containers()]
|
||||
output = run(["docker", "stats", "--no-stream", "--format", STATS_FORMAT, *names])
|
||||
return output.splitlines()
|
||||
|
||||
|
||||
def scenario(
|
||||
args: argparse.Namespace,
|
||||
label: str,
|
||||
action: list[str],
|
||||
restore: list[str] | None,
|
||||
target: str | None = None,
|
||||
env: dict | None = None,
|
||||
) -> dict:
|
||||
client = Client(args.base_url, timeout=REQUEST_TIMEOUT_SEC)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
base = client.base
|
||||
recorder = Recorder()
|
||||
stop = threading.Event()
|
||||
threads = [
|
||||
threading.Thread(
|
||||
target=worker,
|
||||
args=(base, alice["token"], chat["id"], n, recorder, stop),
|
||||
daemon=True,
|
||||
)
|
||||
for n in range(args.workers)
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
time.sleep(args.before / 2)
|
||||
stats = container_stats()
|
||||
time.sleep(args.before / 2)
|
||||
|
||||
instances_before = sorted(
|
||||
{a.instance for a in recorder.attempts if a.ok and a.instance}
|
||||
)
|
||||
recorder.mark("action_start")
|
||||
action_output = run(action, env)
|
||||
recorder.mark("action_end")
|
||||
restore_output = ""
|
||||
if restore is not None:
|
||||
time.sleep(args.outage)
|
||||
recorder.mark("restore_start")
|
||||
restore_output = run(restore)
|
||||
recorder.mark("restore_end")
|
||||
time.sleep(args.after)
|
||||
stop.set()
|
||||
for thread in threads:
|
||||
thread.join(timeout=REQUEST_TIMEOUT_SEC + 1)
|
||||
recorder.mark("end")
|
||||
|
||||
events = recorder.events
|
||||
end = events["end"]
|
||||
after_start = events.get("restore_end", events["action_end"])
|
||||
attempts = recorder.attempts
|
||||
seen_after = [
|
||||
a for a in attempts if a.ok and a.finished >= after_start and a.instance
|
||||
]
|
||||
back_in_rotation = next(
|
||||
(
|
||||
round(a.finished - after_start, 3)
|
||||
for a in sorted(seen_after, key=lambda a: a.finished)
|
||||
if a.instance not in instances_before
|
||||
),
|
||||
None,
|
||||
)
|
||||
served = sorted(
|
||||
(a for a in attempts if a.ok and a.instance), key=lambda a: a.finished
|
||||
)
|
||||
instances_seen: dict[str, list[float]] = {}
|
||||
for a in served:
|
||||
offset = round(a.finished - events["action_start"], 3)
|
||||
instances_seen.setdefault(a.instance, [offset, offset])[1] = offset
|
||||
phases = {
|
||||
"before": phase_stats(attempts, 0, events["action_start"]),
|
||||
"during": phase_stats(attempts, events["action_start"], after_start),
|
||||
"after": phase_stats(attempts, after_start, end),
|
||||
}
|
||||
target_after_stop = None
|
||||
if target is not None:
|
||||
target_after_stop = sum(
|
||||
1
|
||||
for a in attempts
|
||||
if a.ok
|
||||
and a.instance == target
|
||||
and a.finished > events["action_end"]
|
||||
and a.finished < events.get("restore_start", end)
|
||||
)
|
||||
return {
|
||||
"scenario": label,
|
||||
"target_instance": target,
|
||||
"workers": args.workers,
|
||||
"events_seconds": events,
|
||||
"instances_before": instances_before,
|
||||
"instances_after": sorted({a.instance for a in seen_after}),
|
||||
"target_responses_while_stopped": target_after_stop,
|
||||
"restored_replica_in_rotation_seconds": back_in_rotation
|
||||
if restore is not None
|
||||
else None,
|
||||
# alias → [первый, последний] успешный ответ, секунды от начала воздействия
|
||||
"instances_seen_from_action": instances_seen,
|
||||
"phases": phases,
|
||||
"recovery": recovery_stats(attempts, events["action_start"], end),
|
||||
"integrity": integrity(client, alice, chat["id"], recorder),
|
||||
"container_stats_before_action": stats,
|
||||
"action_output": action_output.splitlines(),
|
||||
"restore_output_tail": restore_output.splitlines()[-2:],
|
||||
"timeline": timeline(attempts, end),
|
||||
}
|
||||
|
||||
|
||||
def failover(args: argparse.Namespace) -> list[dict]:
|
||||
results = []
|
||||
for signal in args.signals:
|
||||
for container in api_containers():
|
||||
target = instance_of(container)
|
||||
if signal == "SIGTERM":
|
||||
action = ["docker", "stop", "--timeout", "10", container]
|
||||
else:
|
||||
action = ["docker", "kill", "--signal", "SIGKILL", container]
|
||||
label = f"{signal} {container}"
|
||||
print(f"== {label} (instance {target})", flush=True)
|
||||
result = scenario(
|
||||
args, label, action, ["./scripts/restore.sh"], target=target
|
||||
)
|
||||
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
|
||||
def rolling(args: argparse.Namespace) -> list[dict]:
|
||||
env = dict(os.environ)
|
||||
env.setdefault("APP_REVISION", f"exp-{int(time.time())}")
|
||||
print(f"== rolling update to revision {env['APP_REVISION']}", flush=True)
|
||||
result = scenario(
|
||||
args, "rolling update", ["./scripts/rolling-update.sh"], None, env=env
|
||||
)
|
||||
result["revisions_after"] = run(["./scripts/instances.sh"]).splitlines()
|
||||
print(json.dumps(brief(result), ensure_ascii=False), flush=True)
|
||||
return [result]
|
||||
|
||||
|
||||
def brief(result: dict) -> dict:
|
||||
"""Краткая строка прогона для терминала; полная версия — в --output"""
|
||||
return {
|
||||
"scenario": result["scenario"],
|
||||
"rps_ok_write": {k: v["write"]["ok_rps"] for k, v in result["phases"].items()},
|
||||
"rps_ok_read": {k: v["read"]["ok_rps"] for k, v in result["phases"].items()},
|
||||
"errors": {
|
||||
k: v["write"]["errors"] + v["read"]["errors"]
|
||||
for k, v in result["phases"].items()
|
||||
},
|
||||
"recovery": result["recovery"],
|
||||
"target_responses_while_stopped": result["target_responses_while_stopped"],
|
||||
"restored_replica_in_rotation_seconds": result[
|
||||
"restored_replica_in_rotation_seconds"
|
||||
],
|
||||
"instances_seen_from_action": result["instances_seen_from_action"],
|
||||
"integrity": result["integrity"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("mode", choices=["failover", "rolling"])
|
||||
parser.add_argument("--base-url", default=None)
|
||||
parser.add_argument("--workers", type=int, default=4)
|
||||
parser.add_argument("--before", type=float, default=8, help="нагрузка до отказа")
|
||||
parser.add_argument("--outage", type=float, default=10, help="реплика остановлена")
|
||||
parser.add_argument("--after", type=float, default=8, help="после восстановления")
|
||||
parser.add_argument(
|
||||
"--signals",
|
||||
nargs="+",
|
||||
default=["SIGTERM", "SIGKILL"],
|
||||
choices=["SIGTERM", "SIGKILL"],
|
||||
)
|
||||
parser.add_argument("--output", default=None, help="JSON с полными результатами")
|
||||
args = parser.parse_args()
|
||||
started = time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
||||
results = failover(args) if args.mode == "failover" else rolling(args)
|
||||
report = {
|
||||
"mode": args.mode,
|
||||
"started": started,
|
||||
"profile": {
|
||||
"workers": args.workers,
|
||||
"loop": "closed: POST message (retry same key until 201) + GET limit=1",
|
||||
"client_timeout_seconds": REQUEST_TIMEOUT_SEC,
|
||||
"phases_seconds": {
|
||||
"before": args.before,
|
||||
"outage": args.outage,
|
||||
"after": args.after,
|
||||
},
|
||||
},
|
||||
"runs": results,
|
||||
}
|
||||
if args.output:
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
print(f"saved: {args.output}")
|
||||
failed = [
|
||||
r["scenario"]
|
||||
for r in results
|
||||
if r["integrity"]["confirmed_missing_or_changed"]
|
||||
or r["integrity"]["duplicate_ids"]
|
||||
or r["integrity"]["duplicate_texts"]
|
||||
or r["target_responses_while_stopped"]
|
||||
]
|
||||
if failed:
|
||||
raise SystemExit(f"FAIL: потеря/дубль или отказ не изолирован: {failed}")
|
||||
print(f"PASS: прогонов {len(results)}, подтверждённые сообщения целы, дублей нет")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Операторский вид API-реплик: контейнер ↔ alias X-Instance-Id ↔ ревизия образа.
|
||||
# Нужен, чтобы показать, какой реальный контейнер обслужил запрос или остановлен:
|
||||
# заголовок сам по себе не доказывает разные процессы.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
printf '%-24s %-28s %-14s %-8s %s\n' CONTAINER STATE INSTANCE PID REVISION
|
||||
for container in $(docker compose --env-file "$ENV_FILE" ps -a -q api); do
|
||||
name=$(docker inspect -f '{{.Name}}' "$container")
|
||||
state=$(docker inspect -f '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$container")
|
||||
pid=$(docker inspect -f '{{.State.Pid}}' "$container")
|
||||
revision=$(docker inspect -f '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$container")
|
||||
alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || echo -)
|
||||
printf '%-24s %-28s %-14s %-8s %s\n' "${name#/}" "$state" "$alias" "$pid" "$revision"
|
||||
done
|
||||
+6
-4
@@ -1,9 +1,11 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Перезапуск API и всех используемых хранилищ (PostgreSQL, Valkey) с сохранением
|
||||
# volumes: никаких down -v, flush или повторного seed. Готовность ждёт тест.
|
||||
# Перезапуск всего стенда с сохранением volumes: хранилища (PostgreSQL, Valkey),
|
||||
# затем все API-реплики и балансировщик. Никаких down -v, flush или повторного
|
||||
# seed; migrate не запускается — схема уже в volume. Готовность ждёт тест.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
docker compose --env-file "$ENV_FILE" restart postgres valkey
|
||||
docker compose --env-file "$ENV_FILE" restart api
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps postgres valkey
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps api
|
||||
docker compose --env-file "$ENV_FILE" restart --no-deps nginx
|
||||
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Возврат API-реплик к числу API_REPLICAS после stop-one.sh: запуск остановленных
|
||||
# контейнеров и ожидание healthy. Работающие реплики не пересоздаются
|
||||
# (--no-recreate), хранилища, migrate и LB не перезапускаются.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
|
||||
docker compose --env-file "$ENV_FILE" up -d --no-deps --no-recreate --wait --wait-timeout 60 api
|
||||
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Поочерёдная замена API-реплик новой сборкой без остановки обслуживания.
|
||||
# 1. build нового образа (ревизия видна в метке образа и журнале старта);
|
||||
# 2. migrate однократно: миграции только расширяют схему, старые реплики
|
||||
# продолжают работать на новой схеме;
|
||||
# 3. для каждой старой реплики: +1 новая → ждать healthy → пауза на обновление
|
||||
# DNS в nginx → graceful stop старой (SIGTERM, LB уводит трафик) → удалить.
|
||||
# Во время замены реплик на одну больше, чем API_REPLICAS: ресурс заложен заранее.
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
COMPOSE="docker compose --env-file $ENV_FILE"
|
||||
APP_REVISION="${APP_REVISION:-$(git rev-parse --short HEAD 2>/dev/null || echo dev)-$(date +%s)}"
|
||||
export APP_REVISION
|
||||
# ID образа зависит только от содержимого (без provenance-attestation)
|
||||
export BUILDX_NO_DEFAULT_ATTESTATIONS=1
|
||||
# Больше resolver valid=2s в nginx: новая реплика уже в upstream до остановки старой
|
||||
LB_DNS_REFRESH_SEC=3
|
||||
STOP_TIMEOUT_SEC=10
|
||||
|
||||
log() {
|
||||
echo "[$(date +%H:%M:%S)] $*"
|
||||
}
|
||||
|
||||
log "build revision $APP_REVISION"
|
||||
$COMPOSE build --quiet api
|
||||
log "migrate"
|
||||
$COMPOSE run --rm --no-deps migrate >/dev/null
|
||||
|
||||
old_containers=$($COMPOSE ps -q api)
|
||||
for container in $old_containers; do
|
||||
name=$(docker inspect -f '{{.Name}}' "$container")
|
||||
running=$($COMPOSE ps -q api | wc -l | tr -d ' ')
|
||||
log "scale api to $((running + 1)), wait healthy"
|
||||
$COMPOSE up -d --no-deps --no-recreate --wait --wait-timeout 60 \
|
||||
--scale "api=$((running + 1))" api
|
||||
sleep "$LB_DNS_REFRESH_SEC"
|
||||
log "graceful stop ${name#/}"
|
||||
docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null
|
||||
docker rm "$container" >/dev/null
|
||||
done
|
||||
log "done: all API replicas run revision $APP_REVISION"
|
||||
+78
-57
@@ -1,8 +1,11 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SIGTERM: процесс перестаёт принимать запросы и завершается не дольше чем за 10 секунд.
|
||||
"""SIGTERM одной API-реплики за балансировщиком.
|
||||
|
||||
Во время проверки фоновый клиент непрерывно шлёт GET /health/live: до сигнала
|
||||
ответы 200, после — отказ в соединении; ответов 5xx быть не должно.
|
||||
Реплика перестаёт принимать соединения, дожидается текущих запросов и
|
||||
завершается не дольше чем за 10 секунд; LB уводит трафик на другие реплики.
|
||||
Во время проверки фоновые клиенты непрерывно шлют GET /health/live через LB:
|
||||
ошибок (5xx/обрыв) быть не должно, после выхода реплики её X-Instance-Id
|
||||
в ответах не появляется.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -20,106 +23,124 @@ DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url
|
||||
|
||||
SHUTDOWN_LIMIT_SEC = 10
|
||||
PROBE_INTERVAL_SEC = 0.02
|
||||
PROBERS = 4
|
||||
|
||||
|
||||
class Prober(threading.Thread):
|
||||
"""Фоновый клиент: каждое обращение — новое соединение"""
|
||||
"""Фоновый клиент через LB: время ответа и обслужившая реплика"""
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
super().__init__(daemon=True)
|
||||
self.url = url
|
||||
self.stop = threading.Event()
|
||||
self.ok: list[float] = []
|
||||
self.refused: list[float] = []
|
||||
self.server_errors = 0
|
||||
self.ok: list[tuple[float, str]] = []
|
||||
self.errors: list[tuple[float, str]] = []
|
||||
|
||||
def run(self) -> None:
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
with urllib.request.urlopen(self.url, timeout=2) as response:
|
||||
if response.status == 200:
|
||||
self.ok.append(time.monotonic())
|
||||
self.ok.append(
|
||||
(time.monotonic(), response.headers.get("X-Instance-Id", ""))
|
||||
)
|
||||
except urllib.error.HTTPError as error:
|
||||
if error.code >= 500:
|
||||
self.server_errors += 1
|
||||
except OSError:
|
||||
self.refused.append(time.monotonic())
|
||||
self.errors.append((time.monotonic(), f"HTTP {error.code}"))
|
||||
except OSError as error:
|
||||
self.errors.append((time.monotonic(), type(error).__name__))
|
||||
time.sleep(PROBE_INTERVAL_SEC)
|
||||
|
||||
|
||||
def docker(*args: str) -> str:
|
||||
result = subprocess.run(
|
||||
["docker", *args], check=True, capture_output=True, text=True
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def first_api_container() -> str:
|
||||
ids = docker("compose", "--env-file", ".env", "ps", "-q", "api").split()
|
||||
if not ids:
|
||||
raise SystemExit("FAIL: нет запущенных API-реплик")
|
||||
return docker("inspect", "-f", "{{.Name}}", ids[0]).lstrip("/")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--base-url", default=os.environ.get("BASE_URL") or DEFAULT_BASE_URL
|
||||
)
|
||||
parser.add_argument(
|
||||
"--container", default=os.environ.get("CONTAINER", "messenger-lab3-api-1")
|
||||
)
|
||||
parser.add_argument("--container", default=os.environ.get("CONTAINER"))
|
||||
args = parser.parse_args()
|
||||
os.chdir(ROOT)
|
||||
container = args.container or first_api_container()
|
||||
instance = docker("exec", container, "cat", "/tmp/instance-id")
|
||||
|
||||
logs = subprocess.Popen(
|
||||
["docker", "logs", "--follow", "--since", "0s", args.container],
|
||||
["docker", "logs", "--follow", "--since", "0s", container],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
)
|
||||
waiter = subprocess.Popen(
|
||||
["docker", "wait", args.container], stdout=subprocess.PIPE, text=True
|
||||
["docker", "wait", container], stdout=subprocess.PIPE, text=True
|
||||
)
|
||||
prober = Prober(args.base_url.rstrip("/") + "/health/live")
|
||||
prober.start()
|
||||
probers = [
|
||||
Prober(args.base_url.rstrip("/") + "/health/live") for _ in range(PROBERS)
|
||||
]
|
||||
for prober in probers:
|
||||
prober.start()
|
||||
time.sleep(1)
|
||||
if not prober.ok:
|
||||
raise SystemExit("FAIL: API не отвечает до отправки SIGTERM")
|
||||
before = {alias for prober in probers for _, alias in prober.ok}
|
||||
if instance not in before:
|
||||
raise SystemExit("FAIL: целевая реплика не обслуживает запросы до SIGTERM")
|
||||
|
||||
sent_at = time.monotonic()
|
||||
subprocess.run(
|
||||
["docker", "kill", "--signal", "SIGTERM", args.container],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
docker("kill", "--signal", "SIGTERM", container)
|
||||
exit_code = int(waiter.communicate(timeout=SHUTDOWN_LIMIT_SEC + 5)[0].strip())
|
||||
exited_at = time.monotonic()
|
||||
time.sleep(0.5)
|
||||
prober.stop.set()
|
||||
prober.join()
|
||||
time.sleep(1)
|
||||
for prober in probers:
|
||||
prober.stop.set()
|
||||
prober.join()
|
||||
logs.terminate()
|
||||
shutdown_log = logs.communicate(timeout=5)[0]
|
||||
|
||||
shutdown_sec = exited_at - sent_at
|
||||
ok_after = [t for t in prober.ok if t > sent_at]
|
||||
first_refused = min((t for t in prober.refused if t > sent_at), default=None)
|
||||
print("Логи контейнера после SIGTERM:")
|
||||
ok_after = [(t, a) for p in probers for t, a in p.ok if t > sent_at]
|
||||
errors_after = [(t, e) for p in probers for t, e in p.errors if t > sent_at]
|
||||
target_after_exit = [t for t, a in ok_after if a == instance and t > exited_at]
|
||||
last_target = max((t for t, a in ok_after if a == instance), default=None)
|
||||
print("Журнал реплики после SIGTERM:")
|
||||
print(
|
||||
"\n".join(line for line in shutdown_log.splitlines() if "/health/" not in line)
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"container": args.container,
|
||||
"exit_code": exit_code,
|
||||
"shutdown_seconds": round(shutdown_sec, 3),
|
||||
"limit_seconds": SHUTDOWN_LIMIT_SEC,
|
||||
"probes_ok_before_sigterm": len(prober.ok) - len(ok_after),
|
||||
"probes_ok_after_sigterm": len(ok_after),
|
||||
"first_refused_after_sigterm_seconds": None
|
||||
if first_refused is None
|
||||
else round(first_refused - sent_at, 3),
|
||||
"probes_refused_after_sigterm": len(prober.refused),
|
||||
"server_errors_5xx": prober.server_errors,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
summary = {
|
||||
"container": container,
|
||||
"instance": instance,
|
||||
"exit_code": exit_code,
|
||||
"shutdown_seconds": round(exited_at - sent_at, 3),
|
||||
"limit_seconds": SHUTDOWN_LIMIT_SEC,
|
||||
"instances_before": sorted(before),
|
||||
"probes_ok_after_sigterm": len(ok_after),
|
||||
"instances_after_sigterm": sorted({a for _, a in ok_after}),
|
||||
"last_target_response_after_sigterm_seconds": None
|
||||
if last_target is None
|
||||
else round(last_target - sent_at, 3),
|
||||
"target_responses_after_exit": len(target_after_exit),
|
||||
"errors_after_sigterm": len(errors_after),
|
||||
"error_kinds": sorted({e for _, e in errors_after}),
|
||||
}
|
||||
print(json.dumps(summary, indent=2, ensure_ascii=False))
|
||||
if (
|
||||
exit_code != 0
|
||||
or shutdown_sec > SHUTDOWN_LIMIT_SEC
|
||||
or prober.server_errors
|
||||
or first_refused is None
|
||||
or exited_at - sent_at > SHUTDOWN_LIMIT_SEC
|
||||
or errors_after
|
||||
or target_after_exit
|
||||
or not ok_after
|
||||
):
|
||||
raise SystemExit("FAIL: завершение по SIGTERM не соответствует требованиям")
|
||||
print(
|
||||
"PASS: SIGTERM завершил процесс за %.2f с, новые соединения отклоняются"
|
||||
% shutdown_sec
|
||||
"PASS: SIGTERM завершил %s за %.2f с, LB обслужил %d запросов без ошибок"
|
||||
% (container, exited_at - sent_at, len(ok_after))
|
||||
)
|
||||
|
||||
|
||||
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
# Остановка ОДНОЙ API-реплики, обслужившей тест: TARGET_INSTANCE_ID → контейнер.
|
||||
# Каждый процесс API пишет свой alias (X-Instance-Id) в /tmp/instance-id.
|
||||
# docker stop: SIGTERM → graceful shutdown; restart-политика unless-stopped
|
||||
# не поднимает вручную остановленный контейнер, он остаётся остановленным.
|
||||
# БД, LB и volumes не затрагиваются. Восстановление — scripts/restore.sh.
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
ENV_FILE="${ENV_FILE:-.env}"
|
||||
# Меньше 10 с лимита hook; больше drain + graceful shutdown (1 + 5 с)
|
||||
STOP_TIMEOUT_SEC=8
|
||||
|
||||
for container in $(docker compose --env-file "$ENV_FILE" ps -q api); do
|
||||
alias=$(docker exec "$container" cat /tmp/instance-id 2>/dev/null || true)
|
||||
if [ "$alias" = "$TARGET_INSTANCE_ID" ]; then
|
||||
name=$(docker inspect -f '{{.Name}}' "$container")
|
||||
docker stop --timeout "$STOP_TIMEOUT_SEC" "$container" >/dev/null
|
||||
echo "stopped ${name#/} (instance $alias)" >&2
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
echo "no running API replica with instance $TARGET_INSTANCE_ID" >&2
|
||||
exit 1
|
||||
@@ -1,20 +1,26 @@
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from fastapi import Request, Response
|
||||
from starlette.middleware.base import BaseHTTPMiddleware, RequestResponseEndpoint
|
||||
|
||||
from application.api.utils import REQUEST_ID_HEADER, error_response
|
||||
from config import LOGGER
|
||||
from application.api.utils import INSTANCE_ID_HEADER, REQUEST_ID_HEADER, error_response
|
||||
from config import INSTANCE_ID, LOGGER
|
||||
from domain.errors import AppError
|
||||
|
||||
HEALTH_PATH_PREFIX = "/health/"
|
||||
|
||||
INCOMING_REQUEST_ID = re.compile(r"^[0-9a-f]{32}$")
|
||||
|
||||
|
||||
class RequestContextMiddleware(BaseHTTPMiddleware):
|
||||
"""
|
||||
X-Request-Id в каждом ответе, журнал запросов и защита от необработанных ошибок.
|
||||
X-Request-Id и X-Instance-Id в каждом ответе, журнал запросов и защита
|
||||
от необработанных ошибок.
|
||||
|
||||
X-Request-Id берётся у балансировщика (nginx выставляет $request_id),
|
||||
поэтому одна строка access-лога LB и журнал реплики связаны одним ID.
|
||||
В лог попадают метод, путь, статус и длительность — без тел запросов,
|
||||
query-параметров и текста сообщений.
|
||||
"""
|
||||
@@ -22,7 +28,7 @@ class RequestContextMiddleware(BaseHTTPMiddleware):
|
||||
async def dispatch(
|
||||
self, request: Request, call_next: RequestResponseEndpoint
|
||||
) -> Response:
|
||||
request_id = uuid.uuid4().hex
|
||||
request_id = self._request_id(request)
|
||||
request.state.request_id = request_id
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
@@ -34,9 +40,19 @@ class RequestContextMiddleware(BaseHTTPMiddleware):
|
||||
request_id, error.status_code, error.code, error.message
|
||||
)
|
||||
response.headers[REQUEST_ID_HEADER] = request_id
|
||||
response.headers[INSTANCE_ID_HEADER] = INSTANCE_ID
|
||||
if getattr(request.app.state, "is_draining", False):
|
||||
response.headers["Connection"] = "close"
|
||||
self._log_request(request, response, request_id, started)
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
def _request_id(request: Request) -> str:
|
||||
incoming = request.headers.get(REQUEST_ID_HEADER, "")
|
||||
if INCOMING_REQUEST_ID.fullmatch(incoming):
|
||||
return incoming
|
||||
return uuid.uuid4().hex
|
||||
|
||||
@staticmethod
|
||||
def _log_request(
|
||||
request: Request, response: Response, request_id: str, started: float
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
from typing import Annotated
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Body, Depends, Query, status
|
||||
from fastapi import APIRouter, Body, Depends, Header, Query, status
|
||||
|
||||
from application.api.dependencies import get_current_user
|
||||
from domain import (
|
||||
IDEMPOTENCY_KEY_MAX_LENGTH,
|
||||
IDEMPOTENCY_KEY_PATTERN,
|
||||
CreateMessageSchema,
|
||||
MessagePageSchema,
|
||||
MessageSchema,
|
||||
@@ -26,10 +28,23 @@ messages_router = APIRouter(
|
||||
async def send_message(
|
||||
chat_id: UUID,
|
||||
body: Annotated[CreateMessageSchema, Body(...)],
|
||||
idempotency_key: Annotated[
|
||||
str,
|
||||
Header(
|
||||
alias="Idempotency-Key",
|
||||
min_length=1,
|
||||
max_length=IDEMPOTENCY_KEY_MAX_LENGTH,
|
||||
pattern=IDEMPOTENCY_KEY_PATTERN,
|
||||
),
|
||||
],
|
||||
current_user: UserSchema = Depends(get_current_user),
|
||||
service: MessageService = Depends(ServiceCreator.get_message_service),
|
||||
) -> MessageSchema:
|
||||
return await service.send_message(current_user, chat_id, body)
|
||||
"""
|
||||
Idempotency-Key обязателен: повтор того же тела возвращает тот же 201
|
||||
и то же сообщение, другое тело с тем же ключом — 409. Нет/неверный ключ — 400.
|
||||
"""
|
||||
return await service.send_message(current_user, chat_id, body, idempotency_key)
|
||||
|
||||
|
||||
@messages_router.get(
|
||||
|
||||
@@ -1,3 +1,13 @@
|
||||
from .errors import REQUEST_ID_HEADER, error_response, get_request_id
|
||||
from .errors import (
|
||||
INSTANCE_ID_HEADER,
|
||||
REQUEST_ID_HEADER,
|
||||
error_response,
|
||||
get_request_id,
|
||||
)
|
||||
|
||||
__all__ = ["REQUEST_ID_HEADER", "error_response", "get_request_id"]
|
||||
__all__ = [
|
||||
"REQUEST_ID_HEADER",
|
||||
"INSTANCE_ID_HEADER",
|
||||
"error_response",
|
||||
"get_request_id",
|
||||
]
|
||||
|
||||
@@ -6,6 +6,7 @@ from fastapi.responses import JSONResponse
|
||||
from domain.errors import ErrorBodySchema, ErrorCode, ErrorSchema
|
||||
|
||||
REQUEST_ID_HEADER = "X-Request-Id"
|
||||
INSTANCE_ID_HEADER = "X-Instance-Id"
|
||||
|
||||
|
||||
def get_request_id(request: Request) -> str:
|
||||
|
||||
@@ -2,7 +2,9 @@ from .log_config import LOGGER
|
||||
from .settings import (
|
||||
APP_ENV,
|
||||
APP_HOST,
|
||||
APP_KEEPALIVE_TIMEOUT_SEC,
|
||||
APP_PORT,
|
||||
APP_REVISION,
|
||||
DB_COMMAND_TIMEOUT_SEC,
|
||||
DB_CONNECT_TIMEOUT_SEC,
|
||||
DB_HOST,
|
||||
@@ -13,10 +15,14 @@ from .settings import (
|
||||
DB_PORT,
|
||||
DB_STARTUP_TIMEOUT_SEC,
|
||||
DB_USER,
|
||||
IDEMPOTENCY_TTL_SECONDS,
|
||||
INSTANCE_ID,
|
||||
INSTANCE_ID_FILE,
|
||||
LOG_LEVEL,
|
||||
PASSWORD_HASH_WORKERS,
|
||||
SESSION_CACHE_TTL_SECONDS,
|
||||
SESSION_TTL_SECONDS,
|
||||
SHUTDOWN_DRAIN_SEC,
|
||||
SHUTDOWN_TIMEOUT_SEC,
|
||||
VALKEY_HOST,
|
||||
VALKEY_PASS,
|
||||
@@ -31,8 +37,11 @@ __all__ = [
|
||||
"APP_HOST",
|
||||
"APP_PORT",
|
||||
"APP_ENV",
|
||||
"APP_REVISION",
|
||||
"APP_KEEPALIVE_TIMEOUT_SEC",
|
||||
"LOG_LEVEL",
|
||||
"SHUTDOWN_TIMEOUT_SEC",
|
||||
"SHUTDOWN_DRAIN_SEC",
|
||||
"DB_USER",
|
||||
"DB_PASS",
|
||||
"DB_NAME",
|
||||
@@ -52,4 +61,7 @@ __all__ = [
|
||||
"SESSION_TTL_SECONDS",
|
||||
"SESSION_CACHE_TTL_SECONDS",
|
||||
"PASSWORD_HASH_WORKERS",
|
||||
"IDEMPOTENCY_TTL_SECONDS",
|
||||
"INSTANCE_ID",
|
||||
"INSTANCE_ID_FILE",
|
||||
]
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import os
|
||||
import secrets
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
from envparse import Env
|
||||
@@ -12,9 +14,13 @@ if ENV_FILE.is_file():
|
||||
APP_HOST = env.str("APP_HOST")
|
||||
APP_PORT = env.int("APP_PORT")
|
||||
APP_ENV = env.str("APP_ENV")
|
||||
APP_REVISION = env.str("APP_REVISION", default="dev")
|
||||
# Больше keepalive_timeout балансировщика: idle-соединение закрывает LB, а не API
|
||||
APP_KEEPALIVE_TIMEOUT_SEC = env.int("APP_KEEPALIVE_TIMEOUT_SEC")
|
||||
|
||||
LOG_LEVEL = env.str("LOG_LEVEL")
|
||||
SHUTDOWN_TIMEOUT_SEC = env.int("SHUTDOWN_TIMEOUT_SEC")
|
||||
SHUTDOWN_DRAIN_SEC = env.float("SHUTDOWN_DRAIN_SEC")
|
||||
|
||||
DB_USER = env.str("DB_USER")
|
||||
DB_PASS = env.str("DB_PASS")
|
||||
@@ -37,3 +43,7 @@ VALKEY_RETRY_AFTER_SEC = env.float("VALKEY_RETRY_AFTER_SEC")
|
||||
SESSION_TTL_SECONDS = env.int("SESSION_TTL_SECONDS")
|
||||
SESSION_CACHE_TTL_SECONDS = env.int("SESSION_CACHE_TTL_SECONDS")
|
||||
PASSWORD_HASH_WORKERS = env.int("PASSWORD_HASH_WORKERS")
|
||||
IDEMPOTENCY_TTL_SECONDS = env.int("IDEMPOTENCY_TTL_SECONDS")
|
||||
|
||||
INSTANCE_ID = secrets.token_hex(6)
|
||||
INSTANCE_ID_FILE = Path(tempfile.gettempdir()) / "instance-id"
|
||||
|
||||
+11
-1
@@ -1,7 +1,14 @@
|
||||
from .auth import ActiveSessionSchema, BasicCredentialsSchema, SessionSchema
|
||||
from .chats import ChatPageSchema, ChatSchema, CreateChatSchema
|
||||
from .health import HealthSchema
|
||||
from .messages import CreateMessageSchema, MessagePageSchema, MessageSchema
|
||||
from .messages import (
|
||||
IDEMPOTENCY_KEY_MAX_LENGTH,
|
||||
IDEMPOTENCY_KEY_PATTERN,
|
||||
CreateMessageSchema,
|
||||
IdempotencyKeySchema,
|
||||
MessagePageSchema,
|
||||
MessageSchema,
|
||||
)
|
||||
from .pagination import PageCursorSchema, PageKey
|
||||
from .users import CreateUserSchema, UserCredentialsSchema, UserSchema
|
||||
|
||||
@@ -16,6 +23,9 @@ __all__ = [
|
||||
"ChatSchema",
|
||||
"ChatPageSchema",
|
||||
"CreateMessageSchema",
|
||||
"IdempotencyKeySchema",
|
||||
"IDEMPOTENCY_KEY_PATTERN",
|
||||
"IDEMPOTENCY_KEY_MAX_LENGTH",
|
||||
"MessageSchema",
|
||||
"MessagePageSchema",
|
||||
"HealthSchema",
|
||||
|
||||
@@ -1,7 +1,17 @@
|
||||
from .schemas import CreateMessageSchema, MessagePageSchema, MessageSchema
|
||||
from .schemas import (
|
||||
IDEMPOTENCY_KEY_MAX_LENGTH,
|
||||
IDEMPOTENCY_KEY_PATTERN,
|
||||
CreateMessageSchema,
|
||||
IdempotencyKeySchema,
|
||||
MessagePageSchema,
|
||||
MessageSchema,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"IDEMPOTENCY_KEY_PATTERN",
|
||||
"IDEMPOTENCY_KEY_MAX_LENGTH",
|
||||
"CreateMessageSchema",
|
||||
"IdempotencyKeySchema",
|
||||
"MessageSchema",
|
||||
"MessagePageSchema",
|
||||
]
|
||||
|
||||
@@ -7,6 +7,9 @@ from domain.utils import NotBlankStr, UtcDateTime
|
||||
|
||||
MessageText = Annotated[NotBlankStr, Field(min_length=1, max_length=4000)]
|
||||
|
||||
IDEMPOTENCY_KEY_PATTERN = r"^[A-Za-z0-9._:-]{1,128}$"
|
||||
IDEMPOTENCY_KEY_MAX_LENGTH = 128
|
||||
|
||||
|
||||
class CreateMessageSchema(BaseModel):
|
||||
"""Запрос отправки сообщения: отправитель берётся из текущей сессии"""
|
||||
@@ -33,3 +36,18 @@ class MessagePageSchema(BaseModel):
|
||||
|
||||
items: list[MessageSchema]
|
||||
next_cursor: str | None
|
||||
|
||||
|
||||
class IdempotencyKeySchema(BaseModel):
|
||||
"""
|
||||
Ключ повторной отправки сообщения.
|
||||
|
||||
request_hash — SHA-256 канонического тела: повтор с тем же ключом и тем же
|
||||
телом возвращает исходное сообщение, с другим телом — конфликт.
|
||||
"""
|
||||
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
key: str
|
||||
request_hash: bytes
|
||||
ttl_seconds: int
|
||||
|
||||
+63
-8
@@ -1,3 +1,5 @@
|
||||
import asyncio
|
||||
import socket
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
@@ -10,11 +12,16 @@ from application.api.middlewares import RequestContextMiddleware
|
||||
from config import (
|
||||
APP_ENV,
|
||||
APP_HOST,
|
||||
APP_KEEPALIVE_TIMEOUT_SEC,
|
||||
APP_PORT,
|
||||
APP_REVISION,
|
||||
DB_STARTUP_TIMEOUT_SEC,
|
||||
INSTANCE_ID,
|
||||
INSTANCE_ID_FILE,
|
||||
LOGGER,
|
||||
PASSWORD_HASH_WORKERS,
|
||||
SESSION_TTL_SECONDS,
|
||||
SHUTDOWN_DRAIN_SEC,
|
||||
SHUTDOWN_TIMEOUT_SEC,
|
||||
)
|
||||
from infrastructure import PasswordHasher, cache_manager, database_pool_manager
|
||||
@@ -22,29 +29,35 @@ from infrastructure import PasswordHasher, cache_manager, database_pool_manager
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI) -> AsyncIterator[None]:
|
||||
"""
|
||||
uvicorn открывает порт только после startup: реплика без БД или со старой
|
||||
схемой не принимает соединений, и LB отправляет запросы другим репликам.
|
||||
"""
|
||||
await database_pool_manager.create_pool(DB_STARTUP_TIMEOUT_SEC)
|
||||
await database_pool_manager.apply_migrations()
|
||||
await database_pool_manager.check_migrations()
|
||||
cache_manager.create_client()
|
||||
password_hasher = PasswordHasher(PASSWORD_HASH_WORKERS)
|
||||
app.state.database = database_pool_manager
|
||||
app.state.cache = cache_manager
|
||||
app.state.password_hasher = password_hasher
|
||||
INSTANCE_ID_FILE.write_text(INSTANCE_ID)
|
||||
app.state.is_ready = True
|
||||
LOGGER.info(
|
||||
f"Application started, env={APP_ENV}, session ttl={SESSION_TTL_SECONDS} s"
|
||||
f"Application started, env={APP_ENV}, revision={APP_REVISION}, "
|
||||
f"instance={INSTANCE_ID}, session ttl={SESSION_TTL_SECONDS} s"
|
||||
)
|
||||
yield
|
||||
app.state.is_ready = False
|
||||
await cache_manager.close()
|
||||
await database_pool_manager.close()
|
||||
password_hasher.close()
|
||||
LOGGER.info("Application stopped, connections closed")
|
||||
LOGGER.info(f"Application stopped, instance={INSTANCE_ID}, connections closed")
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="mtuci-labs-messanger",
|
||||
description="мессенджер-конкурент-максу",
|
||||
version="1.2.0",
|
||||
version="1.3.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
@@ -55,17 +68,59 @@ app.include_router(health_endpoints)
|
||||
app.include_router(api_endpoints)
|
||||
|
||||
|
||||
def main(host: str, port: int, shutdown_timeout: int) -> None:
|
||||
"""Функция для запуска api"""
|
||||
uvicorn.run(
|
||||
class DrainingServer(uvicorn.Server):
|
||||
"""
|
||||
SIGTERM: сначала drain, затем штатный graceful shutdown uvicorn.
|
||||
"""
|
||||
|
||||
def __init__(self, config: uvicorn.Config, drain_seconds: float) -> None:
|
||||
super().__init__(config)
|
||||
self._drain_seconds = drain_seconds
|
||||
|
||||
async def shutdown(self, sockets: list[socket.socket] | None = None) -> None:
|
||||
for server in getattr(self, "servers", []):
|
||||
server.close()
|
||||
app.state.is_ready = False
|
||||
app.state.is_draining = True
|
||||
LOGGER.info(
|
||||
f"SIGTERM: drain {self._drain_seconds} с, instance={INSTANCE_ID}: "
|
||||
"порт закрыт, keep-alive соединения закрываются после ответа"
|
||||
)
|
||||
await asyncio.sleep(self._drain_seconds)
|
||||
await super().shutdown(sockets)
|
||||
|
||||
|
||||
def main(
|
||||
host: str,
|
||||
port: int,
|
||||
shutdown_timeout: int,
|
||||
keepalive_timeout: int,
|
||||
drain_seconds: float,
|
||||
) -> None:
|
||||
"""
|
||||
Функция для запуска api.
|
||||
|
||||
SIGTERM: drain_seconds без приёма новых соединений, затем uvicorn ждёт
|
||||
текущих запросов не дольше shutdown_timeout, и lifespan закрывает пулы.
|
||||
Сумма меньше stop_grace_period контейнера (10 с).
|
||||
"""
|
||||
config = uvicorn.Config(
|
||||
app,
|
||||
host=host,
|
||||
port=port,
|
||||
timeout_graceful_shutdown=shutdown_timeout,
|
||||
timeout_keep_alive=keepalive_timeout,
|
||||
access_log=False,
|
||||
server_header=False,
|
||||
)
|
||||
DrainingServer(config, drain_seconds).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(host=APP_HOST, port=APP_PORT, shutdown_timeout=SHUTDOWN_TIMEOUT_SEC)
|
||||
main(
|
||||
host=APP_HOST,
|
||||
port=APP_PORT,
|
||||
shutdown_timeout=SHUTDOWN_TIMEOUT_SEC,
|
||||
keepalive_timeout=APP_KEEPALIVE_TIMEOUT_SEC,
|
||||
drain_seconds=SHUTDOWN_DRAIN_SEC,
|
||||
)
|
||||
|
||||
@@ -106,8 +106,10 @@ class DatabasePoolManager:
|
||||
"""
|
||||
Применение миграций NNNN_name.sql по порядку версий.
|
||||
|
||||
Вызывается однократно отдельным процессом (src/migrate.py, сервис migrate)
|
||||
до запуска API-реплик, а не каждой репликой при старте.
|
||||
Каждая миграция и запись о ней в schema_migrations — одна транзакция.
|
||||
Advisory lock не даёт двум процессам API применять миграции одновременно;
|
||||
Advisory lock — страховка от двух одновременных запусков migrate;
|
||||
уже применённые версии пропускаются, поэтому повторный запуск безопасен.
|
||||
"""
|
||||
async with self.acquire() as conn:
|
||||
@@ -133,8 +135,7 @@ class DatabasePoolManager:
|
||||
"SELECT version FROM schema_migrations;"
|
||||
)
|
||||
}
|
||||
for path in sorted(directory.glob("*.sql")):
|
||||
version = int(path.name.split("_", 1)[0])
|
||||
for version, path in self._migration_files(directory):
|
||||
if version in applied:
|
||||
continue
|
||||
async with conn.transaction():
|
||||
@@ -155,6 +156,35 @@ class DatabasePoolManager:
|
||||
)
|
||||
LOGGER.info("Схема БД в актуальной версии")
|
||||
|
||||
async def check_migrations(self, directory: Path = MIGRATIONS_DIR) -> None:
|
||||
"""
|
||||
Схема БД не старше кода: все миграции из образа уже применены.
|
||||
|
||||
API миграции не применяет. Если migrate ещё не отработал, реплика не
|
||||
стартует и не начинает слушать порт, балансировщик её не использует
|
||||
|
||||
"""
|
||||
expected = {version for version, _ in self._migration_files(directory)}
|
||||
async with self.acquire() as conn:
|
||||
try:
|
||||
records = await conn.fetch("SELECT version FROM schema_migrations;")
|
||||
except asyncpg.UndefinedTableError:
|
||||
records = []
|
||||
applied = {record["version"] for record in records}
|
||||
missing = sorted(expected - applied)
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
f"В БД не применены миграции {missing}: сначала запустите migrate"
|
||||
)
|
||||
LOGGER.info(f"Схема БД совместима: версия {max(applied, default=0)}")
|
||||
|
||||
@staticmethod
|
||||
def _migration_files(directory: Path) -> list[tuple[int, Path]]:
|
||||
"""Файлы NNNN_name.sql по возрастанию версии"""
|
||||
return sorted(
|
||||
(int(path.name.split("_", 1)[0]), path) for path in directory.glob("*.sql")
|
||||
)
|
||||
|
||||
@asynccontextmanager
|
||||
async def acquire(self) -> AsyncIterator[asyncpg.Connection]:
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
from fastapi import Depends
|
||||
|
||||
from config import SESSION_CACHE_TTL_SECONDS, SESSION_TTL_SECONDS
|
||||
from config import (
|
||||
IDEMPOTENCY_TTL_SECONDS,
|
||||
SESSION_CACHE_TTL_SECONDS,
|
||||
SESSION_TTL_SECONDS,
|
||||
)
|
||||
from infrastructure.dependencies.cache import CacheManager, get_cache
|
||||
from infrastructure.dependencies.database import DatabasePoolManager, get_database
|
||||
from infrastructure.repositories import (
|
||||
@@ -90,4 +94,6 @@ class ServiceCreator:
|
||||
chat_repository: ChatRepository = Depends(_get_chat_repository),
|
||||
message_repository: MessageRepository = Depends(_get_message_repository),
|
||||
) -> MessageService:
|
||||
return MessageService(chat_repository, message_repository)
|
||||
return MessageService(
|
||||
chat_repository, message_repository, IDEMPOTENCY_TTL_SECONDS
|
||||
)
|
||||
|
||||
@@ -2,7 +2,8 @@ from uuid import UUID
|
||||
|
||||
import asyncpg
|
||||
|
||||
from domain import MessageSchema, PageKey
|
||||
from domain import IdempotencyKeySchema, MessageSchema, PageKey
|
||||
from domain.errors import ConflictError
|
||||
from infrastructure.utils import Pool, repository_logs, transaction
|
||||
from interfaces import MessageRepository
|
||||
|
||||
@@ -15,11 +16,22 @@ class PostgresMessageRepository(MessageRepository):
|
||||
self._pool = pool
|
||||
|
||||
@repository_logs
|
||||
async def create(self, chat_id: UUID, sender_id: UUID, text: str) -> MessageSchema:
|
||||
async def create(
|
||||
self,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text: str,
|
||||
idempotency: IdempotencyKeySchema,
|
||||
) -> tuple[MessageSchema, bool]:
|
||||
"""
|
||||
Блокировка чата и вставка — одна транзакция; lock снимается на COMMIT.
|
||||
Блокировка чата, проверка ключа, вставка сообщения и ключа — одна
|
||||
транзакция; lock снимается на COMMIT.
|
||||
|
||||
Запись в один чат сериализуется, записи в разные чаты не мешают друг другу.
|
||||
Ключ идемпотентности привязан к чату, поэтому та же блокировка
|
||||
сериализует и одновременные повторы с одним ключом с разных реплик:
|
||||
второй ждёт COMMIT первого и находит его ключ
|
||||
|
||||
created_at строго больше последнего в чате, даже если часы ушли назад:
|
||||
порядок коммитов совпадает с порядком (created_at, id), и читатель
|
||||
с cursor не пропустит сообщение. MAX берётся из индекса
|
||||
@@ -28,6 +40,22 @@ class PostgresMessageRepository(MessageRepository):
|
||||
lock_query = """
|
||||
SELECT pg_advisory_xact_lock($1, hashtext($2::uuid::text));
|
||||
"""
|
||||
find_key_query = """
|
||||
SELECT
|
||||
idem.request_hash,
|
||||
message.id,
|
||||
message.chat_id,
|
||||
message.sender_id,
|
||||
message.text,
|
||||
message.created_at
|
||||
FROM message_idempotency_keys idem
|
||||
JOIN messages message ON message.id = idem.message_id
|
||||
WHERE
|
||||
idem.sender_id = $1
|
||||
AND idem.chat_id = $2
|
||||
AND idem.idempotency_key = $3
|
||||
AND idem.expires_at > clock_timestamp();
|
||||
"""
|
||||
insert_query = """
|
||||
INSERT INTO messages
|
||||
(
|
||||
@@ -45,11 +73,56 @@ class PostgresMessageRepository(MessageRepository):
|
||||
WHERE chat_id = $1
|
||||
RETURNING id, chat_id, sender_id, text, created_at;
|
||||
"""
|
||||
|
||||
save_key_query = """
|
||||
INSERT INTO message_idempotency_keys
|
||||
(
|
||||
sender_id,
|
||||
chat_id,
|
||||
idempotency_key,
|
||||
request_hash,
|
||||
message_id,
|
||||
expires_at
|
||||
)
|
||||
VALUES
|
||||
(
|
||||
$1,
|
||||
$2,
|
||||
$3,
|
||||
$4,
|
||||
$5,
|
||||
clock_timestamp() + make_interval(secs => $6::integer)
|
||||
)
|
||||
ON CONFLICT (sender_id, chat_id, idempotency_key) DO UPDATE
|
||||
SET
|
||||
request_hash = EXCLUDED.request_hash,
|
||||
message_id = EXCLUDED.message_id,
|
||||
created_at = EXCLUDED.created_at,
|
||||
expires_at = EXCLUDED.expires_at;
|
||||
"""
|
||||
async with transaction(self._pool) as conn:
|
||||
await conn.execute(lock_query, CHAT_LOCK_NAMESPACE, chat_id)
|
||||
stored = await conn.fetchrow(
|
||||
find_key_query, sender_id, chat_id, idempotency.key
|
||||
)
|
||||
if stored is not None:
|
||||
if stored["request_hash"] != idempotency.request_hash:
|
||||
raise ConflictError(
|
||||
"Idempotency-Key уже использован с другим телом запроса"
|
||||
)
|
||||
return self._row_to_message(stored), False
|
||||
row = await conn.fetchrow(insert_query, chat_id, sender_id, text)
|
||||
assert row is not None
|
||||
return self._row_to_message(row)
|
||||
assert row is not None
|
||||
await conn.execute(
|
||||
save_key_query,
|
||||
sender_id,
|
||||
chat_id,
|
||||
idempotency.key,
|
||||
idempotency.request_hash,
|
||||
row["id"],
|
||||
idempotency.ttl_seconds,
|
||||
)
|
||||
return self._row_to_message(row), True
|
||||
|
||||
@repository_logs
|
||||
async def list_for_chat(
|
||||
|
||||
@@ -1,7 +1,14 @@
|
||||
import hashlib
|
||||
from uuid import UUID
|
||||
|
||||
from config import LOGGER
|
||||
from domain import CreateMessageSchema, MessagePageSchema, MessageSchema, UserSchema
|
||||
from domain import (
|
||||
CreateMessageSchema,
|
||||
IdempotencyKeySchema,
|
||||
MessagePageSchema,
|
||||
MessageSchema,
|
||||
UserSchema,
|
||||
)
|
||||
from domain.errors import NotFoundError
|
||||
from infrastructure.utils import build_page, decode_cursor
|
||||
from interfaces import ChatRepository, MessageRepository
|
||||
@@ -9,21 +16,47 @@ from interfaces import ChatRepository, MessageRepository
|
||||
|
||||
class MessageService:
|
||||
def __init__(
|
||||
self, chat_repository: ChatRepository, message_repository: MessageRepository
|
||||
self,
|
||||
chat_repository: ChatRepository,
|
||||
message_repository: MessageRepository,
|
||||
idempotency_ttl_seconds: int,
|
||||
) -> None:
|
||||
self._chats = chat_repository
|
||||
self._messages = message_repository
|
||||
self._idempotency_ttl = idempotency_ttl_seconds
|
||||
|
||||
async def send_message(
|
||||
self, current_user: UserSchema, chat_id: UUID, data: CreateMessageSchema
|
||||
self,
|
||||
current_user: UserSchema,
|
||||
chat_id: UUID,
|
||||
data: CreateMessageSchema,
|
||||
idempotency_key: str,
|
||||
) -> MessageSchema:
|
||||
"""Отправитель берётся из текущей идентификации; текст в логи не пишется"""
|
||||
"""
|
||||
Отправитель берётся из текущей идентификации; текст в логи не пишется.
|
||||
|
||||
Повтор с тем же Idempotency-Key заново проверяет доступ к чату
|
||||
и возвращает исходное сообщение.
|
||||
"""
|
||||
await self._ensure_member(current_user, chat_id)
|
||||
message = await self._messages.create(chat_id, current_user.id, data.text)
|
||||
LOGGER.info(
|
||||
f"Сообщение {message.id} отправлено в чат {chat_id} "
|
||||
f"пользователем {current_user.id}"
|
||||
idempotency = IdempotencyKeySchema(
|
||||
key=idempotency_key,
|
||||
request_hash=self._request_hash(data),
|
||||
ttl_seconds=self._idempotency_ttl,
|
||||
)
|
||||
message, created = await self._messages.create(
|
||||
chat_id, current_user.id, data.text, idempotency
|
||||
)
|
||||
if created:
|
||||
LOGGER.info(
|
||||
f"Сообщение {message.id} отправлено в чат {chat_id} "
|
||||
f"пользователем {current_user.id}"
|
||||
)
|
||||
else:
|
||||
LOGGER.info(
|
||||
f"Повтор по Idempotency-Key: возвращено сообщение {message.id} "
|
||||
f"чата {chat_id}"
|
||||
)
|
||||
return message
|
||||
|
||||
async def list_messages(
|
||||
@@ -40,3 +73,11 @@ class MessageService:
|
||||
"""Посторонний получает 404: существование чужого чата не раскрывается"""
|
||||
if await self._chats.get_for_member(chat_id, current_user.id) is None:
|
||||
raise NotFoundError()
|
||||
|
||||
@staticmethod
|
||||
def _request_hash(data: CreateMessageSchema) -> bytes:
|
||||
"""
|
||||
SHA-256 канонического тела. Порядок полей задаёт схема, а не клиент,
|
||||
поэтому JSON с другим порядком ключей считается тем же запросом.
|
||||
"""
|
||||
return hashlib.sha256(data.model_dump_json().encode()).digest()
|
||||
|
||||
@@ -1,18 +1,28 @@
|
||||
from abc import ABC, abstractmethod
|
||||
from uuid import UUID
|
||||
|
||||
from domain import MessageSchema, PageKey
|
||||
from domain import IdempotencyKeySchema, MessageSchema, PageKey
|
||||
|
||||
|
||||
class MessageRepository(ABC):
|
||||
"""Слой абстракции хранилища сообщений"""
|
||||
|
||||
@abstractmethod
|
||||
async def create(self, chat_id: UUID, sender_id: UUID, text: str) -> MessageSchema:
|
||||
async def create(
|
||||
self,
|
||||
chat_id: UUID,
|
||||
sender_id: UUID,
|
||||
text: str,
|
||||
idempotency: IdempotencyKeySchema,
|
||||
) -> tuple[MessageSchema, bool]:
|
||||
"""
|
||||
Добавление сообщения в конец истории чата. id и created_at назначает
|
||||
хранилище; created_at строго возрастает внутри чата, поэтому сообщение
|
||||
не может появиться перед уже выданным cursor.
|
||||
|
||||
Сообщение и запись ключа идемпотентности сохраняются атомарно.
|
||||
Одновременные запросы с одним ключом через любые реплики создают
|
||||
ровно одно сообщение; повтор возвращает его же.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""
|
||||
Однократное применение миграций до запуска API-реплик.
|
||||
|
||||
Запускается сервисом migrate (compose) и завершается; реплики стартуют
|
||||
после его успешного завершения и только проверяют версию схемы.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
from config import DB_STARTUP_TIMEOUT_SEC, LOGGER
|
||||
from infrastructure import database_pool_manager
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
await database_pool_manager.create_pool(DB_STARTUP_TIMEOUT_SEC)
|
||||
try:
|
||||
await database_pool_manager.apply_migrations()
|
||||
finally:
|
||||
await database_pool_manager.close()
|
||||
LOGGER.info("Миграции применены, migrate завершается")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -5,7 +5,7 @@ from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from helpers import auth, basic, unique_password, unique_username
|
||||
from helpers import auth, basic, message_headers, unique_password, unique_username
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
DEFAULT_BASE_URL = json.loads((ROOT / "lab.json").read_text())["default_base_url"]
|
||||
@@ -68,7 +68,7 @@ def send_message(api: httpx.Client) -> Callable[..., dict]:
|
||||
response = api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": text},
|
||||
headers=auth(user),
|
||||
headers=message_headers(user),
|
||||
)
|
||||
assert response.status_code == 201, response.text
|
||||
return response.json()
|
||||
|
||||
@@ -30,6 +30,11 @@ def auth(user: dict) -> dict[str, str]:
|
||||
return {"Authorization": f"Bearer {user['token']}"}
|
||||
|
||||
|
||||
def message_headers(user: dict, key: str | None = None) -> dict[str, str]:
|
||||
"""Bearer и Idempotency-Key (новый, если не задан)"""
|
||||
return {**auth(user), "Idempotency-Key": key or uuid.uuid4().hex}
|
||||
|
||||
|
||||
def public_user(user: dict) -> dict:
|
||||
"""User из ответа API без тестовых полей token/password"""
|
||||
return {field: user[field] for field in USER_FIELDS}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""
|
||||
Балансировка без sticky sessions: запросы одной сессии обслуживают разные
|
||||
реплики, а состояние (сессии, сообщения, logout) общее для всех.
|
||||
"""
|
||||
|
||||
import uuid
|
||||
from collections import Counter
|
||||
|
||||
import httpx
|
||||
from helpers import assert_unauthorized, auth, message_headers, public_user
|
||||
|
||||
REQUESTS = 20
|
||||
|
||||
|
||||
def test_one_session_is_served_by_several_replicas(
|
||||
api: httpx.Client, make_user
|
||||
) -> None:
|
||||
"""Одно keep-alive соединение клиента к LB, без повторного входа"""
|
||||
alice = make_user("alice")
|
||||
instances: Counter[str] = Counter()
|
||||
for _ in range(REQUESTS):
|
||||
response = api.get("/api/v1/users/me", headers=auth(alice))
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == public_user(alice)
|
||||
instances[response.headers["X-Instance-Id"]] += 1
|
||||
print("\nINSTANCES FOR ONE SESSION:", dict(instances))
|
||||
assert len(instances) >= 2
|
||||
|
||||
|
||||
def test_write_is_visible_through_every_replica(
|
||||
api: httpx.Client, make_user, make_chat
|
||||
) -> None:
|
||||
alice = make_user("alice")
|
||||
bob = make_user("bob")
|
||||
chat = make_chat(alice, [alice, bob])
|
||||
sent = api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": f"видно везде {uuid.uuid4().hex}"},
|
||||
headers=message_headers(alice),
|
||||
)
|
||||
assert sent.status_code == 201, sent.text
|
||||
readers: set[str] = set()
|
||||
for _ in range(REQUESTS):
|
||||
page = api.get(f"/api/v1/chats/{chat['id']}/messages", headers=auth(bob))
|
||||
assert page.status_code == 200, page.text
|
||||
assert page.json()["items"] == [sent.json()]
|
||||
readers.add(page.headers["X-Instance-Id"])
|
||||
assert len(readers) >= 2
|
||||
|
||||
|
||||
def test_logout_applies_to_all_replicas(api: httpx.Client, make_user) -> None:
|
||||
"""Сессия попала в общий кэш через обе реплики, затем отозвана на одной"""
|
||||
alice = make_user("alice")
|
||||
for _ in range(4):
|
||||
assert api.get("/api/v1/users/me", headers=auth(alice)).status_code == 200
|
||||
logout = api.delete("/api/v1/auth/sessions/current", headers=auth(alice))
|
||||
assert logout.status_code == 204
|
||||
rejected_by: set[str] = set()
|
||||
for _ in range(REQUESTS):
|
||||
response = api.get("/api/v1/users/me", headers=auth(alice))
|
||||
assert_unauthorized(response)
|
||||
rejected_by.add(response.headers["X-Instance-Id"])
|
||||
assert len(rejected_by) >= 2
|
||||
|
||||
|
||||
def test_every_api_response_has_instance_id(api: httpx.Client) -> None:
|
||||
for response in (
|
||||
api.get("/health/live"),
|
||||
api.get("/health/ready"),
|
||||
api.get("/api/v1/unknown"),
|
||||
api.get("/api/v1/users/me"),
|
||||
):
|
||||
assert response.headers["X-Instance-Id"]
|
||||
assert response.headers["X-Request-Id"]
|
||||
@@ -5,7 +5,7 @@ import json
|
||||
import time
|
||||
|
||||
import httpx
|
||||
from helpers import auth, read_all_messages
|
||||
from helpers import auth, message_headers, read_all_messages
|
||||
|
||||
CLIENTS = 10
|
||||
MESSAGES_PER_CLIENT = 10
|
||||
@@ -29,6 +29,7 @@ async def _send_concurrently(
|
||||
client.post(
|
||||
f"/api/v1/chats/{chat_id}/messages",
|
||||
json={"text": f"{user['username']}#{n}"},
|
||||
headers=message_headers(user),
|
||||
)
|
||||
for n in range(MESSAGES_PER_CLIENT)
|
||||
)
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import uuid
|
||||
|
||||
import httpx
|
||||
from helpers import assert_error, auth
|
||||
from helpers import assert_error, auth, message_headers
|
||||
|
||||
|
||||
def test_outsider_cannot_access_chat(
|
||||
@@ -18,7 +18,9 @@ def test_outsider_cannot_access_chat(
|
||||
|
||||
assert_error(api.get(path, headers=auth(eve)), 404, "not_found")
|
||||
assert_error(api.get(f"{path}/messages", headers=auth(eve)), 404, "not_found")
|
||||
response = api.post(f"{path}/messages", json={"text": "чужое"}, headers=auth(eve))
|
||||
response = api.post(
|
||||
f"{path}/messages", json={"text": "чужое"}, headers=message_headers(eve)
|
||||
)
|
||||
assert_error(response, 404, "not_found")
|
||||
|
||||
history = api.get(f"{path}/messages", headers=auth(bob)).json()
|
||||
|
||||
@@ -0,0 +1,249 @@
|
||||
"""Idempotency-Key отправки сообщения: повтор, конфликт, область ключа, гонки"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from collections import Counter
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from helpers import assert_error, auth, message_headers, psql, read_all_messages
|
||||
|
||||
PARALLEL_REPLAYS = 20
|
||||
|
||||
|
||||
def post_message(
|
||||
api: httpx.Client, user: dict, chat: dict, text: str, key: str
|
||||
) -> httpx.Response:
|
||||
return api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": text},
|
||||
headers=message_headers(user, key),
|
||||
)
|
||||
|
||||
|
||||
def history(api: httpx.Client, user: dict, chat: dict) -> list[dict]:
|
||||
items, _ = read_all_messages(api, user, chat["id"], 100)
|
||||
return items
|
||||
|
||||
|
||||
def test_replay_returns_same_message(api: httpx.Client, make_user, make_chat) -> None:
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
key = uuid.uuid4().hex
|
||||
|
||||
first = post_message(api, alice, chat, "один раз", key)
|
||||
second = post_message(api, alice, chat, "один раз", key)
|
||||
|
||||
assert first.status_code == second.status_code == 201, second.text
|
||||
assert first.json() == second.json()
|
||||
assert history(api, alice, chat) == [first.json()]
|
||||
|
||||
|
||||
def test_replay_ignores_json_formatting(
|
||||
api: httpx.Client, make_user, make_chat
|
||||
) -> None:
|
||||
"""Пробелы и форма записи JSON не меняют смысл тела: это тот же запрос"""
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
headers = {**message_headers(alice), "Content-Type": "application/json"}
|
||||
path = f"/api/v1/chats/{chat['id']}/messages"
|
||||
|
||||
escaped = b'{"text":"\\u041f\\u0440\\u0438"}'
|
||||
compact = api.post(path, content=escaped, headers=headers)
|
||||
spaced = api.post(path, content='{ "text" : "При" }'.encode(), headers=headers)
|
||||
|
||||
assert compact.status_code == spaced.status_code == 201, spaced.text
|
||||
assert compact.json() == spaced.json()
|
||||
assert len(history(api, alice, chat)) == 1
|
||||
|
||||
|
||||
def test_same_key_with_other_body_is_conflict(
|
||||
api: httpx.Client, make_user, make_chat
|
||||
) -> None:
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
key = uuid.uuid4().hex
|
||||
original = post_message(api, alice, chat, "оригинал", key).json()
|
||||
|
||||
assert_error(post_message(api, alice, chat, "другой", key), 409, "conflict")
|
||||
assert history(api, alice, chat) == [original]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"headers",
|
||||
[
|
||||
{},
|
||||
{"Idempotency-Key": ""},
|
||||
{"Idempotency-Key": "with space"},
|
||||
{"Idempotency-Key": "ключ".encode()},
|
||||
{"Idempotency-Key": "a/b"},
|
||||
{"Idempotency-Key": "x" * 129},
|
||||
],
|
||||
)
|
||||
def test_missing_or_invalid_key_is_rejected(
|
||||
api: httpx.Client, make_user, make_chat, headers: dict
|
||||
) -> None:
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
response = api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": "без ключа"},
|
||||
headers={**auth(alice), **headers},
|
||||
)
|
||||
assert_error(response, 400, "invalid_request")
|
||||
assert history(api, alice, chat) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("key", ["x" * 128, "Az09._:-"])
|
||||
def test_key_boundaries_are_accepted(
|
||||
api: httpx.Client, make_user, make_chat, key: str
|
||||
) -> None:
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
assert post_message(api, alice, chat, "граница", key).status_code == 201
|
||||
|
||||
|
||||
def test_key_scope_is_user_and_chat(api: httpx.Client, make_user, make_chat) -> None:
|
||||
"""Один ключ у разных пользователей и в разных чатах — независимые сообщения"""
|
||||
alice = make_user("alice")
|
||||
bob = make_user("bob")
|
||||
chat = make_chat(alice, [alice, bob])
|
||||
other_chat = make_chat(alice, [alice])
|
||||
key = uuid.uuid4().hex
|
||||
|
||||
from_alice = post_message(api, alice, chat, "текст", key).json()
|
||||
from_bob = post_message(api, bob, chat, "текст", key).json()
|
||||
in_other_chat = post_message(api, alice, other_chat, "текст", key).json()
|
||||
|
||||
assert len({from_alice["id"], from_bob["id"], in_other_chat["id"]}) == 3
|
||||
assert from_bob["sender_id"] == bob["id"]
|
||||
assert len(history(api, alice, chat)) == 2
|
||||
assert len(history(api, alice, other_chat)) == 1
|
||||
|
||||
|
||||
def test_replay_rechecks_session_and_access(
|
||||
api: httpx.Client, make_user, make_chat
|
||||
) -> None:
|
||||
alice = make_user("alice")
|
||||
eve = make_user("eve")
|
||||
chat = make_chat(alice, [alice])
|
||||
key = uuid.uuid4().hex
|
||||
post_message(api, alice, chat, "секрет", key)
|
||||
|
||||
# Чужой с тем же ключом не получает сохранённое сообщение
|
||||
assert_error(post_message(api, eve, chat, "секрет", key), 404, "not_found")
|
||||
# Отозванная сессия не даёт повтора
|
||||
logout = api.delete("/api/v1/auth/sessions/current", headers=auth(alice))
|
||||
assert logout.status_code == 204
|
||||
replay = post_message(api, alice, chat, "секрет", key)
|
||||
assert_error(replay, 401, "unauthorized")
|
||||
|
||||
|
||||
def test_expired_key_can_be_reused(api: httpx.Client, make_user, make_chat) -> None:
|
||||
"""После срока хранения (24 ч) ключ снова создаёт сообщение; срок сдвигается в БД"""
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
key = uuid.uuid4().hex
|
||||
first = post_message(api, alice, chat, "вчера", key).json()
|
||||
psql(
|
||||
"UPDATE message_idempotency_keys "
|
||||
"SET created_at = now() - interval '2 days', "
|
||||
"expires_at = now() - interval '1 day' "
|
||||
f"WHERE chat_id = '{chat['id']}' AND idempotency_key = '{key}';"
|
||||
)
|
||||
|
||||
second = post_message(api, alice, chat, "сегодня", key)
|
||||
|
||||
assert second.status_code == 201, second.text
|
||||
assert second.json()["id"] != first["id"]
|
||||
assert [item["text"] for item in history(api, alice, chat)] == ["вчера", "сегодня"]
|
||||
stored = psql(
|
||||
"SELECT count(*) FROM message_idempotency_keys "
|
||||
f"WHERE chat_id = '{chat['id']}' AND idempotency_key = '{key}';"
|
||||
)
|
||||
assert stored == "1"
|
||||
|
||||
|
||||
async def _post_concurrently(
|
||||
base_url: str, user: dict, chat: dict, key: str, texts: list[str]
|
||||
) -> list[httpx.Response]:
|
||||
"""Все запросы стартуют по одному сигналу; отдельные соединения к LB"""
|
||||
start = asyncio.Event()
|
||||
limits = httpx.Limits(max_connections=len(texts))
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
base_url=base_url, timeout=30, limits=limits
|
||||
) as client:
|
||||
|
||||
async def post(text: str) -> httpx.Response:
|
||||
await start.wait()
|
||||
return await client.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": text},
|
||||
headers=message_headers(user, key),
|
||||
)
|
||||
|
||||
tasks = [asyncio.create_task(post(text)) for text in texts]
|
||||
await asyncio.sleep(0)
|
||||
start.set()
|
||||
return list(await asyncio.gather(*tasks))
|
||||
|
||||
|
||||
def test_parallel_replays_through_replicas_create_one_message(
|
||||
api: httpx.Client, base_url: str, make_user, make_chat
|
||||
) -> None:
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
key = uuid.uuid4().hex
|
||||
|
||||
responses = asyncio.run(
|
||||
_post_concurrently(
|
||||
base_url, alice, chat, key, ["одно сообщение"] * PARALLEL_REPLAYS
|
||||
)
|
||||
)
|
||||
|
||||
statuses = Counter(response.status_code for response in responses)
|
||||
instances = Counter(response.headers["X-Instance-Id"] for response in responses)
|
||||
bodies = {json.dumps(response.json(), sort_keys=True) for response in responses}
|
||||
items = history(api, alice, chat)
|
||||
rows = psql(f"SELECT count(*) FROM messages WHERE chat_id = '{chat['id']}';")
|
||||
print(
|
||||
"\nPARALLEL SAME KEY:",
|
||||
json.dumps(
|
||||
{
|
||||
"requests": len(responses),
|
||||
"statuses": dict(statuses),
|
||||
"instances": dict(instances),
|
||||
"distinct_bodies": len(bodies),
|
||||
"messages_in_history": len(items),
|
||||
"rows_in_db": int(rows),
|
||||
}
|
||||
),
|
||||
)
|
||||
assert statuses == {201: PARALLEL_REPLAYS}
|
||||
assert len(instances) >= 2, "запросы не распределились по репликам"
|
||||
assert len(bodies) == 1
|
||||
assert len(items) == 1 and rows == "1"
|
||||
|
||||
|
||||
def test_parallel_same_key_with_different_bodies(
|
||||
api: httpx.Client, base_url: str, make_user, make_chat
|
||||
) -> None:
|
||||
"""Победитель гонки один; совпавшие с ним тела — 201, остальные — 409"""
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
key = uuid.uuid4().hex
|
||||
texts = ["вариант A", "вариант B"] * (PARALLEL_REPLAYS // 2)
|
||||
|
||||
responses = asyncio.run(_post_concurrently(base_url, alice, chat, key, texts))
|
||||
|
||||
items = history(api, alice, chat)
|
||||
assert len(items) == 1
|
||||
winner = items[0]
|
||||
for text, response in zip(texts, responses):
|
||||
if text == winner["text"]:
|
||||
assert response.status_code == 201, response.text
|
||||
assert response.json() == winner
|
||||
else:
|
||||
assert_error(response, 409, "conflict")
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from helpers import assert_error, auth
|
||||
from helpers import assert_error, auth, message_headers
|
||||
|
||||
PASSWORD = "Valid-password-123"
|
||||
|
||||
@@ -32,8 +32,16 @@ def test_malformed_json_on_create_chat_and_message(
|
||||
assert_error(
|
||||
api.post("/api/v1/chats", content=raw, headers=headers), 400, "invalid_request"
|
||||
)
|
||||
message_post_headers = {
|
||||
**message_headers(alice),
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
assert_error(
|
||||
api.post(f"/api/v1/chats/{chat['id']}/messages", content=raw, headers=headers),
|
||||
api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
content=raw,
|
||||
headers=message_post_headers,
|
||||
),
|
||||
400,
|
||||
"invalid_request",
|
||||
)
|
||||
@@ -78,7 +86,9 @@ def test_invalid_message_payload(
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
response = api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", json=body, headers=auth(alice)
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json=body,
|
||||
headers=message_headers(alice),
|
||||
)
|
||||
assert_error(response, 400, "invalid_request")
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from helpers import assert_error, auth
|
||||
from helpers import assert_error, auth, message_headers
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["", " ", " \n\t ", " "])
|
||||
@@ -12,7 +12,9 @@ def test_empty_or_blank_message_is_rejected(
|
||||
alice = make_user("alice")
|
||||
chat = make_chat(alice, [alice])
|
||||
response = api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages", json={"text": text}, headers=auth(alice)
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": text},
|
||||
headers=message_headers(alice),
|
||||
)
|
||||
assert_error(response, 400, "invalid_request")
|
||||
history = api.get(f"/api/v1/chats/{chat['id']}/messages", headers=auth(alice))
|
||||
@@ -29,7 +31,7 @@ def test_message_length_is_counted_in_code_points(
|
||||
response = api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": "😀" * 4001},
|
||||
headers=auth(alice),
|
||||
headers=message_headers(alice),
|
||||
)
|
||||
assert_error(response, 400, "invalid_request")
|
||||
|
||||
@@ -56,7 +58,7 @@ def test_nul_character_is_rejected(api: httpx.Client, make_user, make_chat) -> N
|
||||
response = api.post(
|
||||
f"/api/v1/chats/{chat['id']}/messages",
|
||||
json={"text": "до\u0000после"},
|
||||
headers=auth(alice),
|
||||
headers=message_headers(alice),
|
||||
)
|
||||
assert_error(response, 400, "invalid_request")
|
||||
response = api.post(
|
||||
|
||||
Reference in New Issue
Block a user