diff --git a/.env.example b/.env.example index 9920b3df..1d0d49ab 100644 --- a/.env.example +++ b/.env.example @@ -4,16 +4,21 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: +# Not passed to containers: compose.yaml gives api/migrate DB_HOST/DB_USER/DB_NAME and +# DB_PASSWORD_FILE=/run/secrets/postgres_password (Compose secret built from POSTGRES_PASSWORD below). +# The app also accepts DATABASE_URL_FILE or DATABASE_URL instead. +# Sessions live in PostgreSQL; no Redis/Valkey, CACHE_URL_FILE is unused. SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 -# compose.yaml builds DATABASE_URL from these; pick your own local password. +# Pick your own local password. It reaches containers only as the postgres_password secret file. POSTGRES_USER=messenger POSTGRES_PASSWORD=change-me-locally POSTGRES_DB=messenger -SHUTDOWN_TIMEOUT_SECONDS=5 +# SIGTERM: readiness 503 for DRAIN_SECONDS (> LB detection 3 s), then in-flight requests get +# SHUTDOWN_TIMEOUT_SECONDS. Keep DRAIN + SHUTDOWN < stop_grace_period (10 s). +DRAIN_SECONDS=4 +SHUTDOWN_TIMEOUT_SECONDS=3 +# Image tag messenger-lab3-app:, shown as X-App-Version; scripts/rolling-update.sh sets it. +APP_VERSION=dev LB_PORT=8080 diff --git a/.gitattributes b/.gitattributes index 856422ac..84f24cd7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ *.sh text eol=lf *.conf text eol=lf +*.cfg text eol=lf *.cmd text eol=crlf diff --git a/Dockerfile b/Dockerfile index e7d13f91..7b306b09 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,8 @@ ENV APP_HOST=0.0.0.0 \ APP_PORT=8080 \ APP_ENV=lab \ LOG_LEVEL=info \ - SHUTDOWN_TIMEOUT_SECONDS=5 \ + DRAIN_SECONDS=4 \ + SHUTDOWN_TIMEOUT_SECONDS=3 \ SESSION_TTL_SECONDS=3600 RUN groupadd --gid 10001 app \ @@ -26,6 +27,10 @@ COPY --chown=app:app contracts/openapi.json ./contracts/openapi.json USER app +# Visible as X-App-Version: shows which build served a request during a rolling update. +ARG APP_VERSION=dev +ENV APP_VERSION=${APP_VERSION} + EXPOSE 8080 HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \ diff --git a/Makefile b/Makefile index 6469997d..7207bf28 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,13 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover test-idempotency +APP_VERSION ?= + +.PHONY: help check validate test load test-persistence test-failover test-idempotency failover-series rolling-update lb-status help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "failover-series, rolling-update APP_VERSION=..., lb-status: level 4 experiments (Docker required)" check: $(PYTHON) scripts/check_contract.py @@ -31,3 +34,15 @@ test-failover: test-idempotency: $(PYTHON) scripts/check_idempotency.py --base-url "$(BASE_URL)" + +# Level 4: stop each API replica in turn under load (restoring the previous one); RPS, errors, recovery. +failover-series: + $(PYTHON) scripts/failover_series.py --base-url "$(BASE_URL)" --output evidence/failover-series.json + +# Level 4: one-at-a-time replacement under load. Usage: make rolling-update APP_VERSION=v3 +rolling-update: + @test -n "$(APP_VERSION)" || (echo "Set APP_VERSION of the new build"; exit 2) + APP_VERSION="$(APP_VERSION)" $(PYTHON) scripts/failover_series.py --mode action --action scripts/rolling-update.sh --base-url "$(BASE_URL)" --output evidence/rolling-update.json + +lb-status: + sh scripts/lb-status.sh diff --git a/REPORT.md b/REPORT.md index c0e8c605..f816800b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -7,9 +7,9 @@ ## Запуск из чистого clone -Вариант: **Docker Compose + Nginx**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1), Python 3.9+ для тестов; на Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`). +Вариант: **Docker Compose + HAProxy**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1) и Python 3.9+ для тестов. На Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`). -Образы: `postgres:16.10-alpine`, `nginx:1.28.0-alpine` (закреплены digest в `compose.yaml`), приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…`. +Образы: `postgres:16.10-alpine` и `haproxy:3.2.26-alpine` (LTS), оба закреплены digest в `compose.yaml`. Приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…` в образ `messenger-lab3-app:${APP_VERSION}`. Одна последовательность запуска: @@ -24,6 +24,13 @@ make test-idempotency # один ключ параллельно ч make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd docker compose --env-file .env up -d --wait # вернуть остановленную реплику make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd +# уровень 4 +sh scripts/lb-status.sh # пул HAProxy: реплика, IP, UP/DOWN +make failover-series # поочерёдная остановка каждой реплики под нагрузкой +python scripts/failover_series.py --kill --output evidence/failover-series-kill.json # то же, SIGKILL +docker compose --env-file .env build api migrate # с APP_VERSION=v3 в окружении +make rolling-update APP_VERSION=v3 # Windows: scripts/failover_series.py --mode action --action scripts/rolling-update.cmd +# после rolling update записать APP_VERSION=v3 в .env ``` Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v` (удаляет volume `pgdata`). @@ -31,56 +38,117 @@ make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/res ## Архитектура и решения ``` -client -> lb (nginx :8080, только loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata) - migrate (one-shot) ----------------------^ + edge network data network (internal: true) +client -> lb (HAProxy :8080, loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata) + migrate (one-shot) ------------^ ``` -- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL (`SESSION_STORE=postgres`). -- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик (`depends_on: service_completed_successfully`). Реплики схему не меняют; `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций этой сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`. -- **X-Instance-Id** = hostname контейнера (короткий Docker ID), стабилен на время жизни процесса. `stop-one.sh` по нему находит ровно один running-контейнер сервиса `api` этого проекта и делает `docker stop` (SIGTERM, затем SIGKILL через 8 с). `restart: unless-stopped` не поднимает вручную остановленный контейнер, поэтому реплика остаётся остановленной до `docker compose up -d`. -- **Балансировщик**: round robin без `ip_hash`/sticky. `server api:8080 resolve` + `resolver 127.0.0.11 valid=2s`: Nginx раз в 2 с перечитывает DNS Docker, так что остановленные и добавленные (`--scale api=N`) реплики уходят из пула/входят в него без перезапуска LB. Пассивная проверка: `max_fails=1 fail_timeout=5s`, `proxy_connect_timeout 1s`, `proxy_next_upstream error timeout http_502 http_504`. Активных проверок в open-source Nginx нет; readiness реплик проверяет Docker healthcheck (`/health/ready` каждые 3 с), и LB стартует только после healthy API. -- **Повтор на другой реплике**: для неотправленного запроса Nginx всегда пробует следующую реплику. Уже отправленный POST повторяется только для `/api/v1/chats/{id}/messages` (`non_idempotent`), потому что там обязателен `Idempotency-Key`. POST пользователей/чатов/сессий LB не повторяет. -- **Graceful shutdown**: на SIGTERM uvicorn перестаёт принимать соединения и дорабатывает текущие запросы до `SHUTDOWN_TIMEOUT_SECONDS=5`; `stop_grace_period: 8s` больше этого. `keepalive_timeout 4s` в upstream меньше keep-alive uvicorn (5 с), чтобы LB не переиспользовал закрываемое API соединение. -- **Идемпотентность**: в одной транзакции удаляется истёкший (старше `IDEMPOTENCY_TTL_SECONDS`) ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 канонического JSON тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201 и Message) либо 409 при другом теле. При ошибке транзакция откатывается целиком, ключ не фиксируется, повтор с тем же ключом безопасен. Сессия и членство в чате проверяются при каждом повторе. -- Время сообщений назначает приложение; порядок между репликами опирается на общие часы хоста (учебное упрощение, одна машина). +| Роль | Compose service | Образ | Health/readiness | Сети | Volume / секреты | Лимит CPU / RAM (резерв) | +| -------- | --------------- | ----------------------------- | ----------------------------------------------------- | ---------- | -------------------------------------------------- | ------------------------ | +| LB | `lb` | `haproxy:3.2.26-alpine@…` | `monitor-uri /lb/health`; активные проверки API | edge | только read-only конфиг, секретов нет | 0.5 / 128M (0.1 / 32M) | +| API × N | `api` | `messenger-lab3-app:<версия>` | `/health/live`, `/health/ready` (БД, миграции, drain) | edge, data | `/run/secrets/postgres_password` (uid 10001, 0400) | 1.0 / 256M (0.25 / 128M) | +| Миграции | `migrate` | тот же образ | one-shot, `service_completed_successfully` | data | тот же секрет | 0.5 / 128M | +| БД | `postgres` | `postgres:16.10-alpine@…` | `pg_isready` каждые 2 с | data | `pgdata`, `POSTGRES_PASSWORD_FILE` | 1.0 / 512M (0.25 / 256M) | -Соответствие `.env.example`: `DATABASE_URL` собирается в `compose.yaml` из `POSTGRES_*` (вариант `DATABASE_URL_FILE` тоже поддерживается кодом); `API_REPLICAS` → `deploy.replicas`; `IDEMPOTENCY_TTL_SECONDS`, `SESSION_TTL_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` передаются в API; `CACHE_URL_FILE` не используется. +- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL. +- **Изоляция**: наружу публикуется только `lb` и только на loopback. У API нет `ports`, `docker port` пуст. LB находится только в сети `edge` и не резолвит `postgres`. Сеть `data` объявлена `internal`, без выхода наружу. Volume `pgdata` смонтирован только в `postgres`. `container_name` нигде нет, поэтому `--scale api=N` работает (`evidence/limits.txt`). +- **Секреты**: пароль БД лежит только в незакоммиченном `.env` и попадает в контейнеры как Compose secret `postgres_password` (`environment: POSTGRES_PASSWORD`). В env контейнеров есть только путь `*_PASSWORD_FILE`. API и мигратор собирают `DATABASE_URL` из `DB_HOST/DB_USER/DB_NAME` и файла (`app/config.py`). У LB секретов нет. +- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик, а при rolling update — один раз через `docker compose run --rm migrate`. Реплики схему не меняют. `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций **этой** сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`. +- **X-Instance-Id** = hostname контейнера (короткий Docker ID). **X-App-Version** = `APP_VERSION` образа. По нему видна смена версии при rolling update. +- **Балансировщик**: HAProxy, round robin без sticky. `server-template api 1-10 api:8080` с `resolvers docker` (Docker DNS `127.0.0.11`, `hold valid 1s`), поэтому реплики, добавленные через `--scale` или rolling update и остановленные, входят в пул и выходят из него без перезапуска LB. +- **Повтор на другой реплике**: `retries 2`, `option redispatch`, `retry-on conn-failure empty-response response-timeout`. Для POST, кроме `/api/v1/chats/{id}/messages`, стоит `http-request disable-l7-retry`, то есть они повторяются только если соединение с репликой не было установлено. POST сообщений можно повторять, потому что `Idempotency-Key` обязателен. +- **Идемпотентность**: в одной транзакции удаляется истёкший ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201) либо 409 при другом теле. При ошибке ключ не фиксируется, и повтор безопасен. + +### Health, timeouts и SIGTERM: согласованные тайминги + +``` +SIGTERM -> readiness 503 сразу; запросы ещё обслуживаются + -> HAProxy: 2 неудачные проверки (inter 1s, timeout 1s) -> DOWN, не более 3 с + -> через DRAIN_SECONDS=4 uvicorn перестаёт принимать соединения, дорабатывает текущие за SHUTDOWN_TIMEOUT_SECONDS=3 + -> exit 0 примерно через 5 с; stop_grace_period 10 s (stop-one: docker stop --time 9), SIGKILL не нужен +``` + +| Параметр | Значение | Где | +| -------------------------------------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------- | +| Период активной проверки LB `/health/ready` | 1 с, `timeout check 1s`, `fall 2`, `rise 2` | `infra/haproxy/haproxy.cfg` | +| Обновление DNS реплик | 1 с (`hold valid 1s`, `hold obsolete 0s`) | там же | +| `timeout connect / server / client / http-request` | 1 с / 10 с / 30 с / 10 с | там же | +| Drain после SIGTERM / завершение текущих запросов | 4 с / 3 с | `DRAIN_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` | +| `stop_grace_period` | 10 с (> 4 + 3) | `compose.yaml` | +| Keep-alive uvicorn | 10 с (> `pool-purge-delay` HAProxy 5 с) | `KEEPALIVE_TIMEOUT_SECONDS` | +| Docker healthcheck API | каждые 3 с, 3 попытки; только порядок старта и rolling update, LB его не использует | `compose.yaml` | + +Максимальное время обнаружения отказа: + +- **Плановая остановка (SIGTERM)**: реплика выводится заранее, не более \(2 \cdot 1 + 1 = 3\) с, и раньше, чем закрывается сокет. Замер: последний запрос на останавливаемую реплику пришёл через 1,76–2,05 с после SIGTERM, ошибок 0. +- **Падение процесса (SIGKILL)**: запрос к мёртвому IP завершается по `timeout connect 1s` и уходит на другую реплику (redispatch). Пул очищается по DNS примерно за 1 с или по проверкам не более чем за 3 с. Замер: максимальная пауза без успешного ответа 0,96–0,98 с, ошибок 0. +- **Зависшая реплика, принимающая соединения**: проверки дают DOWN не более чем через 3 с. Запросы, уже отправленные на неё, ждут до `timeout server 10s` и затем повторяются (`response-timeout`) для GET и POST сообщений. Экспериментально не проверялось. +- **Возврат в пул**: `rise 2`, то есть около 2 с после готовности. Замер от `docker start` до первого ответа реплики: 4,5–5,2 с, включая старт приложения. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------- | -| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `docker compose --env-file .env down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy, `migrate` Exited (0); порт публикует только `lb` | `evidence/smoke.txt` (docker compose ps), `compose.yaml` | -| Одна сессия обслуживается разными репликами, вход один раз | `python scripts/show_instances.py` | 6 запросов с одним токеном: 3 на `8b33a91fafee`, 3 на `a87734e887a5`, все 200/201; ID совпадают с hostname контейнеров `api-1`/`api-2` | `evidence/instances.txt` | -| Остановка любой одной API-реплики, запись восстанавливается ≤10 с | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd`, между прогонами `docker compose up -d --wait` | 3 прогона PASS: до отказа 2 реплики, 127–128 успешных записей, 0 ошибок, первая запись через 1,17–1,27 с; остановлены `api-1` (прогоны 1, 2) и `api-2` (прогон 3), статус Exited (0) | `evidence/failover.txt` | -| Повтор POST с тем же Idempotency-Key не создаёт дубль (конкурентно, через разные реплики) | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один и тот же Message, итого 5 сообщений; другое тело с тем же ключом → 409 | `evidence/idempotency.txt` | -| Неподтверждённая запись повторяется с тем же ключом при отказе | `make test-failover` (тест повторяет неопределённый POST с исходным ключом) | дублей нет, подтверждённые сообщения на месте | `evidence/failover.txt` | -| make check, make test | `make check`, `make test` | PASS, exit 0 | `evidence/smoke.txt` | -| Persistence: перезапуск lb, api, postgres с сохранением volume и токена | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; те же ID контейнеров с новым uptime (перезапущены все), volume `pgdata` на месте, прежний токен работает | `evidence/persistence.txt` | +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | +| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy; `migrate` применил 001–003 и Exited (0); в HAProxy 2 реплики UP | `evidence/smoke.txt` | +| Одна сессия обслуживается разными репликами | `python scripts/show_instances.py` | 6 запросов с одним токеном: по 3 на каждую из 2 реплик | `evidence/instances.txt` | +| Остановка любой одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` ×2, между прогонами `up -d --wait` | 2 PASS: остановлены `api-2`, затем `api-1`, Exited (0); 121 и 115 записей, 0 ошибок; HAProxy убрал реплику из пула | `evidence/failover.txt` | +| Повтор POST с тем же Idempotency-Key | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один Message на ключ, другое тело → 409 | `evidence/idempotency.txt` | +| make check, make test, persistence | `make check`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; после рестарта lb, api, postgres те же ID, volume на месте, прежний токен работает | `evidence/smoke.txt`, `evidence/persistence.txt` | +| **4**: health/readiness, timeouts, SIGTERM согласованы; LB исключает недоступный backend | HAProxy active checks + drain; `scripts/lb-status.sh` | см. таблицу таймингов; останавливаемая реплика получает последний запрос через ≤2,05 с после SIGTERM; после выхода её нет в пуле | `infra/haproxy/haproxy.cfg`, `evidence/failover-series.json` | +| **4**: поочерёдная остановка каждой реплики: RPS, ошибки, время восстановления | `make failover-series`; `failover_series.py --kill` | по 2 реплики в каждом режиме, 0 ошибок, все подтверждённые записи на месте, дублей нет (таблица ниже) | `evidence/failover-series*.{txt,json}` | +| **4**: лимиты, нет портов API, секреты и хранилища только где нужно, нет `container_name` | `docker inspect`, `docker stats`, `docker port`, `nslookup` из lb | лимиты у всех 5 сервисов; порт только у lb (127.0.0.1/::1); пароль только файлом 0400; lb без секретов и без доступа к БД; `container_name`: 0 | `evidence/limits.txt` | +| **4**: rolling update под нагрузкой, миграции однократно и совместимо | `make rolling-update APP_VERSION=v3`; ранее `dev → v2` с миграцией 003 | 0 ошибок, максимальная пауза 0,07 с; v2 → v3 без снижения числа реплик ниже 2; при `dev → v2` миграция 003 применена один раз, и старые реплики `dev` продолжали работать | `evidence/rolling-update.{txt,json}`, `evidence/rolling-update-migration.txt` | -Все прогоны 2026-11-07 на commit `8cd25e4` (+ `scripts/show_instances.py`), Docker 29.4.3, Windows 11, один хост. На Windows `make test-failover`/`test-persistence` запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`): GNU Make под cmd не находит `test` в guard-строке, на результат это не влияет. +Все прогоны 2026-11-07 на commit `124c83e` + рабочее дерево (до коммита уровня 4), Docker 29.4.3, Windows 11, один хост. `make test-failover/test-persistence` под Windows запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`), потому что GNU Make под cmd не находит `test` в guard-строке. -Цель failover выбирает тест: реплика, обслужившая первое сообщение. При round robin это детерминированно `api-1`, поэтому перед прогоном 3 через LB отправлен один `GET /health/ready`, чтобы сдвинуть очередь, и тест остановил `api-2`. +Цель `make test-failover` выбирает тест: это реплика, обслужившая первое сообщение. Для второго прогона через LB отправлен один `GET /health/ready`, чтобы сдвинуть round robin, и тест остановил другую реплику. ## Эксперименты и отказы -Профиль failover-теста: 15 с параллельной записи сообщений, пока hook останавливает одну реплику (`docker stop --time 8`, SIGTERM). Нагрузочные эксперименты (RPS, перцентили) — уровень 4/5, здесь не заявлены. +Профиль нагрузки `scripts/failover_series.py`: 4 потока в закрытом цикле, каждый делает POST сообщения (с `Idempotency-Key`; неопределённый результат повторяется с тем же ключом) и GET 20 последних сообщений. Счёт идёт по секундам. Для каждой реплики по очереди: 10 с baseline, затем остановка, 15 с наблюдения, затем `docker start` того же контейнера, ожидание его ответов и 5 с паузы. В конце вся история чата читается заново: каждая подтверждённая запись должна быть ровно один раз. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| ---------------------------------------------------------------------- | ------------------------------------- | ------------------------------------------- | ------------------------------------ | -| `stop-one` реплики, обслужившей тест (`api-1`), под записью, 2 прогона | запись продолжается ≤10 с, дублей нет | 127 и 128 успешных записей, 0 ошибок | 1,27 с и 1,17 с / нет | -| `stop-one` второй реплики (`api-2`) после восстановления первой | то же | 127 записей, 0 ошибок | 1,21 с / нет | -| `restart.cmd`: lb, api, postgres | данные и старый токен сохранены | PASS, volume сохранён | нет потерь | -| 5 ключей × 20 параллельных POST через LB | одно сообщение на ключ | по 10 ответов с каждой реплики, 5 сообщений | — | +### Поочерёдная остановка реплик под нагрузкой -Ошибок в переходный период не было: Nginx при `connect refused` сразу отправляет запрос на другую реплику (`proxy_next_upstream`), а для POST сообщений это безопасно благодаря Idempotency-Key. «Время восстановления» в тесте — это время от старта hook до первой успешной записи, а не время обнаружения отказа. +| Воздействие | Реплика | RPS baseline → во время → после | Ошибки | Макс. пауза без успеха | Последний запрос на реплику после сигнала | Выход контейнера | RPS ≥90% baseline после выхода | Возврат в пул после `docker start` | +| ------------------------------- | ------- | ------------------------------- | ------ | ---------------------- | ----------------------------------------- | ---------------- | ------------------------------ | ---------------------------------- | +| `docker stop` (SIGTERM + drain) | api-1 | 129,0 → 114,8 → 113,1 | 0 | 0,09 с | 2,05 с | 5,06 с, exit 0 | 2,8 с | 5,2 с | +| `docker stop` (SIGTERM + drain) | api-2 | 131,5 → 145,2 → 165,4 | 0 | 0,09 с | 1,76 с | 4,88 с, exit 0 | 1,6 с | 4,9 с | +| `docker kill` (SIGKILL) | api-1 | 174,4 → 145,3 → 158,4 | 0 | 0,96 с | 0,12 с | 0,66 с, exit 137 | 2,3 с | 4,5 с | +| `docker kill` (SIGKILL) | api-2 | 177,9 → 146,5 → 139,0 | 0 | 0,98 с | 0,12 с | 0,63 с, exit 137 | 2,1 с | 5,2 с | + +Итог серий: SIGTERM — 80,5 с, 5312 записей и 5312 чтений, p95 73 мс; SIGKILL — 71,2 с, 5520 записей и 5520 чтений, p95 66 мс. В обоих случаях 0 ошибок, 0 дублей, 0 потерянных подтверждённых записей. В предварительном прогоне `--kill` (не сохранён) была 1 ошибка на 12 890 запросов: запрос, оборванный в момент SIGKILL. Это допустимый переходный эффект, клиент повторяет его с тем же ключом. + +RPS — это пропускная способность закрытого цикла на одном хосте, где рядом работали посторонние контейнеры. Между сериями она колеблется на 30–40 %, а внутри серии падение во время отказа составляет 10–20 %. Иногда RPS с одной репликой выше baseline: узкое место — клиент и общий CPU хоста, а не число реплик. Это не измерение ёмкости. + +### make test-failover и rolling update + +| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных | +| ------------------------------------------------ | ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------- | +| `make test-failover`, `stop-one` → `api-2` | запись продолжается ≤10 с, дублей нет | PASS, 121 запись, 0 ошибок | `first_recovery_seconds` 6,42 / нет | +| то же после восстановления → `api-1` | то же | PASS, 115 записей, 0 ошибок | 6,44 / нет | +| `restart.cmd`: lb, api, postgres | данные и старый токен сохранены | PASS, volume сохранён | нет потерь | +| rolling update v2 → v3 под нагрузкой (2 реплики) | нет ошибок, ёмкость ≥2 реплик, версия сменилась | 42,7 с; 0 ошибок; пауза 0,07 с; RPS 181 → 146 → 186; ответы: до v2 1833; во время v2 3941 / v3 2279; после v3 2787 | 5423 записи, 0 дублей, 0 потерь | +| rolling update dev → v2 с миграцией 003 | старая версия работает на расширенной схеме | 45,7 с; 0 ошибок; пауза 0,10 с; миграция применена один раз в начале, реплики `dev` обслуживали запросы до своей замены | 4458 записей, 0 дублей, 0 потерь | + +`first_recovery_seconds` в `tests/resilience.py` считается до первой успешной записи **после завершения hook**. Hook теперь длится около 6 с: это `docker stop`, который ждёт 4 с drain. Записи при этом не прерывались (0 ошибок), поэтому значение отражает длительность hook, а не простой. Оно укладывается в лимит 10 с, как и сам hook. + +**Rolling update** (`scripts/rolling-update.sh`): + +1. Собрать `messenger-lab3-app:$APP_VERSION`. +2. Один раз выполнить `docker compose run --rm migrate`. +3. Для каждой старой реплики запустить `up -d --no-deps --no-recreate --wait --scale api=N+1`. Новая реплика стартует из нового образа, скрипт ждёт healthy и UP в HAProxy. Затем старая реплика получает `docker stop` (drain) и `docker rm`. + +Ёмкость не опускается ниже N, LB не перезапускается. После обновления версию нужно записать в `.env` (`APP_VERSION`), иначе следующий `docker compose up` пересоздаст реплики в версии из `.env`. + +**Политика миграций (expand/contract)**. Миграция выполняется до замены реплик и должна быть совместима с работающей версией: только новые таблицы, индексы и nullable-колонки. Пример — `003_session_indexes.sql`. Readiness старой сборки проверяет только свои миграции, поэтому расширенная схема её не ломает. Удаление и переименование колонок допустимы только отдельным релизом после ухода старых реплик. Ограничение: `CREATE INDEX` без `CONCURRENTLY` (мигратор работает в транзакции) коротко блокирует запись в индексируемую таблицу. Для `sessions` учебного размера это незаметно, для большой таблицы понадобится `CONCURRENTLY` вне транзакции. ## Ограничения и следующие шаги -- **Выдерживается**: отказ процесса или контейнера одной API-реплики, пока работает хотя бы одна другая. Также полный перезапуск стенда с сохранением volume. -- **Не выдерживается**: отказ PostgreSQL и Nginx. Это единственные точки отказа: БД не реплицирована, LB один. При их остановке API недоступен до перезапуска, подтверждённые данные сохраняются в volume. -- **Один хост**: все контейнеры на одной машине, поэтому отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса, а не узла. -- **Нет активных health-check в LB**: open-source Nginx исключает реплику пассивно (ошибка соединения, `max_fails=1 fail_timeout=5s`) и при обновлении DNS раз в 2 с. Зависшая, но принимающая соединения реплика будет отбрасываться только по `proxy_read_timeout 10s`. -- **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение. На нескольких узлах понадобятся время БД или последовательность. -- **Повтор LB**: Nginx повторяет на другой реплике только POST сообщений. POST пользователей, чатов и сессий после обрыва соединения клиент должен проверить сам, так как ключа идемпотентности у них нет. -- Лимиты CPU/RAM, rolling update и измерения под нагрузкой не делались: это уровень 4. +- **Выдерживается**: плановая остановка, падение или зависание одной API-реплики, пока работает хотя бы одна другая; поочерёдная замена всех реплик; полный перезапуск стенда с сохранением volume. +- **Не выдерживается**: отказ PostgreSQL и HAProxy. Это единственные точки отказа: БД не реплицирована, LB один. При их остановке API недоступен до перезапуска, подтверждённые данные сохраняются в volume. +- **Один хост**: отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса или контейнера, а не узла. +- **Зависание реплики** экспериментально не воспроизводилось. Оценка «не более 3 с до DOWN» следует из конфигурации, а уже отправленные на зависшую реплику запросы ждут до 10 с. +- **Rolling update** в Compose реализован скриптом, а не `deploy.update_config`: Compose его не исполняет. Откат — тот же скрипт с прежней `APP_VERSION`, если схема не прошла contract-фазу. +- **Нагрузка** — закрытый цикл одного клиента на том же хосте (coordinated omission, общий CPU). Цифры RPS сравнимы только внутри одной серии. +- **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение. +- **Повтор LB**: POST пользователей, чатов и сессий повторяется только при неустановленном соединении. После обрыва уже отправленного запроса клиент должен проверить результат сам: ключа идемпотентности у этих POST нет. diff --git a/app/__main__.py b/app/__main__.py index 14beaf1c..5396aabb 100644 --- a/app/__main__.py +++ b/app/__main__.py @@ -1,17 +1,48 @@ +import logging +import threading + import uvicorn from .config import load_settings +from .drain import draining + +logger = logging.getLogger('messenger') + + +class DrainingServer(uvicorn.Server): + """First SIGTERM/SIGINT: keep serving for drain_seconds with readiness 503, then uvicorn's graceful stop. + + drain_seconds must exceed the LB detection time (HAProxy inter 1s x fall 2 + timeout check 1s), + so the replica is out of rotation before it stops accepting connections. + """ + + def __init__(self, config: uvicorn.Config, drain_seconds: float): + super().__init__(config) + self.drain_seconds = drain_seconds + + def handle_exit(self, sig, frame) -> None: + if draining.is_set() or self.drain_seconds <= 0: + draining.set() + super().handle_exit(sig, frame) + return + draining.set() + logger.warning('signal %s: draining for %.1f s before shutdown', sig, self.drain_seconds) + timer = threading.Timer(self.drain_seconds, super().handle_exit, (sig, frame)) + timer.daemon = True + timer.start() def main() -> None: settings = load_settings() - uvicorn.run( + config = uvicorn.Config( 'app.main:app', host=settings.host, port=settings.port, log_level=settings.log_level, timeout_graceful_shutdown=settings.shutdown_timeout, + timeout_keep_alive=settings.keepalive_timeout, ) + DrainingServer(config, settings.drain_seconds).run() if __name__ == '__main__': diff --git a/app/config.py b/app/config.py index 642fd9af..07ac86e5 100644 --- a/app/config.py +++ b/app/config.py @@ -2,6 +2,7 @@ import os import socket from dataclasses import dataclass from pathlib import Path +from urllib.parse import quote @dataclass(frozen=True) @@ -11,39 +12,67 @@ class Settings: env: str log_level: str shutdown_timeout: int + drain_seconds: float + keepalive_timeout: int session_ttl_seconds: int idempotency_ttl_seconds: int database_url: str instance_id: str + app_version: str -def _secret(file_var: str, value_var: str) -> str: - """A *_FILE variable wins. The secret value is not logged.""" - file_name = os.getenv(file_var, '').strip() - if file_name: - try: - value = Path(file_name).read_text(encoding='utf-8').strip() - except OSError: - raise RuntimeError('%s could not be read' % file_var) from None - if not value: - raise RuntimeError('%s is empty' % file_var) - return value - value = os.getenv(value_var, '').strip() +def _read_secret_file(file_var: str, file_name: str) -> str: + try: + value = Path(file_name).read_text(encoding='utf-8').strip() + except OSError: + raise RuntimeError('%s could not be read' % file_var) from None if not value: - raise RuntimeError('%s or %s is required' % (value_var, file_var)) + raise RuntimeError('%s is empty' % file_var) return value +def _database_url() -> str: + """DATABASE_URL_FILE, then DATABASE_URL, then DB_* parts with the password from DB_PASSWORD_FILE. + + Secret values are never logged. + """ + file_name = os.getenv('DATABASE_URL_FILE', '').strip() + if file_name: + return _read_secret_file('DATABASE_URL_FILE', file_name) + value = os.getenv('DATABASE_URL', '').strip() + if value: + return value + password_file = os.getenv('DB_PASSWORD_FILE', '').strip() + if not password_file: + raise RuntimeError('DATABASE_URL, DATABASE_URL_FILE or DB_PASSWORD_FILE is required') + password = _read_secret_file('DB_PASSWORD_FILE', password_file) + parts = {name: os.getenv(name, '').strip() for name in ('DB_HOST', 'DB_USER', 'DB_NAME')} + missing = [name for name, part in parts.items() if not part] + if missing: + raise RuntimeError('%s required with DB_PASSWORD_FILE' % ', '.join(missing)) + return 'postgresql://%s:%s@%s:%s/%s' % ( + quote(parts['DB_USER'], safe=''), + quote(password, safe=''), + parts['DB_HOST'], + int(os.getenv('DB_PORT', '5432')), + quote(parts['DB_NAME'], safe=''), + ) + + def load_settings() -> Settings: return Settings( host=os.getenv('APP_HOST', '0.0.0.0'), port=int(os.getenv('APP_PORT', '8080')), env=os.getenv('APP_ENV', 'lab'), log_level=os.getenv('LOG_LEVEL', 'info').lower(), - shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '5')), + shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '3')), + drain_seconds=float(os.getenv('DRAIN_SECONDS', '4')), + # Longer than HAProxy's idle-connection purge (pool-purge-delay 5 s), so the LB usually closes them first. + keepalive_timeout=int(os.getenv('KEEPALIVE_TIMEOUT_SECONDS', '10')), session_ttl_seconds=int(os.getenv('SESSION_TTL_SECONDS', '3600')), idempotency_ttl_seconds=int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400')), - database_url=_secret('DATABASE_URL_FILE', 'DATABASE_URL'), + database_url=_database_url(), # Docker sets the hostname to the short container ID, which stop-one.sh maps back to a container. instance_id=os.getenv('INSTANCE_ID', '').strip() or socket.gethostname(), + app_version=os.getenv('APP_VERSION', 'dev').strip() or 'dev', ) diff --git a/app/drain.py b/app/drain.py new file mode 100644 index 00000000..f23a67c3 --- /dev/null +++ b/app/drain.py @@ -0,0 +1,4 @@ +"""Set on SIGTERM: readiness turns 503 so the LB stops routing here while requests are still served.""" +import threading + +draining = threading.Event() diff --git a/app/main.py b/app/main.py index 605254eb..052dd11b 100644 --- a/app/main.py +++ b/app/main.py @@ -72,7 +72,9 @@ def create_app() -> FastAPI: version='1.3.0', lifespan=lifespan, ) - instance_id = load_settings().instance_id + settings = load_settings() + instance_id = settings.instance_id + app_version = settings.app_version def openapi(): if app.openapi_schema is None: @@ -83,7 +85,7 @@ def create_app() -> FastAPI: @app.middleware('http') async def request_context(request: Request, call_next): - """Gives every response, including unexpected failures, an X-Request-Id and X-Instance-Id.""" + """Gives every response, including unexpected failures, an X-Request-Id, X-Instance-Id and X-App-Version.""" incoming = request.headers.get('X-Request-Id') request_id = incoming if incoming and _SAFE_REQUEST_ID.match(incoming) else str(uuid.uuid4()) request.state.request_id = request_id @@ -94,6 +96,7 @@ def create_app() -> FastAPI: response = _error(500, 'internal_error', _MESSAGES['internal_error'], request_id) response.headers['X-Request-Id'] = request_id response.headers['X-Instance-Id'] = instance_id + response.headers['X-App-Version'] = app_version return response @app.exception_handler(ApiError) diff --git a/app/routes.py b/app/routes.py index 2221dbd3..065f4f15 100644 --- a/app/routes.py +++ b/app/routes.py @@ -10,7 +10,8 @@ from pydantic import BaseModel, ValidationError from starlette.responses import Response from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer -from .errors import invalid_request, not_found, unauthorized +from .drain import draining +from .errors import invalid_request, not_found, unauthorized, unavailable from .pagination import encode_cursor, parse_page from .schemas import CreateChat, CreateMessage, CreateUser from .store import Chat, Store, User @@ -80,6 +81,8 @@ async def liveness() -> JSONResponse: @router.get('/health/ready') async def readiness(store: Store = Depends(get_store)) -> JSONResponse: + if draining.is_set(): + raise unavailable('Реплика завершает работу') await store.ready() return JSONResponse({'status': 'ok'}) diff --git a/compose.yaml b/compose.yaml index 68d2997d..24961d41 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,57 +1,103 @@ -# Client -> lb (nginx) -> api x API_REPLICAS -> postgres. -# Secrets come from .env, which is not committed. Only the LB publishes a port. +# Client -> lb (HAProxy) -> api x API_REPLICAS -> postgres. +# Only the LB publishes a port. Networks: edge = lb<->api, data = api/migrate<->postgres (no LB, no internet). +# The DB password comes from .env (not committed) as a Compose secret file, never as a container env var. name: messenger-lab3 -x-app-env: &app-env +x-db-client: &db-client APP_ENV: ${APP_ENV:-lab} LOG_LEVEL: ${LOG_LEVEL:-info} - DATABASE_URL: postgresql://${POSTGRES_USER:?set in .env}:${POSTGRES_PASSWORD:?set in .env}@postgres:5432/${POSTGRES_DB:?set in .env} + DB_HOST: postgres + DB_USER: ${POSTGRES_USER:?set in .env} + DB_NAME: ${POSTGRES_DB:?set in .env} + DB_PASSWORD_FILE: /run/secrets/postgres_password + +x-app-build: &app-build + context: . + dockerfile: Dockerfile + args: + APP_VERSION: ${APP_VERSION:-dev} services: postgres: image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 environment: POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password POSTGRES_DB: ${POSTGRES_DB} + secrets: + - postgres_password volumes: - pgdata:/var/lib/postgresql/data + networks: + - data healthcheck: test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] interval: 2s timeout: 5s retries: 30 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M + reservations: + cpus: "0.25" + memory: 256M restart: unless-stopped # One-shot schema step. Replicas never migrate on startup, so there is no race between them; # an advisory lock additionally serializes accidental parallel runs. migrate: - image: messenger-lab3-app:local - build: - context: . - dockerfile: Dockerfile + image: messenger-lab3-app:${APP_VERSION:-dev} + build: *app-build command: ["python", "-m", "app.migrate"] environment: - <<: *app-env + <<: *db-client + secrets: + - source: postgres_password + uid: "10001" + gid: "10001" + mode: 0400 + networks: + - data depends_on: postgres: condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M restart: "no" api: - image: messenger-lab3-app:local - build: - context: . - dockerfile: Dockerfile + image: messenger-lab3-app:${APP_VERSION:-dev} + build: *app-build deploy: replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "1.0" + memory: 256M + reservations: + cpus: "0.25" + memory: 128M environment: - <<: *app-env + <<: *db-client APP_HOST: 0.0.0.0 APP_PORT: "8080" SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} - SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5} + DRAIN_SECONDS: ${DRAIN_SECONDS:-4} + SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3} + secrets: + - source: postgres_password + uid: "10001" + gid: "10001" + mode: 0400 + networks: + - edge + - data expose: - "8080" depends_on: @@ -59,6 +105,7 @@ services: condition: service_healthy migrate: condition: service_completed_successfully + # Docker health gates startup order and rolling update; the LB runs its own checks (infra/haproxy/haproxy.cfg). healthcheck: test: - CMD @@ -69,19 +116,22 @@ services: timeout: 3s retries: 3 start_period: 20s - # SIGTERM -> uvicorn stops accepting, finishes in-flight requests within SHUTDOWN_TIMEOUT_SECONDS. + # SIGTERM -> readiness 503 for DRAIN_SECONDS (LB drops the replica in <= 3 s) -> uvicorn finishes + # in-flight requests within SHUTDOWN_TIMEOUT_SECONDS. 4 + 3 < 10, so SIGKILL is not reached. stop_signal: SIGTERM - stop_grace_period: 8s + stop_grace_period: 10s restart: unless-stopped lb: - image: nginx:1.28.0-alpine@sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284 + image: haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0 # Loopback only. Both families: on Windows "localhost" tries ::1 first and waits ~2 s if nothing listens. ports: - "127.0.0.1:${LB_PORT:-8080}:8080" - "[::1]:${LB_PORT:-8080}:8080" volumes: - - ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: + - edge depends_on: api: condition: service_healthy @@ -90,7 +140,24 @@ services: interval: 5s timeout: 3s retries: 3 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M + reservations: + cpus: "0.1" + memory: 32M restart: unless-stopped +networks: + edge: {} + data: + internal: true + +secrets: + postgres_password: + environment: POSTGRES_PASSWORD + volumes: pgdata: diff --git a/evidence/failover-series-kill.json b/evidence/failover-series-kill.json new file mode 100644 index 00000000..f18a0117 --- /dev/null +++ b/evidence/failover-series-kill.json @@ -0,0 +1,961 @@ +{ + "mode": "failover", + "stop_method": "docker kill (SIGKILL)", + "concurrency": 4, + "workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest", + "replicas_before": [ + "messenger-lab3-api-1", + "messenger-lab3-api-2" + ], + "runs": [ + { + "replica": "messenger-lab3-api-1", + "instance_id": "1b755b53e9fa", + "state_after_stop": "exited exit=137", + "stop_command_seconds": 0.66, + "last_served_by_target_after_sigterm_s": 0.12, + "first_success_after_container_exit_s": 0.67, + "rps_back_to_90pct_after_exit_s": 2.28, + "max_gap_without_success_s": 0.963, + "rejoin_after_docker_start_s": 4.51, + "baseline": { + "seconds": 10.0, + "rps_ok": 174.4, + "writes_ok": 873, + "reads_ok": 871, + "errors": 0, + "p95_ms": 53.8, + "instances": { + "e3303afc01b2": 874, + "1b755b53e9fa": 870 + } + }, + "during_stop_and_outage": { + "seconds": 15.73, + "rps_ok": 145.3, + "writes_ok": 1143, + "reads_ok": 1143, + "errors": 0, + "p95_ms": 65.5, + "instances": { + "1b755b53e9fa": 9, + "e3303afc01b2": 2277 + } + }, + "after_restore": { + "seconds": 9.51, + "rps_ok": 158.4, + "writes_ok": 754, + "reads_ok": 752, + "errors": 0, + "p95_ms": 67.6, + "instances": { + "e3303afc01b2": 1035, + "1b755b53e9fa": 471 + } + }, + "marks": { + "sigterm": 10.06, + "exited": 10.72, + "docker_start": 25.79 + } + }, + { + "replica": "messenger-lab3-api-2", + "instance_id": "e3303afc01b2", + "state_after_stop": "exited exit=137", + "stop_command_seconds": 0.63, + "last_served_by_target_after_sigterm_s": 0.12, + "first_success_after_container_exit_s": 0.755, + "rps_back_to_90pct_after_exit_s": 2.06, + "max_gap_without_success_s": 0.975, + "rejoin_after_docker_start_s": 5.18, + "baseline": { + "seconds": 10.0, + "rps_ok": 177.9, + "writes_ok": 888, + "reads_ok": 891, + "errors": 0, + "p95_ms": 62.7, + "instances": { + "e3303afc01b2": 891, + "1b755b53e9fa": 888 + } + }, + "during_stop_and_outage": { + "seconds": 15.71, + "rps_ok": 146.5, + "writes_ok": 1152, + "reads_ok": 1151, + "errors": 0, + "p95_ms": 67.0, + "instances": { + "1b755b53e9fa": 2291, + "e3303afc01b2": 12 + } + }, + "after_restore": { + "seconds": 10.18, + "rps_ok": 139.0, + "writes_ok": 708, + "reads_ok": 708, + "errors": 0, + "p95_ms": 73.8, + "instances": { + "1b755b53e9fa": 1025, + "e3303afc01b2": 391 + } + }, + "marks": { + "sigterm": 45.31, + "exited": 45.94, + "docker_start": 61.02 + } + } + ], + "total": { + "seconds": 71.23, + "rps_ok": 155.0, + "writes_ok": 5520, + "reads_ok": 5520, + "errors": 0, + "p95_ms": 65.5, + "instances": { + "e3303afc01b2": 5483, + "1b755b53e9fa": 5557 + } + }, + "integrity": { + "confirmed_writes": 5520, + "stored_messages": 5520, + "duplicates": 0, + "missing_confirmed": 0 + }, + "timeline_per_second": [ + { + "t": 0, + "ok": 145, + "err": 0, + "instances": { + "e3303afc01b2": 71, + "1b755b53e9fa": 74 + }, + "versions": { + "v2": 145 + } + }, + { + "t": 1, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 87, + "1b755b53e9fa": 84 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 2, + "ok": 240, + "err": 0, + "instances": { + "e3303afc01b2": 120, + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 240 + } + }, + { + "t": 3, + "ok": 239, + "err": 0, + "instances": { + "e3303afc01b2": 120, + "1b755b53e9fa": 119 + }, + "versions": { + "v2": 239 + } + }, + { + "t": 4, + "ok": 199, + "err": 0, + "instances": { + "e3303afc01b2": 98, + "1b755b53e9fa": 101 + }, + "versions": { + "v2": 199 + } + }, + { + "t": 5, + "ok": 141, + "err": 0, + "instances": { + "1b755b53e9fa": 69, + "e3303afc01b2": 72 + }, + "versions": { + "v2": 141 + } + }, + { + "t": 6, + "ok": 185, + "err": 0, + "instances": { + "1b755b53e9fa": 92, + "e3303afc01b2": 93 + }, + "versions": { + "v2": 185 + } + }, + { + "t": 7, + "ok": 190, + "err": 0, + "instances": { + "1b755b53e9fa": 96, + "e3303afc01b2": 94 + }, + "versions": { + "v2": 190 + } + }, + { + "t": 8, + "ok": 130, + "err": 0, + "instances": { + "e3303afc01b2": 65, + "1b755b53e9fa": 65 + }, + "versions": { + "v2": 130 + } + }, + { + "t": 9, + "ok": 100, + "err": 0, + "instances": { + "1b755b53e9fa": 50, + "e3303afc01b2": 50 + }, + "versions": { + "v2": 100 + } + }, + { + "t": 10, + "ok": 48, + "err": 0, + "instances": { + "e3303afc01b2": 39, + "1b755b53e9fa": 9 + }, + "versions": { + "v2": 48 + } + }, + { + "t": 11, + "ok": 128, + "err": 0, + "instances": { + "e3303afc01b2": 128 + }, + "versions": { + "v2": 128 + } + }, + { + "t": 12, + "ok": 170, + "err": 0, + "instances": { + "e3303afc01b2": 170 + }, + "versions": { + "v2": 170 + } + }, + { + "t": 13, + "ok": 107, + "err": 0, + "instances": { + "e3303afc01b2": 107 + }, + "versions": { + "v2": 107 + } + }, + { + "t": 14, + "ok": 143, + "err": 0, + "instances": { + "e3303afc01b2": 143 + }, + "versions": { + "v2": 143 + } + }, + { + "t": 15, + "ok": 148, + "err": 0, + "instances": { + "e3303afc01b2": 148 + }, + "versions": { + "v2": 148 + } + }, + { + "t": 16, + "ok": 170, + "err": 0, + "instances": { + "e3303afc01b2": 170 + }, + "versions": { + "v2": 170 + } + }, + { + "t": 17, + "ok": 128, + "err": 0, + "instances": { + "e3303afc01b2": 128 + }, + "versions": { + "v2": 128 + } + }, + { + "t": 18, + "ok": 136, + "err": 0, + "instances": { + "e3303afc01b2": 136 + }, + "versions": { + "v2": 136 + } + }, + { + "t": 19, + "ok": 147, + "err": 0, + "instances": { + "e3303afc01b2": 147 + }, + "versions": { + "v2": 147 + } + }, + { + "t": 20, + "ok": 187, + "err": 0, + "instances": { + "e3303afc01b2": 187 + }, + "versions": { + "v2": 187 + } + }, + { + "t": 21, + "ok": 132, + "err": 0, + "instances": { + "e3303afc01b2": 132 + }, + "versions": { + "v2": 132 + } + }, + { + "t": 22, + "ok": 136, + "err": 0, + "instances": { + "e3303afc01b2": 136 + }, + "versions": { + "v2": 136 + } + }, + { + "t": 23, + "ok": 194, + "err": 0, + "instances": { + "e3303afc01b2": 194 + }, + "versions": { + "v2": 194 + } + }, + { + "t": 24, + "ok": 197, + "err": 0, + "instances": { + "e3303afc01b2": 197 + }, + "versions": { + "v2": 197 + } + }, + { + "t": 25, + "ok": 134, + "err": 0, + "instances": { + "e3303afc01b2": 134 + }, + "versions": { + "v2": 134 + } + }, + { + "t": 26, + "ok": 56, + "err": 0, + "instances": { + "e3303afc01b2": 56 + }, + "versions": { + "v2": 56 + } + }, + { + "t": 27, + "ok": 139, + "err": 0, + "instances": { + "e3303afc01b2": 139 + }, + "versions": { + "v2": 139 + } + }, + { + "t": 28, + "ok": 161, + "err": 0, + "instances": { + "e3303afc01b2": 161 + }, + "versions": { + "v2": 161 + } + }, + { + "t": 29, + "ok": 176, + "err": 0, + "instances": { + "e3303afc01b2": 176 + }, + "versions": { + "v2": 176 + } + }, + { + "t": 30, + "ok": 127, + "err": 0, + "instances": { + "e3303afc01b2": 72, + "1b755b53e9fa": 55 + }, + "versions": { + "v2": 127 + } + }, + { + "t": 31, + "ok": 173, + "err": 0, + "instances": { + "e3303afc01b2": 90, + "1b755b53e9fa": 83 + }, + "versions": { + "v2": 173 + } + }, + { + "t": 32, + "ok": 184, + "err": 0, + "instances": { + "1b755b53e9fa": 94, + "e3303afc01b2": 90 + }, + "versions": { + "v2": 184 + } + }, + { + "t": 33, + "ok": 240, + "err": 0, + "instances": { + "e3303afc01b2": 120, + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 240 + } + }, + { + "t": 34, + "ok": 191, + "err": 0, + "instances": { + "e3303afc01b2": 94, + "1b755b53e9fa": 97 + }, + "versions": { + "v2": 191 + } + }, + { + "t": 35, + "ok": 138, + "err": 0, + "instances": { + "e3303afc01b2": 72, + "1b755b53e9fa": 66 + }, + "versions": { + "v2": 138 + } + }, + { + "t": 36, + "ok": 118, + "err": 0, + "instances": { + "e3303afc01b2": 58, + "1b755b53e9fa": 60 + }, + "versions": { + "v2": 118 + } + }, + { + "t": 37, + "ok": 208, + "err": 0, + "instances": { + "e3303afc01b2": 104, + "1b755b53e9fa": 104 + }, + "versions": { + "v2": 208 + } + }, + { + "t": 38, + "ok": 189, + "err": 0, + "instances": { + "1b755b53e9fa": 96, + "e3303afc01b2": 93 + }, + "versions": { + "v2": 189 + } + }, + { + "t": 39, + "ok": 148, + "err": 0, + "instances": { + "e3303afc01b2": 75, + "1b755b53e9fa": 73 + }, + "versions": { + "v2": 148 + } + }, + { + "t": 40, + "ok": 160, + "err": 0, + "instances": { + "1b755b53e9fa": 78, + "e3303afc01b2": 82 + }, + "versions": { + "v2": 160 + } + }, + { + "t": 41, + "ok": 240, + "err": 0, + "instances": { + "1b755b53e9fa": 122, + "e3303afc01b2": 118 + }, + "versions": { + "v2": 240 + } + }, + { + "t": 42, + "ok": 215, + "err": 0, + "instances": { + "1b755b53e9fa": 109, + "e3303afc01b2": 106 + }, + "versions": { + "v2": 215 + } + }, + { + "t": 43, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 87, + "1b755b53e9fa": 84 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 44, + "ok": 174, + "err": 0, + "instances": { + "1b755b53e9fa": 85, + "e3303afc01b2": 89 + }, + "versions": { + "v2": 174 + } + }, + { + "t": 45, + "ok": 116, + "err": 0, + "instances": { + "1b755b53e9fa": 75, + "e3303afc01b2": 41 + }, + "versions": { + "v2": 116 + } + }, + { + "t": 46, + "ok": 62, + "err": 0, + "instances": { + "1b755b53e9fa": 62 + }, + "versions": { + "v2": 62 + } + }, + { + "t": 47, + "ok": 180, + "err": 0, + "instances": { + "1b755b53e9fa": 180 + }, + "versions": { + "v2": 180 + } + }, + { + "t": 48, + "ok": 113, + "err": 0, + "instances": { + "1b755b53e9fa": 113 + }, + "versions": { + "v2": 113 + } + }, + { + "t": 49, + "ok": 141, + "err": 0, + "instances": { + "1b755b53e9fa": 141 + }, + "versions": { + "v2": 141 + } + }, + { + "t": 50, + "ok": 193, + "err": 0, + "instances": { + "1b755b53e9fa": 193 + }, + "versions": { + "v2": 193 + } + }, + { + "t": 51, + "ok": 200, + "err": 0, + "instances": { + "1b755b53e9fa": 200 + }, + "versions": { + "v2": 200 + } + }, + { + "t": 52, + "ok": 139, + "err": 0, + "instances": { + "1b755b53e9fa": 139 + }, + "versions": { + "v2": 139 + } + }, + { + "t": 53, + "ok": 171, + "err": 0, + "instances": { + "1b755b53e9fa": 171 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 54, + "ok": 210, + "err": 0, + "instances": { + "1b755b53e9fa": 210 + }, + "versions": { + "v2": 210 + } + }, + { + "t": 55, + "ok": 151, + "err": 0, + "instances": { + "1b755b53e9fa": 151 + }, + "versions": { + "v2": 151 + } + }, + { + "t": 56, + "ok": 91, + "err": 0, + "instances": { + "1b755b53e9fa": 91 + }, + "versions": { + "v2": 91 + } + }, + { + "t": 57, + "ok": 123, + "err": 0, + "instances": { + "1b755b53e9fa": 123 + }, + "versions": { + "v2": 123 + } + }, + { + "t": 58, + "ok": 180, + "err": 0, + "instances": { + "1b755b53e9fa": 180 + }, + "versions": { + "v2": 180 + } + }, + { + "t": 59, + "ok": 126, + "err": 0, + "instances": { + "1b755b53e9fa": 126 + }, + "versions": { + "v2": 126 + } + }, + { + "t": 60, + "ok": 168, + "err": 0, + "instances": { + "1b755b53e9fa": 168 + }, + "versions": { + "v2": 168 + } + }, + { + "t": 61, + "ok": 78, + "err": 0, + "instances": { + "1b755b53e9fa": 78 + }, + "versions": { + "v2": 78 + } + }, + { + "t": 62, + "ok": 163, + "err": 0, + "instances": { + "1b755b53e9fa": 163 + }, + "versions": { + "v2": 163 + } + }, + { + "t": 63, + "ok": 144, + "err": 0, + "instances": { + "1b755b53e9fa": 144 + }, + "versions": { + "v2": 144 + } + }, + { + "t": 64, + "ok": 179, + "err": 0, + "instances": { + "1b755b53e9fa": 179 + }, + "versions": { + "v2": 179 + } + }, + { + "t": 65, + "ok": 68, + "err": 0, + "instances": { + "1b755b53e9fa": 68 + }, + "versions": { + "v2": 68 + } + }, + { + "t": 66, + "ok": 71, + "err": 0, + "instances": { + "1b755b53e9fa": 38, + "e3303afc01b2": 33 + }, + "versions": { + "v2": 71 + } + }, + { + "t": 67, + "ok": 145, + "err": 0, + "instances": { + "e3303afc01b2": 73, + "1b755b53e9fa": 72 + }, + "versions": { + "v2": 145 + } + }, + { + "t": 68, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 86, + "1b755b53e9fa": 85 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 69, + "ok": 206, + "err": 0, + "instances": { + "1b755b53e9fa": 103, + "e3303afc01b2": 103 + }, + "versions": { + "v2": 206 + } + }, + { + "t": 70, + "ok": 163, + "err": 0, + "instances": { + "1b755b53e9fa": 80, + "e3303afc01b2": 83 + }, + "versions": { + "v2": 163 + } + }, + { + "t": 71, + "ok": 35, + "err": 0, + "instances": { + "1b755b53e9fa": 19, + "e3303afc01b2": 16 + }, + "versions": { + "v2": 35 + } + } + ] +} diff --git a/evidence/failover-series-kill.txt b/evidence/failover-series-kill.txt new file mode 100644 index 00000000..0cffa7ec --- /dev/null +++ b/evidence/failover-series-kill.txt @@ -0,0 +1,7 @@ +# failover series 2026-11-07T18:10:53.0538421+03:00: python scripts/failover_series.py --kill (docker kill, SIGKILL) +killing messenger-lab3-api-1 (1b755b53e9fa) under load... + errors=0 gap=0.96s first_ok_after_exit=0.67 rps 174.4 -> 145.3 -> 158.4 rejoin=4.51s +killing messenger-lab3-api-2 (e3303afc01b2) under load... + errors=0 gap=0.97s first_ok_after_exit=0.755 rps 177.9 -> 146.5 -> 139.0 rejoin=5.18s +{"total": {"seconds": 71.23, "rps_ok": 155.0, "writes_ok": 5520, "reads_ok": 5520, "errors": 0, "p95_ms": 65.5, "instances": {"e3303afc01b2": 5483, "1b755b53e9fa": 5557}}, "integrity": {"confirmed_writes": 5520, "stored_messages": 5520, "duplicates": 0, "missing_confirmed": 0}} +PASS: failover, 5520 confirmed writes, no duplicates, nothing lost diff --git a/evidence/failover-series.json b/evidence/failover-series.json new file mode 100644 index 00000000..d0593889 --- /dev/null +++ b/evidence/failover-series.json @@ -0,0 +1,1064 @@ +{ + "mode": "failover", + "stop_method": "docker stop --time 9 (SIGTERM, drain)", + "concurrency": 4, + "workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest", + "replicas_before": [ + "messenger-lab3-api-1", + "messenger-lab3-api-2" + ], + "runs": [ + { + "replica": "messenger-lab3-api-1", + "instance_id": "1b755b53e9fa", + "state_after_stop": "exited exit=0", + "stop_command_seconds": 5.06, + "last_served_by_target_after_sigterm_s": 2.05, + "first_success_after_container_exit_s": 0.018, + "rps_back_to_90pct_after_exit_s": 2.76, + "max_gap_without_success_s": 0.091, + "rejoin_after_docker_start_s": 5.15, + "baseline": { + "seconds": 10.0, + "rps_ok": 129.0, + "writes_ok": 646, + "reads_ok": 644, + "errors": 0, + "p95_ms": 70.3, + "instances": { + "1b755b53e9fa": 645, + "e3303afc01b2": 645 + } + }, + "during_stop_and_outage": { + "seconds": 20.19, + "rps_ok": 114.8, + "writes_ok": 1159, + "reads_ok": 1158, + "errors": 0, + "p95_ms": 77.4, + "instances": { + "1b755b53e9fa": 123, + "e3303afc01b2": 2194 + } + }, + "after_restore": { + "seconds": 10.16, + "rps_ok": 113.1, + "writes_ok": 574, + "reads_ok": 575, + "errors": 0, + "p95_ms": 81.2, + "instances": { + "e3303afc01b2": 824, + "1b755b53e9fa": 325 + } + }, + "marks": { + "sigterm": 10.19, + "exited": 15.24, + "docker_start": 30.37 + } + }, + { + "replica": "messenger-lab3-api-2", + "instance_id": "e3303afc01b2", + "state_after_stop": "exited exit=0", + "stop_command_seconds": 4.88, + "last_served_by_target_after_sigterm_s": 1.76, + "first_success_after_container_exit_s": 0.012, + "rps_back_to_90pct_after_exit_s": 1.58, + "max_gap_without_success_s": 0.088, + "rejoin_after_docker_start_s": 4.91, + "baseline": { + "seconds": 10.0, + "rps_ok": 131.5, + "writes_ok": 658, + "reads_ok": 657, + "errors": 0, + "p95_ms": 74.2, + "instances": { + "e3303afc01b2": 658, + "1b755b53e9fa": 657 + } + }, + "during_stop_and_outage": { + "seconds": 19.99, + "rps_ok": 145.2, + "writes_ok": 1451, + "reads_ok": 1453, + "errors": 0, + "p95_ms": 68.7, + "instances": { + "1b755b53e9fa": 2767, + "e3303afc01b2": 137 + } + }, + "after_restore": { + "seconds": 9.92, + "rps_ok": 165.4, + "writes_ok": 820, + "reads_ok": 821, + "errors": 0, + "p95_ms": 60.3, + "instances": { + "1b755b53e9fa": 1184, + "e3303afc01b2": 457 + } + }, + "marks": { + "sigterm": 50.54, + "exited": 55.42, + "docker_start": 70.54 + } + } + ], + "total": { + "seconds": 80.52, + "rps_ok": 131.9, + "writes_ok": 5312, + "reads_ok": 5312, + "errors": 0, + "p95_ms": 73.4, + "instances": { + "1b755b53e9fa": 5703, + "e3303afc01b2": 4921 + } + }, + "integrity": { + "confirmed_writes": 5312, + "stored_messages": 5312, + "duplicates": 0, + "missing_confirmed": 0 + }, + "timeline_per_second": [ + { + "t": 0, + "ok": 90, + "err": 0, + "instances": { + "1b755b53e9fa": 45, + "e3303afc01b2": 45 + }, + "versions": { + "v2": 90 + } + }, + { + "t": 1, + "ok": 111, + "err": 0, + "instances": { + "1b755b53e9fa": 54, + "e3303afc01b2": 57 + }, + "versions": { + "v2": 111 + } + }, + { + "t": 2, + "ok": 103, + "err": 0, + "instances": { + "1b755b53e9fa": 53, + "e3303afc01b2": 50 + }, + "versions": { + "v2": 103 + } + }, + { + "t": 3, + "ok": 112, + "err": 0, + "instances": { + "1b755b53e9fa": 56, + "e3303afc01b2": 56 + }, + "versions": { + "v2": 112 + } + }, + { + "t": 4, + "ok": 121, + "err": 0, + "instances": { + "e3303afc01b2": 60, + "1b755b53e9fa": 61 + }, + "versions": { + "v2": 121 + } + }, + { + "t": 5, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 87, + "1b755b53e9fa": 84 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 6, + "ok": 130, + "err": 0, + "instances": { + "e3303afc01b2": 65, + "1b755b53e9fa": 65 + }, + "versions": { + "v2": 130 + } + }, + { + "t": 7, + "ok": 143, + "err": 0, + "instances": { + "e3303afc01b2": 70, + "1b755b53e9fa": 73 + }, + "versions": { + "v2": 143 + } + }, + { + "t": 8, + "ok": 152, + "err": 0, + "instances": { + "e3303afc01b2": 77, + "1b755b53e9fa": 75 + }, + "versions": { + "v2": 152 + } + }, + { + "t": 9, + "ok": 120, + "err": 0, + "instances": { + "1b755b53e9fa": 61, + "e3303afc01b2": 59 + }, + "versions": { + "v2": 120 + } + }, + { + "t": 10, + "ok": 149, + "err": 0, + "instances": { + "e3303afc01b2": 76, + "1b755b53e9fa": 73 + }, + "versions": { + "v2": 149 + } + }, + { + "t": 11, + "ok": 102, + "err": 0, + "instances": { + "1b755b53e9fa": 51, + "e3303afc01b2": 51 + }, + "versions": { + "v2": 102 + } + }, + { + "t": 12, + "ok": 128, + "err": 0, + "instances": { + "1b755b53e9fa": 17, + "e3303afc01b2": 111 + }, + "versions": { + "v2": 128 + } + }, + { + "t": 13, + "ok": 117, + "err": 0, + "instances": { + "e3303afc01b2": 117 + }, + "versions": { + "v2": 117 + } + }, + { + "t": 14, + "ok": 143, + "err": 0, + "instances": { + "e3303afc01b2": 143 + }, + "versions": { + "v2": 143 + } + }, + { + "t": 15, + "ok": 64, + "err": 0, + "instances": { + "e3303afc01b2": 64 + }, + "versions": { + "v2": 64 + } + }, + { + "t": 16, + "ok": 92, + "err": 0, + "instances": { + "e3303afc01b2": 92 + }, + "versions": { + "v2": 92 + } + }, + { + "t": 17, + "ok": 151, + "err": 0, + "instances": { + "e3303afc01b2": 151 + }, + "versions": { + "v2": 151 + } + }, + { + "t": 18, + "ok": 129, + "err": 0, + "instances": { + "e3303afc01b2": 129 + }, + "versions": { + "v2": 129 + } + }, + { + "t": 19, + "ok": 108, + "err": 0, + "instances": { + "e3303afc01b2": 108 + }, + "versions": { + "v2": 108 + } + }, + { + "t": 20, + "ok": 72, + "err": 0, + "instances": { + "e3303afc01b2": 72 + }, + "versions": { + "v2": 72 + } + }, + { + "t": 21, + "ok": 109, + "err": 0, + "instances": { + "e3303afc01b2": 109 + }, + "versions": { + "v2": 109 + } + }, + { + "t": 22, + "ok": 138, + "err": 0, + "instances": { + "e3303afc01b2": 138 + }, + "versions": { + "v2": 138 + } + }, + { + "t": 23, + "ok": 150, + "err": 0, + "instances": { + "e3303afc01b2": 150 + }, + "versions": { + "v2": 150 + } + }, + { + "t": 24, + "ok": 115, + "err": 0, + "instances": { + "e3303afc01b2": 115 + }, + "versions": { + "v2": 115 + } + }, + { + "t": 25, + "ok": 92, + "err": 0, + "instances": { + "e3303afc01b2": 92 + }, + "versions": { + "v2": 92 + } + }, + { + "t": 26, + "ok": 150, + "err": 0, + "instances": { + "e3303afc01b2": 150 + }, + "versions": { + "v2": 150 + } + }, + { + "t": 27, + "ok": 146, + "err": 0, + "instances": { + "e3303afc01b2": 146 + }, + "versions": { + "v2": 146 + } + }, + { + "t": 28, + "ok": 111, + "err": 0, + "instances": { + "e3303afc01b2": 111 + }, + "versions": { + "v2": 111 + } + }, + { + "t": 29, + "ok": 59, + "err": 0, + "instances": { + "e3303afc01b2": 59 + }, + "versions": { + "v2": 59 + } + }, + { + "t": 30, + "ok": 87, + "err": 0, + "instances": { + "e3303afc01b2": 87 + }, + "versions": { + "v2": 87 + } + }, + { + "t": 31, + "ok": 95, + "err": 0, + "instances": { + "e3303afc01b2": 95 + }, + "versions": { + "v2": 95 + } + }, + { + "t": 32, + "ok": 122, + "err": 0, + "instances": { + "e3303afc01b2": 122 + }, + "versions": { + "v2": 122 + } + }, + { + "t": 33, + "ok": 119, + "err": 0, + "instances": { + "e3303afc01b2": 119 + }, + "versions": { + "v2": 119 + } + }, + { + "t": 34, + "ok": 72, + "err": 0, + "instances": { + "e3303afc01b2": 72 + }, + "versions": { + "v2": 72 + } + }, + { + "t": 35, + "ok": 151, + "err": 0, + "instances": { + "e3303afc01b2": 92, + "1b755b53e9fa": 59 + }, + "versions": { + "v2": 151 + } + }, + { + "t": 36, + "ok": 134, + "err": 0, + "instances": { + "1b755b53e9fa": 65, + "e3303afc01b2": 69 + }, + "versions": { + "v2": 134 + } + }, + { + "t": 37, + "ok": 142, + "err": 0, + "instances": { + "1b755b53e9fa": 73, + "e3303afc01b2": 69 + }, + "versions": { + "v2": 142 + } + }, + { + "t": 38, + "ok": 94, + "err": 0, + "instances": { + "e3303afc01b2": 47, + "1b755b53e9fa": 47 + }, + "versions": { + "v2": 94 + } + }, + { + "t": 39, + "ok": 82, + "err": 0, + "instances": { + "e3303afc01b2": 42, + "1b755b53e9fa": 40 + }, + "versions": { + "v2": 82 + } + }, + { + "t": 40, + "ok": 139, + "err": 0, + "instances": { + "1b755b53e9fa": 69, + "e3303afc01b2": 70 + }, + "versions": { + "v2": 139 + } + }, + { + "t": 41, + "ok": 111, + "err": 0, + "instances": { + "e3303afc01b2": 55, + "1b755b53e9fa": 56 + }, + "versions": { + "v2": 111 + } + }, + { + "t": 42, + "ok": 156, + "err": 0, + "instances": { + "e3303afc01b2": 77, + "1b755b53e9fa": 79 + }, + "versions": { + "v2": 156 + } + }, + { + "t": 43, + "ok": 143, + "err": 0, + "instances": { + "e3303afc01b2": 71, + "1b755b53e9fa": 72 + }, + "versions": { + "v2": 143 + } + }, + { + "t": 44, + "ok": 144, + "err": 0, + "instances": { + "1b755b53e9fa": 71, + "e3303afc01b2": 73 + }, + "versions": { + "v2": 144 + } + }, + { + "t": 45, + "ok": 127, + "err": 0, + "instances": { + "e3303afc01b2": 65, + "1b755b53e9fa": 62 + }, + "versions": { + "v2": 127 + } + }, + { + "t": 46, + "ok": 142, + "err": 0, + "instances": { + "1b755b53e9fa": 73, + "e3303afc01b2": 69 + }, + "versions": { + "v2": 142 + } + }, + { + "t": 47, + "ok": 158, + "err": 0, + "instances": { + "e3303afc01b2": 79, + "1b755b53e9fa": 79 + }, + "versions": { + "v2": 158 + } + }, + { + "t": 48, + "ok": 109, + "err": 0, + "instances": { + "1b755b53e9fa": 54, + "e3303afc01b2": 55 + }, + "versions": { + "v2": 109 + } + }, + { + "t": 49, + "ok": 104, + "err": 0, + "instances": { + "1b755b53e9fa": 51, + "e3303afc01b2": 53 + }, + "versions": { + "v2": 104 + } + }, + { + "t": 50, + "ok": 111, + "err": 0, + "instances": { + "1b755b53e9fa": 55, + "e3303afc01b2": 56 + }, + "versions": { + "v2": 111 + } + }, + { + "t": 51, + "ok": 167, + "err": 0, + "instances": { + "1b755b53e9fa": 85, + "e3303afc01b2": 82 + }, + "versions": { + "v2": 167 + } + }, + { + "t": 52, + "ok": 156, + "err": 0, + "instances": { + "e3303afc01b2": 29, + "1b755b53e9fa": 127 + }, + "versions": { + "v2": 156 + } + }, + { + "t": 53, + "ok": 158, + "err": 0, + "instances": { + "1b755b53e9fa": 158 + }, + "versions": { + "v2": 158 + } + }, + { + "t": 54, + "ok": 120, + "err": 0, + "instances": { + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 120 + } + }, + { + "t": 55, + "ok": 79, + "err": 0, + "instances": { + "1b755b53e9fa": 79 + }, + "versions": { + "v2": 79 + } + }, + { + "t": 56, + "ok": 157, + "err": 0, + "instances": { + "1b755b53e9fa": 157 + }, + "versions": { + "v2": 157 + } + }, + { + "t": 57, + "ok": 160, + "err": 0, + "instances": { + "1b755b53e9fa": 160 + }, + "versions": { + "v2": 160 + } + }, + { + "t": 58, + "ok": 120, + "err": 0, + "instances": { + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 120 + } + }, + { + "t": 59, + "ok": 62, + "err": 0, + "instances": { + "1b755b53e9fa": 62 + }, + "versions": { + "v2": 62 + } + }, + { + "t": 60, + "ok": 138, + "err": 0, + "instances": { + "1b755b53e9fa": 138 + }, + "versions": { + "v2": 138 + } + }, + { + "t": 61, + "ok": 179, + "err": 0, + "instances": { + "1b755b53e9fa": 179 + }, + "versions": { + "v2": 179 + } + }, + { + "t": 62, + "ok": 166, + "err": 0, + "instances": { + "1b755b53e9fa": 166 + }, + "versions": { + "v2": 166 + } + }, + { + "t": 63, + "ok": 134, + "err": 0, + "instances": { + "1b755b53e9fa": 134 + }, + "versions": { + "v2": 134 + } + }, + { + "t": 64, + "ok": 161, + "err": 0, + "instances": { + "1b755b53e9fa": 161 + }, + "versions": { + "v2": 161 + } + }, + { + "t": 65, + "ok": 194, + "err": 0, + "instances": { + "1b755b53e9fa": 194 + }, + "versions": { + "v2": 194 + } + }, + { + "t": 66, + "ok": 191, + "err": 0, + "instances": { + "1b755b53e9fa": 191 + }, + "versions": { + "v2": 191 + } + }, + { + "t": 67, + "ok": 152, + "err": 0, + "instances": { + "1b755b53e9fa": 152 + }, + "versions": { + "v2": 152 + } + }, + { + "t": 68, + "ok": 120, + "err": 0, + "instances": { + "1b755b53e9fa": 120 + }, + "versions": { + "v2": 120 + } + }, + { + "t": 69, + "ok": 146, + "err": 0, + "instances": { + "1b755b53e9fa": 146 + }, + "versions": { + "v2": 146 + } + }, + { + "t": 70, + "ok": 154, + "err": 0, + "instances": { + "1b755b53e9fa": 154 + }, + "versions": { + "v2": 154 + } + }, + { + "t": 71, + "ok": 117, + "err": 0, + "instances": { + "1b755b53e9fa": 117 + }, + "versions": { + "v2": 117 + } + }, + { + "t": 72, + "ok": 89, + "err": 0, + "instances": { + "1b755b53e9fa": 89 + }, + "versions": { + "v2": 89 + } + }, + { + "t": 73, + "ok": 181, + "err": 0, + "instances": { + "1b755b53e9fa": 181 + }, + "versions": { + "v2": 181 + } + }, + { + "t": 74, + "ok": 206, + "err": 0, + "instances": { + "1b755b53e9fa": 206 + }, + "versions": { + "v2": 206 + } + }, + { + "t": 75, + "ok": 197, + "err": 0, + "instances": { + "1b755b53e9fa": 134, + "e3303afc01b2": 63 + }, + "versions": { + "v2": 197 + } + }, + { + "t": 76, + "ok": 145, + "err": 0, + "instances": { + "1b755b53e9fa": 72, + "e3303afc01b2": 73 + }, + "versions": { + "v2": 145 + } + }, + { + "t": 77, + "ok": 197, + "err": 0, + "instances": { + "1b755b53e9fa": 99, + "e3303afc01b2": 98 + }, + "versions": { + "v2": 197 + } + }, + { + "t": 78, + "ok": 203, + "err": 0, + "instances": { + "1b755b53e9fa": 101, + "e3303afc01b2": 102 + }, + "versions": { + "v2": 203 + } + }, + { + "t": 79, + "ok": 184, + "err": 0, + "instances": { + "1b755b53e9fa": 93, + "e3303afc01b2": 91 + }, + "versions": { + "v2": 184 + } + }, + { + "t": 80, + "ok": 71, + "err": 0, + "instances": { + "1b755b53e9fa": 35, + "e3303afc01b2": 36 + }, + "versions": { + "v2": 71 + } + } + ] +} diff --git a/evidence/failover-series.txt b/evidence/failover-series.txt new file mode 100644 index 00000000..35ac2865 --- /dev/null +++ b/evidence/failover-series.txt @@ -0,0 +1,7 @@ +# failover series 2026-11-07T18:09:39.2795947+03:00: make failover-series (docker stop, SIGTERM + drain) +stopping messenger-lab3-api-1 (1b755b53e9fa) under load... + errors=0 gap=0.09s first_ok_after_exit=0.018 rps 129.0 -> 114.8 -> 113.1 rejoin=5.15s +stopping messenger-lab3-api-2 (e3303afc01b2) under load... + errors=0 gap=0.09s first_ok_after_exit=0.012 rps 131.5 -> 145.2 -> 165.4 rejoin=4.91s +{"total": {"seconds": 80.52, "rps_ok": 131.9, "writes_ok": 5312, "reads_ok": 5312, "errors": 0, "p95_ms": 73.4, "instances": {"1b755b53e9fa": 5703, "e3303afc01b2": 4921}}, "integrity": {"confirmed_writes": 5312, "stored_messages": 5312, "duplicates": 0, "missing_confirmed": 0}} +PASS: failover, 5312 confirmed writes, no duplicates, nothing lost diff --git a/evidence/failover.txt b/evidence/failover.txt index 8778b332..3ce6c5d7 100644 --- a/evidence/failover.txt +++ b/evidence/failover.txt @@ -1,41 +1,38 @@ -# failover run 2026-11-07T17:39:07.3980307+03:00 +# failover runs 2026-11-07T18:03:34.5946145+03:00 (HAProxy, drain 4 s) # command (= make test-failover ACTION_SCRIPT=scripts/stop-one.cmd): python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080 ## run 1 - before (shift requests through LB: 0) -messenger-lab3-api-1 8b33a91fafee Up 20 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) +messenger-lab3-api-1 1b755b53e9fa Up 53 seconds (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 53 seconds (healthy) +api1 172.25.0.2 UP +api2 172.25.0.3 UP Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 1.27} +{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 6.421} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 ## run 1 - after -messenger-lab3-api-1 8b33a91fafee Exited (0) 14 seconds ago -messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) +messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy) +messenger-lab3-api-2 e3303afc01b2 Exited (0) 9 seconds ago +api1 172.25.0.2 UP ## restored (docker compose --env-file .env up -d --wait) -messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) -## run 2 - before (shift requests through LB: 0) -messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy) +messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 10 seconds (healthy) +api1 172.25.0.2 UP +api2 172.25.0.3 UP +## run 2 - before (shift requests through LB: 1) +messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 11 seconds (healthy) +api1 172.25.0.2 UP +api2 172.25.0.3 UP Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.166} +{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 0, "first_recovery_seconds": 6.437} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 ## run 2 - after -messenger-lab3-api-1 8b33a91fafee Exited (0) 14 seconds ago -messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) +messenger-lab3-api-1 1b755b53e9fa Exited (0) 9 seconds ago +messenger-lab3-api-2 e3303afc01b2 Up 30 seconds (healthy) +api2 172.25.0.3 UP ## restored (docker compose --env-file .env up -d --wait) -messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) -## run 3 - before (shift requests through LB: 1) -messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy) -Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 1.213} -PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. -exit=0 -## run 3 - after -messenger-lab3-api-1 8b33a91fafee Up 22 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Exited (0) 14 seconds ago -## restored (docker compose --env-file .env up -d --wait) -messenger-lab3-api-1 8b33a91fafee Up 31 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 6 seconds (healthy) +messenger-lab3-api-1 1b755b53e9fa Up 9 seconds (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 43 seconds (healthy) +api1 172.25.0.2 UP +api2 172.25.0.3 UP diff --git a/evidence/idempotency.txt b/evidence/idempotency.txt index 41104879..5e96baac 100644 --- a/evidence/idempotency.txt +++ b/evidence/idempotency.txt @@ -1,10 +1,7 @@ -# idempotency run 2026-11-07T17:36:58.5045267+03:00 -## make test-idempotency PYTHON=python -python scripts/check_idempotency.py --base-url "http://localhost:8080" -round 0: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10} -round 1: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10} -round 2: 20 responses, 1 message, replicas {'8b33a91fafee': 10, 'a87734e887a5': 10} -round 3: 20 responses, 1 message, replicas {'8b33a91fafee': 10, 'a87734e887a5': 10} -round 4: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10} -PASS: 5 keys x 20 parallel requests -> 5 messages via 2 replicas {'a87734e887a5': 50, '8b33a91fafee': 50} -exit=0 +# make test-idempotency 2026-11-07T18:03:22.9365380+03:00 via HAProxy +round 0: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +round 1: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +round 2: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +round 3: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +round 4: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10} +PASS: 5 keys x 20 parallel requests -> 5 messages via 2 replicas {'e3303afc01b2': 50, '1b755b53e9fa': 50} diff --git a/evidence/instances.txt b/evidence/instances.txt index 09f89bca..6f0b9904 100644 --- a/evidence/instances.txt +++ b/evidence/instances.txt @@ -1,16 +1,8 @@ -# X-Instance-Id run 2026-11-07T17:41:06.5911429+03:00 -## python scripts/show_instances.py --base-url http://localhost:8080 -login + chat via LB http://localhost:8080: one session token for user 2ec10789-7814-4751-9e23-7b85215c262e (token not printed) -1. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee -2. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 -3. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee -4. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 -5. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee -6. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5 -PASS: same session served by 2 replicas without re-login {'8b33a91fafee': 3, 'a87734e887a5': 3} -exit=0 -## docker compose ps api (X-Instance-Id = container hostname = short container ID) -messenger-lab3-api-1 8b33a91fafee Up 31 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 31 seconds (healthy) -/messenger-lab3-api-1 hostname=8b33a91fafee -/messenger-lab3-api-2 hostname=a87734e887a5 +login + chat via LB http://localhost:8080: one session token for user 5757ef5e-391e-4369-85c5-19499d5a4930 (token not printed) +1. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2 +2. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa +3. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2 +4. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa +5. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2 +6. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa +PASS: same session served by 2 replicas without re-login {'e3303afc01b2': 3, '1b755b53e9fa': 3} diff --git a/evidence/limits.txt b/evidence/limits.txt new file mode 100644 index 00000000..f7d5d7d9 --- /dev/null +++ b/evidence/limits.txt @@ -0,0 +1,30 @@ +# isolation/limits 2026-11-07T18:03:06.9670680+03:00 +## per container: CPU limit (NanoCpus), memory limit, published ports, networks, mounts +/messenger-lab3-api-1 cpus=1000000000 mem=268435456 reserv_mem=134217728 ports= networks=messenger-lab3_data messenger-lab3_edge mounts= +/messenger-lab3-api-2 cpus=1000000000 mem=268435456 reserv_mem=134217728 ports= networks=messenger-lab3_data messenger-lab3_edge mounts= +/messenger-lab3-lb-1 cpus=500000000 mem=134217728 reserv_mem=33554432 ports=8080/tcp->127.0.0.1:8080 ::1:8080 networks=messenger-lab3_edge mounts=bind:/usr/local/etc/haproxy/haproxy.cfg +/messenger-lab3-migrate-1 cpus=500000000 mem=134217728 reserv_mem=0 ports= networks=messenger-lab3_data mounts= +/messenger-lab3-postgres-1 cpus=1000000000 mem=536870912 reserv_mem=268435456 ports= networks=messenger-lab3_data mounts=volume:/var/lib/postgresql/data +## docker stats --no-stream (this project) +NAME CPU % MEM USAGE / LIMIT +messenger-lab3-api-1 0.87% 41.18MiB / 256MiB +messenger-lab3-api-2 0.62% 41.18MiB / 256MiB +messenger-lab3-lb-1 0.21% 14.71MiB / 128MiB +messenger-lab3-postgres-1 0.15% 38.04MiB / 512MiB +## container env mentioning PASSWORD (expect only *_PASSWORD_FILE paths, no values) +messenger-lab3-api-1 DB_PASSWORD_FILE=/run/secrets/postgres_password +messenger-lab3-api-2 DB_PASSWORD_FILE=/run/secrets/postgres_password +messenger-lab3-migrate-1 DB_PASSWORD_FILE=/run/secrets/postgres_password +messenger-lab3-postgres-1 POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password +## secret file inside api: owner/mode only, value not printed +total 4 +-r-------- 1 10001 10001 18 Nov 7 15:02 postgres_password +## lb has no secrets and cannot resolve postgres (not on the data network) +no /run/secrets +postgres not resolvable from lb +## networks +messenger-lab3_data internal=true +messenger-lab3_edge internal=false +## docker port messenger-lab3-api-1 (empty = no published port) +## occurrences of container_name in compose.yaml +0 diff --git a/evidence/persistence.txt b/evidence/persistence.txt index 7586ea5e..1841f6e4 100644 --- a/evidence/persistence.txt +++ b/evidence/persistence.txt @@ -1,18 +1,21 @@ -# persistence run 2026-11-07T17:40:29.1339292+03:00 +# persistence run 2026-11-07T18:04:54.7155080+03:00 # command (= make test-persistence ACTION_SCRIPT=scripts/restart.cmd): python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080 ## before -messenger-lab3-api-1 8b33a91fafee Up 38 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 14 seconds (healthy) -messenger-lab3-lb-1 4ba96e32076f Up 3 minutes (healthy) -messenger-lab3-migrate-1 18f3889ef9a7 Exited (0) 14 seconds ago -messenger-lab3-postgres-1 4c60c6522907 Up 3 minutes (healthy) +NAME CONTAINER ID STATUS +messenger-lab3-api-1 1b755b53e9fa Up 22 seconds (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 56 seconds (healthy) +messenger-lab3-lb-1 760a68ddce9c Up 3 minutes (healthy) +messenger-lab3-postgres-1 2265aa5e595d Up 3 minutes (healthy) +volume messenger-lab3_pgdata Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 -## after (same container IDs, fresh uptime = restarted; volume pgdata kept) -messenger-lab3-api-1 8b33a91fafee Up 11 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 11 seconds (healthy) -messenger-lab3-lb-1 4ba96e32076f Up 5 seconds (healthy) -messenger-lab3-migrate-1 18f3889ef9a7 Exited (0) 12 seconds ago -messenger-lab3-postgres-1 4c60c6522907 Up 15 seconds (healthy) +## after (same IDs, new uptime) +NAME CONTAINER ID STATUS +messenger-lab3-api-1 1b755b53e9fa Up 12 seconds (healthy) +messenger-lab3-api-2 e3303afc01b2 Up 11 seconds (healthy) +messenger-lab3-lb-1 760a68ddce9c Up 5 seconds (healthy) +messenger-lab3-postgres-1 2265aa5e595d Up 16 seconds (healthy) volume messenger-lab3_pgdata +api1 172.25.0.2 UP +api2 172.25.0.3 UP diff --git a/evidence/rolling-update-migration.txt b/evidence/rolling-update-migration.txt new file mode 100644 index 00000000..5fff24ea --- /dev/null +++ b/evidence/rolling-update-migration.txt @@ -0,0 +1,24 @@ +# rolling update dev -> v2 with migration 003 applied under load, 2026-11-07 17:59 +# Console output copied verbatim; the JSON of this run was overwritten by the later v2 -> v3 run (rolling-update.json). +# Old replicas: image messenger-lab3-app:dev, built before migrations/003_session_indexes.sql existed +# (its readiness requires only 001, 002). New image messenger-lab3-app:v2 contains 001-003. +# Command: APP_VERSION=v2 python scripts/failover_series.py --mode action --action scripts/rolling-update.cmd --output evidence/rolling-update.json +running scripts/rolling-update.cmd under load... +{"total": {"seconds": 70.91, "rps_ok": 125.7, "writes_ok": 4458, "reads_ok": 4458, "errors": 0, "p95_ms": 78.1, "instances": {"419aeb0c6c36": 2714, "e90dced210a7": 1695, "a8021c6dc466": 2752, "a2bb8bd69281": 1755}}, "integrity": {"confirmed_writes": 4458, "stored_messages": 4458, "duplicates": 0, "missing_confirmed": 0}} +PASS: action, 4458 confirmed writes, no duplicates, nothing lost +## rolling-update.sh output and metrics +17:59:39 rolling 2 replica(s) to messenger-lab3-app:v2 +17:59:48 migrations applied once +17:59:59 new replica(s) UP in LB: a8021c6dc466 ; stopping old e90dced210a7 +18:00:05 old e90dced210a7 drained and removed +18:00:17 new replica(s) UP in LB: a2bb8bd69281 a8021c6dc466 ; stopping old 419aeb0c6c36 +18:00:24 old 419aeb0c6c36 drained and removed +messenger-lab3-api-3 messenger-lab3-app:v2 Up 33 seconds (healthy) +messenger-lab3-api-4 messenger-lab3-app:v2 Up 15 seconds (healthy) +18:00:24 done; set APP_VERSION=v2 in .env to keep this version on the next compose up +action_seconds 45.73 max_gap 0.097 +versions before {'dev': 1001} during {'dev': 3408, 'v2': 2234} after {'v2': 2265} +during {'seconds': 45.73, 'rps_ok': 123.4, 'writes_ok': 2821, 'reads_ok': 2821, 'errors': 0, 'p95_ms': 78.5, 'instances': {'419aeb0c6c36': 2214, 'e90dced210a7': 1194, 'a8021c6dc466': 1615, 'a2bb8bd69281': 619}} +# After 003 was applied at 17:59:48 both dev replicas stayed in rotation until they were replaced +# (17:59:59 and 18:00:17); dev served 3408 successful responses during the action with 0 errors overall, +# i.e. the old build stayed ready and correct on the expanded schema. diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..4ea65bdb --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,955 @@ +{ + "mode": "action", + "stop_method": "docker stop --time 9 (SIGTERM, drain)", + "concurrency": 4, + "workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest", + "replicas_before": [ + "messenger-lab3-api-1", + "messenger-lab3-api-2" + ], + "runs": [ + { + "action": "rolling-update.cmd", + "action_exit_code": 0, + "action_output": [ + "18:11:19 rolling 2 replica(s) to messenger-lab3-app:v3", + "18:11:26 migrations applied once", + "18:11:37 new replica(s) UP in LB: 3cfcdcac70c7 ; stopping old 1b755b53e9fa", + "18:11:43 old 1b755b53e9fa drained and removed", + "18:11:54 new replica(s) UP in LB: 13c61b35afd7 3cfcdcac70c7 ; stopping old e3303afc01b2", + "18:12:00 old e3303afc01b2 drained and removed", + "messenger-lab3-api-3 messenger-lab3-app:v3 Up 32 seconds (healthy)", + "messenger-lab3-api-4 messenger-lab3-app:v3 Up 15 seconds (healthy)", + "18:12:01 done; set APP_VERSION=v3 in .env to keep this version on the next compose up" + ], + "action_seconds": 42.67, + "max_gap_without_success_s": 0.073, + "baseline": { + "seconds": 10.1, + "rps_ok": 181.4, + "writes_ok": 917, + "reads_ok": 916, + "errors": 0, + "p95_ms": 53.4, + "instances": { + "1b755b53e9fa": 915, + "e3303afc01b2": 918 + } + }, + "during_action": { + "seconds": 42.67, + "rps_ok": 145.8, + "writes_ok": 3110, + "reads_ok": 3110, + "errors": 0, + "p95_ms": 68.3, + "instances": { + "e3303afc01b2": 2471, + "1b755b53e9fa": 1470, + "3cfcdcac70c7": 1689, + "13c61b35afd7": 590 + } + }, + "after_action": { + "seconds": 15.0, + "rps_ok": 185.8, + "writes_ok": 1394, + "reads_ok": 1393, + "errors": 0, + "p95_ms": 54.4, + "instances": { + "3cfcdcac70c7": 1394, + "13c61b35afd7": 1393 + } + }, + "versions_before": { + "v2": 1833 + }, + "versions_during": { + "v2": 3941, + "v3": 2279 + }, + "versions_after": { + "v3": 2787 + }, + "marks": { + "action_start": 10.1, + "action_end": 52.77 + } + } + ], + "total": { + "seconds": 67.8, + "rps_ok": 160.0, + "writes_ok": 5423, + "reads_ok": 5423, + "errors": 0, + "p95_ms": 62.6, + "instances": { + "1b755b53e9fa": 2385, + "e3303afc01b2": 3389, + "3cfcdcac70c7": 3086, + "13c61b35afd7": 1986 + } + }, + "integrity": { + "confirmed_writes": 5423, + "stored_messages": 5423, + "duplicates": 0, + "missing_confirmed": 0 + }, + "timeline_per_second": [ + { + "t": 0, + "ok": 174, + "err": 0, + "instances": { + "1b755b53e9fa": 87, + "e3303afc01b2": 87 + }, + "versions": { + "v2": 174 + } + }, + { + "t": 1, + "ok": 149, + "err": 0, + "instances": { + "e3303afc01b2": 76, + "1b755b53e9fa": 73 + }, + "versions": { + "v2": 149 + } + }, + { + "t": 2, + "ok": 148, + "err": 0, + "instances": { + "e3303afc01b2": 73, + "1b755b53e9fa": 75 + }, + "versions": { + "v2": 148 + } + }, + { + "t": 3, + "ok": 204, + "err": 0, + "instances": { + "1b755b53e9fa": 102, + "e3303afc01b2": 102 + }, + "versions": { + "v2": 204 + } + }, + { + "t": 4, + "ok": 226, + "err": 0, + "instances": { + "1b755b53e9fa": 113, + "e3303afc01b2": 113 + }, + "versions": { + "v2": 226 + } + }, + { + "t": 5, + "ok": 192, + "err": 0, + "instances": { + "1b755b53e9fa": 95, + "e3303afc01b2": 97 + }, + "versions": { + "v2": 192 + } + }, + { + "t": 6, + "ok": 156, + "err": 0, + "instances": { + "e3303afc01b2": 77, + "1b755b53e9fa": 79 + }, + "versions": { + "v2": 156 + } + }, + { + "t": 7, + "ok": 212, + "err": 0, + "instances": { + "1b755b53e9fa": 106, + "e3303afc01b2": 106 + }, + "versions": { + "v2": 212 + } + }, + { + "t": 8, + "ok": 193, + "err": 0, + "instances": { + "e3303afc01b2": 96, + "1b755b53e9fa": 97 + }, + "versions": { + "v2": 193 + } + }, + { + "t": 9, + "ok": 164, + "err": 0, + "instances": { + "1b755b53e9fa": 80, + "e3303afc01b2": 84 + }, + "versions": { + "v2": 164 + } + }, + { + "t": 10, + "ok": 100, + "err": 0, + "instances": { + "1b755b53e9fa": 53, + "e3303afc01b2": 47 + }, + "versions": { + "v2": 100 + } + }, + { + "t": 11, + "ok": 136, + "err": 0, + "instances": { + "e3303afc01b2": 69, + "1b755b53e9fa": 67 + }, + "versions": { + "v2": 136 + } + }, + { + "t": 12, + "ok": 161, + "err": 0, + "instances": { + "e3303afc01b2": 81, + "1b755b53e9fa": 80 + }, + "versions": { + "v2": 161 + } + }, + { + "t": 13, + "ok": 133, + "err": 0, + "instances": { + "1b755b53e9fa": 67, + "e3303afc01b2": 66 + }, + "versions": { + "v2": 133 + } + }, + { + "t": 14, + "ok": 99, + "err": 0, + "instances": { + "e3303afc01b2": 49, + "1b755b53e9fa": 50 + }, + "versions": { + "v2": 99 + } + }, + { + "t": 15, + "ok": 130, + "err": 0, + "instances": { + "e3303afc01b2": 66, + "1b755b53e9fa": 64 + }, + "versions": { + "v2": 130 + } + }, + { + "t": 16, + "ok": 215, + "err": 0, + "instances": { + "e3303afc01b2": 107, + "1b755b53e9fa": 108 + }, + "versions": { + "v2": 215 + } + }, + { + "t": 17, + "ok": 171, + "err": 0, + "instances": { + "e3303afc01b2": 86, + "1b755b53e9fa": 85 + }, + "versions": { + "v2": 171 + } + }, + { + "t": 18, + "ok": 102, + "err": 0, + "instances": { + "e3303afc01b2": 51, + "1b755b53e9fa": 51 + }, + "versions": { + "v2": 102 + } + }, + { + "t": 19, + "ok": 85, + "err": 0, + "instances": { + "e3303afc01b2": 42, + "1b755b53e9fa": 43 + }, + "versions": { + "v2": 85 + } + }, + { + "t": 20, + "ok": 124, + "err": 0, + "instances": { + "e3303afc01b2": 62, + "1b755b53e9fa": 62 + }, + "versions": { + "v2": 124 + } + }, + { + "t": 21, + "ok": 176, + "err": 0, + "instances": { + "1b755b53e9fa": 88, + "e3303afc01b2": 88 + }, + "versions": { + "v2": 176 + } + }, + { + "t": 22, + "ok": 198, + "err": 0, + "instances": { + "1b755b53e9fa": 99, + "e3303afc01b2": 99 + }, + "versions": { + "v2": 198 + } + }, + { + "t": 23, + "ok": 152, + "err": 0, + "instances": { + "e3303afc01b2": 76, + "1b755b53e9fa": 76 + }, + "versions": { + "v2": 152 + } + }, + { + "t": 24, + "ok": 176, + "err": 0, + "instances": { + "e3303afc01b2": 88, + "1b755b53e9fa": 88 + }, + "versions": { + "v2": 176 + } + }, + { + "t": 25, + "ok": 203, + "err": 0, + "instances": { + "e3303afc01b2": 91, + "1b755b53e9fa": 91, + "3cfcdcac70c7": 21 + }, + "versions": { + "v2": 182, + "v3": 21 + } + }, + { + "t": 26, + "ok": 194, + "err": 0, + "instances": { + "3cfcdcac70c7": 64, + "1b755b53e9fa": 64, + "e3303afc01b2": 66 + }, + "versions": { + "v3": 64, + "v2": 130 + } + }, + { + "t": 27, + "ok": 161, + "err": 0, + "instances": { + "3cfcdcac70c7": 55, + "1b755b53e9fa": 54, + "e3303afc01b2": 52 + }, + "versions": { + "v3": 55, + "v2": 106 + } + }, + { + "t": 28, + "ok": 104, + "err": 0, + "instances": { + "e3303afc01b2": 35, + "3cfcdcac70c7": 34, + "1b755b53e9fa": 35 + }, + "versions": { + "v2": 70, + "v3": 34 + } + }, + { + "t": 29, + "ok": 156, + "err": 0, + "instances": { + "3cfcdcac70c7": 52, + "e3303afc01b2": 52, + "1b755b53e9fa": 52 + }, + "versions": { + "v3": 52, + "v2": 104 + } + }, + { + "t": 30, + "ok": 174, + "err": 0, + "instances": { + "e3303afc01b2": 58, + "1b755b53e9fa": 58, + "3cfcdcac70c7": 58 + }, + "versions": { + "v2": 116, + "v3": 58 + } + }, + { + "t": 31, + "ok": 191, + "err": 0, + "instances": { + "e3303afc01b2": 74, + "3cfcdcac70c7": 74, + "1b755b53e9fa": 43 + }, + "versions": { + "v2": 117, + "v3": 74 + } + }, + { + "t": 32, + "ok": 162, + "err": 0, + "instances": { + "3cfcdcac70c7": 81, + "e3303afc01b2": 81 + }, + "versions": { + "v3": 81, + "v2": 81 + } + }, + { + "t": 33, + "ok": 166, + "err": 0, + "instances": { + "e3303afc01b2": 83, + "3cfcdcac70c7": 83 + }, + "versions": { + "v2": 83, + "v3": 83 + } + }, + { + "t": 34, + "ok": 175, + "err": 0, + "instances": { + "3cfcdcac70c7": 87, + "e3303afc01b2": 88 + }, + "versions": { + "v3": 87, + "v2": 88 + } + }, + { + "t": 35, + "ok": 114, + "err": 0, + "instances": { + "e3303afc01b2": 57, + "3cfcdcac70c7": 57 + }, + "versions": { + "v2": 57, + "v3": 57 + } + }, + { + "t": 36, + "ok": 179, + "err": 0, + "instances": { + "3cfcdcac70c7": 91, + "e3303afc01b2": 88 + }, + "versions": { + "v3": 91, + "v2": 88 + } + }, + { + "t": 37, + "ok": 92, + "err": 0, + "instances": { + "e3303afc01b2": 46, + "3cfcdcac70c7": 46 + }, + "versions": { + "v2": 46, + "v3": 46 + } + }, + { + "t": 38, + "ok": 101, + "err": 0, + "instances": { + "3cfcdcac70c7": 49, + "e3303afc01b2": 52 + }, + "versions": { + "v3": 49, + "v2": 52 + } + }, + { + "t": 39, + "ok": 121, + "err": 0, + "instances": { + "e3303afc01b2": 60, + "3cfcdcac70c7": 61 + }, + "versions": { + "v2": 60, + "v3": 61 + } + }, + { + "t": 40, + "ok": 128, + "err": 0, + "instances": { + "e3303afc01b2": 64, + "3cfcdcac70c7": 64 + }, + "versions": { + "v2": 64, + "v3": 64 + } + }, + { + "t": 41, + "ok": 148, + "err": 0, + "instances": { + "e3303afc01b2": 73, + "3cfcdcac70c7": 75 + }, + "versions": { + "v2": 73, + "v3": 75 + } + }, + { + "t": 42, + "ok": 115, + "err": 0, + "instances": { + "e3303afc01b2": 56, + "3cfcdcac70c7": 54, + "13c61b35afd7": 5 + }, + "versions": { + "v2": 56, + "v3": 59 + } + }, + { + "t": 43, + "ok": 150, + "err": 0, + "instances": { + "e3303afc01b2": 50, + "13c61b35afd7": 51, + "3cfcdcac70c7": 49 + }, + "versions": { + "v2": 50, + "v3": 100 + } + }, + { + "t": 44, + "ok": 156, + "err": 0, + "instances": { + "e3303afc01b2": 51, + "3cfcdcac70c7": 53, + "13c61b35afd7": 52 + }, + "versions": { + "v2": 51, + "v3": 105 + } + }, + { + "t": 45, + "ok": 140, + "err": 0, + "instances": { + "e3303afc01b2": 48, + "13c61b35afd7": 46, + "3cfcdcac70c7": 46 + }, + "versions": { + "v2": 48, + "v3": 92 + } + }, + { + "t": 46, + "ok": 134, + "err": 0, + "instances": { + "3cfcdcac70c7": 45, + "13c61b35afd7": 45, + "e3303afc01b2": 44 + }, + "versions": { + "v3": 90, + "v2": 44 + } + }, + { + "t": 47, + "ok": 136, + "err": 0, + "instances": { + "e3303afc01b2": 32, + "3cfcdcac70c7": 52, + "13c61b35afd7": 52 + }, + "versions": { + "v2": 32, + "v3": 104 + } + }, + { + "t": 48, + "ok": 117, + "err": 0, + "instances": { + "3cfcdcac70c7": 58, + "13c61b35afd7": 59 + }, + "versions": { + "v3": 117 + } + }, + { + "t": 49, + "ok": 163, + "err": 0, + "instances": { + "3cfcdcac70c7": 82, + "13c61b35afd7": 81 + }, + "versions": { + "v3": 163 + } + }, + { + "t": 50, + "ok": 187, + "err": 0, + "instances": { + "13c61b35afd7": 94, + "3cfcdcac70c7": 93 + }, + "versions": { + "v3": 187 + } + }, + { + "t": 51, + "ok": 140, + "err": 0, + "instances": { + "13c61b35afd7": 69, + "3cfcdcac70c7": 71 + }, + "versions": { + "v3": 140 + } + }, + { + "t": 52, + "ok": 101, + "err": 0, + "instances": { + "3cfcdcac70c7": 50, + "13c61b35afd7": 51 + }, + "versions": { + "v3": 101 + } + }, + { + "t": 53, + "ok": 165, + "err": 0, + "instances": { + "3cfcdcac70c7": 82, + "13c61b35afd7": 83 + }, + "versions": { + "v3": 165 + } + }, + { + "t": 54, + "ok": 240, + "err": 0, + "instances": { + "3cfcdcac70c7": 120, + "13c61b35afd7": 120 + }, + "versions": { + "v3": 240 + } + }, + { + "t": 55, + "ok": 253, + "err": 0, + "instances": { + "13c61b35afd7": 126, + "3cfcdcac70c7": 127 + }, + "versions": { + "v3": 253 + } + }, + { + "t": 56, + "ok": 150, + "err": 0, + "instances": { + "3cfcdcac70c7": 75, + "13c61b35afd7": 75 + }, + "versions": { + "v3": 150 + } + }, + { + "t": 57, + "ok": 142, + "err": 0, + "instances": { + "3cfcdcac70c7": 71, + "13c61b35afd7": 71 + }, + "versions": { + "v3": 142 + } + }, + { + "t": 58, + "ok": 197, + "err": 0, + "instances": { + "3cfcdcac70c7": 98, + "13c61b35afd7": 99 + }, + "versions": { + "v3": 197 + } + }, + { + "t": 59, + "ok": 202, + "err": 0, + "instances": { + "3cfcdcac70c7": 101, + "13c61b35afd7": 101 + }, + "versions": { + "v3": 202 + } + }, + { + "t": 60, + "ok": 175, + "err": 0, + "instances": { + "13c61b35afd7": 87, + "3cfcdcac70c7": 88 + }, + "versions": { + "v3": 175 + } + }, + { + "t": 61, + "ok": 126, + "err": 0, + "instances": { + "3cfcdcac70c7": 62, + "13c61b35afd7": 64 + }, + "versions": { + "v3": 126 + } + }, + { + "t": 62, + "ok": 190, + "err": 0, + "instances": { + "3cfcdcac70c7": 96, + "13c61b35afd7": 94 + }, + "versions": { + "v3": 190 + } + }, + { + "t": 63, + "ok": 209, + "err": 0, + "instances": { + "3cfcdcac70c7": 104, + "13c61b35afd7": 105 + }, + "versions": { + "v3": 209 + } + }, + { + "t": 64, + "ok": 214, + "err": 0, + "instances": { + "13c61b35afd7": 107, + "3cfcdcac70c7": 107 + }, + "versions": { + "v3": 214 + } + }, + { + "t": 65, + "ok": 145, + "err": 0, + "instances": { + "13c61b35afd7": 72, + "3cfcdcac70c7": 73 + }, + "versions": { + "v3": 145 + } + }, + { + "t": 66, + "ok": 166, + "err": 0, + "instances": { + "3cfcdcac70c7": 83, + "13c61b35afd7": 83 + }, + "versions": { + "v3": 166 + } + }, + { + "t": 67, + "ok": 188, + "err": 0, + "instances": { + "3cfcdcac70c7": 94, + "13c61b35afd7": 94 + }, + "versions": { + "v3": 188 + } + } + ] +} diff --git a/evidence/rolling-update.txt b/evidence/rolling-update.txt new file mode 100644 index 00000000..fac58478 --- /dev/null +++ b/evidence/rolling-update.txt @@ -0,0 +1,19 @@ +# rolling update 2026-11-07T18:12:17.4627477+03:00: APP_VERSION=v3 (image prebuilt), python scripts/failover_series.py --mode action --action scripts/rolling-update.cmd +running scripts/rolling-update.cmd under load... +{"total": {"seconds": 67.8, "rps_ok": 160.0, "writes_ok": 5423, "reads_ok": 5423, "errors": 0, "p95_ms": 62.6, "instances": {"1b755b53e9fa": 2385, "e3303afc01b2": 3389, "3cfcdcac70c7": 3086, "13c61b35afd7": 1986}}, "integrity": {"confirmed_writes": 5423, "stored_messages": 5423, "duplicates": 0, "missing_confirmed": 0}} +PASS: action, 5423 confirmed writes, no duplicates, nothing lost +## rolling-update.sh output and metrics +18:11:19 rolling 2 replica(s) to messenger-lab3-app:v3 +18:11:26 migrations applied once +18:11:37 new replica(s) UP in LB: 3cfcdcac70c7 ; stopping old 1b755b53e9fa +18:11:43 old 1b755b53e9fa drained and removed +18:11:54 new replica(s) UP in LB: 13c61b35afd7 3cfcdcac70c7 ; stopping old e3303afc01b2 +18:12:00 old e3303afc01b2 drained and removed +messenger-lab3-api-3 messenger-lab3-app:v3 Up 32 seconds (healthy) +messenger-lab3-api-4 messenger-lab3-app:v3 Up 15 seconds (healthy) +18:12:01 done; set APP_VERSION=v3 in .env to keep this version on the next compose up +action_seconds 42.67 max_gap 0.073 +versions before {'v2': 1833} during {'v2': 3941, 'v3': 2279} after {'v3': 2787} +baseline {'seconds': 10.1, 'rps_ok': 181.4, 'writes_ok': 917, 'reads_ok': 916, 'errors': 0, 'p95_ms': 53.4, 'instances': {'1b755b53e9fa': 915, 'e3303afc01b2': 918}} +during {'seconds': 42.67, 'rps_ok': 145.8, 'writes_ok': 3110, 'reads_ok': 3110, 'errors': 0, 'p95_ms': 68.3, 'instances': {'e3303afc01b2': 2471, '1b755b53e9fa': 1470, '3cfcdcac70c7': 1689, '13c61b35afd7': 590}} +after {'seconds': 15.0, 'rps_ok': 185.8, 'writes_ok': 1394, 'reads_ok': 1393, 'errors': 0, 'p95_ms': 54.4, 'instances': {'3cfcdcac70c7': 1394, '13c61b35afd7': 1393}} diff --git a/evidence/smoke.txt b/evidence/smoke.txt index dec53a71..09308293 100644 --- a/evidence/smoke.txt +++ b/evidence/smoke.txt @@ -1,14 +1,48 @@ -# smoke run 2026-11-07T17:36:58.5045267+03:00 -## docker compose ps -messenger-lab3-api-1 8b33a91fafee Up 19 seconds (healthy) -messenger-lab3-api-2 a87734e887a5 Up 19 seconds (healthy) -messenger-lab3-lb-1 4ba96e32076f Up 13 seconds (healthy) -messenger-lab3-postgres-1 4c60c6522907 Up 23 seconds (healthy) -## make check PYTHON=python -python scripts/check_contract.py +# smoke run 2026-11-07T18:01:21.4109350+03:00, commit 124c83e + working tree +## docker compose --env-file .env down -v + + Container messenger-lab3-lb-1 Removed + Container messenger-lab3-api-4 Removed + Container messenger-lab3-api-3 Removed + Container messenger-lab3-migrate-1 Removed + Container messenger-lab3-postgres-1 Removed + Volume messenger-lab3_pgdata Removing + Volume messenger-lab3_pgdata Removed + Network messenger-lab3_data Removed + Network messenger-lab3_edge Removed +## docker compose --env-file .env config --quiet +exit=0 +## docker compose --env-file .env up -d --build --wait + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-migrate-1 Exited + Container messenger-lab3-postgres-1 Healthy + Container messenger-lab3-api-1 Healthy + Container messenger-lab3-api-2 Healthy + Container messenger-lab3-lb-1 Healthy +exit=0 +## docker compose ps -a +NAME IMAGE STATUS PORTS +messenger-lab3-api-1 messenger-lab3-app:v2 Up 12 seconds (healthy) 8080/tcp +messenger-lab3-api-2 messenger-lab3-app:v2 Up 12 seconds (healthy) 8080/tcp +messenger-lab3-lb-1 haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0 Up 5 seconds (healthy) 127.0.0.1:8080->8080/tcp, [::1]:8080->8080/tcp +messenger-lab3-migrate-1 messenger-lab3-app:v2 Exited (0) 12 seconds ago +messenger-lab3-postgres-1 postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 Up 19 seconds (healthy) 5432/tcp +## migrate log (one-shot) +INFO applied migration 001_init.sql +INFO applied migration 002_idempotency.sql +INFO applied migration 003_session_indexes.sql +## HAProxy pool (scripts/lb-status.sh) +api1 172.25.0.2 UP +api2 172.25.0.3 UP +## make check PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. exit=0 -## make test PYTHON=python -python tests/smoke.py --base-url "http://localhost:8080" +## make test PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. exit=0 + + diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..96c3dcdb --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,79 @@ +# Client -> HAProxy -> api x N. Replicas are discovered through Docker DNS and checked actively. +# +# Timing contract (keep in sync with compose.yaml and app settings): +# check period inter 1s, timeout check 1s +# marked DOWN after fall 2 failed checks -> hung/draining replica leaves the pool in <= 3 s +# marked UP after rise 2 good checks -> new replica gets traffic ~2 s after it is ready +# DNS refresh every 1 s (hold valid 1s); a removed container disappears within ~1-2 s +# crashed process connect refused -> request redispatched to another replica at once +# app DRAIN_SECONDS=4 > 3 s, so a replica receiving SIGTERM is out of rotation before it closes its socket. + +global + log stdout format raw local0 info + maxconn 2048 + # Runtime API for evidence (scripts/lb-status.sh); loopback inside the container only. + stats socket ipv4@127.0.0.1:9999 level operator + +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 0s + +defaults + mode http + log global + option httplog + option dontlognull + option forwardfor + timeout connect 1s + timeout client 30s + timeout server 10s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 2s + timeout check 1s + # A request whose connection to a replica failed is sent to another replica. + retries 2 + option redispatch 1 + +frontend public + bind :8080 + monitor-uri /lb/health + http-request deny deny_status 413 if { req.hdr_val(content-length) gt 1048576 } + + # Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe. + # Other POSTs (users, chats, sessions) are retried only when the connection was never established. + acl message_post method POST + acl message_post_path path_reg ^/api/v1/chats/[^/]+/messages$ + http-request disable-l7-retry if message_post !message_post_path + + default_backend api + +backend api + # Round robin, no stickiness: any replica serves any session from PostgreSQL. + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb + http-check expect status 200 + retry-on conn-failure empty-response response-timeout + http-error status 503 content-type application/json string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API replica is available\",\"request_id\":\"lb\"}}" + http-error status 502 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Bad gateway\",\"request_id\":\"lb\"}}" + http-error status 504 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Gateway timeout\",\"request_id\":\"lb\"}}" + default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none + server-template api 1-10 api:8080 + +# Read-only stats for evidence; not published, reachable only inside the container. +frontend stats + bind 127.0.0.1:8404 + stats enable + stats uri /stats + stats refresh 5s + http-request set-log-level silent diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf deleted file mode 100644 index 0ebbd32b..00000000 --- a/infra/nginx/nginx.conf +++ /dev/null @@ -1,66 +0,0 @@ -worker_processes auto; -pid /tmp/nginx.pid; - -events { - worker_connections 1024; -} - -http { - access_log /dev/stdout; - error_log /dev/stderr warn; - - # Docker embedded DNS. "api" resolves to every running replica of the Compose service; - # with "resolve" below nginx re-reads the set every 2 s, so stopped/scaled replicas leave and join the pool. - resolver 127.0.0.11 valid=2s ipv6=off; - resolver_timeout 2s; - - log_format upstream '$remote_addr "$request" $status upstream=$upstream_addr ' - 'upstream_status=$upstream_status rt=$request_time'; - - upstream api { - zone api_upstream 64k; - # Round robin, no ip_hash/sticky: any replica serves any session from PostgreSQL. - server api:8080 resolve max_fails=1 fail_timeout=5s; - keepalive 32; - # Shorter than uvicorn's keep-alive (5 s), so nginx never reuses a connection the API is closing. - keepalive_timeout 4s; - } - - server { - listen 8080; - access_log /dev/stdout upstream; - client_max_body_size 1m; - - proxy_http_version 1.1; - proxy_set_header Connection ""; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_connect_timeout 1s; - proxy_send_timeout 10s; - proxy_read_timeout 10s; - - # Not-yet-sent requests and idempotent methods go to the next replica on a dead backend. - # Non-idempotent POSTs (users, chats, sessions) are never replayed by the LB. - proxy_next_upstream error timeout http_502 http_504; - proxy_next_upstream_tries 3; - proxy_next_upstream_timeout 5s; - - location = /lb/health { - access_log off; - default_type application/json; - return 200 '{"status":"ok"}'; - } - - # Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe. - location ~ ^/api/v1/chats/[^/]+/messages$ { - proxy_next_upstream error timeout http_502 http_504 non_idempotent; - proxy_pass http://api; - } - - location / { - proxy_pass http://api; - } - } -} diff --git a/migrations/003_session_indexes.sql b/migrations/003_session_indexes.sql new file mode 100644 index 00000000..0b7eb784 --- /dev/null +++ b/migrations/003_session_indexes.sql @@ -0,0 +1,4 @@ +-- Expand-only: new indexes, no column/table removal or rename, so replicas of the previous build keep working +-- while the rolling update is in progress. Destructive changes go to a later release, after old replicas are gone. +CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id); +CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions (expires_at); diff --git a/scripts/failover_series.py b/scripts/failover_series.py new file mode 100755 index 00000000..c9750b45 --- /dev/null +++ b/scripts/failover_series.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs). + +failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe -> + docker start the same container -> wait until it serves again -> settle. Restores before the next one. +action: baseline -> run --action script (e.g. rolling-update) under load -> observe. + +Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body) +and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded. +At the end the whole chat history is read back: every confirmed message must be present exactly once. +Session tokens stay in memory and are never printed. +""" +import argparse +import json +import subprocess +import sys +import threading +import time +import uuid +from collections import Counter, defaultdict +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check, run # noqa: E402 +from resilience import history # noqa: E402 + +PROJECT = 'messenger-lab3' + + +def docker(*args, timeout=60): + return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip() + + +def api_containers(): + """(name, short id) of running api containers of this Compose project, sorted by name.""" + out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT, + '--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}') + return sorted(tuple(line.split()) for line in out.splitlines() if line.strip()) + + +class Load: + def __init__(self, base_url, concurrency): + self.base_url = base_url + self.concurrency = concurrency + self.events = [] # (t, op, ok, instance, version, latency) + self.confirmed = {} # message id -> text + self.lock = threading.Lock() + self.stop = threading.Event() + self.t0 = None + + def now(self): + return time.monotonic() - self.t0 + + def record(self, op, ok, response=None, latency=0.0): + headers = response['headers'] if response else {} + event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None, + headers.get('X-App-Version') if response else None, latency) + with self.lock: + self.events.append(event) + + def worker(self, worker_id, alice, chat): + c = Client(self.base_url, timeout=5) + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20' + index = 0 + key = text = None + while not self.stop.is_set(): + if key is None: + key = uuid.uuid4().hex + text = 'series-%d-%d-%s' % (worker_id, index, key[:8]) + start = time.monotonic() + try: + response = c.message(alice, chat, text, key=key) + self.record('write', True, response, time.monotonic() - start) + with self.lock: + self.confirmed[response['json']['id']] = text + key = None + index += 1 + except Failure: + self.record('write', False, latency=time.monotonic() - start) + time.sleep(0.05) + continue # same key and text: the first attempt may have committed + start = time.monotonic() + try: + response = c.call('GET', path, actor=alice, schema='MessagePage') + self.record('read', True, response, time.monotonic() - start) + except Failure: + self.record('read', False, latency=time.monotonic() - start) + time.sleep(0.05) + + def start(self, alice, chat): + self.t0 = time.monotonic() + self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True) + for i in range(self.concurrency)] + for t in self.threads: + t.start() + + def finish(self): + self.stop.set() + for t in self.threads: + t.join(timeout=10) + + def window(self, start, end): + return [e for e in self.events if start <= e[0] < end] + + +def summarize(events, start, end): + ok = [e for e in events if e[2]] + duration = max(end - start, 1e-9) + latencies = sorted(e[5] for e in ok) + p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None + return { + 'seconds': round(duration, 2), + 'rps_ok': round(len(ok) / duration, 1), + 'writes_ok': sum(1 for e in ok if e[1] == 'write'), + 'reads_ok': sum(1 for e in ok if e[1] == 'read'), + 'errors': sum(1 for e in events if not e[2]), + 'p95_ms': p95, + 'instances': dict(Counter(e[3] for e in ok)), + } + + +def timeline(load, end): + buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()}) + for t, _, ok, instance, version, _ in load.events: + if t >= end: + continue + b = buckets[int(t)] + if ok: + b['ok'] += 1 + b['instances'][instance] += 1 + b['versions'][version] += 1 + else: + b['err'] += 1 + return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'], + 'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])} + for s in range(int(end) + 1)] + + +def max_gap(events, start, end): + """Longest interval without any successful response inside [start, end).""" + times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end] + return round(max(b - a for a, b in zip(times, times[1:])), 3) + + +def wait_serving(load, instance, since, limit=30): + deadline = time.monotonic() + limit + while time.monotonic() < deadline: + with load.lock: + hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None) + if hit is not None: + return hit + time.sleep(0.2) + return None + + +def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill): + time.sleep(baseline) + t_stop = load.now() + if kill: + docker('kill', cid) + else: + docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30) + t_stopped = load.now() + state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid) + time.sleep(observe) + t_start = load.now() + docker('start', cid) + rejoined = wait_serving(load, cid, t_start) + time.sleep(settle) + t_end = load.now() + + with load.lock: + events = list(load.events) + base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop) + outage = [e for e in events if t_stop <= e[0] < t_start] + last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None) + first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None) + # Recovery: first full second after the container stopped whose throughput is >= 90% of baseline. + recovered = None + for s in range(int(t_stopped), int(t_start)): + n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1) + if n >= 0.9 * base['rps_ok']: + recovered = s + 1 - t_stopped + break + return { + 'replica': name, + 'instance_id': cid, + 'state_after_stop': state, + 'stop_command_seconds': round(t_stopped - t_stop, 2), + 'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None, + 'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None, + 'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None, + 'max_gap_without_success_s': max_gap(events, t_stop, t_start), + 'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None, + 'baseline': base, + 'during_stop_and_outage': summarize(outage, t_stop, t_start), + 'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end), + 'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)}, + } + + +def action_run(load, action, baseline, observe): + time.sleep(baseline) + t_action = load.now() + proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600) + t_done = load.now() + time.sleep(observe) + t_end = load.now() + with load.lock: + events = list(load.events) + during = [e for e in events if t_action <= e[0] < t_done] + return { + 'action': Path(action).name, + 'action_exit_code': proc.returncode, + 'action_output': proc.stdout.strip().splitlines()[-30:], + 'action_seconds': round(t_done - t_action, 2), + 'max_gap_without_success_s': max_gap(events, t_action, t_done), + 'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action), + 'during_action': summarize(during, t_action, t_done), + 'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end), + 'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)), + 'versions_during': dict(Counter(e[4] for e in during if e[2])), + 'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)), + 'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)}, + } + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument('--base-url', default=None) + p.add_argument('--mode', choices=['failover', 'action'], default='failover') + p.add_argument('--action', help='script to run under load in --mode action') + p.add_argument('--concurrency', type=int, default=4) + p.add_argument('--baseline', type=float, default=10) + p.add_argument('--observe', type=float, default=15) + p.add_argument('--settle', type=float, default=5) + p.add_argument('--stop-timeout', type=int, default=9) + p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop') + p.add_argument('--output', help='JSON result file (no tokens)') + args = p.parse_args() + check(args.mode != 'action' or args.action, '--action is required in action mode') + + c = Client(args.base_url) + alice, bob, _, chat = c.bootstrap() + replicas = api_containers() + if args.mode == 'failover': + check(len(replicas) >= 2, 'need at least two running api containers') + load = Load(args.base_url, args.concurrency) + load.start(alice, chat) + runs = [] + try: + if args.mode == 'failover': + for name, cid in replicas: + print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True) + runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle, + args.stop_timeout, args.kill)) + r = runs[-1] + print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % ( + r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'], + r['first_success_after_container_exit_s'], r['baseline']['rps_ok'], + r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'], + r['rejoin_after_docker_start_s']), flush=True) + else: + print('running %s under load...' % args.action, flush=True) + runs.append(action_run(load, args.action, args.baseline, args.observe)) + finally: + load.finish() + end = load.now() + + confirmed = dict(load.confirmed) + stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')] + counts = Counter(m['text'] for m in stored) + duplicates = sum(n - 1 for n in counts.values() if n > 1) + stored_ids = {m['id'] for m in stored} + missing = [mid for mid in confirmed if mid not in stored_ids] + integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored), + 'duplicates': duplicates, 'missing_confirmed': len(missing)} + report = { + 'mode': args.mode, + 'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout, + 'concurrency': args.concurrency, + 'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest', + 'replicas_before': [name for name, _ in replicas], + 'runs': runs, + 'total': summarize(load.events, 0, end), + 'integrity': integrity, + 'timeline_per_second': timeline(load, end), + } + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + out = Path(args.output) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(text + '\n', encoding='utf-8') + print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False)) + check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity) + if args.mode == 'action': + check(runs[0]['action_exit_code'] == 0, 'action script failed') + print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed))) + + +if __name__ == '__main__': + run(main) diff --git a/scripts/lb-status.cmd b/scripts/lb-status.cmd new file mode 100644 index 00000000..f1a81fe0 --- /dev/null +++ b/scripts/lb-status.cmd @@ -0,0 +1,3 @@ +@echo off +rem Windows entry point: runs lb-status.sh with Git Bash. +"%ProgramFiles%\Git\bin\sh.exe" "%~dp0lb-status.sh" diff --git a/scripts/lb-status.sh b/scripts/lb-status.sh new file mode 100755 index 00000000..e8277685 --- /dev/null +++ b/scripts/lb-status.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# HAProxy view of the api pool: slot, replica IP, state (2=UP, 0=DOWN), check result. Empty template slots are hidden. +docker compose --env-file .env exec -T lb sh -c \ + "echo 'show servers state api' | nc 127.0.0.1 9999 | awk '\$1 !~ /^#/ && NF > 10 && \$5 != \"-\" {print \$4, \$5, (\$6 == 2 ? \"UP\" : \"DOWN\")}'" diff --git a/scripts/rolling-update.cmd b/scripts/rolling-update.cmd new file mode 100644 index 00000000..9d6cae27 --- /dev/null +++ b/scripts/rolling-update.cmd @@ -0,0 +1,3 @@ +@echo off +rem Windows entry point: runs rolling-update.sh with Git Bash. Set APP_VERSION first. +"%ProgramFiles%\Git\bin\sh.exe" "%~dp0rolling-update.sh" diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..27224d07 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,47 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +# Replace API replicas one at a time with a new build while the LB keeps serving. +# APP_VERSION=v2 scripts/rolling-update.sh +# 1. Build messenger-lab3-app:$APP_VERSION. +# 2. Run migrations ONCE (expand-only, compatible with replicas still on the old build). +# 3. For each old replica: start one new replica (N+1), wait until Docker reports it healthy AND HAProxy +# marks it UP, then docker stop the old one (SIGTERM -> drain -> exit) and remove it. +# Capacity never drops below N. Afterwards put APP_VERSION=$APP_VERSION into .env, otherwise a later +# `docker compose up` recreates replicas from the version written there. +: "${APP_VERSION:?set APP_VERSION of the new build}" +export APP_VERSION +project="${COMPOSE_PROJECT_NAME:-messenger-lab3}" +compose() { docker compose --env-file .env "$@"; } +api_ids() { + docker ps -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.service=api" +} +lb_up() { sh scripts/lb-status.sh | grep -c ' UP$' || true; } + +old=$(api_ids) +count=$(printf '%s\n' "$old" | grep -c . || true) +[ "$count" -ge 1 ] || { echo "no running api replicas" >&2; exit 1; } +echo "$(date +%T) rolling $count replica(s) to messenger-lab3-app:$APP_VERSION" + +compose build --quiet api migrate +compose run --rm --no-deps -T migrate +echo "$(date +%T) migrations applied once" + +for id in $old; do + before=$(lb_up) + compose up -d --no-deps --no-recreate --wait --scale "api=$((count + 1))" api >/dev/null 2>&1 + new=$(api_ids | grep -v -F "$old" || true) + i=0 + until [ "$(lb_up)" -gt "$before" ]; do + i=$((i + 1)) + [ "$i" -le 60 ] || { echo "new replica did not become UP in HAProxy" >&2; exit 1; } + sleep 0.5 + done + echo "$(date +%T) new replica(s) UP in LB: $(printf '%s ' $new); stopping old $id" + docker stop --time 9 "$id" >/dev/null + docker rm "$id" >/dev/null + echo "$(date +%T) old $id drained and removed" +done + +compose ps api --format '{{.Name}} {{.Image}} {{.Status}}' +echo "$(date +%T) done; set APP_VERSION=$APP_VERSION in .env to keep this version on the next compose up" diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index 6301907c..196469e9 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -23,6 +23,7 @@ if [ "$count" -ne 1 ]; then exit 1 fi name=$(docker inspect --format '{{.Name}}' "$matches") -# Graceful: SIGTERM, then SIGKILL after 8 s; returns before the test's 10 s limit. -docker stop --time 8 "$matches" >/dev/null +# Graceful: SIGTERM -> 4 s drain (readiness 503, LB drops it) -> <= 3 s in-flight finish. +# SIGKILL only after 9 s, so the hook still returns before the test's 10 s limit. +docker stop --time 9 "$matches" >/dev/null echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)"