level 4, changed to HAProxy
This commit is contained in:
+11
-6
@@ -4,16 +4,21 @@ APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
# Not passed to containers: compose.yaml gives api/migrate DB_HOST/DB_USER/DB_NAME and
|
||||
# DB_PASSWORD_FILE=/run/secrets/postgres_password (Compose secret built from POSTGRES_PASSWORD below).
|
||||
# The app also accepts DATABASE_URL_FILE or DATABASE_URL instead.
|
||||
# Sessions live in PostgreSQL; no Redis/Valkey, CACHE_URL_FILE is unused.
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# compose.yaml builds DATABASE_URL from these; pick your own local password.
|
||||
# Pick your own local password. It reaches containers only as the postgres_password secret file.
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change-me-locally
|
||||
POSTGRES_DB=messenger
|
||||
SHUTDOWN_TIMEOUT_SECONDS=5
|
||||
# SIGTERM: readiness 503 for DRAIN_SECONDS (> LB detection 3 s), then in-flight requests get
|
||||
# SHUTDOWN_TIMEOUT_SECONDS. Keep DRAIN + SHUTDOWN < stop_grace_period (10 s).
|
||||
DRAIN_SECONDS=4
|
||||
SHUTDOWN_TIMEOUT_SECONDS=3
|
||||
# Image tag messenger-lab3-app:<APP_VERSION>, shown as X-App-Version; scripts/rolling-update.sh sets it.
|
||||
APP_VERSION=dev
|
||||
LB_PORT=8080
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
*.sh text eol=lf
|
||||
*.conf text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
|
||||
+6
-1
@@ -9,7 +9,8 @@ ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
LOG_LEVEL=info \
|
||||
SHUTDOWN_TIMEOUT_SECONDS=5 \
|
||||
DRAIN_SECONDS=4 \
|
||||
SHUTDOWN_TIMEOUT_SECONDS=3 \
|
||||
SESSION_TTL_SECONDS=3600
|
||||
|
||||
RUN groupadd --gid 10001 app \
|
||||
@@ -26,6 +27,10 @@ COPY --chown=app:app contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
USER app
|
||||
|
||||
# Visible as X-App-Version: shows which build served a request during a rolling update.
|
||||
ARG APP_VERSION=dev
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \
|
||||
|
||||
@@ -4,10 +4,13 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-idempotency
|
||||
APP_VERSION ?=
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-idempotency failover-series rolling-update lb-status
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "failover-series, rolling-update APP_VERSION=..., lb-status: level 4 experiments (Docker required)"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -31,3 +34,15 @@ test-failover:
|
||||
|
||||
test-idempotency:
|
||||
$(PYTHON) scripts/check_idempotency.py --base-url "$(BASE_URL)"
|
||||
|
||||
# Level 4: stop each API replica in turn under load (restoring the previous one); RPS, errors, recovery.
|
||||
failover-series:
|
||||
$(PYTHON) scripts/failover_series.py --base-url "$(BASE_URL)" --output evidence/failover-series.json
|
||||
|
||||
# Level 4: one-at-a-time replacement under load. Usage: make rolling-update APP_VERSION=v3
|
||||
rolling-update:
|
||||
@test -n "$(APP_VERSION)" || (echo "Set APP_VERSION of the new build"; exit 2)
|
||||
APP_VERSION="$(APP_VERSION)" $(PYTHON) scripts/failover_series.py --mode action --action scripts/rolling-update.sh --base-url "$(BASE_URL)" --output evidence/rolling-update.json
|
||||
|
||||
lb-status:
|
||||
sh scripts/lb-status.sh
|
||||
|
||||
@@ -7,9 +7,9 @@
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
Вариант: **Docker Compose + Nginx**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1), Python 3.9+ для тестов; на Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`).
|
||||
Вариант: **Docker Compose + HAProxy**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1) и Python 3.9+ для тестов. На Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`).
|
||||
|
||||
Образы: `postgres:16.10-alpine`, `nginx:1.28.0-alpine` (закреплены digest в `compose.yaml`), приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…`.
|
||||
Образы: `postgres:16.10-alpine` и `haproxy:3.2.26-alpine` (LTS), оба закреплены digest в `compose.yaml`. Приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…` в образ `messenger-lab3-app:${APP_VERSION}`.
|
||||
|
||||
Одна последовательность запуска:
|
||||
|
||||
@@ -24,6 +24,13 @@ make test-idempotency # один ключ параллельно ч
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd
|
||||
docker compose --env-file .env up -d --wait # вернуть остановленную реплику
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd
|
||||
# уровень 4
|
||||
sh scripts/lb-status.sh # пул HAProxy: реплика, IP, UP/DOWN
|
||||
make failover-series # поочерёдная остановка каждой реплики под нагрузкой
|
||||
python scripts/failover_series.py --kill --output evidence/failover-series-kill.json # то же, SIGKILL
|
||||
docker compose --env-file .env build api migrate # с APP_VERSION=v3 в окружении
|
||||
make rolling-update APP_VERSION=v3 # Windows: scripts/failover_series.py --mode action --action scripts/rolling-update.cmd
|
||||
# после rolling update записать APP_VERSION=v3 в .env
|
||||
```
|
||||
|
||||
Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v` (удаляет volume `pgdata`).
|
||||
@@ -31,56 +38,117 @@ make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/res
|
||||
## Архитектура и решения
|
||||
|
||||
```
|
||||
client -> lb (nginx :8080, только loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata)
|
||||
migrate (one-shot) ----------------------^
|
||||
edge network data network (internal: true)
|
||||
client -> lb (HAProxy :8080, loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata)
|
||||
migrate (one-shot) ------------^
|
||||
```
|
||||
|
||||
- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL (`SESSION_STORE=postgres`).
|
||||
- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик (`depends_on: service_completed_successfully`). Реплики схему не меняют; `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций этой сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`.
|
||||
- **X-Instance-Id** = hostname контейнера (короткий Docker ID), стабилен на время жизни процесса. `stop-one.sh` по нему находит ровно один running-контейнер сервиса `api` этого проекта и делает `docker stop` (SIGTERM, затем SIGKILL через 8 с). `restart: unless-stopped` не поднимает вручную остановленный контейнер, поэтому реплика остаётся остановленной до `docker compose up -d`.
|
||||
- **Балансировщик**: round robin без `ip_hash`/sticky. `server api:8080 resolve` + `resolver 127.0.0.11 valid=2s`: Nginx раз в 2 с перечитывает DNS Docker, так что остановленные и добавленные (`--scale api=N`) реплики уходят из пула/входят в него без перезапуска LB. Пассивная проверка: `max_fails=1 fail_timeout=5s`, `proxy_connect_timeout 1s`, `proxy_next_upstream error timeout http_502 http_504`. Активных проверок в open-source Nginx нет; readiness реплик проверяет Docker healthcheck (`/health/ready` каждые 3 с), и LB стартует только после healthy API.
|
||||
- **Повтор на другой реплике**: для неотправленного запроса Nginx всегда пробует следующую реплику. Уже отправленный POST повторяется только для `/api/v1/chats/{id}/messages` (`non_idempotent`), потому что там обязателен `Idempotency-Key`. POST пользователей/чатов/сессий LB не повторяет.
|
||||
- **Graceful shutdown**: на SIGTERM uvicorn перестаёт принимать соединения и дорабатывает текущие запросы до `SHUTDOWN_TIMEOUT_SECONDS=5`; `stop_grace_period: 8s` больше этого. `keepalive_timeout 4s` в upstream меньше keep-alive uvicorn (5 с), чтобы LB не переиспользовал закрываемое API соединение.
|
||||
- **Идемпотентность**: в одной транзакции удаляется истёкший (старше `IDEMPOTENCY_TTL_SECONDS`) ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 канонического JSON тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201 и Message) либо 409 при другом теле. При ошибке транзакция откатывается целиком, ключ не фиксируется, повтор с тем же ключом безопасен. Сессия и членство в чате проверяются при каждом повторе.
|
||||
- Время сообщений назначает приложение; порядок между репликами опирается на общие часы хоста (учебное упрощение, одна машина).
|
||||
| Роль | Compose service | Образ | Health/readiness | Сети | Volume / секреты | Лимит CPU / RAM (резерв) |
|
||||
| -------- | --------------- | ----------------------------- | ----------------------------------------------------- | ---------- | -------------------------------------------------- | ------------------------ |
|
||||
| LB | `lb` | `haproxy:3.2.26-alpine@…` | `monitor-uri /lb/health`; активные проверки API | edge | только read-only конфиг, секретов нет | 0.5 / 128M (0.1 / 32M) |
|
||||
| API × N | `api` | `messenger-lab3-app:<версия>` | `/health/live`, `/health/ready` (БД, миграции, drain) | edge, data | `/run/secrets/postgres_password` (uid 10001, 0400) | 1.0 / 256M (0.25 / 128M) |
|
||||
| Миграции | `migrate` | тот же образ | one-shot, `service_completed_successfully` | data | тот же секрет | 0.5 / 128M |
|
||||
| БД | `postgres` | `postgres:16.10-alpine@…` | `pg_isready` каждые 2 с | data | `pgdata`, `POSTGRES_PASSWORD_FILE` | 1.0 / 512M (0.25 / 256M) |
|
||||
|
||||
Соответствие `.env.example`: `DATABASE_URL` собирается в `compose.yaml` из `POSTGRES_*` (вариант `DATABASE_URL_FILE` тоже поддерживается кодом); `API_REPLICAS` → `deploy.replicas`; `IDEMPOTENCY_TTL_SECONDS`, `SESSION_TTL_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` передаются в API; `CACHE_URL_FILE` не используется.
|
||||
- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL.
|
||||
- **Изоляция**: наружу публикуется только `lb` и только на loopback. У API нет `ports`, `docker port` пуст. LB находится только в сети `edge` и не резолвит `postgres`. Сеть `data` объявлена `internal`, без выхода наружу. Volume `pgdata` смонтирован только в `postgres`. `container_name` нигде нет, поэтому `--scale api=N` работает (`evidence/limits.txt`).
|
||||
- **Секреты**: пароль БД лежит только в незакоммиченном `.env` и попадает в контейнеры как Compose secret `postgres_password` (`environment: POSTGRES_PASSWORD`). В env контейнеров есть только путь `*_PASSWORD_FILE`. API и мигратор собирают `DATABASE_URL` из `DB_HOST/DB_USER/DB_NAME` и файла (`app/config.py`). У LB секретов нет.
|
||||
- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик, а при rolling update — один раз через `docker compose run --rm migrate`. Реплики схему не меняют. `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций **этой** сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`.
|
||||
- **X-Instance-Id** = hostname контейнера (короткий Docker ID). **X-App-Version** = `APP_VERSION` образа. По нему видна смена версии при rolling update.
|
||||
- **Балансировщик**: HAProxy, round robin без sticky. `server-template api 1-10 api:8080` с `resolvers docker` (Docker DNS `127.0.0.11`, `hold valid 1s`), поэтому реплики, добавленные через `--scale` или rolling update и остановленные, входят в пул и выходят из него без перезапуска LB.
|
||||
- **Повтор на другой реплике**: `retries 2`, `option redispatch`, `retry-on conn-failure empty-response response-timeout`. Для POST, кроме `/api/v1/chats/{id}/messages`, стоит `http-request disable-l7-retry`, то есть они повторяются только если соединение с репликой не было установлено. POST сообщений можно повторять, потому что `Idempotency-Key` обязателен.
|
||||
- **Идемпотентность**: в одной транзакции удаляется истёкший ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201) либо 409 при другом теле. При ошибке ключ не фиксируется, и повтор безопасен.
|
||||
|
||||
### Health, timeouts и SIGTERM: согласованные тайминги
|
||||
|
||||
```
|
||||
SIGTERM -> readiness 503 сразу; запросы ещё обслуживаются
|
||||
-> HAProxy: 2 неудачные проверки (inter 1s, timeout 1s) -> DOWN, не более 3 с
|
||||
-> через DRAIN_SECONDS=4 uvicorn перестаёт принимать соединения, дорабатывает текущие за SHUTDOWN_TIMEOUT_SECONDS=3
|
||||
-> exit 0 примерно через 5 с; stop_grace_period 10 s (stop-one: docker stop --time 9), SIGKILL не нужен
|
||||
```
|
||||
|
||||
| Параметр | Значение | Где |
|
||||
| -------------------------------------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------- |
|
||||
| Период активной проверки LB `/health/ready` | 1 с, `timeout check 1s`, `fall 2`, `rise 2` | `infra/haproxy/haproxy.cfg` |
|
||||
| Обновление DNS реплик | 1 с (`hold valid 1s`, `hold obsolete 0s`) | там же |
|
||||
| `timeout connect / server / client / http-request` | 1 с / 10 с / 30 с / 10 с | там же |
|
||||
| Drain после SIGTERM / завершение текущих запросов | 4 с / 3 с | `DRAIN_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` |
|
||||
| `stop_grace_period` | 10 с (> 4 + 3) | `compose.yaml` |
|
||||
| Keep-alive uvicorn | 10 с (> `pool-purge-delay` HAProxy 5 с) | `KEEPALIVE_TIMEOUT_SECONDS` |
|
||||
| Docker healthcheck API | каждые 3 с, 3 попытки; только порядок старта и rolling update, LB его не использует | `compose.yaml` |
|
||||
|
||||
Максимальное время обнаружения отказа:
|
||||
|
||||
- **Плановая остановка (SIGTERM)**: реплика выводится заранее, не более \(2 \cdot 1 + 1 = 3\) с, и раньше, чем закрывается сокет. Замер: последний запрос на останавливаемую реплику пришёл через 1,76–2,05 с после SIGTERM, ошибок 0.
|
||||
- **Падение процесса (SIGKILL)**: запрос к мёртвому IP завершается по `timeout connect 1s` и уходит на другую реплику (redispatch). Пул очищается по DNS примерно за 1 с или по проверкам не более чем за 3 с. Замер: максимальная пауза без успешного ответа 0,96–0,98 с, ошибок 0.
|
||||
- **Зависшая реплика, принимающая соединения**: проверки дают DOWN не более чем через 3 с. Запросы, уже отправленные на неё, ждут до `timeout server 10s` и затем повторяются (`response-timeout`) для GET и POST сообщений. Экспериментально не проверялось.
|
||||
- **Возврат в пул**: `rise 2`, то есть около 2 с после готовности. Замер от `docker start` до первого ответа реплики: 4,5–5,2 с, включая старт приложения.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------- |
|
||||
| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `docker compose --env-file .env down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy, `migrate` Exited (0); порт публикует только `lb` | `evidence/smoke.txt` (docker compose ps), `compose.yaml` |
|
||||
| Одна сессия обслуживается разными репликами, вход один раз | `python scripts/show_instances.py` | 6 запросов с одним токеном: 3 на `8b33a91fafee`, 3 на `a87734e887a5`, все 200/201; ID совпадают с hostname контейнеров `api-1`/`api-2` | `evidence/instances.txt` |
|
||||
| Остановка любой одной API-реплики, запись восстанавливается ≤10 с | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd`, между прогонами `docker compose up -d --wait` | 3 прогона PASS: до отказа 2 реплики, 127–128 успешных записей, 0 ошибок, первая запись через 1,17–1,27 с; остановлены `api-1` (прогоны 1, 2) и `api-2` (прогон 3), статус Exited (0) | `evidence/failover.txt` |
|
||||
| Повтор POST с тем же Idempotency-Key не создаёт дубль (конкурентно, через разные реплики) | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один и тот же Message, итого 5 сообщений; другое тело с тем же ключом → 409 | `evidence/idempotency.txt` |
|
||||
| Неподтверждённая запись повторяется с тем же ключом при отказе | `make test-failover` (тест повторяет неопределённый POST с исходным ключом) | дублей нет, подтверждённые сообщения на месте | `evidence/failover.txt` |
|
||||
| make check, make test | `make check`, `make test` | PASS, exit 0 | `evidence/smoke.txt` |
|
||||
| Persistence: перезапуск lb, api, postgres с сохранением volume и токена | `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; те же ID контейнеров с новым uptime (перезапущены все), volume `pgdata` на месте, прежний токен работает | `evidence/persistence.txt` |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| ----------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- |
|
||||
| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy; `migrate` применил 001–003 и Exited (0); в HAProxy 2 реплики UP | `evidence/smoke.txt` |
|
||||
| Одна сессия обслуживается разными репликами | `python scripts/show_instances.py` | 6 запросов с одним токеном: по 3 на каждую из 2 реплик | `evidence/instances.txt` |
|
||||
| Остановка любой одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` ×2, между прогонами `up -d --wait` | 2 PASS: остановлены `api-2`, затем `api-1`, Exited (0); 121 и 115 записей, 0 ошибок; HAProxy убрал реплику из пула | `evidence/failover.txt` |
|
||||
| Повтор POST с тем же Idempotency-Key | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один Message на ключ, другое тело → 409 | `evidence/idempotency.txt` |
|
||||
| make check, make test, persistence | `make check`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; после рестарта lb, api, postgres те же ID, volume на месте, прежний токен работает | `evidence/smoke.txt`, `evidence/persistence.txt` |
|
||||
| **4**: health/readiness, timeouts, SIGTERM согласованы; LB исключает недоступный backend | HAProxy active checks + drain; `scripts/lb-status.sh` | см. таблицу таймингов; останавливаемая реплика получает последний запрос через ≤2,05 с после SIGTERM; после выхода её нет в пуле | `infra/haproxy/haproxy.cfg`, `evidence/failover-series.json` |
|
||||
| **4**: поочерёдная остановка каждой реплики: RPS, ошибки, время восстановления | `make failover-series`; `failover_series.py --kill` | по 2 реплики в каждом режиме, 0 ошибок, все подтверждённые записи на месте, дублей нет (таблица ниже) | `evidence/failover-series*.{txt,json}` |
|
||||
| **4**: лимиты, нет портов API, секреты и хранилища только где нужно, нет `container_name` | `docker inspect`, `docker stats`, `docker port`, `nslookup` из lb | лимиты у всех 5 сервисов; порт только у lb (127.0.0.1/::1); пароль только файлом 0400; lb без секретов и без доступа к БД; `container_name`: 0 | `evidence/limits.txt` |
|
||||
| **4**: rolling update под нагрузкой, миграции однократно и совместимо | `make rolling-update APP_VERSION=v3`; ранее `dev → v2` с миграцией 003 | 0 ошибок, максимальная пауза 0,07 с; v2 → v3 без снижения числа реплик ниже 2; при `dev → v2` миграция 003 применена один раз, и старые реплики `dev` продолжали работать | `evidence/rolling-update.{txt,json}`, `evidence/rolling-update-migration.txt` |
|
||||
|
||||
Все прогоны 2026-11-07 на commit `8cd25e4` (+ `scripts/show_instances.py`), Docker 29.4.3, Windows 11, один хост. На Windows `make test-failover`/`test-persistence` запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`): GNU Make под cmd не находит `test` в guard-строке, на результат это не влияет.
|
||||
Все прогоны 2026-11-07 на commit `124c83e` + рабочее дерево (до коммита уровня 4), Docker 29.4.3, Windows 11, один хост. `make test-failover/test-persistence` под Windows запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`), потому что GNU Make под cmd не находит `test` в guard-строке.
|
||||
|
||||
Цель failover выбирает тест: реплика, обслужившая первое сообщение. При round robin это детерминированно `api-1`, поэтому перед прогоном 3 через LB отправлен один `GET /health/ready`, чтобы сдвинуть очередь, и тест остановил `api-2`.
|
||||
Цель `make test-failover` выбирает тест: это реплика, обслужившая первое сообщение. Для второго прогона через LB отправлен один `GET /health/ready`, чтобы сдвинуть round robin, и тест остановил другую реплику.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
Профиль failover-теста: 15 с параллельной записи сообщений, пока hook останавливает одну реплику (`docker stop --time 8`, SIGTERM). Нагрузочные эксперименты (RPS, перцентили) — уровень 4/5, здесь не заявлены.
|
||||
Профиль нагрузки `scripts/failover_series.py`: 4 потока в закрытом цикле, каждый делает POST сообщения (с `Idempotency-Key`; неопределённый результат повторяется с тем же ключом) и GET 20 последних сообщений. Счёт идёт по секундам. Для каждой реплики по очереди: 10 с baseline, затем остановка, 15 с наблюдения, затем `docker start` того же контейнера, ожидание его ответов и 5 с паузы. В конце вся история чата читается заново: каждая подтверждённая запись должна быть ровно один раз.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| ---------------------------------------------------------------------- | ------------------------------------- | ------------------------------------------- | ------------------------------------ |
|
||||
| `stop-one` реплики, обслужившей тест (`api-1`), под записью, 2 прогона | запись продолжается ≤10 с, дублей нет | 127 и 128 успешных записей, 0 ошибок | 1,27 с и 1,17 с / нет |
|
||||
| `stop-one` второй реплики (`api-2`) после восстановления первой | то же | 127 записей, 0 ошибок | 1,21 с / нет |
|
||||
| `restart.cmd`: lb, api, postgres | данные и старый токен сохранены | PASS, volume сохранён | нет потерь |
|
||||
| 5 ключей × 20 параллельных POST через LB | одно сообщение на ключ | по 10 ответов с каждой реплики, 5 сообщений | — |
|
||||
### Поочерёдная остановка реплик под нагрузкой
|
||||
|
||||
Ошибок в переходный период не было: Nginx при `connect refused` сразу отправляет запрос на другую реплику (`proxy_next_upstream`), а для POST сообщений это безопасно благодаря Idempotency-Key. «Время восстановления» в тесте — это время от старта hook до первой успешной записи, а не время обнаружения отказа.
|
||||
| Воздействие | Реплика | RPS baseline → во время → после | Ошибки | Макс. пауза без успеха | Последний запрос на реплику после сигнала | Выход контейнера | RPS ≥90% baseline после выхода | Возврат в пул после `docker start` |
|
||||
| ------------------------------- | ------- | ------------------------------- | ------ | ---------------------- | ----------------------------------------- | ---------------- | ------------------------------ | ---------------------------------- |
|
||||
| `docker stop` (SIGTERM + drain) | api-1 | 129,0 → 114,8 → 113,1 | 0 | 0,09 с | 2,05 с | 5,06 с, exit 0 | 2,8 с | 5,2 с |
|
||||
| `docker stop` (SIGTERM + drain) | api-2 | 131,5 → 145,2 → 165,4 | 0 | 0,09 с | 1,76 с | 4,88 с, exit 0 | 1,6 с | 4,9 с |
|
||||
| `docker kill` (SIGKILL) | api-1 | 174,4 → 145,3 → 158,4 | 0 | 0,96 с | 0,12 с | 0,66 с, exit 137 | 2,3 с | 4,5 с |
|
||||
| `docker kill` (SIGKILL) | api-2 | 177,9 → 146,5 → 139,0 | 0 | 0,98 с | 0,12 с | 0,63 с, exit 137 | 2,1 с | 5,2 с |
|
||||
|
||||
Итог серий: SIGTERM — 80,5 с, 5312 записей и 5312 чтений, p95 73 мс; SIGKILL — 71,2 с, 5520 записей и 5520 чтений, p95 66 мс. В обоих случаях 0 ошибок, 0 дублей, 0 потерянных подтверждённых записей. В предварительном прогоне `--kill` (не сохранён) была 1 ошибка на 12 890 запросов: запрос, оборванный в момент SIGKILL. Это допустимый переходный эффект, клиент повторяет его с тем же ключом.
|
||||
|
||||
RPS — это пропускная способность закрытого цикла на одном хосте, где рядом работали посторонние контейнеры. Между сериями она колеблется на 30–40 %, а внутри серии падение во время отказа составляет 10–20 %. Иногда RPS с одной репликой выше baseline: узкое место — клиент и общий CPU хоста, а не число реплик. Это не измерение ёмкости.
|
||||
|
||||
### make test-failover и rolling update
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных |
|
||||
| ------------------------------------------------ | ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------- |
|
||||
| `make test-failover`, `stop-one` → `api-2` | запись продолжается ≤10 с, дублей нет | PASS, 121 запись, 0 ошибок | `first_recovery_seconds` 6,42 / нет |
|
||||
| то же после восстановления → `api-1` | то же | PASS, 115 записей, 0 ошибок | 6,44 / нет |
|
||||
| `restart.cmd`: lb, api, postgres | данные и старый токен сохранены | PASS, volume сохранён | нет потерь |
|
||||
| rolling update v2 → v3 под нагрузкой (2 реплики) | нет ошибок, ёмкость ≥2 реплик, версия сменилась | 42,7 с; 0 ошибок; пауза 0,07 с; RPS 181 → 146 → 186; ответы: до v2 1833; во время v2 3941 / v3 2279; после v3 2787 | 5423 записи, 0 дублей, 0 потерь |
|
||||
| rolling update dev → v2 с миграцией 003 | старая версия работает на расширенной схеме | 45,7 с; 0 ошибок; пауза 0,10 с; миграция применена один раз в начале, реплики `dev` обслуживали запросы до своей замены | 4458 записей, 0 дублей, 0 потерь |
|
||||
|
||||
`first_recovery_seconds` в `tests/resilience.py` считается до первой успешной записи **после завершения hook**. Hook теперь длится около 6 с: это `docker stop`, который ждёт 4 с drain. Записи при этом не прерывались (0 ошибок), поэтому значение отражает длительность hook, а не простой. Оно укладывается в лимит 10 с, как и сам hook.
|
||||
|
||||
**Rolling update** (`scripts/rolling-update.sh`):
|
||||
|
||||
1. Собрать `messenger-lab3-app:$APP_VERSION`.
|
||||
2. Один раз выполнить `docker compose run --rm migrate`.
|
||||
3. Для каждой старой реплики запустить `up -d --no-deps --no-recreate --wait --scale api=N+1`. Новая реплика стартует из нового образа, скрипт ждёт healthy и UP в HAProxy. Затем старая реплика получает `docker stop` (drain) и `docker rm`.
|
||||
|
||||
Ёмкость не опускается ниже N, LB не перезапускается. После обновления версию нужно записать в `.env` (`APP_VERSION`), иначе следующий `docker compose up` пересоздаст реплики в версии из `.env`.
|
||||
|
||||
**Политика миграций (expand/contract)**. Миграция выполняется до замены реплик и должна быть совместима с работающей версией: только новые таблицы, индексы и nullable-колонки. Пример — `003_session_indexes.sql`. Readiness старой сборки проверяет только свои миграции, поэтому расширенная схема её не ломает. Удаление и переименование колонок допустимы только отдельным релизом после ухода старых реплик. Ограничение: `CREATE INDEX` без `CONCURRENTLY` (мигратор работает в транзакции) коротко блокирует запись в индексируемую таблицу. Для `sessions` учебного размера это незаметно, для большой таблицы понадобится `CONCURRENTLY` вне транзакции.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
- **Выдерживается**: отказ процесса или контейнера одной API-реплики, пока работает хотя бы одна другая. Также полный перезапуск стенда с сохранением volume.
|
||||
- **Не выдерживается**: отказ PostgreSQL и Nginx. Это единственные точки отказа: БД не реплицирована, LB один. При их остановке API недоступен до перезапуска, подтверждённые данные сохраняются в volume.
|
||||
- **Один хост**: все контейнеры на одной машине, поэтому отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса, а не узла.
|
||||
- **Нет активных health-check в LB**: open-source Nginx исключает реплику пассивно (ошибка соединения, `max_fails=1 fail_timeout=5s`) и при обновлении DNS раз в 2 с. Зависшая, но принимающая соединения реплика будет отбрасываться только по `proxy_read_timeout 10s`.
|
||||
- **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение. На нескольких узлах понадобятся время БД или последовательность.
|
||||
- **Повтор LB**: Nginx повторяет на другой реплике только POST сообщений. POST пользователей, чатов и сессий после обрыва соединения клиент должен проверить сам, так как ключа идемпотентности у них нет.
|
||||
- Лимиты CPU/RAM, rolling update и измерения под нагрузкой не делались: это уровень 4.
|
||||
- **Выдерживается**: плановая остановка, падение или зависание одной API-реплики, пока работает хотя бы одна другая; поочерёдная замена всех реплик; полный перезапуск стенда с сохранением volume.
|
||||
- **Не выдерживается**: отказ PostgreSQL и HAProxy. Это единственные точки отказа: БД не реплицирована, LB один. При их остановке API недоступен до перезапуска, подтверждённые данные сохраняются в volume.
|
||||
- **Один хост**: отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса или контейнера, а не узла.
|
||||
- **Зависание реплики** экспериментально не воспроизводилось. Оценка «не более 3 с до DOWN» следует из конфигурации, а уже отправленные на зависшую реплику запросы ждут до 10 с.
|
||||
- **Rolling update** в Compose реализован скриптом, а не `deploy.update_config`: Compose его не исполняет. Откат — тот же скрипт с прежней `APP_VERSION`, если схема не прошла contract-фазу.
|
||||
- **Нагрузка** — закрытый цикл одного клиента на том же хосте (coordinated omission, общий CPU). Цифры RPS сравнимы только внутри одной серии.
|
||||
- **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение.
|
||||
- **Повтор LB**: POST пользователей, чатов и сессий повторяется только при неустановленном соединении. После обрыва уже отправленного запроса клиент должен проверить результат сам: ключа идемпотентности у этих POST нет.
|
||||
|
||||
+32
-1
@@ -1,17 +1,48 @@
|
||||
import logging
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from .config import load_settings
|
||||
from .drain import draining
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
"""First SIGTERM/SIGINT: keep serving for drain_seconds with readiness 503, then uvicorn's graceful stop.
|
||||
|
||||
drain_seconds must exceed the LB detection time (HAProxy inter 1s x fall 2 + timeout check 1s),
|
||||
so the replica is out of rotation before it stops accepting connections.
|
||||
"""
|
||||
|
||||
def __init__(self, config: uvicorn.Config, drain_seconds: float):
|
||||
super().__init__(config)
|
||||
self.drain_seconds = drain_seconds
|
||||
|
||||
def handle_exit(self, sig, frame) -> None:
|
||||
if draining.is_set() or self.drain_seconds <= 0:
|
||||
draining.set()
|
||||
super().handle_exit(sig, frame)
|
||||
return
|
||||
draining.set()
|
||||
logger.warning('signal %s: draining for %.1f s before shutdown', sig, self.drain_seconds)
|
||||
timer = threading.Timer(self.drain_seconds, super().handle_exit, (sig, frame))
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
settings = load_settings()
|
||||
uvicorn.run(
|
||||
config = uvicorn.Config(
|
||||
'app.main:app',
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
log_level=settings.log_level,
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout,
|
||||
timeout_keep_alive=settings.keepalive_timeout,
|
||||
)
|
||||
DrainingServer(config, settings.drain_seconds).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
+44
-15
@@ -2,6 +2,7 @@ import os
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -11,39 +12,67 @@ class Settings:
|
||||
env: str
|
||||
log_level: str
|
||||
shutdown_timeout: int
|
||||
drain_seconds: float
|
||||
keepalive_timeout: int
|
||||
session_ttl_seconds: int
|
||||
idempotency_ttl_seconds: int
|
||||
database_url: str
|
||||
instance_id: str
|
||||
app_version: str
|
||||
|
||||
|
||||
def _secret(file_var: str, value_var: str) -> str:
|
||||
"""A *_FILE variable wins. The secret value is not logged."""
|
||||
file_name = os.getenv(file_var, '').strip()
|
||||
if file_name:
|
||||
try:
|
||||
value = Path(file_name).read_text(encoding='utf-8').strip()
|
||||
except OSError:
|
||||
raise RuntimeError('%s could not be read' % file_var) from None
|
||||
if not value:
|
||||
raise RuntimeError('%s is empty' % file_var)
|
||||
return value
|
||||
value = os.getenv(value_var, '').strip()
|
||||
def _read_secret_file(file_var: str, file_name: str) -> str:
|
||||
try:
|
||||
value = Path(file_name).read_text(encoding='utf-8').strip()
|
||||
except OSError:
|
||||
raise RuntimeError('%s could not be read' % file_var) from None
|
||||
if not value:
|
||||
raise RuntimeError('%s or %s is required' % (value_var, file_var))
|
||||
raise RuntimeError('%s is empty' % file_var)
|
||||
return value
|
||||
|
||||
|
||||
def _database_url() -> str:
|
||||
"""DATABASE_URL_FILE, then DATABASE_URL, then DB_* parts with the password from DB_PASSWORD_FILE.
|
||||
|
||||
Secret values are never logged.
|
||||
"""
|
||||
file_name = os.getenv('DATABASE_URL_FILE', '').strip()
|
||||
if file_name:
|
||||
return _read_secret_file('DATABASE_URL_FILE', file_name)
|
||||
value = os.getenv('DATABASE_URL', '').strip()
|
||||
if value:
|
||||
return value
|
||||
password_file = os.getenv('DB_PASSWORD_FILE', '').strip()
|
||||
if not password_file:
|
||||
raise RuntimeError('DATABASE_URL, DATABASE_URL_FILE or DB_PASSWORD_FILE is required')
|
||||
password = _read_secret_file('DB_PASSWORD_FILE', password_file)
|
||||
parts = {name: os.getenv(name, '').strip() for name in ('DB_HOST', 'DB_USER', 'DB_NAME')}
|
||||
missing = [name for name, part in parts.items() if not part]
|
||||
if missing:
|
||||
raise RuntimeError('%s required with DB_PASSWORD_FILE' % ', '.join(missing))
|
||||
return 'postgresql://%s:%s@%s:%s/%s' % (
|
||||
quote(parts['DB_USER'], safe=''),
|
||||
quote(password, safe=''),
|
||||
parts['DB_HOST'],
|
||||
int(os.getenv('DB_PORT', '5432')),
|
||||
quote(parts['DB_NAME'], safe=''),
|
||||
)
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings(
|
||||
host=os.getenv('APP_HOST', '0.0.0.0'),
|
||||
port=int(os.getenv('APP_PORT', '8080')),
|
||||
env=os.getenv('APP_ENV', 'lab'),
|
||||
log_level=os.getenv('LOG_LEVEL', 'info').lower(),
|
||||
shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '5')),
|
||||
shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '3')),
|
||||
drain_seconds=float(os.getenv('DRAIN_SECONDS', '4')),
|
||||
# Longer than HAProxy's idle-connection purge (pool-purge-delay 5 s), so the LB usually closes them first.
|
||||
keepalive_timeout=int(os.getenv('KEEPALIVE_TIMEOUT_SECONDS', '10')),
|
||||
session_ttl_seconds=int(os.getenv('SESSION_TTL_SECONDS', '3600')),
|
||||
idempotency_ttl_seconds=int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400')),
|
||||
database_url=_secret('DATABASE_URL_FILE', 'DATABASE_URL'),
|
||||
database_url=_database_url(),
|
||||
# Docker sets the hostname to the short container ID, which stop-one.sh maps back to a container.
|
||||
instance_id=os.getenv('INSTANCE_ID', '').strip() or socket.gethostname(),
|
||||
app_version=os.getenv('APP_VERSION', 'dev').strip() or 'dev',
|
||||
)
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Set on SIGTERM: readiness turns 503 so the LB stops routing here while requests are still served."""
|
||||
import threading
|
||||
|
||||
draining = threading.Event()
|
||||
+5
-2
@@ -72,7 +72,9 @@ def create_app() -> FastAPI:
|
||||
version='1.3.0',
|
||||
lifespan=lifespan,
|
||||
)
|
||||
instance_id = load_settings().instance_id
|
||||
settings = load_settings()
|
||||
instance_id = settings.instance_id
|
||||
app_version = settings.app_version
|
||||
|
||||
def openapi():
|
||||
if app.openapi_schema is None:
|
||||
@@ -83,7 +85,7 @@ def create_app() -> FastAPI:
|
||||
|
||||
@app.middleware('http')
|
||||
async def request_context(request: Request, call_next):
|
||||
"""Gives every response, including unexpected failures, an X-Request-Id and X-Instance-Id."""
|
||||
"""Gives every response, including unexpected failures, an X-Request-Id, X-Instance-Id and X-App-Version."""
|
||||
incoming = request.headers.get('X-Request-Id')
|
||||
request_id = incoming if incoming and _SAFE_REQUEST_ID.match(incoming) else str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
@@ -94,6 +96,7 @@ def create_app() -> FastAPI:
|
||||
response = _error(500, 'internal_error', _MESSAGES['internal_error'], request_id)
|
||||
response.headers['X-Request-Id'] = request_id
|
||||
response.headers['X-Instance-Id'] = instance_id
|
||||
response.headers['X-App-Version'] = app_version
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
|
||||
+4
-1
@@ -10,7 +10,8 @@ from pydantic import BaseModel, ValidationError
|
||||
from starlette.responses import Response
|
||||
|
||||
from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer
|
||||
from .errors import invalid_request, not_found, unauthorized
|
||||
from .drain import draining
|
||||
from .errors import invalid_request, not_found, unauthorized, unavailable
|
||||
from .pagination import encode_cursor, parse_page
|
||||
from .schemas import CreateChat, CreateMessage, CreateUser
|
||||
from .store import Chat, Store, User
|
||||
@@ -80,6 +81,8 @@ async def liveness() -> JSONResponse:
|
||||
|
||||
@router.get('/health/ready')
|
||||
async def readiness(store: Store = Depends(get_store)) -> JSONResponse:
|
||||
if draining.is_set():
|
||||
raise unavailable('Реплика завершает работу')
|
||||
await store.ready()
|
||||
return JSONResponse({'status': 'ok'})
|
||||
|
||||
|
||||
+87
-20
@@ -1,57 +1,103 @@
|
||||
# Client -> lb (nginx) -> api x API_REPLICAS -> postgres.
|
||||
# Secrets come from .env, which is not committed. Only the LB publishes a port.
|
||||
# Client -> lb (HAProxy) -> api x API_REPLICAS -> postgres.
|
||||
# Only the LB publishes a port. Networks: edge = lb<->api, data = api/migrate<->postgres (no LB, no internet).
|
||||
# The DB password comes from .env (not committed) as a Compose secret file, never as a container env var.
|
||||
name: messenger-lab3
|
||||
|
||||
x-app-env: &app-env
|
||||
x-db-client: &db-client
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
DATABASE_URL: postgresql://${POSTGRES_USER:?set in .env}:${POSTGRES_PASSWORD:?set in .env}@postgres:5432/${POSTGRES_DB:?set in .env}
|
||||
DB_HOST: postgres
|
||||
DB_USER: ${POSTGRES_USER:?set in .env}
|
||||
DB_NAME: ${POSTGRES_DB:?set in .env}
|
||||
DB_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
|
||||
x-app-build: &app-build
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-dev}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks:
|
||||
- data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
# One-shot schema step. Replicas never migrate on startup, so there is no race between them;
|
||||
# an advisory lock additionally serializes accidental parallel runs.
|
||||
migrate:
|
||||
image: messenger-lab3-app:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: messenger-lab3-app:${APP_VERSION:-dev}
|
||||
build: *app-build
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
<<: *app-env
|
||||
<<: *db-client
|
||||
secrets:
|
||||
- source: postgres_password
|
||||
uid: "10001"
|
||||
gid: "10001"
|
||||
mode: 0400
|
||||
networks:
|
||||
- data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
image: messenger-lab3-app:local
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: messenger-lab3-app:${APP_VERSION:-dev}
|
||||
build: *app-build
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
environment:
|
||||
<<: *app-env
|
||||
<<: *db-client
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-5}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-4}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3}
|
||||
secrets:
|
||||
- source: postgres_password
|
||||
uid: "10001"
|
||||
gid: "10001"
|
||||
mode: 0400
|
||||
networks:
|
||||
- edge
|
||||
- data
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
@@ -59,6 +105,7 @@ services:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
# Docker health gates startup order and rolling update; the LB runs its own checks (infra/haproxy/haproxy.cfg).
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
@@ -69,19 +116,22 @@ services:
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
# SIGTERM -> uvicorn stops accepting, finishes in-flight requests within SHUTDOWN_TIMEOUT_SECONDS.
|
||||
# SIGTERM -> readiness 503 for DRAIN_SECONDS (LB drops the replica in <= 3 s) -> uvicorn finishes
|
||||
# in-flight requests within SHUTDOWN_TIMEOUT_SECONDS. 4 + 3 < 10, so SIGKILL is not reached.
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 8s
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: nginx:1.28.0-alpine@sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284
|
||||
image: haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0
|
||||
# Loopback only. Both families: on Windows "localhost" tries ::1 first and waits ~2 s if nothing listens.
|
||||
ports:
|
||||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||||
- "[::1]:${LB_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
networks:
|
||||
- edge
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
@@ -90,7 +140,24 @@ services:
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 32M
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
edge: {}
|
||||
data:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
environment: POSTGRES_PASSWORD
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
||||
@@ -0,0 +1,961 @@
|
||||
{
|
||||
"mode": "failover",
|
||||
"stop_method": "docker kill (SIGKILL)",
|
||||
"concurrency": 4,
|
||||
"workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest",
|
||||
"replicas_before": [
|
||||
"messenger-lab3-api-1",
|
||||
"messenger-lab3-api-2"
|
||||
],
|
||||
"runs": [
|
||||
{
|
||||
"replica": "messenger-lab3-api-1",
|
||||
"instance_id": "1b755b53e9fa",
|
||||
"state_after_stop": "exited exit=137",
|
||||
"stop_command_seconds": 0.66,
|
||||
"last_served_by_target_after_sigterm_s": 0.12,
|
||||
"first_success_after_container_exit_s": 0.67,
|
||||
"rps_back_to_90pct_after_exit_s": 2.28,
|
||||
"max_gap_without_success_s": 0.963,
|
||||
"rejoin_after_docker_start_s": 4.51,
|
||||
"baseline": {
|
||||
"seconds": 10.0,
|
||||
"rps_ok": 174.4,
|
||||
"writes_ok": 873,
|
||||
"reads_ok": 871,
|
||||
"errors": 0,
|
||||
"p95_ms": 53.8,
|
||||
"instances": {
|
||||
"e3303afc01b2": 874,
|
||||
"1b755b53e9fa": 870
|
||||
}
|
||||
},
|
||||
"during_stop_and_outage": {
|
||||
"seconds": 15.73,
|
||||
"rps_ok": 145.3,
|
||||
"writes_ok": 1143,
|
||||
"reads_ok": 1143,
|
||||
"errors": 0,
|
||||
"p95_ms": 65.5,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 9,
|
||||
"e3303afc01b2": 2277
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 9.51,
|
||||
"rps_ok": 158.4,
|
||||
"writes_ok": 754,
|
||||
"reads_ok": 752,
|
||||
"errors": 0,
|
||||
"p95_ms": 67.6,
|
||||
"instances": {
|
||||
"e3303afc01b2": 1035,
|
||||
"1b755b53e9fa": 471
|
||||
}
|
||||
},
|
||||
"marks": {
|
||||
"sigterm": 10.06,
|
||||
"exited": 10.72,
|
||||
"docker_start": 25.79
|
||||
}
|
||||
},
|
||||
{
|
||||
"replica": "messenger-lab3-api-2",
|
||||
"instance_id": "e3303afc01b2",
|
||||
"state_after_stop": "exited exit=137",
|
||||
"stop_command_seconds": 0.63,
|
||||
"last_served_by_target_after_sigterm_s": 0.12,
|
||||
"first_success_after_container_exit_s": 0.755,
|
||||
"rps_back_to_90pct_after_exit_s": 2.06,
|
||||
"max_gap_without_success_s": 0.975,
|
||||
"rejoin_after_docker_start_s": 5.18,
|
||||
"baseline": {
|
||||
"seconds": 10.0,
|
||||
"rps_ok": 177.9,
|
||||
"writes_ok": 888,
|
||||
"reads_ok": 891,
|
||||
"errors": 0,
|
||||
"p95_ms": 62.7,
|
||||
"instances": {
|
||||
"e3303afc01b2": 891,
|
||||
"1b755b53e9fa": 888
|
||||
}
|
||||
},
|
||||
"during_stop_and_outage": {
|
||||
"seconds": 15.71,
|
||||
"rps_ok": 146.5,
|
||||
"writes_ok": 1152,
|
||||
"reads_ok": 1151,
|
||||
"errors": 0,
|
||||
"p95_ms": 67.0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 2291,
|
||||
"e3303afc01b2": 12
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 10.18,
|
||||
"rps_ok": 139.0,
|
||||
"writes_ok": 708,
|
||||
"reads_ok": 708,
|
||||
"errors": 0,
|
||||
"p95_ms": 73.8,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 1025,
|
||||
"e3303afc01b2": 391
|
||||
}
|
||||
},
|
||||
"marks": {
|
||||
"sigterm": 45.31,
|
||||
"exited": 45.94,
|
||||
"docker_start": 61.02
|
||||
}
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"seconds": 71.23,
|
||||
"rps_ok": 155.0,
|
||||
"writes_ok": 5520,
|
||||
"reads_ok": 5520,
|
||||
"errors": 0,
|
||||
"p95_ms": 65.5,
|
||||
"instances": {
|
||||
"e3303afc01b2": 5483,
|
||||
"1b755b53e9fa": 5557
|
||||
}
|
||||
},
|
||||
"integrity": {
|
||||
"confirmed_writes": 5520,
|
||||
"stored_messages": 5520,
|
||||
"duplicates": 0,
|
||||
"missing_confirmed": 0
|
||||
},
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 145,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 71,
|
||||
"1b755b53e9fa": 74
|
||||
},
|
||||
"versions": {
|
||||
"v2": 145
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 87,
|
||||
"1b755b53e9fa": 84
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 240,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 120,
|
||||
"1b755b53e9fa": 120
|
||||
},
|
||||
"versions": {
|
||||
"v2": 240
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 239,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 120,
|
||||
"1b755b53e9fa": 119
|
||||
},
|
||||
"versions": {
|
||||
"v2": 239
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 199,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 98,
|
||||
"1b755b53e9fa": 101
|
||||
},
|
||||
"versions": {
|
||||
"v2": 199
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 141,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 69,
|
||||
"e3303afc01b2": 72
|
||||
},
|
||||
"versions": {
|
||||
"v2": 141
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 185,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 92,
|
||||
"e3303afc01b2": 93
|
||||
},
|
||||
"versions": {
|
||||
"v2": 185
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 190,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 96,
|
||||
"e3303afc01b2": 94
|
||||
},
|
||||
"versions": {
|
||||
"v2": 190
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 130,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 65,
|
||||
"1b755b53e9fa": 65
|
||||
},
|
||||
"versions": {
|
||||
"v2": 130
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 100,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 50,
|
||||
"e3303afc01b2": 50
|
||||
},
|
||||
"versions": {
|
||||
"v2": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 48,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 39,
|
||||
"1b755b53e9fa": 9
|
||||
},
|
||||
"versions": {
|
||||
"v2": 48
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 128,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 128
|
||||
},
|
||||
"versions": {
|
||||
"v2": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 170,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 170
|
||||
},
|
||||
"versions": {
|
||||
"v2": 170
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 107,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 107
|
||||
},
|
||||
"versions": {
|
||||
"v2": 107
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 143,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 143
|
||||
},
|
||||
"versions": {
|
||||
"v2": 143
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 148,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 148
|
||||
},
|
||||
"versions": {
|
||||
"v2": 148
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 170,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 170
|
||||
},
|
||||
"versions": {
|
||||
"v2": 170
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 128,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 128
|
||||
},
|
||||
"versions": {
|
||||
"v2": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 136,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 136
|
||||
},
|
||||
"versions": {
|
||||
"v2": 136
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 147,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 147
|
||||
},
|
||||
"versions": {
|
||||
"v2": 147
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 187,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 187
|
||||
},
|
||||
"versions": {
|
||||
"v2": 187
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 132,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 132
|
||||
},
|
||||
"versions": {
|
||||
"v2": 132
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 136,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 136
|
||||
},
|
||||
"versions": {
|
||||
"v2": 136
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 194,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 194
|
||||
},
|
||||
"versions": {
|
||||
"v2": 194
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 197,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 197
|
||||
},
|
||||
"versions": {
|
||||
"v2": 197
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 134,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 134
|
||||
},
|
||||
"versions": {
|
||||
"v2": 134
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 56,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 56
|
||||
},
|
||||
"versions": {
|
||||
"v2": 56
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 139,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 139
|
||||
},
|
||||
"versions": {
|
||||
"v2": 139
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 161,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 161
|
||||
},
|
||||
"versions": {
|
||||
"v2": 161
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 176,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 176
|
||||
},
|
||||
"versions": {
|
||||
"v2": 176
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 127,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 72,
|
||||
"1b755b53e9fa": 55
|
||||
},
|
||||
"versions": {
|
||||
"v2": 127
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 173,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 90,
|
||||
"1b755b53e9fa": 83
|
||||
},
|
||||
"versions": {
|
||||
"v2": 173
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 184,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 94,
|
||||
"e3303afc01b2": 90
|
||||
},
|
||||
"versions": {
|
||||
"v2": 184
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 240,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 120,
|
||||
"1b755b53e9fa": 120
|
||||
},
|
||||
"versions": {
|
||||
"v2": 240
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 191,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 94,
|
||||
"1b755b53e9fa": 97
|
||||
},
|
||||
"versions": {
|
||||
"v2": 191
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 138,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 72,
|
||||
"1b755b53e9fa": 66
|
||||
},
|
||||
"versions": {
|
||||
"v2": 138
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 118,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 58,
|
||||
"1b755b53e9fa": 60
|
||||
},
|
||||
"versions": {
|
||||
"v2": 118
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 208,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 104,
|
||||
"1b755b53e9fa": 104
|
||||
},
|
||||
"versions": {
|
||||
"v2": 208
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 189,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 96,
|
||||
"e3303afc01b2": 93
|
||||
},
|
||||
"versions": {
|
||||
"v2": 189
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 148,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 75,
|
||||
"1b755b53e9fa": 73
|
||||
},
|
||||
"versions": {
|
||||
"v2": 148
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 160,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 78,
|
||||
"e3303afc01b2": 82
|
||||
},
|
||||
"versions": {
|
||||
"v2": 160
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 240,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 122,
|
||||
"e3303afc01b2": 118
|
||||
},
|
||||
"versions": {
|
||||
"v2": 240
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 215,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 109,
|
||||
"e3303afc01b2": 106
|
||||
},
|
||||
"versions": {
|
||||
"v2": 215
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 87,
|
||||
"1b755b53e9fa": 84
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 174,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 85,
|
||||
"e3303afc01b2": 89
|
||||
},
|
||||
"versions": {
|
||||
"v2": 174
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 116,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 75,
|
||||
"e3303afc01b2": 41
|
||||
},
|
||||
"versions": {
|
||||
"v2": 116
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 62,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 62
|
||||
},
|
||||
"versions": {
|
||||
"v2": 62
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 180,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 180
|
||||
},
|
||||
"versions": {
|
||||
"v2": 180
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 113,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 113
|
||||
},
|
||||
"versions": {
|
||||
"v2": 113
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 141,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 141
|
||||
},
|
||||
"versions": {
|
||||
"v2": 141
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 193,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 193
|
||||
},
|
||||
"versions": {
|
||||
"v2": 193
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 200,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 200
|
||||
},
|
||||
"versions": {
|
||||
"v2": 200
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 139,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 139
|
||||
},
|
||||
"versions": {
|
||||
"v2": 139
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 171
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 210,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 210
|
||||
},
|
||||
"versions": {
|
||||
"v2": 210
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 151,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 151
|
||||
},
|
||||
"versions": {
|
||||
"v2": 151
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 56,
|
||||
"ok": 91,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 91
|
||||
},
|
||||
"versions": {
|
||||
"v2": 91
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 57,
|
||||
"ok": 123,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 123
|
||||
},
|
||||
"versions": {
|
||||
"v2": 123
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 58,
|
||||
"ok": 180,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 180
|
||||
},
|
||||
"versions": {
|
||||
"v2": 180
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 59,
|
||||
"ok": 126,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 126
|
||||
},
|
||||
"versions": {
|
||||
"v2": 126
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 60,
|
||||
"ok": 168,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 168
|
||||
},
|
||||
"versions": {
|
||||
"v2": 168
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 61,
|
||||
"ok": 78,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 78
|
||||
},
|
||||
"versions": {
|
||||
"v2": 78
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 62,
|
||||
"ok": 163,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 163
|
||||
},
|
||||
"versions": {
|
||||
"v2": 163
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 63,
|
||||
"ok": 144,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 144
|
||||
},
|
||||
"versions": {
|
||||
"v2": 144
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 64,
|
||||
"ok": 179,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 179
|
||||
},
|
||||
"versions": {
|
||||
"v2": 179
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 65,
|
||||
"ok": 68,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 68
|
||||
},
|
||||
"versions": {
|
||||
"v2": 68
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 66,
|
||||
"ok": 71,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 38,
|
||||
"e3303afc01b2": 33
|
||||
},
|
||||
"versions": {
|
||||
"v2": 71
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 67,
|
||||
"ok": 145,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 73,
|
||||
"1b755b53e9fa": 72
|
||||
},
|
||||
"versions": {
|
||||
"v2": 145
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 68,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 86,
|
||||
"1b755b53e9fa": 85
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 69,
|
||||
"ok": 206,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 103,
|
||||
"e3303afc01b2": 103
|
||||
},
|
||||
"versions": {
|
||||
"v2": 206
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 70,
|
||||
"ok": 163,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 80,
|
||||
"e3303afc01b2": 83
|
||||
},
|
||||
"versions": {
|
||||
"v2": 163
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 71,
|
||||
"ok": 35,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 19,
|
||||
"e3303afc01b2": 16
|
||||
},
|
||||
"versions": {
|
||||
"v2": 35
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# failover series 2026-11-07T18:10:53.0538421+03:00: python scripts/failover_series.py --kill (docker kill, SIGKILL)
|
||||
killing messenger-lab3-api-1 (1b755b53e9fa) under load...
|
||||
errors=0 gap=0.96s first_ok_after_exit=0.67 rps 174.4 -> 145.3 -> 158.4 rejoin=4.51s
|
||||
killing messenger-lab3-api-2 (e3303afc01b2) under load...
|
||||
errors=0 gap=0.97s first_ok_after_exit=0.755 rps 177.9 -> 146.5 -> 139.0 rejoin=5.18s
|
||||
{"total": {"seconds": 71.23, "rps_ok": 155.0, "writes_ok": 5520, "reads_ok": 5520, "errors": 0, "p95_ms": 65.5, "instances": {"e3303afc01b2": 5483, "1b755b53e9fa": 5557}}, "integrity": {"confirmed_writes": 5520, "stored_messages": 5520, "duplicates": 0, "missing_confirmed": 0}}
|
||||
PASS: failover, 5520 confirmed writes, no duplicates, nothing lost
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,7 @@
|
||||
# failover series 2026-11-07T18:09:39.2795947+03:00: make failover-series (docker stop, SIGTERM + drain)
|
||||
stopping messenger-lab3-api-1 (1b755b53e9fa) under load...
|
||||
errors=0 gap=0.09s first_ok_after_exit=0.018 rps 129.0 -> 114.8 -> 113.1 rejoin=5.15s
|
||||
stopping messenger-lab3-api-2 (e3303afc01b2) under load...
|
||||
errors=0 gap=0.09s first_ok_after_exit=0.012 rps 131.5 -> 145.2 -> 165.4 rejoin=4.91s
|
||||
{"total": {"seconds": 80.52, "rps_ok": 131.9, "writes_ok": 5312, "reads_ok": 5312, "errors": 0, "p95_ms": 73.4, "instances": {"1b755b53e9fa": 5703, "e3303afc01b2": 4921}}, "integrity": {"confirmed_writes": 5312, "stored_messages": 5312, "duplicates": 0, "missing_confirmed": 0}}
|
||||
PASS: failover, 5312 confirmed writes, no duplicates, nothing lost
|
||||
+26
-29
@@ -1,41 +1,38 @@
|
||||
# failover run 2026-11-07T17:39:07.3980307+03:00
|
||||
# failover runs 2026-11-07T18:03:34.5946145+03:00 (HAProxy, drain 4 s)
|
||||
# command (= make test-failover ACTION_SCRIPT=scripts/stop-one.cmd): python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080
|
||||
## run 1 - before (shift requests through LB: 0)
|
||||
messenger-lab3-api-1 8b33a91fafee Up 20 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up 53 seconds (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 53 seconds (healthy)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 1.27}
|
||||
{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 6.421}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## run 1 - after
|
||||
messenger-lab3-api-1 8b33a91fafee Exited (0) 14 seconds ago
|
||||
messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Exited (0) 9 seconds ago
|
||||
api1 172.25.0.2 UP
|
||||
## restored (docker compose --env-file .env up -d --wait)
|
||||
messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy)
|
||||
## run 2 - before (shift requests through LB: 0)
|
||||
messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 2 minutes (healthy)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 10 seconds (healthy)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
## run 2 - before (shift requests through LB: 1)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 11 seconds (healthy)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 128, "failed_attempts": 0, "first_recovery_seconds": 1.166}
|
||||
{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 0, "first_recovery_seconds": 6.437}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## run 2 - after
|
||||
messenger-lab3-api-1 8b33a91fafee Exited (0) 14 seconds ago
|
||||
messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy)
|
||||
messenger-lab3-api-1 1b755b53e9fa Exited (0) 9 seconds ago
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 30 seconds (healthy)
|
||||
api2 172.25.0.3 UP
|
||||
## restored (docker compose --env-file .env up -d --wait)
|
||||
messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy)
|
||||
## run 3 - before (shift requests through LB: 1)
|
||||
messenger-lab3-api-1 8b33a91fafee Up 6 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 3 minutes (healthy)
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 127, "failed_attempts": 0, "first_recovery_seconds": 1.213}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## run 3 - after
|
||||
messenger-lab3-api-1 8b33a91fafee Up 22 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Exited (0) 14 seconds ago
|
||||
## restored (docker compose --env-file .env up -d --wait)
|
||||
messenger-lab3-api-1 8b33a91fafee Up 31 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 6 seconds (healthy)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up 9 seconds (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 43 seconds (healthy)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
|
||||
@@ -1,10 +1,7 @@
|
||||
# idempotency run 2026-11-07T17:36:58.5045267+03:00
|
||||
## make test-idempotency PYTHON=python
|
||||
python scripts/check_idempotency.py --base-url "http://localhost:8080"
|
||||
round 0: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10}
|
||||
round 1: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10}
|
||||
round 2: 20 responses, 1 message, replicas {'8b33a91fafee': 10, 'a87734e887a5': 10}
|
||||
round 3: 20 responses, 1 message, replicas {'8b33a91fafee': 10, 'a87734e887a5': 10}
|
||||
round 4: 20 responses, 1 message, replicas {'a87734e887a5': 10, '8b33a91fafee': 10}
|
||||
PASS: 5 keys x 20 parallel requests -> 5 messages via 2 replicas {'a87734e887a5': 50, '8b33a91fafee': 50}
|
||||
exit=0
|
||||
# make test-idempotency 2026-11-07T18:03:22.9365380+03:00 via HAProxy
|
||||
round 0: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
round 1: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
round 2: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
round 3: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
round 4: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
PASS: 5 keys x 20 parallel requests -> 5 messages via 2 replicas {'e3303afc01b2': 50, '1b755b53e9fa': 50}
|
||||
|
||||
+8
-16
@@ -1,16 +1,8 @@
|
||||
# X-Instance-Id run 2026-11-07T17:41:06.5911429+03:00
|
||||
## python scripts/show_instances.py --base-url http://localhost:8080
|
||||
login + chat via LB http://localhost:8080: one session token for user 2ec10789-7814-4751-9e23-7b85215c262e (token not printed)
|
||||
1. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee
|
||||
2. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5
|
||||
3. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee
|
||||
4. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5
|
||||
5. GET /api/v1/users/me -> 200 X-Instance-Id=8b33a91fafee
|
||||
6. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=a87734e887a5
|
||||
PASS: same session served by 2 replicas without re-login {'8b33a91fafee': 3, 'a87734e887a5': 3}
|
||||
exit=0
|
||||
## docker compose ps api (X-Instance-Id = container hostname = short container ID)
|
||||
messenger-lab3-api-1 8b33a91fafee Up 31 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 31 seconds (healthy)
|
||||
/messenger-lab3-api-1 hostname=8b33a91fafee
|
||||
/messenger-lab3-api-2 hostname=a87734e887a5
|
||||
login + chat via LB http://localhost:8080: one session token for user 5757ef5e-391e-4369-85c5-19499d5a4930 (token not printed)
|
||||
1. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2
|
||||
2. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa
|
||||
3. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2
|
||||
4. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa
|
||||
5. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2
|
||||
6. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa
|
||||
PASS: same session served by 2 replicas without re-login {'e3303afc01b2': 3, '1b755b53e9fa': 3}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
# isolation/limits 2026-11-07T18:03:06.9670680+03:00
|
||||
## per container: CPU limit (NanoCpus), memory limit, published ports, networks, mounts
|
||||
/messenger-lab3-api-1 cpus=1000000000 mem=268435456 reserv_mem=134217728 ports= networks=messenger-lab3_data messenger-lab3_edge mounts=
|
||||
/messenger-lab3-api-2 cpus=1000000000 mem=268435456 reserv_mem=134217728 ports= networks=messenger-lab3_data messenger-lab3_edge mounts=
|
||||
/messenger-lab3-lb-1 cpus=500000000 mem=134217728 reserv_mem=33554432 ports=8080/tcp->127.0.0.1:8080 ::1:8080 networks=messenger-lab3_edge mounts=bind:/usr/local/etc/haproxy/haproxy.cfg
|
||||
/messenger-lab3-migrate-1 cpus=500000000 mem=134217728 reserv_mem=0 ports= networks=messenger-lab3_data mounts=
|
||||
/messenger-lab3-postgres-1 cpus=1000000000 mem=536870912 reserv_mem=268435456 ports= networks=messenger-lab3_data mounts=volume:/var/lib/postgresql/data
|
||||
## docker stats --no-stream (this project)
|
||||
NAME CPU % MEM USAGE / LIMIT
|
||||
messenger-lab3-api-1 0.87% 41.18MiB / 256MiB
|
||||
messenger-lab3-api-2 0.62% 41.18MiB / 256MiB
|
||||
messenger-lab3-lb-1 0.21% 14.71MiB / 128MiB
|
||||
messenger-lab3-postgres-1 0.15% 38.04MiB / 512MiB
|
||||
## container env mentioning PASSWORD (expect only *_PASSWORD_FILE paths, no values)
|
||||
messenger-lab3-api-1 DB_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
messenger-lab3-api-2 DB_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
messenger-lab3-migrate-1 DB_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
messenger-lab3-postgres-1 POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
## secret file inside api: owner/mode only, value not printed
|
||||
total 4
|
||||
-r-------- 1 10001 10001 18 Nov 7 15:02 postgres_password
|
||||
## lb has no secrets and cannot resolve postgres (not on the data network)
|
||||
no /run/secrets
|
||||
postgres not resolvable from lb
|
||||
## networks
|
||||
messenger-lab3_data internal=true
|
||||
messenger-lab3_edge internal=false
|
||||
## docker port messenger-lab3-api-1 (empty = no published port)
|
||||
## occurrences of container_name in compose.yaml
|
||||
0
|
||||
+15
-12
@@ -1,18 +1,21 @@
|
||||
# persistence run 2026-11-07T17:40:29.1339292+03:00
|
||||
# persistence run 2026-11-07T18:04:54.7155080+03:00
|
||||
# command (= make test-persistence ACTION_SCRIPT=scripts/restart.cmd): python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080
|
||||
## before
|
||||
messenger-lab3-api-1 8b33a91fafee Up 38 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 14 seconds (healthy)
|
||||
messenger-lab3-lb-1 4ba96e32076f Up 3 minutes (healthy)
|
||||
messenger-lab3-migrate-1 18f3889ef9a7 Exited (0) 14 seconds ago
|
||||
messenger-lab3-postgres-1 4c60c6522907 Up 3 minutes (healthy)
|
||||
NAME CONTAINER ID STATUS
|
||||
messenger-lab3-api-1 1b755b53e9fa Up 22 seconds (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 56 seconds (healthy)
|
||||
messenger-lab3-lb-1 760a68ddce9c Up 3 minutes (healthy)
|
||||
messenger-lab3-postgres-1 2265aa5e595d Up 3 minutes (healthy)
|
||||
volume messenger-lab3_pgdata
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## after (same container IDs, fresh uptime = restarted; volume pgdata kept)
|
||||
messenger-lab3-api-1 8b33a91fafee Up 11 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 11 seconds (healthy)
|
||||
messenger-lab3-lb-1 4ba96e32076f Up 5 seconds (healthy)
|
||||
messenger-lab3-migrate-1 18f3889ef9a7 Exited (0) 12 seconds ago
|
||||
messenger-lab3-postgres-1 4c60c6522907 Up 15 seconds (healthy)
|
||||
## after (same IDs, new uptime)
|
||||
NAME CONTAINER ID STATUS
|
||||
messenger-lab3-api-1 1b755b53e9fa Up 12 seconds (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 11 seconds (healthy)
|
||||
messenger-lab3-lb-1 760a68ddce9c Up 5 seconds (healthy)
|
||||
messenger-lab3-postgres-1 2265aa5e595d Up 16 seconds (healthy)
|
||||
volume messenger-lab3_pgdata
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# rolling update dev -> v2 with migration 003 applied under load, 2026-11-07 17:59
|
||||
# Console output copied verbatim; the JSON of this run was overwritten by the later v2 -> v3 run (rolling-update.json).
|
||||
# Old replicas: image messenger-lab3-app:dev, built before migrations/003_session_indexes.sql existed
|
||||
# (its readiness requires only 001, 002). New image messenger-lab3-app:v2 contains 001-003.
|
||||
# Command: APP_VERSION=v2 python scripts/failover_series.py --mode action --action scripts/rolling-update.cmd --output evidence/rolling-update.json
|
||||
running scripts/rolling-update.cmd under load...
|
||||
{"total": {"seconds": 70.91, "rps_ok": 125.7, "writes_ok": 4458, "reads_ok": 4458, "errors": 0, "p95_ms": 78.1, "instances": {"419aeb0c6c36": 2714, "e90dced210a7": 1695, "a8021c6dc466": 2752, "a2bb8bd69281": 1755}}, "integrity": {"confirmed_writes": 4458, "stored_messages": 4458, "duplicates": 0, "missing_confirmed": 0}}
|
||||
PASS: action, 4458 confirmed writes, no duplicates, nothing lost
|
||||
## rolling-update.sh output and metrics
|
||||
17:59:39 rolling 2 replica(s) to messenger-lab3-app:v2
|
||||
17:59:48 migrations applied once
|
||||
17:59:59 new replica(s) UP in LB: a8021c6dc466 ; stopping old e90dced210a7
|
||||
18:00:05 old e90dced210a7 drained and removed
|
||||
18:00:17 new replica(s) UP in LB: a2bb8bd69281 a8021c6dc466 ; stopping old 419aeb0c6c36
|
||||
18:00:24 old 419aeb0c6c36 drained and removed
|
||||
messenger-lab3-api-3 messenger-lab3-app:v2 Up 33 seconds (healthy)
|
||||
messenger-lab3-api-4 messenger-lab3-app:v2 Up 15 seconds (healthy)
|
||||
18:00:24 done; set APP_VERSION=v2 in .env to keep this version on the next compose up
|
||||
action_seconds 45.73 max_gap 0.097
|
||||
versions before {'dev': 1001} during {'dev': 3408, 'v2': 2234} after {'v2': 2265}
|
||||
during {'seconds': 45.73, 'rps_ok': 123.4, 'writes_ok': 2821, 'reads_ok': 2821, 'errors': 0, 'p95_ms': 78.5, 'instances': {'419aeb0c6c36': 2214, 'e90dced210a7': 1194, 'a8021c6dc466': 1615, 'a2bb8bd69281': 619}}
|
||||
# After 003 was applied at 17:59:48 both dev replicas stayed in rotation until they were replaced
|
||||
# (17:59:59 and 18:00:17); dev served 3408 successful responses during the action with 0 errors overall,
|
||||
# i.e. the old build stayed ready and correct on the expanded schema.
|
||||
@@ -0,0 +1,955 @@
|
||||
{
|
||||
"mode": "action",
|
||||
"stop_method": "docker stop --time 9 (SIGTERM, drain)",
|
||||
"concurrency": 4,
|
||||
"workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest",
|
||||
"replicas_before": [
|
||||
"messenger-lab3-api-1",
|
||||
"messenger-lab3-api-2"
|
||||
],
|
||||
"runs": [
|
||||
{
|
||||
"action": "rolling-update.cmd",
|
||||
"action_exit_code": 0,
|
||||
"action_output": [
|
||||
"18:11:19 rolling 2 replica(s) to messenger-lab3-app:v3",
|
||||
"18:11:26 migrations applied once",
|
||||
"18:11:37 new replica(s) UP in LB: 3cfcdcac70c7 ; stopping old 1b755b53e9fa",
|
||||
"18:11:43 old 1b755b53e9fa drained and removed",
|
||||
"18:11:54 new replica(s) UP in LB: 13c61b35afd7 3cfcdcac70c7 ; stopping old e3303afc01b2",
|
||||
"18:12:00 old e3303afc01b2 drained and removed",
|
||||
"messenger-lab3-api-3 messenger-lab3-app:v3 Up 32 seconds (healthy)",
|
||||
"messenger-lab3-api-4 messenger-lab3-app:v3 Up 15 seconds (healthy)",
|
||||
"18:12:01 done; set APP_VERSION=v3 in .env to keep this version on the next compose up"
|
||||
],
|
||||
"action_seconds": 42.67,
|
||||
"max_gap_without_success_s": 0.073,
|
||||
"baseline": {
|
||||
"seconds": 10.1,
|
||||
"rps_ok": 181.4,
|
||||
"writes_ok": 917,
|
||||
"reads_ok": 916,
|
||||
"errors": 0,
|
||||
"p95_ms": 53.4,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 915,
|
||||
"e3303afc01b2": 918
|
||||
}
|
||||
},
|
||||
"during_action": {
|
||||
"seconds": 42.67,
|
||||
"rps_ok": 145.8,
|
||||
"writes_ok": 3110,
|
||||
"reads_ok": 3110,
|
||||
"errors": 0,
|
||||
"p95_ms": 68.3,
|
||||
"instances": {
|
||||
"e3303afc01b2": 2471,
|
||||
"1b755b53e9fa": 1470,
|
||||
"3cfcdcac70c7": 1689,
|
||||
"13c61b35afd7": 590
|
||||
}
|
||||
},
|
||||
"after_action": {
|
||||
"seconds": 15.0,
|
||||
"rps_ok": 185.8,
|
||||
"writes_ok": 1394,
|
||||
"reads_ok": 1393,
|
||||
"errors": 0,
|
||||
"p95_ms": 54.4,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 1394,
|
||||
"13c61b35afd7": 1393
|
||||
}
|
||||
},
|
||||
"versions_before": {
|
||||
"v2": 1833
|
||||
},
|
||||
"versions_during": {
|
||||
"v2": 3941,
|
||||
"v3": 2279
|
||||
},
|
||||
"versions_after": {
|
||||
"v3": 2787
|
||||
},
|
||||
"marks": {
|
||||
"action_start": 10.1,
|
||||
"action_end": 52.77
|
||||
}
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"seconds": 67.8,
|
||||
"rps_ok": 160.0,
|
||||
"writes_ok": 5423,
|
||||
"reads_ok": 5423,
|
||||
"errors": 0,
|
||||
"p95_ms": 62.6,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 2385,
|
||||
"e3303afc01b2": 3389,
|
||||
"3cfcdcac70c7": 3086,
|
||||
"13c61b35afd7": 1986
|
||||
}
|
||||
},
|
||||
"integrity": {
|
||||
"confirmed_writes": 5423,
|
||||
"stored_messages": 5423,
|
||||
"duplicates": 0,
|
||||
"missing_confirmed": 0
|
||||
},
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 174,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 87,
|
||||
"e3303afc01b2": 87
|
||||
},
|
||||
"versions": {
|
||||
"v2": 174
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 149,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 76,
|
||||
"1b755b53e9fa": 73
|
||||
},
|
||||
"versions": {
|
||||
"v2": 149
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 148,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 73,
|
||||
"1b755b53e9fa": 75
|
||||
},
|
||||
"versions": {
|
||||
"v2": 148
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 204,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 102,
|
||||
"e3303afc01b2": 102
|
||||
},
|
||||
"versions": {
|
||||
"v2": 204
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 226,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 113,
|
||||
"e3303afc01b2": 113
|
||||
},
|
||||
"versions": {
|
||||
"v2": 226
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 192,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 95,
|
||||
"e3303afc01b2": 97
|
||||
},
|
||||
"versions": {
|
||||
"v2": 192
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 156,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 77,
|
||||
"1b755b53e9fa": 79
|
||||
},
|
||||
"versions": {
|
||||
"v2": 156
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 212,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 106,
|
||||
"e3303afc01b2": 106
|
||||
},
|
||||
"versions": {
|
||||
"v2": 212
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 193,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 96,
|
||||
"1b755b53e9fa": 97
|
||||
},
|
||||
"versions": {
|
||||
"v2": 193
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 164,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 80,
|
||||
"e3303afc01b2": 84
|
||||
},
|
||||
"versions": {
|
||||
"v2": 164
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 100,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 53,
|
||||
"e3303afc01b2": 47
|
||||
},
|
||||
"versions": {
|
||||
"v2": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 136,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 69,
|
||||
"1b755b53e9fa": 67
|
||||
},
|
||||
"versions": {
|
||||
"v2": 136
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 161,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 81,
|
||||
"1b755b53e9fa": 80
|
||||
},
|
||||
"versions": {
|
||||
"v2": 161
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 133,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 67,
|
||||
"e3303afc01b2": 66
|
||||
},
|
||||
"versions": {
|
||||
"v2": 133
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 99,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 49,
|
||||
"1b755b53e9fa": 50
|
||||
},
|
||||
"versions": {
|
||||
"v2": 99
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 130,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 66,
|
||||
"1b755b53e9fa": 64
|
||||
},
|
||||
"versions": {
|
||||
"v2": 130
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 215,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 107,
|
||||
"1b755b53e9fa": 108
|
||||
},
|
||||
"versions": {
|
||||
"v2": 215
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 86,
|
||||
"1b755b53e9fa": 85
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 102,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 51,
|
||||
"1b755b53e9fa": 51
|
||||
},
|
||||
"versions": {
|
||||
"v2": 102
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 85,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 42,
|
||||
"1b755b53e9fa": 43
|
||||
},
|
||||
"versions": {
|
||||
"v2": 85
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 124,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 62,
|
||||
"1b755b53e9fa": 62
|
||||
},
|
||||
"versions": {
|
||||
"v2": 124
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 176,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 88,
|
||||
"e3303afc01b2": 88
|
||||
},
|
||||
"versions": {
|
||||
"v2": 176
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 198,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 99,
|
||||
"e3303afc01b2": 99
|
||||
},
|
||||
"versions": {
|
||||
"v2": 198
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 152,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 76,
|
||||
"1b755b53e9fa": 76
|
||||
},
|
||||
"versions": {
|
||||
"v2": 152
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 176,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 88,
|
||||
"1b755b53e9fa": 88
|
||||
},
|
||||
"versions": {
|
||||
"v2": 176
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 203,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 91,
|
||||
"1b755b53e9fa": 91,
|
||||
"3cfcdcac70c7": 21
|
||||
},
|
||||
"versions": {
|
||||
"v2": 182,
|
||||
"v3": 21
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 194,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 64,
|
||||
"1b755b53e9fa": 64,
|
||||
"e3303afc01b2": 66
|
||||
},
|
||||
"versions": {
|
||||
"v3": 64,
|
||||
"v2": 130
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 161,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 55,
|
||||
"1b755b53e9fa": 54,
|
||||
"e3303afc01b2": 52
|
||||
},
|
||||
"versions": {
|
||||
"v3": 55,
|
||||
"v2": 106
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 104,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 35,
|
||||
"3cfcdcac70c7": 34,
|
||||
"1b755b53e9fa": 35
|
||||
},
|
||||
"versions": {
|
||||
"v2": 70,
|
||||
"v3": 34
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 156,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 52,
|
||||
"e3303afc01b2": 52,
|
||||
"1b755b53e9fa": 52
|
||||
},
|
||||
"versions": {
|
||||
"v3": 52,
|
||||
"v2": 104
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 174,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 58,
|
||||
"1b755b53e9fa": 58,
|
||||
"3cfcdcac70c7": 58
|
||||
},
|
||||
"versions": {
|
||||
"v2": 116,
|
||||
"v3": 58
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 191,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 74,
|
||||
"3cfcdcac70c7": 74,
|
||||
"1b755b53e9fa": 43
|
||||
},
|
||||
"versions": {
|
||||
"v2": 117,
|
||||
"v3": 74
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 162,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 81,
|
||||
"e3303afc01b2": 81
|
||||
},
|
||||
"versions": {
|
||||
"v3": 81,
|
||||
"v2": 81
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 166,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 83,
|
||||
"3cfcdcac70c7": 83
|
||||
},
|
||||
"versions": {
|
||||
"v2": 83,
|
||||
"v3": 83
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 175,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 87,
|
||||
"e3303afc01b2": 88
|
||||
},
|
||||
"versions": {
|
||||
"v3": 87,
|
||||
"v2": 88
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 114,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 57,
|
||||
"3cfcdcac70c7": 57
|
||||
},
|
||||
"versions": {
|
||||
"v2": 57,
|
||||
"v3": 57
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 179,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 91,
|
||||
"e3303afc01b2": 88
|
||||
},
|
||||
"versions": {
|
||||
"v3": 91,
|
||||
"v2": 88
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 92,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 46,
|
||||
"3cfcdcac70c7": 46
|
||||
},
|
||||
"versions": {
|
||||
"v2": 46,
|
||||
"v3": 46
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 101,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 49,
|
||||
"e3303afc01b2": 52
|
||||
},
|
||||
"versions": {
|
||||
"v3": 49,
|
||||
"v2": 52
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 121,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 60,
|
||||
"3cfcdcac70c7": 61
|
||||
},
|
||||
"versions": {
|
||||
"v2": 60,
|
||||
"v3": 61
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 128,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 64,
|
||||
"3cfcdcac70c7": 64
|
||||
},
|
||||
"versions": {
|
||||
"v2": 64,
|
||||
"v3": 64
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 148,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 73,
|
||||
"3cfcdcac70c7": 75
|
||||
},
|
||||
"versions": {
|
||||
"v2": 73,
|
||||
"v3": 75
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 115,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 56,
|
||||
"3cfcdcac70c7": 54,
|
||||
"13c61b35afd7": 5
|
||||
},
|
||||
"versions": {
|
||||
"v2": 56,
|
||||
"v3": 59
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 150,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 50,
|
||||
"13c61b35afd7": 51,
|
||||
"3cfcdcac70c7": 49
|
||||
},
|
||||
"versions": {
|
||||
"v2": 50,
|
||||
"v3": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 156,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 51,
|
||||
"3cfcdcac70c7": 53,
|
||||
"13c61b35afd7": 52
|
||||
},
|
||||
"versions": {
|
||||
"v2": 51,
|
||||
"v3": 105
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 140,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 48,
|
||||
"13c61b35afd7": 46,
|
||||
"3cfcdcac70c7": 46
|
||||
},
|
||||
"versions": {
|
||||
"v2": 48,
|
||||
"v3": 92
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 134,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 45,
|
||||
"13c61b35afd7": 45,
|
||||
"e3303afc01b2": 44
|
||||
},
|
||||
"versions": {
|
||||
"v3": 90,
|
||||
"v2": 44
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 136,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 32,
|
||||
"3cfcdcac70c7": 52,
|
||||
"13c61b35afd7": 52
|
||||
},
|
||||
"versions": {
|
||||
"v2": 32,
|
||||
"v3": 104
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 117,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 58,
|
||||
"13c61b35afd7": 59
|
||||
},
|
||||
"versions": {
|
||||
"v3": 117
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 163,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 82,
|
||||
"13c61b35afd7": 81
|
||||
},
|
||||
"versions": {
|
||||
"v3": 163
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 187,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 94,
|
||||
"3cfcdcac70c7": 93
|
||||
},
|
||||
"versions": {
|
||||
"v3": 187
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 140,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 69,
|
||||
"3cfcdcac70c7": 71
|
||||
},
|
||||
"versions": {
|
||||
"v3": 140
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 101,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 50,
|
||||
"13c61b35afd7": 51
|
||||
},
|
||||
"versions": {
|
||||
"v3": 101
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 165,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 82,
|
||||
"13c61b35afd7": 83
|
||||
},
|
||||
"versions": {
|
||||
"v3": 165
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 240,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 120,
|
||||
"13c61b35afd7": 120
|
||||
},
|
||||
"versions": {
|
||||
"v3": 240
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 253,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 126,
|
||||
"3cfcdcac70c7": 127
|
||||
},
|
||||
"versions": {
|
||||
"v3": 253
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 56,
|
||||
"ok": 150,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 75,
|
||||
"13c61b35afd7": 75
|
||||
},
|
||||
"versions": {
|
||||
"v3": 150
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 57,
|
||||
"ok": 142,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 71,
|
||||
"13c61b35afd7": 71
|
||||
},
|
||||
"versions": {
|
||||
"v3": 142
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 58,
|
||||
"ok": 197,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 98,
|
||||
"13c61b35afd7": 99
|
||||
},
|
||||
"versions": {
|
||||
"v3": 197
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 59,
|
||||
"ok": 202,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 101,
|
||||
"13c61b35afd7": 101
|
||||
},
|
||||
"versions": {
|
||||
"v3": 202
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 60,
|
||||
"ok": 175,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 87,
|
||||
"3cfcdcac70c7": 88
|
||||
},
|
||||
"versions": {
|
||||
"v3": 175
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 61,
|
||||
"ok": 126,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 62,
|
||||
"13c61b35afd7": 64
|
||||
},
|
||||
"versions": {
|
||||
"v3": 126
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 62,
|
||||
"ok": 190,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 96,
|
||||
"13c61b35afd7": 94
|
||||
},
|
||||
"versions": {
|
||||
"v3": 190
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 63,
|
||||
"ok": 209,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 104,
|
||||
"13c61b35afd7": 105
|
||||
},
|
||||
"versions": {
|
||||
"v3": 209
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 64,
|
||||
"ok": 214,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 107,
|
||||
"3cfcdcac70c7": 107
|
||||
},
|
||||
"versions": {
|
||||
"v3": 214
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 65,
|
||||
"ok": 145,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 72,
|
||||
"3cfcdcac70c7": 73
|
||||
},
|
||||
"versions": {
|
||||
"v3": 145
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 66,
|
||||
"ok": 166,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 83,
|
||||
"13c61b35afd7": 83
|
||||
},
|
||||
"versions": {
|
||||
"v3": 166
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 67,
|
||||
"ok": 188,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 94,
|
||||
"13c61b35afd7": 94
|
||||
},
|
||||
"versions": {
|
||||
"v3": 188
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
# rolling update 2026-11-07T18:12:17.4627477+03:00: APP_VERSION=v3 (image prebuilt), python scripts/failover_series.py --mode action --action scripts/rolling-update.cmd
|
||||
running scripts/rolling-update.cmd under load...
|
||||
{"total": {"seconds": 67.8, "rps_ok": 160.0, "writes_ok": 5423, "reads_ok": 5423, "errors": 0, "p95_ms": 62.6, "instances": {"1b755b53e9fa": 2385, "e3303afc01b2": 3389, "3cfcdcac70c7": 3086, "13c61b35afd7": 1986}}, "integrity": {"confirmed_writes": 5423, "stored_messages": 5423, "duplicates": 0, "missing_confirmed": 0}}
|
||||
PASS: action, 5423 confirmed writes, no duplicates, nothing lost
|
||||
## rolling-update.sh output and metrics
|
||||
18:11:19 rolling 2 replica(s) to messenger-lab3-app:v3
|
||||
18:11:26 migrations applied once
|
||||
18:11:37 new replica(s) UP in LB: 3cfcdcac70c7 ; stopping old 1b755b53e9fa
|
||||
18:11:43 old 1b755b53e9fa drained and removed
|
||||
18:11:54 new replica(s) UP in LB: 13c61b35afd7 3cfcdcac70c7 ; stopping old e3303afc01b2
|
||||
18:12:00 old e3303afc01b2 drained and removed
|
||||
messenger-lab3-api-3 messenger-lab3-app:v3 Up 32 seconds (healthy)
|
||||
messenger-lab3-api-4 messenger-lab3-app:v3 Up 15 seconds (healthy)
|
||||
18:12:01 done; set APP_VERSION=v3 in .env to keep this version on the next compose up
|
||||
action_seconds 42.67 max_gap 0.073
|
||||
versions before {'v2': 1833} during {'v2': 3941, 'v3': 2279} after {'v3': 2787}
|
||||
baseline {'seconds': 10.1, 'rps_ok': 181.4, 'writes_ok': 917, 'reads_ok': 916, 'errors': 0, 'p95_ms': 53.4, 'instances': {'1b755b53e9fa': 915, 'e3303afc01b2': 918}}
|
||||
during {'seconds': 42.67, 'rps_ok': 145.8, 'writes_ok': 3110, 'reads_ok': 3110, 'errors': 0, 'p95_ms': 68.3, 'instances': {'e3303afc01b2': 2471, '1b755b53e9fa': 1470, '3cfcdcac70c7': 1689, '13c61b35afd7': 590}}
|
||||
after {'seconds': 15.0, 'rps_ok': 185.8, 'writes_ok': 1394, 'reads_ok': 1393, 'errors': 0, 'p95_ms': 54.4, 'instances': {'3cfcdcac70c7': 1394, '13c61b35afd7': 1393}}
|
||||
+44
-10
@@ -1,14 +1,48 @@
|
||||
# smoke run 2026-11-07T17:36:58.5045267+03:00
|
||||
## docker compose ps
|
||||
messenger-lab3-api-1 8b33a91fafee Up 19 seconds (healthy)
|
||||
messenger-lab3-api-2 a87734e887a5 Up 19 seconds (healthy)
|
||||
messenger-lab3-lb-1 4ba96e32076f Up 13 seconds (healthy)
|
||||
messenger-lab3-postgres-1 4c60c6522907 Up 23 seconds (healthy)
|
||||
## make check PYTHON=python
|
||||
python scripts/check_contract.py
|
||||
# smoke run 2026-11-07T18:01:21.4109350+03:00, commit 124c83e + working tree
|
||||
## docker compose --env-file .env down -v
|
||||
|
||||
Container messenger-lab3-lb-1 Removed
|
||||
Container messenger-lab3-api-4 Removed
|
||||
Container messenger-lab3-api-3 Removed
|
||||
Container messenger-lab3-migrate-1 Removed
|
||||
Container messenger-lab3-postgres-1 Removed
|
||||
Volume messenger-lab3_pgdata Removing
|
||||
Volume messenger-lab3_pgdata Removed
|
||||
Network messenger-lab3_data Removed
|
||||
Network messenger-lab3_edge Removed
|
||||
## docker compose --env-file .env config --quiet
|
||||
exit=0
|
||||
## docker compose --env-file .env up -d --build --wait
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-migrate-1 Exited
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
Container messenger-lab3-migrate-1 Exited
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
Container messenger-lab3-lb-1 Healthy
|
||||
exit=0
|
||||
## docker compose ps -a
|
||||
NAME IMAGE STATUS PORTS
|
||||
messenger-lab3-api-1 messenger-lab3-app:v2 Up 12 seconds (healthy) 8080/tcp
|
||||
messenger-lab3-api-2 messenger-lab3-app:v2 Up 12 seconds (healthy) 8080/tcp
|
||||
messenger-lab3-lb-1 haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0 Up 5 seconds (healthy) 127.0.0.1:8080->8080/tcp, [::1]:8080->8080/tcp
|
||||
messenger-lab3-migrate-1 messenger-lab3-app:v2 Exited (0) 12 seconds ago
|
||||
messenger-lab3-postgres-1 postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 Up 19 seconds (healthy) 5432/tcp
|
||||
## migrate log (one-shot)
|
||||
INFO applied migration 001_init.sql
|
||||
INFO applied migration 002_idempotency.sql
|
||||
INFO applied migration 003_session_indexes.sql
|
||||
## HAProxy pool (scripts/lb-status.sh)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
## make check
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
## make test PYTHON=python
|
||||
python tests/smoke.py --base-url "http://localhost:8080"
|
||||
## make test
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# Client -> HAProxy -> api x N. Replicas are discovered through Docker DNS and checked actively.
|
||||
#
|
||||
# Timing contract (keep in sync with compose.yaml and app settings):
|
||||
# check period inter 1s, timeout check 1s
|
||||
# marked DOWN after fall 2 failed checks -> hung/draining replica leaves the pool in <= 3 s
|
||||
# marked UP after rise 2 good checks -> new replica gets traffic ~2 s after it is ready
|
||||
# DNS refresh every 1 s (hold valid 1s); a removed container disappears within ~1-2 s
|
||||
# crashed process connect refused -> request redispatched to another replica at once
|
||||
# app DRAIN_SECONDS=4 > 3 s, so a replica receiving SIGTERM is out of rotation before it closes its socket.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2048
|
||||
# Runtime API for evidence (scripts/lb-status.sh); loopback inside the container only.
|
||||
stats socket ipv4@127.0.0.1:9999 level operator
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold other 1s
|
||||
hold refused 1s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold obsolete 0s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
option forwardfor
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 2s
|
||||
timeout check 1s
|
||||
# A request whose connection to a replica failed is sent to another replica.
|
||||
retries 2
|
||||
option redispatch 1
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
monitor-uri /lb/health
|
||||
http-request deny deny_status 413 if { req.hdr_val(content-length) gt 1048576 }
|
||||
|
||||
# Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe.
|
||||
# Other POSTs (users, chats, sessions) are retried only when the connection was never established.
|
||||
acl message_post method POST
|
||||
acl message_post_path path_reg ^/api/v1/chats/[^/]+/messages$
|
||||
http-request disable-l7-retry if message_post !message_post_path
|
||||
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
# Round robin, no stickiness: any replica serves any session from PostgreSQL.
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
|
||||
http-check expect status 200
|
||||
retry-on conn-failure empty-response response-timeout
|
||||
http-error status 503 content-type application/json string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API replica is available\",\"request_id\":\"lb\"}}"
|
||||
http-error status 502 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Bad gateway\",\"request_id\":\"lb\"}}"
|
||||
http-error status 504 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Gateway timeout\",\"request_id\":\"lb\"}}"
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
|
||||
server-template api 1-10 api:8080
|
||||
|
||||
# Read-only stats for evidence; not published, reachable only inside the container.
|
||||
frontend stats
|
||||
bind 127.0.0.1:8404
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 5s
|
||||
http-request set-log-level silent
|
||||
@@ -1,66 +0,0 @@
|
||||
worker_processes auto;
|
||||
pid /tmp/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
access_log /dev/stdout;
|
||||
error_log /dev/stderr warn;
|
||||
|
||||
# Docker embedded DNS. "api" resolves to every running replica of the Compose service;
|
||||
# with "resolve" below nginx re-reads the set every 2 s, so stopped/scaled replicas leave and join the pool.
|
||||
resolver 127.0.0.11 valid=2s ipv6=off;
|
||||
resolver_timeout 2s;
|
||||
|
||||
log_format upstream '$remote_addr "$request" $status upstream=$upstream_addr '
|
||||
'upstream_status=$upstream_status rt=$request_time';
|
||||
|
||||
upstream api {
|
||||
zone api_upstream 64k;
|
||||
# Round robin, no ip_hash/sticky: any replica serves any session from PostgreSQL.
|
||||
server api:8080 resolve max_fails=1 fail_timeout=5s;
|
||||
keepalive 32;
|
||||
# Shorter than uvicorn's keep-alive (5 s), so nginx never reuses a connection the API is closing.
|
||||
keepalive_timeout 4s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
access_log /dev/stdout upstream;
|
||||
client_max_body_size 1m;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 10s;
|
||||
proxy_read_timeout 10s;
|
||||
|
||||
# Not-yet-sent requests and idempotent methods go to the next replica on a dead backend.
|
||||
# Non-idempotent POSTs (users, chats, sessions) are never replayed by the LB.
|
||||
proxy_next_upstream error timeout http_502 http_504;
|
||||
proxy_next_upstream_tries 3;
|
||||
proxy_next_upstream_timeout 5s;
|
||||
|
||||
location = /lb/health {
|
||||
access_log off;
|
||||
default_type application/json;
|
||||
return 200 '{"status":"ok"}';
|
||||
}
|
||||
|
||||
# Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe.
|
||||
location ~ ^/api/v1/chats/[^/]+/messages$ {
|
||||
proxy_next_upstream error timeout http_502 http_504 non_idempotent;
|
||||
proxy_pass http://api;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Expand-only: new indexes, no column/table removal or rename, so replicas of the previous build keep working
|
||||
-- while the rolling update is in progress. Destructive changes go to a later release, after old replicas are gone.
|
||||
CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id);
|
||||
CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions (expires_at);
|
||||
Executable
+303
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs).
|
||||
|
||||
failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe ->
|
||||
docker start the same container -> wait until it serves again -> settle. Restores before the next one.
|
||||
action: baseline -> run --action script (e.g. rolling-update) under load -> observe.
|
||||
|
||||
Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body)
|
||||
and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded.
|
||||
At the end the whole chat history is read back: every confirmed message must be present exactly once.
|
||||
Session tokens stay in memory and are never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
PROJECT = 'messenger-lab3'
|
||||
|
||||
|
||||
def docker(*args, timeout=60):
|
||||
return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip()
|
||||
|
||||
|
||||
def api_containers():
|
||||
"""(name, short id) of running api containers of this Compose project, sorted by name."""
|
||||
out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}')
|
||||
return sorted(tuple(line.split()) for line in out.splitlines() if line.strip())
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, base_url, concurrency):
|
||||
self.base_url = base_url
|
||||
self.concurrency = concurrency
|
||||
self.events = [] # (t, op, ok, instance, version, latency)
|
||||
self.confirmed = {} # message id -> text
|
||||
self.lock = threading.Lock()
|
||||
self.stop = threading.Event()
|
||||
self.t0 = None
|
||||
|
||||
def now(self):
|
||||
return time.monotonic() - self.t0
|
||||
|
||||
def record(self, op, ok, response=None, latency=0.0):
|
||||
headers = response['headers'] if response else {}
|
||||
event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None,
|
||||
headers.get('X-App-Version') if response else None, latency)
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
|
||||
def worker(self, worker_id, alice, chat):
|
||||
c = Client(self.base_url, timeout=5)
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20'
|
||||
index = 0
|
||||
key = text = None
|
||||
while not self.stop.is_set():
|
||||
if key is None:
|
||||
key = uuid.uuid4().hex
|
||||
text = 'series-%d-%d-%s' % (worker_id, index, key[:8])
|
||||
start = time.monotonic()
|
||||
try:
|
||||
response = c.message(alice, chat, text, key=key)
|
||||
self.record('write', True, response, time.monotonic() - start)
|
||||
with self.lock:
|
||||
self.confirmed[response['json']['id']] = text
|
||||
key = None
|
||||
index += 1
|
||||
except Failure:
|
||||
self.record('write', False, latency=time.monotonic() - start)
|
||||
time.sleep(0.05)
|
||||
continue # same key and text: the first attempt may have committed
|
||||
start = time.monotonic()
|
||||
try:
|
||||
response = c.call('GET', path, actor=alice, schema='MessagePage')
|
||||
self.record('read', True, response, time.monotonic() - start)
|
||||
except Failure:
|
||||
self.record('read', False, latency=time.monotonic() - start)
|
||||
time.sleep(0.05)
|
||||
|
||||
def start(self, alice, chat):
|
||||
self.t0 = time.monotonic()
|
||||
self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True)
|
||||
for i in range(self.concurrency)]
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=10)
|
||||
|
||||
def window(self, start, end):
|
||||
return [e for e in self.events if start <= e[0] < end]
|
||||
|
||||
|
||||
def summarize(events, start, end):
|
||||
ok = [e for e in events if e[2]]
|
||||
duration = max(end - start, 1e-9)
|
||||
latencies = sorted(e[5] for e in ok)
|
||||
p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None
|
||||
return {
|
||||
'seconds': round(duration, 2),
|
||||
'rps_ok': round(len(ok) / duration, 1),
|
||||
'writes_ok': sum(1 for e in ok if e[1] == 'write'),
|
||||
'reads_ok': sum(1 for e in ok if e[1] == 'read'),
|
||||
'errors': sum(1 for e in events if not e[2]),
|
||||
'p95_ms': p95,
|
||||
'instances': dict(Counter(e[3] for e in ok)),
|
||||
}
|
||||
|
||||
|
||||
def timeline(load, end):
|
||||
buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()})
|
||||
for t, _, ok, instance, version, _ in load.events:
|
||||
if t >= end:
|
||||
continue
|
||||
b = buckets[int(t)]
|
||||
if ok:
|
||||
b['ok'] += 1
|
||||
b['instances'][instance] += 1
|
||||
b['versions'][version] += 1
|
||||
else:
|
||||
b['err'] += 1
|
||||
return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'],
|
||||
'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])}
|
||||
for s in range(int(end) + 1)]
|
||||
|
||||
|
||||
def max_gap(events, start, end):
|
||||
"""Longest interval without any successful response inside [start, end)."""
|
||||
times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end]
|
||||
return round(max(b - a for a, b in zip(times, times[1:])), 3)
|
||||
|
||||
|
||||
def wait_serving(load, instance, since, limit=30):
|
||||
deadline = time.monotonic() + limit
|
||||
while time.monotonic() < deadline:
|
||||
with load.lock:
|
||||
hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None)
|
||||
if hit is not None:
|
||||
return hit
|
||||
time.sleep(0.2)
|
||||
return None
|
||||
|
||||
|
||||
def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill):
|
||||
time.sleep(baseline)
|
||||
t_stop = load.now()
|
||||
if kill:
|
||||
docker('kill', cid)
|
||||
else:
|
||||
docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30)
|
||||
t_stopped = load.now()
|
||||
state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid)
|
||||
time.sleep(observe)
|
||||
t_start = load.now()
|
||||
docker('start', cid)
|
||||
rejoined = wait_serving(load, cid, t_start)
|
||||
time.sleep(settle)
|
||||
t_end = load.now()
|
||||
|
||||
with load.lock:
|
||||
events = list(load.events)
|
||||
base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop)
|
||||
outage = [e for e in events if t_stop <= e[0] < t_start]
|
||||
last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None)
|
||||
first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None)
|
||||
# Recovery: first full second after the container stopped whose throughput is >= 90% of baseline.
|
||||
recovered = None
|
||||
for s in range(int(t_stopped), int(t_start)):
|
||||
n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1)
|
||||
if n >= 0.9 * base['rps_ok']:
|
||||
recovered = s + 1 - t_stopped
|
||||
break
|
||||
return {
|
||||
'replica': name,
|
||||
'instance_id': cid,
|
||||
'state_after_stop': state,
|
||||
'stop_command_seconds': round(t_stopped - t_stop, 2),
|
||||
'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None,
|
||||
'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None,
|
||||
'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None,
|
||||
'max_gap_without_success_s': max_gap(events, t_stop, t_start),
|
||||
'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None,
|
||||
'baseline': base,
|
||||
'during_stop_and_outage': summarize(outage, t_stop, t_start),
|
||||
'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end),
|
||||
'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)},
|
||||
}
|
||||
|
||||
|
||||
def action_run(load, action, baseline, observe):
|
||||
time.sleep(baseline)
|
||||
t_action = load.now()
|
||||
proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600)
|
||||
t_done = load.now()
|
||||
time.sleep(observe)
|
||||
t_end = load.now()
|
||||
with load.lock:
|
||||
events = list(load.events)
|
||||
during = [e for e in events if t_action <= e[0] < t_done]
|
||||
return {
|
||||
'action': Path(action).name,
|
||||
'action_exit_code': proc.returncode,
|
||||
'action_output': proc.stdout.strip().splitlines()[-30:],
|
||||
'action_seconds': round(t_done - t_action, 2),
|
||||
'max_gap_without_success_s': max_gap(events, t_action, t_done),
|
||||
'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action),
|
||||
'during_action': summarize(during, t_action, t_done),
|
||||
'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end),
|
||||
'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)),
|
||||
'versions_during': dict(Counter(e[4] for e in during if e[2])),
|
||||
'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)),
|
||||
'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument('--base-url', default=None)
|
||||
p.add_argument('--mode', choices=['failover', 'action'], default='failover')
|
||||
p.add_argument('--action', help='script to run under load in --mode action')
|
||||
p.add_argument('--concurrency', type=int, default=4)
|
||||
p.add_argument('--baseline', type=float, default=10)
|
||||
p.add_argument('--observe', type=float, default=15)
|
||||
p.add_argument('--settle', type=float, default=5)
|
||||
p.add_argument('--stop-timeout', type=int, default=9)
|
||||
p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop')
|
||||
p.add_argument('--output', help='JSON result file (no tokens)')
|
||||
args = p.parse_args()
|
||||
check(args.mode != 'action' or args.action, '--action is required in action mode')
|
||||
|
||||
c = Client(args.base_url)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
replicas = api_containers()
|
||||
if args.mode == 'failover':
|
||||
check(len(replicas) >= 2, 'need at least two running api containers')
|
||||
load = Load(args.base_url, args.concurrency)
|
||||
load.start(alice, chat)
|
||||
runs = []
|
||||
try:
|
||||
if args.mode == 'failover':
|
||||
for name, cid in replicas:
|
||||
print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True)
|
||||
runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle,
|
||||
args.stop_timeout, args.kill))
|
||||
r = runs[-1]
|
||||
print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % (
|
||||
r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'],
|
||||
r['first_success_after_container_exit_s'], r['baseline']['rps_ok'],
|
||||
r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'],
|
||||
r['rejoin_after_docker_start_s']), flush=True)
|
||||
else:
|
||||
print('running %s under load...' % args.action, flush=True)
|
||||
runs.append(action_run(load, args.action, args.baseline, args.observe))
|
||||
finally:
|
||||
load.finish()
|
||||
end = load.now()
|
||||
|
||||
confirmed = dict(load.confirmed)
|
||||
stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')]
|
||||
counts = Counter(m['text'] for m in stored)
|
||||
duplicates = sum(n - 1 for n in counts.values() if n > 1)
|
||||
stored_ids = {m['id'] for m in stored}
|
||||
missing = [mid for mid in confirmed if mid not in stored_ids]
|
||||
integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored),
|
||||
'duplicates': duplicates, 'missing_confirmed': len(missing)}
|
||||
report = {
|
||||
'mode': args.mode,
|
||||
'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout,
|
||||
'concurrency': args.concurrency,
|
||||
'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest',
|
||||
'replicas_before': [name for name, _ in replicas],
|
||||
'runs': runs,
|
||||
'total': summarize(load.events, 0, end),
|
||||
'integrity': integrity,
|
||||
'timeline_per_second': timeline(load, end),
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
out = Path(args.output)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(text + '\n', encoding='utf-8')
|
||||
print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False))
|
||||
check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity)
|
||||
if args.mode == 'action':
|
||||
check(runs[0]['action_exit_code'] == 0, 'action script failed')
|
||||
print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
rem Windows entry point: runs lb-status.sh with Git Bash.
|
||||
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0lb-status.sh"
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# HAProxy view of the api pool: slot, replica IP, state (2=UP, 0=DOWN), check result. Empty template slots are hidden.
|
||||
docker compose --env-file .env exec -T lb sh -c \
|
||||
"echo 'show servers state api' | nc 127.0.0.1 9999 | awk '\$1 !~ /^#/ && NF > 10 && \$5 != \"-\" {print \$4, \$5, (\$6 == 2 ? \"UP\" : \"DOWN\")}'"
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
rem Windows entry point: runs rolling-update.sh with Git Bash. Set APP_VERSION first.
|
||||
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0rolling-update.sh"
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Replace API replicas one at a time with a new build while the LB keeps serving.
|
||||
# APP_VERSION=v2 scripts/rolling-update.sh
|
||||
# 1. Build messenger-lab3-app:$APP_VERSION.
|
||||
# 2. Run migrations ONCE (expand-only, compatible with replicas still on the old build).
|
||||
# 3. For each old replica: start one new replica (N+1), wait until Docker reports it healthy AND HAProxy
|
||||
# marks it UP, then docker stop the old one (SIGTERM -> drain -> exit) and remove it.
|
||||
# Capacity never drops below N. Afterwards put APP_VERSION=$APP_VERSION into .env, otherwise a later
|
||||
# `docker compose up` recreates replicas from the version written there.
|
||||
: "${APP_VERSION:?set APP_VERSION of the new build}"
|
||||
export APP_VERSION
|
||||
project="${COMPOSE_PROJECT_NAME:-messenger-lab3}"
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
api_ids() {
|
||||
docker ps -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.service=api"
|
||||
}
|
||||
lb_up() { sh scripts/lb-status.sh | grep -c ' UP$' || true; }
|
||||
|
||||
old=$(api_ids)
|
||||
count=$(printf '%s\n' "$old" | grep -c . || true)
|
||||
[ "$count" -ge 1 ] || { echo "no running api replicas" >&2; exit 1; }
|
||||
echo "$(date +%T) rolling $count replica(s) to messenger-lab3-app:$APP_VERSION"
|
||||
|
||||
compose build --quiet api migrate
|
||||
compose run --rm --no-deps -T migrate
|
||||
echo "$(date +%T) migrations applied once"
|
||||
|
||||
for id in $old; do
|
||||
before=$(lb_up)
|
||||
compose up -d --no-deps --no-recreate --wait --scale "api=$((count + 1))" api >/dev/null 2>&1
|
||||
new=$(api_ids | grep -v -F "$old" || true)
|
||||
i=0
|
||||
until [ "$(lb_up)" -gt "$before" ]; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -le 60 ] || { echo "new replica did not become UP in HAProxy" >&2; exit 1; }
|
||||
sleep 0.5
|
||||
done
|
||||
echo "$(date +%T) new replica(s) UP in LB: $(printf '%s ' $new); stopping old $id"
|
||||
docker stop --time 9 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
echo "$(date +%T) old $id drained and removed"
|
||||
done
|
||||
|
||||
compose ps api --format '{{.Name}} {{.Image}} {{.Status}}'
|
||||
echo "$(date +%T) done; set APP_VERSION=$APP_VERSION in .env to keep this version on the next compose up"
|
||||
+3
-2
@@ -23,6 +23,7 @@ if [ "$count" -ne 1 ]; then
|
||||
exit 1
|
||||
fi
|
||||
name=$(docker inspect --format '{{.Name}}' "$matches")
|
||||
# Graceful: SIGTERM, then SIGKILL after 8 s; returns before the test's 10 s limit.
|
||||
docker stop --time 8 "$matches" >/dev/null
|
||||
# Graceful: SIGTERM -> 4 s drain (readiness 503, LB drops it) -> <= 3 s in-flight finish.
|
||||
# SIGKILL only after 9 s, so the hook still returns before the test's 10 s limit.
|
||||
docker stop --time 9 "$matches" >/dev/null
|
||||
echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)"
|
||||
|
||||
Reference in New Issue
Block a user