ЛР 3 - БПИ2302 - Тимаева - на 5 #129
+22
-4
@@ -4,10 +4,28 @@ APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
# Not passed to containers: compose.yaml gives api/migrate DB_HOST/DB_USER/DB_NAME and
|
||||
# DB_PASSWORD_FILE=/run/secrets/postgres_password (Compose secret built from POSTGRES_PASSWORD below).
|
||||
# The app also accepts DATABASE_URL_FILE or DATABASE_URL instead.
|
||||
# Sessions live in PostgreSQL; no Redis/Valkey, CACHE_URL_FILE is unused.
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
# Per-replica API limits; total budget in track 1 is replicas × these values. Defaults match level 4.
|
||||
API_CPUS=1.0
|
||||
API_MEMORY=256M
|
||||
POSTGRES_CPUS=1.0
|
||||
POSTGRES_MEMORY=512M
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# Pick your own local password. It reaches containers only as the postgres_password secret file.
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_PASSWORD=change-me-locally
|
||||
POSTGRES_DB=messenger
|
||||
# SIGTERM: readiness 503 for DRAIN_SECONDS (> LB detection 3 s), then in-flight requests get
|
||||
# SHUTDOWN_TIMEOUT_SECONDS. Keep DRAIN + SHUTDOWN < stop_grace_period (10 s).
|
||||
DRAIN_SECONDS=4
|
||||
SHUTDOWN_TIMEOUT_SECONDS=3
|
||||
# Image tag messenger-lab3-app:<APP_VERSION>, shown as X-App-Version; scripts/rolling-update.sh sets it.
|
||||
APP_VERSION=dev
|
||||
LB_PORT=8080
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
*.sh text eol=lf
|
||||
*.conf text eol=lf
|
||||
*.cfg text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
# Закреплённая среда выполнения. Тег называет версию, digest — то, что Docker реально скачает.
|
||||
FROM python:3.14.8-slim-bookworm@sha256:c8137f4c460908c8763f281c8f22c431eb5c538514ba9553fc3a89c06b7cfb88
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1
|
||||
|
||||
# Значения по умолчанию, секретов нет. `docker run --env-file .env` перекрывает их при запуске.
|
||||
ENV APP_HOST=0.0.0.0 \
|
||||
APP_PORT=8080 \
|
||||
APP_ENV=lab \
|
||||
LOG_LEVEL=info \
|
||||
DRAIN_SECONDS=4 \
|
||||
SHUTDOWN_TIMEOUT_SECONDS=3 \
|
||||
SESSION_TTL_SECONDS=3600
|
||||
|
||||
RUN groupadd --gid 10001 app \
|
||||
&& useradd --uid 10001 --gid app --create-home --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN python -m pip install --no-cache-dir --disable-pip-version-check --root-user-action=ignore -r requirements.txt
|
||||
|
||||
COPY --chown=app:app app ./app
|
||||
COPY --chown=app:app migrations ./migrations
|
||||
COPY --chown=app:app contracts/openapi.json ./contracts/openapi.json
|
||||
|
||||
USER app
|
||||
|
||||
# Visible as X-App-Version: shows which build served a request during a rolling update.
|
||||
ARG APP_VERSION=dev
|
||||
ENV APP_VERSION=${APP_VERSION}
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=15s --retries=3 \
|
||||
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ['APP_PORT'], timeout=2)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
CMD ["python", "-m", "app"]
|
||||
@@ -4,10 +4,14 @@ CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover
|
||||
APP_VERSION ?=
|
||||
|
||||
.PHONY: help check validate test load test-persistence test-failover test-idempotency failover-series rolling-update lb-status scale-cost
|
||||
|
||||
help:
|
||||
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
|
||||
@echo "failover-series, rolling-update APP_VERSION=..., lb-status: level 4 experiments (Docker required)"
|
||||
@echo "scale-cost: level 5 track 1 (1/2/4 replicas at a fixed CPU/RAM budget, then grow resources)"
|
||||
|
||||
check:
|
||||
$(PYTHON) scripts/check_contract.py
|
||||
@@ -28,3 +32,22 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-idempotency:
|
||||
$(PYTHON) scripts/check_idempotency.py --base-url "$(BASE_URL)"
|
||||
|
||||
# Level 4: stop each API replica in turn under load (restoring the previous one); RPS, errors, recovery.
|
||||
failover-series:
|
||||
$(PYTHON) scripts/failover_series.py --base-url "$(BASE_URL)" --output evidence/failover-series.json
|
||||
|
||||
# Level 4: one-at-a-time replacement under load. Usage: make rolling-update APP_VERSION=v3
|
||||
rolling-update:
|
||||
@test -n "$(APP_VERSION)" || (echo "Set APP_VERSION of the new build"; exit 2)
|
||||
APP_VERSION="$(APP_VERSION)" $(PYTHON) scripts/failover_series.py --mode action --action scripts/rolling-update.sh --base-url "$(BASE_URL)" --output evidence/rolling-update.json
|
||||
|
||||
lb-status:
|
||||
sh scripts/lb-status.sh
|
||||
|
||||
# Level 5 track 1: 1/2/4 API replicas at a fixed total CPU/RAM budget, then a growing-resource series.
|
||||
scale-cost:
|
||||
$(PYTHON) scripts/scale_cost.py --base-url "$(BASE_URL)"
|
||||
|
||||
@@ -1,37 +1,195 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Автор, группа: Тимаева М.П., БПИ2302
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 5, трек 1 (цена масштабирования)
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Использованные источники, библиотеки и помощь ИИ: документация Docker Compose / HAProxy / asyncpg; ИИ использовалось для написания отчета и помощи с написанием скриптов, цифры с прогона `scripts/scale_cost.py` 2026-11-08.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Вариант: **Docker Compose + HAProxy**. Нужны Docker Engine с Compose v2 (проверено на Docker 29.4, Compose 5.1) и Python 3.9+ для тестов. На Windows скрипты `.sh` выполняются через Git Bash (обёртки `scripts/*.cmd`).
|
||||
|
||||
Образы: `postgres:16.10-alpine` и `haproxy:3.2.26-alpine` (LTS), оба закреплены digest в `compose.yaml`. Приложение собирается из `Dockerfile` на `python:3.14.8-slim-bookworm@sha256:…` в образ `messenger-lab3-app:${APP_VERSION}`.
|
||||
|
||||
Одна последовательность запуска:
|
||||
|
||||
```sh
|
||||
cp .env.example .env # задать свой POSTGRES_PASSWORD
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
# postgres (healthy) -> migrate (один раз, exit 0) -> api x API_REPLICAS (healthy) -> lb (healthy)
|
||||
make test # smoke через LB, http://localhost:8080
|
||||
python scripts/show_instances.py # одна сессия, разные X-Instance-Id
|
||||
make test-idempotency # один ключ параллельно через разные реплики
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh # Windows: scripts/stop-one.cmd
|
||||
docker compose --env-file .env up -d --wait # вернуть остановленную реплику
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh # Windows: scripts/restart.cmd
|
||||
# уровень 4
|
||||
sh scripts/lb-status.sh # пул HAProxy: реплика, IP, UP/DOWN
|
||||
make failover-series # поочерёдная остановка каждой реплики под нагрузкой
|
||||
python scripts/failover_series.py --kill --output evidence/failover-series-kill.json # то же, SIGKILL
|
||||
docker compose --env-file .env build api migrate # с APP_VERSION=v3 в окружении
|
||||
make rolling-update APP_VERSION=v3 # Windows: scripts/failover_series.py --mode action --action scripts/rolling-update.cmd
|
||||
# после rolling update записать APP_VERSION=v3 в .env
|
||||
# уровень 5, трек 1 (после прогона стенд возвращается к 2×1.0 CPU)
|
||||
make scale-cost PYTHON=python # Windows; GNU Make: make scale-cost
|
||||
```
|
||||
|
||||
Остановка без потери данных: `docker compose --env-file .env down`. Намеренное удаление учебных данных: `docker compose --env-file .env down -v` (удаляет volume `pgdata`).
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
edge network data network (internal: true)
|
||||
client -> lb (HAProxy :8080, loopback) -> api-1, api-2, … (порт не публикуется) -> postgres (volume pgdata)
|
||||
migrate (one-shot) ------------^
|
||||
```
|
||||
|
||||
| Роль | Compose service | Образ | Health/readiness | Сети | Volume / секреты | Лимит CPU / RAM (резерв) |
|
||||
| -------- | --------------- | ----------------------------- | ----------------------------------------------------- | ---------- | -------------------------------------------------- | ---------------------------------------------------------------------------------- |
|
||||
| LB | `lb` | `haproxy:3.2.26-alpine@…` | `monitor-uri /lb/health`; активные проверки API | edge | только read-only конфиг, секретов нет | 0.5 / 128M (0.1 / 32M) |
|
||||
| API × N | `api` | `messenger-lab3-app:<версия>` | `/health/live`, `/health/ready` (БД, миграции, drain) | edge, data | `/run/secrets/postgres_password` (uid 10001, 0400) | `${API_CPUS:-1.0}` / `${API_MEMORY:-256M}` (0.25 / 128M); пул `${DB_POOL_MAX:-10}` |
|
||||
| Миграции | `migrate` | тот же образ | one-shot, `service_completed_successfully` | data | тот же секрет | 0.5 / 128M |
|
||||
| БД | `postgres` | `postgres:16.10-alpine@…` | `pg_isready` каждые 2 с | data | `pgdata`, `POSTGRES_PASSWORD_FILE` | `${POSTGRES_CPUS:-1.0}` / `${POSTGRES_MEMORY:-512M}` (0.25 / 256M) |
|
||||
|
||||
- **Источник истины — PostgreSQL**: пользователи, чаты, сообщения, сессии (`sessions`, токен хранится как SHA-256) и результаты идемпотентности (`idempotency_keys`). В API нет локального состояния, от которого зависит корректность. Redis из lab2 убран: он был только кэшем, а для общего хранилища сессий достаточно PostgreSQL.
|
||||
- **Изоляция**: наружу публикуется только `lb` и только на loopback. У API нет `ports`, `docker port` пуст. LB находится только в сети `edge` и не резолвит `postgres`. Сеть `data` объявлена `internal`, без выхода наружу. Volume `pgdata` смонтирован только в `postgres`. `container_name` нигде нет, поэтому `--scale api=N` работает (`evidence/limits.txt`).
|
||||
- **Секреты**: пароль БД лежит только в незакоммиченном `.env` и попадает в контейнеры как Compose secret `postgres_password` (`environment: POSTGRES_PASSWORD`). В env контейнеров есть только путь `*_PASSWORD_FILE`. API и мигратор собирают `DATABASE_URL` из `DB_HOST/DB_USER/DB_NAME` и файла (`app/config.py`). У LB секретов нет.
|
||||
- **Миграции** выполняет отдельный сервис `migrate` (`python -m app.migrate`) до старта реплик, а при rolling update — один раз через `docker compose run --rm migrate`. Реплики схему не меняют. `/health/ready` возвращает 503, пока в `schema_migrations` нет всех миграций **этой** сборки. Параллельный запуск мигратора сериализуется `pg_advisory_lock`.
|
||||
- **X-Instance-Id** = hostname контейнера (короткий Docker ID). **X-App-Version** = `APP_VERSION` образа. По нему видна смена версии при rolling update.
|
||||
- **Балансировщик**: HAProxy, round robin без sticky. `server-template api 1-10 api:8080` с `resolvers docker` (Docker DNS `127.0.0.11`, `hold valid 1s`), поэтому реплики, добавленные через `--scale` или rolling update и остановленные, входят в пул и выходят из него без перезапуска LB.
|
||||
- **Повтор на другой реплике**: `retries 2`, `option redispatch`, `retry-on conn-failure empty-response response-timeout`. Для POST, кроме `/api/v1/chats/{id}/messages`, стоит `http-request disable-l7-retry`, то есть они повторяются только если соединение с репликой не было установлено. POST сообщений можно повторять, потому что `Idempotency-Key` обязателен.
|
||||
- **Идемпотентность**: в одной транзакции удаляется истёкший ключ, затем `INSERT … ON CONFLICT DO NOTHING` в `idempotency_keys` (PK: пользователь + метод + путь + ключ) вместе с готовым ответом и SHA-256 тела, затем вставка сообщения. Конкурентный дубль на другой реплике ждёт на уникальном индексе до COMMIT первой транзакции и получает сохранённый ответ (тот же 201) либо 409 при другом теле. При ошибке ключ не фиксируется, и повтор безопасен.
|
||||
|
||||
### Health, timeouts и SIGTERM: согласованные тайминги
|
||||
|
||||
```
|
||||
SIGTERM -> readiness 503 сразу; запросы ещё обслуживаются
|
||||
-> HAProxy: 2 неудачные проверки (inter 1s, timeout 1s) -> DOWN, не более 3 с
|
||||
-> через DRAIN_SECONDS=4 uvicorn перестаёт принимать соединения, дорабатывает текущие за SHUTDOWN_TIMEOUT_SECONDS=3
|
||||
-> exit 0 примерно через 5 с; stop_grace_period 10 s (stop-one: docker stop --time 9), SIGKILL не нужен
|
||||
```
|
||||
|
||||
| Параметр | Значение | Где |
|
||||
| -------------------------------------------------- | ----------------------------------------------------------------------------------- | ------------------------------------------- |
|
||||
| Период активной проверки LB `/health/ready` | 1 с, `timeout check 1s`, `fall 2`, `rise 2` | `infra/haproxy/haproxy.cfg` |
|
||||
| Обновление DNS реплик | 1 с (`hold valid 1s`, `hold obsolete 0s`) | там же |
|
||||
| `timeout connect / server / client / http-request` | 1 с / 10 с / 30 с / 10 с | там же |
|
||||
| Drain после SIGTERM / завершение текущих запросов | 4 с / 3 с | `DRAIN_SECONDS`, `SHUTDOWN_TIMEOUT_SECONDS` |
|
||||
| `stop_grace_period` | 10 с (> 4 + 3) | `compose.yaml` |
|
||||
| Keep-alive uvicorn | 10 с (> `pool-purge-delay` HAProxy 5 с) | `KEEPALIVE_TIMEOUT_SECONDS` |
|
||||
| Docker healthcheck API | каждые 3 с, 3 попытки; только порядок старта и rolling update, LB его не использует | `compose.yaml` |
|
||||
|
||||
Максимальное время обнаружения отказа:
|
||||
|
||||
- **Плановая остановка (SIGTERM)**: реплика выводится заранее, не более \(2 \cdot 1 + 1 = 3\) с, и раньше, чем закрывается сокет. Замер: последний запрос на останавливаемую реплику пришёл через 1,76–2,05 с после SIGTERM, ошибок 0.
|
||||
- **Падение процесса (SIGKILL)**: запрос к мёртвому IP завершается по `timeout connect 1s` и уходит на другую реплику (redispatch). Пул очищается по DNS примерно за 1 с или по проверкам не более чем за 3 с. Замер: максимальная пауза без успешного ответа 0,96–0,98 с, ошибок 0.
|
||||
- **Зависшая реплика, принимающая соединения**: проверки дают DOWN не более чем через 3 с. Запросы, уже отправленные на неё, ждут до `timeout server 10s` и затем повторяются (`response-timeout`) для GET и POST сообщений. Экспериментально не проверялось.
|
||||
- **Возврат в пул**: `rise 2`, то есть около 2 с после готовности. Замер от `docker start` до первого ответа реплики: 4,5–5,2 с, включая старт приложения.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- |
|
||||
| Стенд в compose.yaml, ≥2 API, одна точка входа, общая БД и сессии | `down -v`, `config --quiet`, `up -d --build --wait` | config OK; `api-1`, `api-2`, `lb`, `postgres` healthy; `migrate` применил 001–003 и Exited (0); в HAProxy 2 реплики UP | `evidence/smoke.txt` |
|
||||
| Одна сессия обслуживается разными репликами | `python scripts/show_instances.py` | 6 запросов с одним токеном: по 3 на каждую из 2 реплик | `evidence/instances.txt` |
|
||||
| Остановка любой одной API-реплики | `make test-failover ACTION_SCRIPT=scripts/stop-one.cmd` ×2, между прогонами `up -d --wait` | 2 PASS: остановлены `api-2`, затем `api-1`, Exited (0); 121 и 115 записей, 0 ошибок; HAProxy убрал реплику из пула | `evidence/failover.txt` |
|
||||
| Повтор POST с тем же Idempotency-Key | `make test-idempotency` | 5 ключей × 20 параллельных POST: по 10 ответов с каждой реплики, один Message на ключ, другое тело → 409 | `evidence/idempotency.txt` |
|
||||
| make check, make test, persistence | `make check`, `make test`, `make test-persistence ACTION_SCRIPT=scripts/restart.cmd` | PASS; после рестарта lb, api, postgres те же ID, volume на месте, прежний токен работает | `evidence/smoke.txt`, `evidence/persistence.txt` |
|
||||
| **4**: health/readiness, timeouts, SIGTERM согласованы; LB исключает недоступный backend | HAProxy active checks + drain; `scripts/lb-status.sh` | см. таблицу таймингов; останавливаемая реплика получает последний запрос через ≤2,05 с после SIGTERM; после выхода её нет в пуле | `infra/haproxy/haproxy.cfg`, `evidence/failover-series.json` |
|
||||
| **4**: поочерёдная остановка каждой реплики: RPS, ошибки, время восстановления | `make failover-series`; `failover_series.py --kill` | по 2 реплики в каждом режиме, 0 ошибок, все подтверждённые записи на месте, дублей нет (таблица ниже) | `evidence/failover-series*.{txt,json}` |
|
||||
| **4**: лимиты, нет портов API, секреты и хранилища только где нужно, нет `container_name` | `docker inspect`, `docker stats`, `docker port`, `nslookup` из lb | лимиты у всех 5 сервисов; порт только у lb (127.0.0.1/::1); пароль только файлом 0400; lb без секретов и без доступа к БД; `container_name`: 0 | `evidence/limits.txt` |
|
||||
| **4**: rolling update под нагрузкой, миграции однократно и совместимо | `make rolling-update APP_VERSION=v3`; ранее `dev → v2` с миграцией 003 | 0 ошибок, максимальная пауза 0,07 с; v2 → v3 без снижения числа реплик ниже 2; при `dev → v2` миграция 003 применена один раз, и старые реплики `dev` продолжали работать | `evidence/rolling-update.{txt,json}`, `evidence/rolling-update-migration.txt` |
|
||||
| **5 трек 1**: 1/2/4 реплики при бюджете 2.0 CPU/512M, рост CPU, saturation, одно улучшение | `make scale-cost PYTHON=python` | 187.5 / 249.5 / 185.8 RPS; 4×0.5 → 2×1.0 даёт +34% RPS и p95 465→197; пул 20 отвергнут; LB не насыщен | `evidence/scale-cost.{json,txt,svg}` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Все прогоны 2026-11-07 на commit `124c83e` + рабочее дерево (до коммита уровня 4), Docker 29.4.3, Windows 11, один хост. `make test-failover/test-persistence` под Windows запускались той же командой из `Makefile` напрямую (`python tests/resilience.py ...`), потому что GNU Make под cmd не находит `test` в guard-строке.
|
||||
|
||||
Цель `make test-failover` выбирает тест: это реплика, обслужившая первое сообщение. Для второго прогона через LB отправлен один `GET /health/ready`, чтобы сдвинуть round robin, и тест остановил другую реплику.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Профиль нагрузки `scripts/failover_series.py`: 4 потока в закрытом цикле, каждый делает POST сообщения (с `Idempotency-Key`; неопределённый результат повторяется с тем же ключом) и GET 20 последних сообщений. Счёт идёт по секундам. Для каждой реплики по очереди: 10 с baseline, затем остановка, 15 с наблюдения, затем `docker start` того же контейнера, ожидание его ответов и 5 с паузы. В конце вся история чата читается заново: каждая подтверждённая запись должна быть ровно один раз.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
### Поочерёдная остановка реплик под нагрузкой
|
||||
|
||||
| Воздействие | Реплика | RPS baseline → во время → после | Ошибки | Макс. пауза без успеха | Последний запрос на реплику после сигнала | Выход контейнера | RPS ≥90% baseline после выхода | Возврат в пул после `docker start` |
|
||||
| ------------------------------- | ------- | ------------------------------- | ------ | ---------------------- | ----------------------------------------- | ---------------- | ------------------------------ | ---------------------------------- |
|
||||
| `docker stop` (SIGTERM + drain) | api-1 | 129,0 → 114,8 → 113,1 | 0 | 0,09 с | 2,05 с | 5,06 с, exit 0 | 2,8 с | 5,2 с |
|
||||
| `docker stop` (SIGTERM + drain) | api-2 | 131,5 → 145,2 → 165,4 | 0 | 0,09 с | 1,76 с | 4,88 с, exit 0 | 1,6 с | 4,9 с |
|
||||
| `docker kill` (SIGKILL) | api-1 | 174,4 → 145,3 → 158,4 | 0 | 0,96 с | 0,12 с | 0,66 с, exit 137 | 2,3 с | 4,5 с |
|
||||
| `docker kill` (SIGKILL) | api-2 | 177,9 → 146,5 → 139,0 | 0 | 0,98 с | 0,12 с | 0,63 с, exit 137 | 2,1 с | 5,2 с |
|
||||
|
||||
Итог серий: SIGTERM — 80,5 с, 5312 записей и 5312 чтений, p95 73 мс; SIGKILL — 71,2 с, 5520 записей и 5520 чтений, p95 66 мс. В обоих случаях 0 ошибок, 0 дублей, 0 потерянных подтверждённых записей. В предварительном прогоне `--kill` (не сохранён) была 1 ошибка на 12 890 запросов: запрос, оборванный в момент SIGKILL. Это допустимый переходный эффект, клиент повторяет его с тем же ключом.
|
||||
|
||||
RPS — это пропускная способность закрытого цикла на одном хосте, где рядом работали посторонние контейнеры. Между сериями она колеблется на 30–40 %, а внутри серии падение во время отказа составляет 10–20 %. Иногда RPS с одной репликой выше baseline: узкое место — клиент и общий CPU хоста, а не число реплик. Это не измерение ёмкости.
|
||||
|
||||
### make test-failover и rolling update
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Восстановление / потеря данных |
|
||||
| ------------------------------------------------ | ----------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------- | ----------------------------------- |
|
||||
| `make test-failover`, `stop-one` → `api-2` | запись продолжается ≤10 с, дублей нет | PASS, 121 запись, 0 ошибок | `first_recovery_seconds` 6,42 / нет |
|
||||
| то же после восстановления → `api-1` | то же | PASS, 115 записей, 0 ошибок | 6,44 / нет |
|
||||
| `restart.cmd`: lb, api, postgres | данные и старый токен сохранены | PASS, volume сохранён | нет потерь |
|
||||
| rolling update v2 → v3 под нагрузкой (2 реплики) | нет ошибок, ёмкость ≥2 реплик, версия сменилась | 42,7 с; 0 ошибок; пауза 0,07 с; RPS 181 → 146 → 186; ответы: до v2 1833; во время v2 3941 / v3 2279; после v3 2787 | 5423 записи, 0 дублей, 0 потерь |
|
||||
| rolling update dev → v2 с миграцией 003 | старая версия работает на расширенной схеме | 45,7 с; 0 ошибок; пауза 0,10 с; миграция применена один раз в начале, реплики `dev` обслуживали запросы до своей замены | 4458 записей, 0 дублей, 0 потерь |
|
||||
|
||||
`first_recovery_seconds` в `tests/resilience.py` считается до первой успешной записи **после завершения hook**. Hook теперь длится около 6 с: это `docker stop`, который ждёт 4 с drain. Записи при этом не прерывались (0 ошибок), поэтому значение отражает длительность hook, а не простой. Оно укладывается в лимит 10 с, как и сам hook.
|
||||
|
||||
**Rolling update** (`scripts/rolling-update.sh`):
|
||||
|
||||
1. Собрать `messenger-lab3-app:$APP_VERSION`.
|
||||
2. Один раз выполнить `docker compose run --rm migrate`.
|
||||
3. Для каждой старой реплики запустить `up -d --no-deps --no-recreate --wait --scale api=N+1`. Новая реплика стартует из нового образа, скрипт ждёт healthy и UP в HAProxy. Затем старая реплика получает `docker stop` (drain) и `docker rm`.
|
||||
|
||||
Ёмкость не опускается ниже N, LB не перезапускается. После обновления версию нужно записать в `.env` (`APP_VERSION`), иначе следующий `docker compose up` пересоздаст реплики в версии из `.env`.
|
||||
|
||||
**Политика миграций (expand/contract)**. Миграция выполняется до замены реплик и должна быть совместима с работающей версией: только новые таблицы, индексы и nullable-колонки. Пример — `003_session_indexes.sql`. Readiness старой сборки проверяет только свои миграции, поэтому расширенная схема её не ломает. Удаление и переименование колонок допустимы только отдельным релизом после ухода старых реплик. Ограничение: `CREATE INDEX` без `CONCURRENTLY` (мигратор работает в транзакции) коротко блокирует запись в индексируемую таблицу. Для `sessions` учебного размера это незаметно, для большой таблицы понадобится `CONCURRENTLY` вне транзакции.
|
||||
|
||||
## Трек 1. Цена масштабирования
|
||||
|
||||
**Гипотеза.** При фиксированном суммарном бюджете API 2.0 CPU / 512M рост числа реплик 1 → 2 → 4 не увеличивает RPS пропорционально: один uvicorn-процесс не утилизирует 2.0 CPU, а четыре процесса по 0.5 CPU режут квант каждого воркера. Узкое место после двух реплик — не HAProxy, а общая PostgreSQL (лимит 1.0 CPU) плюс пул соединений. Подтверждённое улучшение: тот же бюджет отдать двум репликам по 1.0 CPU, а не четырём по 0.5.
|
||||
|
||||
**Методика.** Хост: Windows 11, Docker 29.4, 12 CPU, ≈11.6 GiB RAM, один Docker Engine. Профиль один на все точки: 24 потока, закрытый цикл POST `/api/v1/chats/{id}/messages` через LB (`http://localhost:8080`), прогрев 5 с, замер 20 с. Между точками Compose пересоздаёт только API (лимиты `API_CPUS` / `API_MEMORY` / `DB_POOL_MAX`); PostgreSQL остаётся 1.0 CPU / 512M. Раз в секунду снимаются `docker stats` и `pg_stat_activity`. Утилизация CPU считается как (сумма % docker stats) / лимит контейнера: 100% stats ≈ 1 host CPU. Повтор: `make scale-cost PYTHON=python`. Исходные JSON/таблица/графики: `evidence/scale-cost.{json,txt,svg}`. Дата прогона: 2026-11-08.
|
||||
|
||||
Серии:
|
||||
|
||||
1. Фиксированный бюджет API **2.0 CPU / 512M**: 1×2.0/512M, 2×1.0/256M, 4×0.5/128M; пул 10 на реплику.
|
||||
2. Рост ресурсов, 2 реплики: 0.25 → 0.50 → 1.0 CPU на процесс (postgres 1.0 не меняется).
|
||||
3. Одна попытка улучшения, отвергнутая: 2×1.0, `DB_POOL_MAX` 10 → 20.
|
||||
|
||||
**Результаты.** Ошибок 0 во всех точках. Лимиты совпали с `docker inspect` (`NanoCpus` / `Memory`).
|
||||
|
||||
| id | N | CPU/RAM реплики | пул | RPS | p95, мс | API util | PG util | LB util | backends / ёмкость пула |
|
||||
| -------------- | --- | --------------- | --- | --------- | ------- | -------- | ------- | ------- | ----------------------- |
|
||||
| fixed-1 | 1 | 2.0 / 512M | 10 | 187.5 | 204 | 49% | 47% | 40% | 11 / 10 |
|
||||
| fixed-2 | 2 | 1.0 / 256M | 10 | **249.5** | 197 | 85% | 66% | 47% | 21 / 20 |
|
||||
| fixed-4 | 4 | 0.5 / 128M | 10 | 185.8 | 465 | 83% | 62% | 38% | 39 / 40 |
|
||||
| grow-0.25 | 2 | 0.25 / 256M | 10 | 25.1 | 1916 | **96%** | 26% | 12% | 21 / 20 |
|
||||
| grow-0.50 | 2 | 0.50 / 256M | 10 | 93.8 | 593 | **97%** | 38% | 24% | 21 / 20 |
|
||||
| improve-pool20 | 2 | 1.0 / 256M | 20 | 140.7 | 397 | 89% | 63% | 45% | 35 / 40 |
|
||||
|
||||
График: `evidence/scale-cost.svg` (верх — фиксированный бюджет, низ — рост CPU).
|
||||
|
||||
**Saturation.**
|
||||
|
||||
- **API CPU** — saturation при срезе 0.25 и 0.50 (util 96–97%, p95 0.6–1.9 с). На 1×2.0 util только 49%: один процесс uvicorn ≈ один core, второй CPU бюджета не используется. На 4×0.5 каждый воркер упирается в квоту 0.5, как grow-0.50, только процессов больше.
|
||||
- **Пул.** `backends` почти равны `N × DB_POOL_MAX` (плюс соединение sampler). Это занятость слотов, не очередь в PostgreSQL: `pg_active` ≈ 1–2, `waiting` ≈ 0. На N=1 слотов 10 при 24 клиентах, часть потоков ждёт `pool.acquire()`. На N=4 слотов 40 > 24 клиентов, пул уже не ограничивает.
|
||||
- **PostgreSQL** — максимум 66% своего 1.0 CPU, saturation нет.
|
||||
- **LB** — максимум 47% своего 0.5 CPU, saturation нет. `timeout queue 2s` не срабатывал (0 ошибок).
|
||||
|
||||
Поэтому рост реплик при фиксированном бюджете перестаёт помогать на N=4: суммарные 2.0 CPU режутся на кванты по 0.5, p95 растёт в 2.4 раза, а лишние соединения (40 вместо 20) бьют в ту же одноядерную БД и RPS падает ниже N=2.
|
||||
|
||||
**Подтверждённое улучшение.** Тот же бюджет 2.0 CPU / 512M: **4×0.5 → 2×1.0**. RPS 185.8 → 249.5 (**+34%**), p95 465 → 197 мс. Это и есть рабочая топология стенда (`API_REPLICAS=2`, `API_CPUS=1.0`). Увеличение пула 10 → 20 на тех же 2×1.0 отвергнуто: 249.5 → 140.7 RPS (больше сессий на 1 CPU Postgres, p95 197 → 397).
|
||||
|
||||
**Ограничения трека.** Один прогон по 20 с, закрытый цикл на том же хосте (coordinated omission, общий CPU с Docker). `pool_util` считает idle-слоты asyncpg, не wait. WAL/диск не снимались. Open-loop / wrk не использовались: для минимума трека достаточно сравнить одинаковый профиль.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **Выдерживается**: плановая остановка, падение или зависание одной API-реплики, пока работает хотя бы одна другая; поочерёдная замена всех реплик; полный перезапуск стенда с сохранением volume.
|
||||
- **Не выдерживается**: отказ PostgreSQL и HAProxy. Это единственные точки отказа: БД не реплицирована, LB один. При их остановке API недоступен до перезапуска, подтверждённые данные сохраняются в volume.
|
||||
- **Один хост**: отказ узла или Docker Engine останавливает весь стенд. Проверен отказ процесса или контейнера, а не узла.
|
||||
- **Зависание реплики** экспериментально не воспроизводилось. Оценка «не более 3 с до DOWN» следует из конфигурации, а уже отправленные на зависшую реплику запросы ждут до 10 с.
|
||||
- **Rolling update** в Compose реализован скриптом, а не `deploy.update_config`: Compose его не исполняет. Откат — тот же скрипт с прежней `APP_VERSION`, если схема не прошла contract-фазу.
|
||||
- **Нагрузка** — закрытый цикл одного клиента на том же хосте (coordinated omission, общий CPU). Цифры RPS сравнимы только внутри одной серии. В треке 1 сравнивались только точки той же матрицы `scale_cost.py`.
|
||||
- **Порядок сообщений** между репликами зависит от общих часов хоста: `created_at` назначает приложение.
|
||||
- **Повтор LB**: POST пользователей, чатов и сессий повторяется только при неустановленном соединении. После обрыва уже отправленного запроса клиент должен проверить результат сам: ключа идемпотентности у этих POST нет.
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
import logging
|
||||
import threading
|
||||
|
||||
import uvicorn
|
||||
|
||||
from .config import load_settings
|
||||
from .drain import draining
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
"""First SIGTERM/SIGINT: keep serving for drain_seconds with readiness 503, then uvicorn's graceful stop.
|
||||
|
||||
drain_seconds must exceed the LB detection time (HAProxy inter 1s x fall 2 + timeout check 1s),
|
||||
so the replica is out of rotation before it stops accepting connections.
|
||||
"""
|
||||
|
||||
def __init__(self, config: uvicorn.Config, drain_seconds: float):
|
||||
super().__init__(config)
|
||||
self.drain_seconds = drain_seconds
|
||||
|
||||
def handle_exit(self, sig, frame) -> None:
|
||||
if draining.is_set() or self.drain_seconds <= 0:
|
||||
draining.set()
|
||||
super().handle_exit(sig, frame)
|
||||
return
|
||||
draining.set()
|
||||
logger.warning('signal %s: draining for %.1f s before shutdown', sig, self.drain_seconds)
|
||||
timer = threading.Timer(self.drain_seconds, super().handle_exit, (sig, frame))
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
settings = load_settings()
|
||||
config = uvicorn.Config(
|
||||
'app.main:app',
|
||||
host=settings.host,
|
||||
port=settings.port,
|
||||
log_level=settings.log_level,
|
||||
timeout_graceful_shutdown=settings.shutdown_timeout,
|
||||
timeout_keep_alive=settings.keepalive_timeout,
|
||||
)
|
||||
DrainingServer(config, settings.drain_seconds).run()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
"""Basic is only for creating a session. Every other route accepts only Bearer."""
|
||||
import base64
|
||||
import binascii
|
||||
from typing import Optional, Tuple
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def parse_basic(header: str) -> Optional[Tuple[str, str]]:
|
||||
"""username:password. The password may itself contain a colon."""
|
||||
scheme, separator, param = header.partition(' ')
|
||||
if separator != ' ' or scheme.lower() != 'basic' or not param or param != param.strip():
|
||||
return None
|
||||
try:
|
||||
decoded = base64.b64decode(param, validate=True)
|
||||
text = decoded.decode('utf-8')
|
||||
except (binascii.Error, UnicodeDecodeError, ValueError):
|
||||
return None
|
||||
username, separator, password = text.partition(':')
|
||||
if separator != ':':
|
||||
return None
|
||||
return username, password
|
||||
|
||||
|
||||
def parse_bearer(header: str) -> Optional[str]:
|
||||
scheme, separator, token = header.partition(' ')
|
||||
if separator != ' ' or scheme.lower() != 'bearer' or not token or token != token.strip() or ' ' in token:
|
||||
return None
|
||||
return token
|
||||
@@ -0,0 +1,78 @@
|
||||
import os
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Settings:
|
||||
host: str
|
||||
port: int
|
||||
env: str
|
||||
log_level: str
|
||||
shutdown_timeout: int
|
||||
drain_seconds: float
|
||||
keepalive_timeout: int
|
||||
session_ttl_seconds: int
|
||||
idempotency_ttl_seconds: int
|
||||
database_url: str
|
||||
instance_id: str
|
||||
app_version: str
|
||||
|
||||
|
||||
def _read_secret_file(file_var: str, file_name: str) -> str:
|
||||
try:
|
||||
value = Path(file_name).read_text(encoding='utf-8').strip()
|
||||
except OSError:
|
||||
raise RuntimeError('%s could not be read' % file_var) from None
|
||||
if not value:
|
||||
raise RuntimeError('%s is empty' % file_var)
|
||||
return value
|
||||
|
||||
|
||||
def _database_url() -> str:
|
||||
"""DATABASE_URL_FILE, then DATABASE_URL, then DB_* parts with the password from DB_PASSWORD_FILE.
|
||||
|
||||
Secret values are never logged.
|
||||
"""
|
||||
file_name = os.getenv('DATABASE_URL_FILE', '').strip()
|
||||
if file_name:
|
||||
return _read_secret_file('DATABASE_URL_FILE', file_name)
|
||||
value = os.getenv('DATABASE_URL', '').strip()
|
||||
if value:
|
||||
return value
|
||||
password_file = os.getenv('DB_PASSWORD_FILE', '').strip()
|
||||
if not password_file:
|
||||
raise RuntimeError('DATABASE_URL, DATABASE_URL_FILE or DB_PASSWORD_FILE is required')
|
||||
password = _read_secret_file('DB_PASSWORD_FILE', password_file)
|
||||
parts = {name: os.getenv(name, '').strip() for name in ('DB_HOST', 'DB_USER', 'DB_NAME')}
|
||||
missing = [name for name, part in parts.items() if not part]
|
||||
if missing:
|
||||
raise RuntimeError('%s required with DB_PASSWORD_FILE' % ', '.join(missing))
|
||||
return 'postgresql://%s:%s@%s:%s/%s' % (
|
||||
quote(parts['DB_USER'], safe=''),
|
||||
quote(password, safe=''),
|
||||
parts['DB_HOST'],
|
||||
int(os.getenv('DB_PORT', '5432')),
|
||||
quote(parts['DB_NAME'], safe=''),
|
||||
)
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
return Settings(
|
||||
host=os.getenv('APP_HOST', '0.0.0.0'),
|
||||
port=int(os.getenv('APP_PORT', '8080')),
|
||||
env=os.getenv('APP_ENV', 'lab'),
|
||||
log_level=os.getenv('LOG_LEVEL', 'info').lower(),
|
||||
shutdown_timeout=int(os.getenv('SHUTDOWN_TIMEOUT_SECONDS', '3')),
|
||||
drain_seconds=float(os.getenv('DRAIN_SECONDS', '4')),
|
||||
# Longer than HAProxy's idle-connection purge (pool-purge-delay 5 s), so the LB usually closes them first.
|
||||
keepalive_timeout=int(os.getenv('KEEPALIVE_TIMEOUT_SECONDS', '10')),
|
||||
session_ttl_seconds=int(os.getenv('SESSION_TTL_SECONDS', '3600')),
|
||||
idempotency_ttl_seconds=int(os.getenv('IDEMPOTENCY_TTL_SECONDS', '86400')),
|
||||
database_url=_database_url(),
|
||||
# Docker sets the hostname to the short container ID, which stop-one.sh maps back to a container.
|
||||
instance_id=os.getenv('INSTANCE_ID', '').strip() or socket.gethostname(),
|
||||
app_version=os.getenv('APP_VERSION', 'dev').strip() or 'dev',
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
"""Connection pool. The URL is never written to the log."""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
|
||||
import asyncpg
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
_ATTEMPTS = 40
|
||||
|
||||
|
||||
def _pool_size(name: str, default: int) -> int:
|
||||
raw = os.getenv(name, '').strip()
|
||||
return int(raw) if raw else default
|
||||
|
||||
|
||||
async def open_pool(database_url: str) -> asyncpg.Pool:
|
||||
delay = 0.5
|
||||
last_type = 'error'
|
||||
min_size = _pool_size('DB_POOL_MIN', 1)
|
||||
max_size = max(min_size, _pool_size('DB_POOL_MAX', 10))
|
||||
for attempt in range(1, _ATTEMPTS + 1):
|
||||
pool = None
|
||||
try:
|
||||
pool = await asyncpg.create_pool(
|
||||
database_url,
|
||||
min_size=min_size,
|
||||
max_size=max_size,
|
||||
timeout=5,
|
||||
command_timeout=5,
|
||||
server_settings={'statement_timeout': '5000'},
|
||||
)
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute('SELECT 1')
|
||||
return pool
|
||||
except Exception as exc:
|
||||
last_type = type(exc).__name__
|
||||
if pool is not None:
|
||||
await pool.close()
|
||||
if attempt == _ATTEMPTS:
|
||||
break
|
||||
logger.warning('database is not ready (%s), retrying', last_type)
|
||||
await asyncio.sleep(delay)
|
||||
delay = min(delay * 1.5, 2)
|
||||
raise RuntimeError('database did not become reachable (%s)' % last_type)
|
||||
@@ -0,0 +1,4 @@
|
||||
"""Set on SIGTERM: readiness turns 503 so the LB stops routing here while requests are still served."""
|
||||
import threading
|
||||
|
||||
draining = threading.Event()
|
||||
@@ -0,0 +1,29 @@
|
||||
class ApiError(Exception):
|
||||
"""Expected failure: the message never contains internal details."""
|
||||
|
||||
def __init__(self, status: int, code: str, message: str, challenge: str | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.challenge = challenge
|
||||
|
||||
|
||||
def invalid_request(message: str) -> ApiError:
|
||||
return ApiError(400, 'invalid_request', message)
|
||||
|
||||
|
||||
def unauthorized(message: str, challenge: str) -> ApiError:
|
||||
return ApiError(401, 'unauthorized', message, challenge)
|
||||
|
||||
|
||||
def not_found(message: str) -> ApiError:
|
||||
return ApiError(404, 'not_found', message)
|
||||
|
||||
|
||||
def conflict(message: str) -> ApiError:
|
||||
return ApiError(409, 'conflict', message)
|
||||
|
||||
|
||||
def unavailable(message: str) -> ApiError:
|
||||
return ApiError(503, 'unavailable', message)
|
||||
+120
@@ -0,0 +1,120 @@
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from .config import load_settings
|
||||
from .db import open_pool
|
||||
from .errors import ApiError
|
||||
from .routes import router
|
||||
from .store import Store
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
_SAFE_REQUEST_ID = re.compile(r'^[A-Za-z0-9._:-]{1,128}$')
|
||||
_CODES = {
|
||||
400: 'invalid_request',
|
||||
401: 'unauthorized',
|
||||
404: 'not_found',
|
||||
409: 'conflict',
|
||||
413: 'payload_too_large',
|
||||
415: 'unsupported_media_type',
|
||||
429: 'rate_limited',
|
||||
503: 'unavailable',
|
||||
}
|
||||
_MESSAGES = {
|
||||
'invalid_request': 'Некорректный запрос',
|
||||
'unauthorized': 'Требуется идентификация',
|
||||
'not_found': 'Объект не найден',
|
||||
'internal_error': 'Внутренняя ошибка',
|
||||
}
|
||||
_BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
def _error(status: int, code: str, message: str, request_id: str, challenge: str | None = None) -> JSONResponse:
|
||||
body = {'error': {'code': code, 'message': message, 'request_id': request_id}}
|
||||
headers = {'X-Request-Id': request_id}
|
||||
if challenge:
|
||||
headers['WWW-Authenticate'] = challenge
|
||||
return JSONResponse(body, status_code=status, headers=headers)
|
||||
|
||||
|
||||
def _request_id(request: Request) -> str:
|
||||
return getattr(request.state, 'request_id', None) or str(uuid.uuid4())
|
||||
|
||||
|
||||
_OPENAPI_PATH = Path(__file__).resolve().parents[1] / 'contracts' / 'openapi.json'
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Migrations are not applied here: the one-shot migrate service runs them before replicas start."""
|
||||
settings = load_settings()
|
||||
pool = await open_pool(settings.database_url)
|
||||
try:
|
||||
app.state.store = Store(pool, settings.session_ttl_seconds, settings.idempotency_ttl_seconds)
|
||||
logger.info('instance %s started', settings.instance_id)
|
||||
yield
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
app = FastAPI(
|
||||
title='MTUSI Messenger lab3',
|
||||
version='1.3.0',
|
||||
lifespan=lifespan,
|
||||
)
|
||||
settings = load_settings()
|
||||
instance_id = settings.instance_id
|
||||
app_version = settings.app_version
|
||||
|
||||
def openapi():
|
||||
if app.openapi_schema is None:
|
||||
app.openapi_schema = json.loads(_OPENAPI_PATH.read_text(encoding='utf-8'))
|
||||
return app.openapi_schema
|
||||
|
||||
app.openapi = openapi
|
||||
|
||||
@app.middleware('http')
|
||||
async def request_context(request: Request, call_next):
|
||||
"""Gives every response, including unexpected failures, an X-Request-Id, X-Instance-Id and X-App-Version."""
|
||||
incoming = request.headers.get('X-Request-Id')
|
||||
request_id = incoming if incoming and _SAFE_REQUEST_ID.match(incoming) else str(uuid.uuid4())
|
||||
request.state.request_id = request_id
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
logger.exception('request %s failed: %s %s', request_id, request.method, request.url.path)
|
||||
response = _error(500, 'internal_error', _MESSAGES['internal_error'], request_id)
|
||||
response.headers['X-Request-Id'] = request_id
|
||||
response.headers['X-Instance-Id'] = instance_id
|
||||
response.headers['X-App-Version'] = app_version
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def handle_api_error(request: Request, exc: ApiError) -> JSONResponse:
|
||||
return _error(exc.status, exc.code, exc.message, _request_id(request), exc.challenge)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def handle_validation_error(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
return _error(400, 'invalid_request', _MESSAGES['invalid_request'], _request_id(request))
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def handle_http_error(request: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||
code = _CODES.get(exc.status_code, 'internal_error' if exc.status_code >= 500 else 'invalid_request')
|
||||
challenge = _BEARER_CHALLENGE if exc.status_code == 401 else None
|
||||
return _error(exc.status_code, code, _MESSAGES.get(code, _MESSAGES['invalid_request']), _request_id(request), challenge)
|
||||
|
||||
app.include_router(router)
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Apply numbered SQL files once. Run as a separate one-shot step: python -m app.migrate."""
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import asyncpg
|
||||
|
||||
from .config import load_settings
|
||||
from .db import open_pool
|
||||
|
||||
logger = logging.getLogger('messenger')
|
||||
|
||||
_MIGRATIONS = Path(__file__).resolve().parents[1] / 'migrations'
|
||||
# Any constant shared by all runners; serializes concurrent migrators across processes.
|
||||
_LOCK_ID = 3_2026_0003
|
||||
|
||||
_BOOTSTRAP = """
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def expected_versions() -> list[str]:
|
||||
return sorted(path.name for path in _MIGRATIONS.glob('*.sql'))
|
||||
|
||||
|
||||
async def apply_migrations(pool: asyncpg.Pool) -> None:
|
||||
async with pool.acquire() as conn:
|
||||
await conn.execute('SET statement_timeout = 0')
|
||||
await conn.execute('SELECT pg_advisory_lock($1)', _LOCK_ID)
|
||||
try:
|
||||
await conn.execute(_BOOTSTRAP)
|
||||
rows = await conn.fetch('SELECT version FROM schema_migrations')
|
||||
applied = {row['version'] for row in rows}
|
||||
for version in expected_versions():
|
||||
if version in applied:
|
||||
logger.info('migration %s already applied', version)
|
||||
continue
|
||||
sql = (_MIGRATIONS / version).read_text(encoding='utf-8')
|
||||
async with conn.transaction():
|
||||
await conn.execute(sql)
|
||||
await conn.execute(
|
||||
'INSERT INTO schema_migrations (version) VALUES ($1)',
|
||||
version,
|
||||
)
|
||||
logger.info('applied migration %s', version)
|
||||
finally:
|
||||
await conn.execute('SELECT pg_advisory_unlock($1)', _LOCK_ID)
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
settings = load_settings()
|
||||
pool = await open_pool(settings.database_url)
|
||||
try:
|
||||
await apply_migrations(pool)
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,43 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from typing import Mapping, Optional, Tuple
|
||||
|
||||
from .errors import invalid_request
|
||||
|
||||
# Position of the last item returned to the client.
|
||||
Key = Tuple[str, str]
|
||||
|
||||
DEFAULT_LIMIT = 50
|
||||
MAX_LIMIT = 100
|
||||
_LIMIT = re.compile(r'^\d{1,3}$')
|
||||
|
||||
|
||||
def encode_cursor(scope: str, key: Key) -> str:
|
||||
"""Scope binds the cursor to one resource, so a foreign cursor is detectable."""
|
||||
payload = json.dumps({'s': scope, 't': key[0], 'i': key[1]}, separators=(',', ':'))
|
||||
return base64.urlsafe_b64encode(payload.encode('utf-8')).decode('ascii').rstrip('=')
|
||||
|
||||
|
||||
def decode_cursor(value: str, scope: str) -> Key:
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(value + '=' * (-len(value) % 4))
|
||||
payload = json.loads(raw)
|
||||
created_at, item_id = payload['t'], payload['i']
|
||||
belongs = payload['s'] == scope
|
||||
except (ValueError, TypeError, KeyError):
|
||||
raise invalid_request('Некорректный cursor') from None
|
||||
if not belongs or not isinstance(created_at, str) or not isinstance(item_id, str):
|
||||
raise invalid_request('Cursor относится к другому запросу') from None
|
||||
return created_at, item_id
|
||||
|
||||
|
||||
def parse_page(params: Mapping[str, str], scope: str) -> Tuple[int, Optional[Key]]:
|
||||
raw_limit = params.get('limit')
|
||||
limit = DEFAULT_LIMIT
|
||||
if raw_limit is not None:
|
||||
if not _LIMIT.match(raw_limit) or not 1 <= int(raw_limit) <= MAX_LIMIT:
|
||||
raise invalid_request('limit должен быть целым числом от 1 до %d' % MAX_LIMIT)
|
||||
limit = int(raw_limit)
|
||||
raw_cursor = params.get('cursor')
|
||||
return limit, decode_cursor(raw_cursor, scope) if raw_cursor is not None else None
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Argon2id password hashes. The hash never leaves this module in an API response."""
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError
|
||||
|
||||
_hasher = PasswordHasher()
|
||||
# Same cost as a real hash, so a missing user is not faster than a wrong password.
|
||||
_DUMMY_HASH = _hasher.hash('not-a-user-password')
|
||||
_DUMMY_PASSWORD = 'not-a-user-password'
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return _hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str, password: str) -> bool:
|
||||
try:
|
||||
return _hasher.verify(password_hash, password)
|
||||
except VerificationError:
|
||||
return False
|
||||
|
||||
|
||||
def password_matches(stored_hash: str | None, password: str) -> bool:
|
||||
"""One Argon2 check either way. An out-of-range secret is never the stored password."""
|
||||
if stored_hash is None or not 12 <= len(password) <= 128 or not password.strip():
|
||||
verify_password(_DUMMY_HASH, _DUMMY_PASSWORD)
|
||||
return False
|
||||
return verify_password(stored_hash, password)
|
||||
+196
@@ -0,0 +1,196 @@
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from typing import Optional, Sequence, Type, TypeVar
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from starlette.responses import Response
|
||||
|
||||
from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, parse_basic, parse_bearer
|
||||
from .drain import draining
|
||||
from .errors import invalid_request, not_found, unauthorized, unavailable
|
||||
from .pagination import encode_cursor, parse_page
|
||||
from .schemas import CreateChat, CreateMessage, CreateUser
|
||||
from .store import Chat, Store, User
|
||||
|
||||
router = APIRouter()
|
||||
Body = TypeVar('Body', bound=BaseModel)
|
||||
|
||||
_IDEMPOTENCY_KEY = re.compile(r'[A-Za-z0-9._:-]{1,128}')
|
||||
|
||||
|
||||
async def get_store(request: Request) -> Store:
|
||||
return request.app.state.store
|
||||
|
||||
|
||||
async def current_user(request: Request) -> User:
|
||||
"""Bearer only. X-User-Id is ignored and does not identify anyone."""
|
||||
token = parse_bearer(request.headers.get('Authorization', ''))
|
||||
if token is None:
|
||||
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
|
||||
user = await request.app.state.store.user_for_token(token)
|
||||
if user is None:
|
||||
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
|
||||
return user
|
||||
|
||||
|
||||
async def read_body(request: Request, model: Type[Body]) -> Body:
|
||||
try:
|
||||
payload = json.loads(await request.body())
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
raise invalid_request('Тело запроса не является корректным JSON') from None
|
||||
if not isinstance(payload, dict):
|
||||
raise invalid_request('Тело запроса должно быть JSON-объектом')
|
||||
try:
|
||||
return model.model_validate(payload)
|
||||
except ValidationError:
|
||||
raise invalid_request('Тело запроса не соответствует контракту') from None
|
||||
|
||||
|
||||
def _request_hash(payload: BaseModel) -> str:
|
||||
"""Key order in the client JSON does not matter: the validated body is re-serialized canonically."""
|
||||
canonical = json.dumps(payload.model_dump(), sort_keys=True, separators=(',', ':'), ensure_ascii=False)
|
||||
return hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
async def _member_chat(store: Store, chat_id: str, actor: User) -> Chat:
|
||||
try:
|
||||
canonical = str(uuid.UUID(chat_id))
|
||||
except ValueError:
|
||||
raise invalid_request('Некорректный chat_id') from None
|
||||
chat = await store.find_chat(canonical, actor.id)
|
||||
if chat is None:
|
||||
raise not_found('Чат не найден')
|
||||
return chat
|
||||
|
||||
|
||||
def _page(items: Sequence, has_more: bool, scope: str) -> dict:
|
||||
cursor: Optional[str] = None
|
||||
if has_more and items:
|
||||
cursor = encode_cursor(scope, (items[-1].created_at, items[-1].id))
|
||||
return {'items': [item.to_dict() for item in items], 'next_cursor': cursor}
|
||||
|
||||
|
||||
@router.get('/health/live')
|
||||
async def liveness() -> JSONResponse:
|
||||
return JSONResponse({'status': 'ok'})
|
||||
|
||||
|
||||
@router.get('/health/ready')
|
||||
async def readiness(store: Store = Depends(get_store)) -> JSONResponse:
|
||||
if draining.is_set():
|
||||
raise unavailable('Реплика завершает работу')
|
||||
await store.ready()
|
||||
return JSONResponse({'status': 'ok'})
|
||||
|
||||
|
||||
@router.post('/api/v1/users')
|
||||
async def register_user(request: Request, store: Store = Depends(get_store)) -> JSONResponse:
|
||||
payload = await read_body(request, CreateUser)
|
||||
user = await store.create_user(payload.username, payload.display_name, payload.password)
|
||||
return JSONResponse(user.to_dict(), status_code=201)
|
||||
|
||||
|
||||
@router.post('/api/v1/auth/sessions')
|
||||
async def create_session(request: Request, store: Store = Depends(get_store)) -> JSONResponse:
|
||||
parsed = parse_basic(request.headers.get('Authorization', ''))
|
||||
if parsed is None:
|
||||
raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE)
|
||||
user = await store.authenticate(*parsed)
|
||||
if user is None:
|
||||
raise unauthorized('Неверные учётные данные', BASIC_CHALLENGE)
|
||||
token, expires_at = await store.create_session(user.id)
|
||||
body = {
|
||||
'token': token,
|
||||
'token_type': 'Bearer',
|
||||
'expires_at': expires_at,
|
||||
'user': user.to_dict(),
|
||||
}
|
||||
return JSONResponse(body, status_code=201)
|
||||
|
||||
|
||||
@router.delete('/api/v1/auth/sessions/current')
|
||||
async def revoke_session(request: Request, actor: User = Depends(current_user)) -> Response:
|
||||
token = parse_bearer(request.headers.get('Authorization', ''))
|
||||
if token is None or not await request.app.state.store.revoke_session(token):
|
||||
raise unauthorized('Требуется действительная сессия', BEARER_CHALLENGE)
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@router.get('/api/v1/users/me')
|
||||
async def show_current_user(actor: User = Depends(current_user)) -> JSONResponse:
|
||||
return JSONResponse(actor.to_dict())
|
||||
|
||||
|
||||
@router.post('/api/v1/chats')
|
||||
async def create_chat(
|
||||
request: Request,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
payload = await read_body(request, CreateChat)
|
||||
if actor.id not in payload.member_ids:
|
||||
raise invalid_request('Создатель чата должен быть указан в member_ids')
|
||||
chat = await store.create_chat(payload.title, payload.member_ids)
|
||||
return JSONResponse(chat.to_dict(), status_code=201)
|
||||
|
||||
|
||||
@router.get('/api/v1/chats')
|
||||
async def list_chats(
|
||||
request: Request,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
scope = 'chats:' + actor.id
|
||||
limit, after = parse_page(request.query_params, scope)
|
||||
chats, has_more = await store.list_chats(actor.id, limit, after)
|
||||
return JSONResponse(_page(chats, has_more, scope))
|
||||
|
||||
|
||||
@router.get('/api/v1/chats/{chat_id}')
|
||||
async def show_chat(
|
||||
chat_id: str,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
return JSONResponse((await _member_chat(store, chat_id, actor)).to_dict())
|
||||
|
||||
|
||||
@router.post('/api/v1/chats/{chat_id}/messages')
|
||||
async def send_message(
|
||||
chat_id: str,
|
||||
request: Request,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
chat = await _member_chat(store, chat_id, actor)
|
||||
payload = await read_body(request, CreateMessage)
|
||||
key = request.headers.get('Idempotency-Key')
|
||||
if key is None or not _IDEMPOTENCY_KEY.fullmatch(key):
|
||||
raise invalid_request('Требуется корректный Idempotency-Key')
|
||||
message = await store.add_message(
|
||||
chat.id,
|
||||
actor.id,
|
||||
payload.text,
|
||||
path='/api/v1/chats/%s/messages' % chat.id,
|
||||
key=key,
|
||||
request_hash=_request_hash(payload),
|
||||
)
|
||||
return JSONResponse(message.to_dict(), status_code=201)
|
||||
|
||||
|
||||
@router.get('/api/v1/chats/{chat_id}/messages')
|
||||
async def list_messages(
|
||||
chat_id: str,
|
||||
request: Request,
|
||||
actor: User = Depends(current_user),
|
||||
store: Store = Depends(get_store),
|
||||
) -> JSONResponse:
|
||||
chat = await _member_chat(store, chat_id, actor)
|
||||
scope = 'messages:' + chat.id
|
||||
limit, after = parse_page(request.query_params, scope)
|
||||
messages, has_more = await store.list_messages(chat.id, limit, after)
|
||||
return JSONResponse(_page(messages, has_more, scope))
|
||||
@@ -0,0 +1,55 @@
|
||||
"""Request bodies. The contract forbids unknown fields, wrong types and blank strings."""
|
||||
import uuid
|
||||
from typing import List
|
||||
|
||||
from pydantic import AfterValidator, BaseModel, ConfigDict, Field, field_validator
|
||||
from typing_extensions import Annotated
|
||||
|
||||
|
||||
def _not_blank(value: str) -> str:
|
||||
if not value.strip():
|
||||
raise ValueError('value must not be whitespace only')
|
||||
return value
|
||||
|
||||
|
||||
def _canonical_uuid(value: str) -> str:
|
||||
parsed = uuid.UUID(value)
|
||||
if str(parsed) != value.lower():
|
||||
raise ValueError('value must be a canonical UUID')
|
||||
return str(parsed)
|
||||
|
||||
|
||||
Username = Annotated[str, Field(pattern=r'^[a-z0-9_]{3,32}$')]
|
||||
DisplayName = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)]
|
||||
Title = Annotated[str, Field(min_length=1, max_length=100), AfterValidator(_not_blank)]
|
||||
Text = Annotated[str, Field(min_length=1, max_length=4000), AfterValidator(_not_blank)]
|
||||
Password = Annotated[str, Field(min_length=12, max_length=128), AfterValidator(_not_blank)]
|
||||
UserId = Annotated[str, AfterValidator(_canonical_uuid)]
|
||||
|
||||
|
||||
class CreateUser(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
|
||||
username: Username
|
||||
display_name: DisplayName
|
||||
password: Password
|
||||
|
||||
|
||||
class CreateChat(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
|
||||
title: Title
|
||||
member_ids: Annotated[List[UserId], Field(min_length=1, max_length=100)]
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def _unique(cls, value: List[str]) -> List[str]:
|
||||
if len(set(value)) != len(value):
|
||||
raise ValueError('member_ids must be unique')
|
||||
return value
|
||||
|
||||
|
||||
class CreateMessage(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid')
|
||||
|
||||
text: Text
|
||||
+467
@@ -0,0 +1,467 @@
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, List, Optional, Sequence, Tuple
|
||||
|
||||
import asyncpg
|
||||
|
||||
from .errors import conflict, invalid_request, not_found, unavailable
|
||||
from .migrate import expected_versions
|
||||
from .pagination import Key
|
||||
from .passwords import hash_password, password_matches
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class User:
|
||||
id: str
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'username': self.username,
|
||||
'display_name': self.display_name,
|
||||
'created_at': self.created_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Chat:
|
||||
id: str
|
||||
title: str
|
||||
member_ids: Tuple[str, ...]
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'title': self.title,
|
||||
'member_ids': list(self.member_ids),
|
||||
'created_at': self.created_at,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Message:
|
||||
id: str
|
||||
chat_id: str
|
||||
sender_id: str
|
||||
text: str
|
||||
created_at: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return {
|
||||
'id': self.id,
|
||||
'chat_id': self.chat_id,
|
||||
'sender_id': self.sender_id,
|
||||
'text': self.text,
|
||||
'created_at': self.created_at,
|
||||
}
|
||||
|
||||
|
||||
def _token_key(token: str) -> str:
|
||||
return hashlib.sha256(token.encode('utf-8')).hexdigest()
|
||||
|
||||
|
||||
def _stamp(moment: datetime) -> str:
|
||||
return moment.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
|
||||
def _parse_stamp(value: str) -> datetime:
|
||||
if value.endswith('Z'):
|
||||
value = value[:-1] + '+00:00'
|
||||
return datetime.fromisoformat(value)
|
||||
|
||||
|
||||
def _user(row: asyncpg.Record) -> User:
|
||||
return User(str(row['id']), row['username'], row['display_name'], _stamp(row['created_at']))
|
||||
|
||||
|
||||
def _as_uuid(value: str, message: str) -> uuid.UUID:
|
||||
try:
|
||||
return uuid.UUID(value)
|
||||
except ValueError:
|
||||
raise invalid_request(message) from None
|
||||
|
||||
|
||||
def _db_down(exc: BaseException) -> bool:
|
||||
return isinstance(exc, (
|
||||
asyncpg.PostgresConnectionError,
|
||||
asyncpg.InterfaceError,
|
||||
asyncpg.CannotConnectNowError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
))
|
||||
|
||||
|
||||
class Store:
|
||||
"""PostgreSQL is the only state: users, chats, sessions and idempotency results are shared by all replicas."""
|
||||
|
||||
def __init__(self, pool: asyncpg.Pool, session_ttl_seconds: int = 3600, idempotency_ttl_seconds: int = 86400) -> None:
|
||||
self._pool = pool
|
||||
self._session_ttl = timedelta(seconds=session_ttl_seconds)
|
||||
self._idempotency_ttl = timedelta(seconds=idempotency_ttl_seconds)
|
||||
self._lock = threading.Lock()
|
||||
self._last_moment: Optional[datetime] = None
|
||||
self._migrations = expected_versions()
|
||||
|
||||
def _now(self) -> datetime:
|
||||
"""Strictly increasing within one process; across replicas order relies on the shared host clock."""
|
||||
with self._lock:
|
||||
moment = datetime.now(timezone.utc)
|
||||
if self._last_moment is not None and moment <= self._last_moment:
|
||||
moment = self._last_moment + timedelta(microseconds=1)
|
||||
self._last_moment = moment
|
||||
return moment
|
||||
|
||||
async def ready(self) -> None:
|
||||
"""Ready only when PostgreSQL answers and the separate migrator has applied this build's schema."""
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
applied = await conn.fetchval(
|
||||
'SELECT count(*) FROM schema_migrations WHERE version = ANY($1::text[])',
|
||||
self._migrations,
|
||||
)
|
||||
except Exception:
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
if applied != len(self._migrations):
|
||||
raise unavailable('Схема БД ещё не применена')
|
||||
|
||||
async def create_user(self, username: str, display_name: str, password: str) -> User:
|
||||
password_hash = hash_password(password)
|
||||
user_id = uuid.uuid4()
|
||||
created = self._now()
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
''',
|
||||
user_id,
|
||||
username,
|
||||
display_name,
|
||||
password_hash,
|
||||
created,
|
||||
)
|
||||
except asyncpg.UniqueViolationError:
|
||||
raise conflict('Username уже занят') from None
|
||||
return User(str(user_id), username, display_name, _stamp(created))
|
||||
|
||||
async def authenticate(self, username: str, password: str) -> Optional[User]:
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
SELECT id, username, display_name, created_at, password_hash
|
||||
FROM users
|
||||
WHERE username = $1
|
||||
''',
|
||||
username,
|
||||
)
|
||||
except Exception as exc:
|
||||
if _db_down(exc):
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
raise
|
||||
stored_hash = row['password_hash'] if row is not None else None
|
||||
if not password_matches(stored_hash, password) or row is None:
|
||||
return None
|
||||
return _user(row)
|
||||
|
||||
async def create_session(self, user_id: str) -> Tuple[str, str]:
|
||||
token = secrets.token_urlsafe(32)
|
||||
expires = datetime.now(timezone.utc) + self._session_ttl
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
'INSERT INTO sessions (token_hash, user_id, expires_at) VALUES ($1, $2, $3)',
|
||||
_token_key(token),
|
||||
_as_uuid(user_id, 'Некорректный user_id'),
|
||||
expires,
|
||||
)
|
||||
except Exception as exc:
|
||||
if _db_down(exc):
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
raise
|
||||
return token, _stamp(expires)
|
||||
|
||||
async def user_for_token(self, token: str) -> Optional[User]:
|
||||
now = datetime.now(timezone.utc)
|
||||
key = _token_key(token)
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at
|
||||
FROM sessions s
|
||||
JOIN users u ON u.id = s.user_id
|
||||
WHERE s.token_hash = $1
|
||||
''',
|
||||
key,
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
if row['expires_at'] <= now:
|
||||
await conn.execute('DELETE FROM sessions WHERE token_hash = $1', key)
|
||||
return None
|
||||
return _user(row)
|
||||
except Exception as exc:
|
||||
if _db_down(exc):
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
raise
|
||||
|
||||
async def revoke_session(self, token: str) -> bool:
|
||||
try:
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
DELETE FROM sessions
|
||||
WHERE token_hash = $1 AND expires_at > $2
|
||||
RETURNING token_hash
|
||||
''',
|
||||
_token_key(token),
|
||||
datetime.now(timezone.utc),
|
||||
)
|
||||
except Exception as exc:
|
||||
if _db_down(exc):
|
||||
raise unavailable('Хранилище недоступно') from None
|
||||
raise
|
||||
return row is not None
|
||||
|
||||
async def create_chat(self, title: str, member_ids: Sequence[str]) -> Chat:
|
||||
ids = [_as_uuid(member_id, 'Некорректный user_id') for member_id in member_ids]
|
||||
chat_id = uuid.uuid4()
|
||||
created = self._now()
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
found = await conn.fetch('SELECT id FROM users WHERE id = ANY($1::uuid[])', ids)
|
||||
if len(found) != len(ids):
|
||||
raise invalid_request('Все участники должны быть существующими пользователями')
|
||||
await conn.execute(
|
||||
'INSERT INTO chats (id, title, created_at) VALUES ($1, $2, $3)',
|
||||
chat_id,
|
||||
title,
|
||||
created,
|
||||
)
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT $1, member_id, (ordinality - 1)::integer
|
||||
FROM unnest($2::uuid[]) WITH ORDINALITY AS members(member_id, ordinality)
|
||||
''',
|
||||
chat_id,
|
||||
ids,
|
||||
)
|
||||
return Chat(str(chat_id), title, tuple(member_ids), _stamp(created))
|
||||
|
||||
async def find_chat(self, chat_id: str, member_id: str) -> Optional[Chat]:
|
||||
"""A chat of somebody else is indistinguishable from a missing one."""
|
||||
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
|
||||
member_uuid = _as_uuid(member_id, 'Некорректный user_id')
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $2
|
||||
WHERE c.id = $1
|
||||
''',
|
||||
chat_uuid,
|
||||
member_uuid,
|
||||
)
|
||||
if row is None:
|
||||
return None
|
||||
members = await self._members(conn, [chat_uuid])
|
||||
return Chat(str(row['id']), row['title'], members.get(chat_uuid, ()), _stamp(row['created_at']))
|
||||
|
||||
async def list_chats(self, member_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Chat], bool]:
|
||||
member_uuid = _as_uuid(member_id, 'Некорректный user_id')
|
||||
after_at, after_id = self._cursor(after)
|
||||
async with self._pool.acquire() as conn:
|
||||
if after_at is None:
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $2
|
||||
''',
|
||||
member_uuid,
|
||||
limit + 1,
|
||||
)
|
||||
else:
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT c.id, c.title, c.created_at
|
||||
FROM chats c
|
||||
JOIN chat_members mine ON mine.chat_id = c.id AND mine.user_id = $1
|
||||
WHERE (c.created_at, c.id) > ($2::timestamptz, $3::uuid)
|
||||
ORDER BY c.created_at, c.id
|
||||
LIMIT $4
|
||||
''',
|
||||
member_uuid,
|
||||
after_at,
|
||||
after_id,
|
||||
limit + 1,
|
||||
)
|
||||
page = rows[:limit]
|
||||
members = await self._members(conn, [row['id'] for row in page])
|
||||
chats = [
|
||||
Chat(str(row['id']), row['title'], members.get(row['id'], ()), _stamp(row['created_at']))
|
||||
for row in page
|
||||
]
|
||||
return chats, len(rows) > limit
|
||||
|
||||
async def add_message(
|
||||
self,
|
||||
chat_id: str,
|
||||
sender_id: str,
|
||||
text: str,
|
||||
*,
|
||||
path: str,
|
||||
key: str,
|
||||
request_hash: str,
|
||||
) -> Message:
|
||||
"""Exactly one message per (sender, POST, path, key), whichever replica receives the request.
|
||||
|
||||
Concurrent duplicates block on the primary key of idempotency_keys until the first
|
||||
transaction commits or rolls back, so a rolled-back attempt leaves the key free.
|
||||
"""
|
||||
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
|
||||
sender_uuid = _as_uuid(sender_id, 'Некорректный user_id')
|
||||
message = Message(str(uuid.uuid4()), str(chat_uuid), sender_id, text, _stamp(self._now()))
|
||||
cutoff = datetime.now(timezone.utc) - self._idempotency_ttl
|
||||
async with self._pool.acquire() as conn:
|
||||
async with conn.transaction():
|
||||
await conn.execute(
|
||||
'''
|
||||
DELETE FROM idempotency_keys
|
||||
WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3 AND created_at < $4
|
||||
''',
|
||||
sender_uuid,
|
||||
path,
|
||||
key,
|
||||
cutoff,
|
||||
)
|
||||
claimed = await conn.fetchval(
|
||||
'''
|
||||
INSERT INTO idempotency_keys (user_id, method, path, key, request_hash, status_code, response)
|
||||
VALUES ($1, 'POST', $2, $3, $4, 201, $5::jsonb)
|
||||
ON CONFLICT (user_id, method, path, key) DO NOTHING
|
||||
RETURNING 1
|
||||
''',
|
||||
sender_uuid,
|
||||
path,
|
||||
key,
|
||||
request_hash,
|
||||
json.dumps(message.to_dict()),
|
||||
)
|
||||
if claimed is None:
|
||||
return await self._replay(conn, sender_uuid, path, key, request_hash)
|
||||
await conn.execute(
|
||||
'''
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
''',
|
||||
uuid.UUID(message.id),
|
||||
chat_uuid,
|
||||
sender_uuid,
|
||||
text,
|
||||
_parse_stamp(message.created_at),
|
||||
)
|
||||
return message
|
||||
|
||||
async def _replay(
|
||||
self,
|
||||
conn: asyncpg.Connection,
|
||||
sender_uuid: uuid.UUID,
|
||||
path: str,
|
||||
key: str,
|
||||
request_hash: str,
|
||||
) -> Message:
|
||||
row = await conn.fetchrow(
|
||||
'''
|
||||
SELECT request_hash, response
|
||||
FROM idempotency_keys
|
||||
WHERE user_id = $1 AND method = 'POST' AND path = $2 AND key = $3
|
||||
''',
|
||||
sender_uuid,
|
||||
path,
|
||||
key,
|
||||
)
|
||||
if row is None:
|
||||
raise unavailable('Повторите запрос с тем же Idempotency-Key')
|
||||
if row['request_hash'] != request_hash:
|
||||
raise conflict('Idempotency-Key уже использован с другим сообщением')
|
||||
data = json.loads(row['response'])
|
||||
return Message(data['id'], data['chat_id'], data['sender_id'], data['text'], data['created_at'])
|
||||
|
||||
async def list_messages(self, chat_id: str, limit: int, after: Optional[Key]) -> Tuple[List[Message], bool]:
|
||||
chat_uuid = _as_uuid(chat_id, 'Некорректный chat_id')
|
||||
after_at, after_id = self._cursor(after)
|
||||
async with self._pool.acquire() as conn:
|
||||
if after_at is None:
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
ORDER BY created_at, id
|
||||
LIMIT $2
|
||||
''',
|
||||
chat_uuid,
|
||||
limit + 1,
|
||||
)
|
||||
else:
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = $1
|
||||
AND (created_at, id) > ($2::timestamptz, $3::uuid)
|
||||
ORDER BY created_at, id
|
||||
LIMIT $4
|
||||
''',
|
||||
chat_uuid,
|
||||
after_at,
|
||||
after_id,
|
||||
limit + 1,
|
||||
)
|
||||
page = rows[:limit]
|
||||
messages = [
|
||||
Message(str(row['id']), str(row['chat_id']), str(row['sender_id']), row['text'], _stamp(row['created_at']))
|
||||
for row in page
|
||||
]
|
||||
return messages, len(rows) > limit
|
||||
|
||||
async def _members(self, conn: asyncpg.Connection, chat_ids: Sequence[uuid.UUID]) -> Dict[uuid.UUID, Tuple[str, ...]]:
|
||||
if not chat_ids:
|
||||
return {}
|
||||
rows = await conn.fetch(
|
||||
'''
|
||||
SELECT chat_id, user_id
|
||||
FROM chat_members
|
||||
WHERE chat_id = ANY($1::uuid[])
|
||||
ORDER BY chat_id, position
|
||||
''',
|
||||
list(chat_ids),
|
||||
)
|
||||
grouped: Dict[uuid.UUID, List[str]] = {}
|
||||
for row in rows:
|
||||
grouped.setdefault(row['chat_id'], []).append(str(row['user_id']))
|
||||
return {chat_id: tuple(member_ids) for chat_id, member_ids in grouped.items()}
|
||||
|
||||
def _cursor(self, after: Optional[Key]) -> Tuple[Optional[datetime], Optional[uuid.UUID]]:
|
||||
if after is None:
|
||||
return None, None
|
||||
try:
|
||||
return _parse_stamp(after[0]), uuid.UUID(after[1])
|
||||
except ValueError:
|
||||
raise invalid_request('Некорректный cursor') from None
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
# Client -> lb (HAProxy) -> api x API_REPLICAS -> postgres.
|
||||
# Only the LB publishes a port. Networks: edge = lb<->api, data = api/migrate<->postgres (no LB, no internet).
|
||||
# The DB password comes from .env (not committed) as a Compose secret file, never as a container env var.
|
||||
name: messenger-lab3
|
||||
|
||||
x-db-client: &db-client
|
||||
APP_ENV: ${APP_ENV:-lab}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-info}
|
||||
DB_HOST: postgres
|
||||
DB_USER: ${POSTGRES_USER:?set in .env}
|
||||
DB_NAME: ${POSTGRES_DB:?set in .env}
|
||||
DB_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
|
||||
x-app-build: &app-build
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-dev}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297
|
||||
environment:
|
||||
POSTGRES_USER: ${POSTGRES_USER}
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_DB: ${POSTGRES_DB}
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
networks:
|
||||
- data
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${POSTGRES_CPUS:-1.0}"
|
||||
memory: "${POSTGRES_MEMORY:-512M}"
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 256M
|
||||
restart: unless-stopped
|
||||
|
||||
# One-shot schema step. Replicas never migrate on startup, so there is no race between them;
|
||||
# an advisory lock additionally serializes accidental parallel runs.
|
||||
migrate:
|
||||
image: messenger-lab3-app:${APP_VERSION:-dev}
|
||||
build: *app-build
|
||||
command: ["python", "-m", "app.migrate"]
|
||||
environment:
|
||||
<<: *db-client
|
||||
secrets:
|
||||
- source: postgres_password
|
||||
uid: "10001"
|
||||
gid: "10001"
|
||||
mode: 0400
|
||||
networks:
|
||||
- data
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
image: messenger-lab3-app:${APP_VERSION:-dev}
|
||||
build: *app-build
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPUS:-1.0}"
|
||||
memory: "${API_MEMORY:-256M}"
|
||||
reservations:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
environment:
|
||||
<<: *db-client
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
|
||||
DRAIN_SECONDS: ${DRAIN_SECONDS:-4}
|
||||
SHUTDOWN_TIMEOUT_SECONDS: ${SHUTDOWN_TIMEOUT_SECONDS:-3}
|
||||
DB_POOL_MIN: ${DB_POOL_MIN:-1}
|
||||
DB_POOL_MAX: ${DB_POOL_MAX:-10}
|
||||
secrets:
|
||||
- source: postgres_password
|
||||
uid: "10001"
|
||||
gid: "10001"
|
||||
mode: 0400
|
||||
networks:
|
||||
- edge
|
||||
- data
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
# Docker health gates startup order and rolling update; the LB runs its own checks (infra/haproxy/haproxy.cfg).
|
||||
healthcheck:
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)
|
||||
interval: 3s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 20s
|
||||
# SIGTERM -> readiness 503 for DRAIN_SECONDS (LB drops the replica in <= 3 s) -> uvicorn finishes
|
||||
# in-flight requests within SHUTDOWN_TIMEOUT_SECONDS. 4 + 3 < 10, so SIGKILL is not reached.
|
||||
stop_signal: SIGTERM
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0
|
||||
# Loopback only. Both families: on Windows "localhost" tries ::1 first and waits ~2 s if nothing listens.
|
||||
ports:
|
||||
- "127.0.0.1:${LB_PORT:-8080}:8080"
|
||||
- "[::1]:${LB_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
networks:
|
||||
- edge
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-qO-", "http://127.0.0.1:8080/lb/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
reservations:
|
||||
cpus: "0.1"
|
||||
memory: 32M
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
edge: {}
|
||||
data:
|
||||
internal: true
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
environment: POSTGRES_PASSWORD
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
@@ -0,0 +1,961 @@
|
||||
{
|
||||
"mode": "failover",
|
||||
"stop_method": "docker kill (SIGKILL)",
|
||||
"concurrency": 4,
|
||||
"workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest",
|
||||
"replicas_before": [
|
||||
"messenger-lab3-api-1",
|
||||
"messenger-lab3-api-2"
|
||||
],
|
||||
"runs": [
|
||||
{
|
||||
"replica": "messenger-lab3-api-1",
|
||||
"instance_id": "1b755b53e9fa",
|
||||
"state_after_stop": "exited exit=137",
|
||||
"stop_command_seconds": 0.66,
|
||||
"last_served_by_target_after_sigterm_s": 0.12,
|
||||
"first_success_after_container_exit_s": 0.67,
|
||||
"rps_back_to_90pct_after_exit_s": 2.28,
|
||||
"max_gap_without_success_s": 0.963,
|
||||
"rejoin_after_docker_start_s": 4.51,
|
||||
"baseline": {
|
||||
"seconds": 10.0,
|
||||
"rps_ok": 174.4,
|
||||
"writes_ok": 873,
|
||||
"reads_ok": 871,
|
||||
"errors": 0,
|
||||
"p95_ms": 53.8,
|
||||
"instances": {
|
||||
"e3303afc01b2": 874,
|
||||
"1b755b53e9fa": 870
|
||||
}
|
||||
},
|
||||
"during_stop_and_outage": {
|
||||
"seconds": 15.73,
|
||||
"rps_ok": 145.3,
|
||||
"writes_ok": 1143,
|
||||
"reads_ok": 1143,
|
||||
"errors": 0,
|
||||
"p95_ms": 65.5,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 9,
|
||||
"e3303afc01b2": 2277
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 9.51,
|
||||
"rps_ok": 158.4,
|
||||
"writes_ok": 754,
|
||||
"reads_ok": 752,
|
||||
"errors": 0,
|
||||
"p95_ms": 67.6,
|
||||
"instances": {
|
||||
"e3303afc01b2": 1035,
|
||||
"1b755b53e9fa": 471
|
||||
}
|
||||
},
|
||||
"marks": {
|
||||
"sigterm": 10.06,
|
||||
"exited": 10.72,
|
||||
"docker_start": 25.79
|
||||
}
|
||||
},
|
||||
{
|
||||
"replica": "messenger-lab3-api-2",
|
||||
"instance_id": "e3303afc01b2",
|
||||
"state_after_stop": "exited exit=137",
|
||||
"stop_command_seconds": 0.63,
|
||||
"last_served_by_target_after_sigterm_s": 0.12,
|
||||
"first_success_after_container_exit_s": 0.755,
|
||||
"rps_back_to_90pct_after_exit_s": 2.06,
|
||||
"max_gap_without_success_s": 0.975,
|
||||
"rejoin_after_docker_start_s": 5.18,
|
||||
"baseline": {
|
||||
"seconds": 10.0,
|
||||
"rps_ok": 177.9,
|
||||
"writes_ok": 888,
|
||||
"reads_ok": 891,
|
||||
"errors": 0,
|
||||
"p95_ms": 62.7,
|
||||
"instances": {
|
||||
"e3303afc01b2": 891,
|
||||
"1b755b53e9fa": 888
|
||||
}
|
||||
},
|
||||
"during_stop_and_outage": {
|
||||
"seconds": 15.71,
|
||||
"rps_ok": 146.5,
|
||||
"writes_ok": 1152,
|
||||
"reads_ok": 1151,
|
||||
"errors": 0,
|
||||
"p95_ms": 67.0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 2291,
|
||||
"e3303afc01b2": 12
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 10.18,
|
||||
"rps_ok": 139.0,
|
||||
"writes_ok": 708,
|
||||
"reads_ok": 708,
|
||||
"errors": 0,
|
||||
"p95_ms": 73.8,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 1025,
|
||||
"e3303afc01b2": 391
|
||||
}
|
||||
},
|
||||
"marks": {
|
||||
"sigterm": 45.31,
|
||||
"exited": 45.94,
|
||||
"docker_start": 61.02
|
||||
}
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"seconds": 71.23,
|
||||
"rps_ok": 155.0,
|
||||
"writes_ok": 5520,
|
||||
"reads_ok": 5520,
|
||||
"errors": 0,
|
||||
"p95_ms": 65.5,
|
||||
"instances": {
|
||||
"e3303afc01b2": 5483,
|
||||
"1b755b53e9fa": 5557
|
||||
}
|
||||
},
|
||||
"integrity": {
|
||||
"confirmed_writes": 5520,
|
||||
"stored_messages": 5520,
|
||||
"duplicates": 0,
|
||||
"missing_confirmed": 0
|
||||
},
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 145,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 71,
|
||||
"1b755b53e9fa": 74
|
||||
},
|
||||
"versions": {
|
||||
"v2": 145
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 87,
|
||||
"1b755b53e9fa": 84
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 240,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 120,
|
||||
"1b755b53e9fa": 120
|
||||
},
|
||||
"versions": {
|
||||
"v2": 240
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 239,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 120,
|
||||
"1b755b53e9fa": 119
|
||||
},
|
||||
"versions": {
|
||||
"v2": 239
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 199,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 98,
|
||||
"1b755b53e9fa": 101
|
||||
},
|
||||
"versions": {
|
||||
"v2": 199
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 141,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 69,
|
||||
"e3303afc01b2": 72
|
||||
},
|
||||
"versions": {
|
||||
"v2": 141
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 185,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 92,
|
||||
"e3303afc01b2": 93
|
||||
},
|
||||
"versions": {
|
||||
"v2": 185
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 190,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 96,
|
||||
"e3303afc01b2": 94
|
||||
},
|
||||
"versions": {
|
||||
"v2": 190
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 130,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 65,
|
||||
"1b755b53e9fa": 65
|
||||
},
|
||||
"versions": {
|
||||
"v2": 130
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 100,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 50,
|
||||
"e3303afc01b2": 50
|
||||
},
|
||||
"versions": {
|
||||
"v2": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 48,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 39,
|
||||
"1b755b53e9fa": 9
|
||||
},
|
||||
"versions": {
|
||||
"v2": 48
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 128,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 128
|
||||
},
|
||||
"versions": {
|
||||
"v2": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 170,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 170
|
||||
},
|
||||
"versions": {
|
||||
"v2": 170
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 107,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 107
|
||||
},
|
||||
"versions": {
|
||||
"v2": 107
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 143,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 143
|
||||
},
|
||||
"versions": {
|
||||
"v2": 143
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 148,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 148
|
||||
},
|
||||
"versions": {
|
||||
"v2": 148
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 170,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 170
|
||||
},
|
||||
"versions": {
|
||||
"v2": 170
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 128,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 128
|
||||
},
|
||||
"versions": {
|
||||
"v2": 128
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 136,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 136
|
||||
},
|
||||
"versions": {
|
||||
"v2": 136
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 147,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 147
|
||||
},
|
||||
"versions": {
|
||||
"v2": 147
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 187,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 187
|
||||
},
|
||||
"versions": {
|
||||
"v2": 187
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 132,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 132
|
||||
},
|
||||
"versions": {
|
||||
"v2": 132
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 136,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 136
|
||||
},
|
||||
"versions": {
|
||||
"v2": 136
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 194,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 194
|
||||
},
|
||||
"versions": {
|
||||
"v2": 194
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 197,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 197
|
||||
},
|
||||
"versions": {
|
||||
"v2": 197
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 134,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 134
|
||||
},
|
||||
"versions": {
|
||||
"v2": 134
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 56,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 56
|
||||
},
|
||||
"versions": {
|
||||
"v2": 56
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 139,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 139
|
||||
},
|
||||
"versions": {
|
||||
"v2": 139
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 161,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 161
|
||||
},
|
||||
"versions": {
|
||||
"v2": 161
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 176,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 176
|
||||
},
|
||||
"versions": {
|
||||
"v2": 176
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 127,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 72,
|
||||
"1b755b53e9fa": 55
|
||||
},
|
||||
"versions": {
|
||||
"v2": 127
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 173,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 90,
|
||||
"1b755b53e9fa": 83
|
||||
},
|
||||
"versions": {
|
||||
"v2": 173
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 184,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 94,
|
||||
"e3303afc01b2": 90
|
||||
},
|
||||
"versions": {
|
||||
"v2": 184
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 240,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 120,
|
||||
"1b755b53e9fa": 120
|
||||
},
|
||||
"versions": {
|
||||
"v2": 240
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 191,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 94,
|
||||
"1b755b53e9fa": 97
|
||||
},
|
||||
"versions": {
|
||||
"v2": 191
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 138,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 72,
|
||||
"1b755b53e9fa": 66
|
||||
},
|
||||
"versions": {
|
||||
"v2": 138
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 118,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 58,
|
||||
"1b755b53e9fa": 60
|
||||
},
|
||||
"versions": {
|
||||
"v2": 118
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 208,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 104,
|
||||
"1b755b53e9fa": 104
|
||||
},
|
||||
"versions": {
|
||||
"v2": 208
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 189,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 96,
|
||||
"e3303afc01b2": 93
|
||||
},
|
||||
"versions": {
|
||||
"v2": 189
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 148,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 75,
|
||||
"1b755b53e9fa": 73
|
||||
},
|
||||
"versions": {
|
||||
"v2": 148
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 160,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 78,
|
||||
"e3303afc01b2": 82
|
||||
},
|
||||
"versions": {
|
||||
"v2": 160
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 240,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 122,
|
||||
"e3303afc01b2": 118
|
||||
},
|
||||
"versions": {
|
||||
"v2": 240
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 215,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 109,
|
||||
"e3303afc01b2": 106
|
||||
},
|
||||
"versions": {
|
||||
"v2": 215
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 87,
|
||||
"1b755b53e9fa": 84
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 174,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 85,
|
||||
"e3303afc01b2": 89
|
||||
},
|
||||
"versions": {
|
||||
"v2": 174
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 116,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 75,
|
||||
"e3303afc01b2": 41
|
||||
},
|
||||
"versions": {
|
||||
"v2": 116
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 62,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 62
|
||||
},
|
||||
"versions": {
|
||||
"v2": 62
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 180,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 180
|
||||
},
|
||||
"versions": {
|
||||
"v2": 180
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 113,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 113
|
||||
},
|
||||
"versions": {
|
||||
"v2": 113
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 141,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 141
|
||||
},
|
||||
"versions": {
|
||||
"v2": 141
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 193,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 193
|
||||
},
|
||||
"versions": {
|
||||
"v2": 193
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 200,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 200
|
||||
},
|
||||
"versions": {
|
||||
"v2": 200
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 139,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 139
|
||||
},
|
||||
"versions": {
|
||||
"v2": 139
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 171
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 210,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 210
|
||||
},
|
||||
"versions": {
|
||||
"v2": 210
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 151,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 151
|
||||
},
|
||||
"versions": {
|
||||
"v2": 151
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 56,
|
||||
"ok": 91,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 91
|
||||
},
|
||||
"versions": {
|
||||
"v2": 91
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 57,
|
||||
"ok": 123,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 123
|
||||
},
|
||||
"versions": {
|
||||
"v2": 123
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 58,
|
||||
"ok": 180,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 180
|
||||
},
|
||||
"versions": {
|
||||
"v2": 180
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 59,
|
||||
"ok": 126,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 126
|
||||
},
|
||||
"versions": {
|
||||
"v2": 126
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 60,
|
||||
"ok": 168,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 168
|
||||
},
|
||||
"versions": {
|
||||
"v2": 168
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 61,
|
||||
"ok": 78,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 78
|
||||
},
|
||||
"versions": {
|
||||
"v2": 78
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 62,
|
||||
"ok": 163,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 163
|
||||
},
|
||||
"versions": {
|
||||
"v2": 163
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 63,
|
||||
"ok": 144,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 144
|
||||
},
|
||||
"versions": {
|
||||
"v2": 144
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 64,
|
||||
"ok": 179,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 179
|
||||
},
|
||||
"versions": {
|
||||
"v2": 179
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 65,
|
||||
"ok": 68,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 68
|
||||
},
|
||||
"versions": {
|
||||
"v2": 68
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 66,
|
||||
"ok": 71,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 38,
|
||||
"e3303afc01b2": 33
|
||||
},
|
||||
"versions": {
|
||||
"v2": 71
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 67,
|
||||
"ok": 145,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 73,
|
||||
"1b755b53e9fa": 72
|
||||
},
|
||||
"versions": {
|
||||
"v2": 145
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 68,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 86,
|
||||
"1b755b53e9fa": 85
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 69,
|
||||
"ok": 206,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 103,
|
||||
"e3303afc01b2": 103
|
||||
},
|
||||
"versions": {
|
||||
"v2": 206
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 70,
|
||||
"ok": 163,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 80,
|
||||
"e3303afc01b2": 83
|
||||
},
|
||||
"versions": {
|
||||
"v2": 163
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 71,
|
||||
"ok": 35,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 19,
|
||||
"e3303afc01b2": 16
|
||||
},
|
||||
"versions": {
|
||||
"v2": 35
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# failover series 2026-11-07T18:10:53.0538421+03:00: python scripts/failover_series.py --kill (docker kill, SIGKILL)
|
||||
killing messenger-lab3-api-1 (1b755b53e9fa) under load...
|
||||
errors=0 gap=0.96s first_ok_after_exit=0.67 rps 174.4 -> 145.3 -> 158.4 rejoin=4.51s
|
||||
killing messenger-lab3-api-2 (e3303afc01b2) under load...
|
||||
errors=0 gap=0.97s first_ok_after_exit=0.755 rps 177.9 -> 146.5 -> 139.0 rejoin=5.18s
|
||||
{"total": {"seconds": 71.23, "rps_ok": 155.0, "writes_ok": 5520, "reads_ok": 5520, "errors": 0, "p95_ms": 65.5, "instances": {"e3303afc01b2": 5483, "1b755b53e9fa": 5557}}, "integrity": {"confirmed_writes": 5520, "stored_messages": 5520, "duplicates": 0, "missing_confirmed": 0}}
|
||||
PASS: failover, 5520 confirmed writes, no duplicates, nothing lost
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,7 @@
|
||||
# failover series 2026-11-07T18:09:39.2795947+03:00: make failover-series (docker stop, SIGTERM + drain)
|
||||
stopping messenger-lab3-api-1 (1b755b53e9fa) under load...
|
||||
errors=0 gap=0.09s first_ok_after_exit=0.018 rps 129.0 -> 114.8 -> 113.1 rejoin=5.15s
|
||||
stopping messenger-lab3-api-2 (e3303afc01b2) under load...
|
||||
errors=0 gap=0.09s first_ok_after_exit=0.012 rps 131.5 -> 145.2 -> 165.4 rejoin=4.91s
|
||||
{"total": {"seconds": 80.52, "rps_ok": 131.9, "writes_ok": 5312, "reads_ok": 5312, "errors": 0, "p95_ms": 73.4, "instances": {"1b755b53e9fa": 5703, "e3303afc01b2": 4921}}, "integrity": {"confirmed_writes": 5312, "stored_messages": 5312, "duplicates": 0, "missing_confirmed": 0}}
|
||||
PASS: failover, 5312 confirmed writes, no duplicates, nothing lost
|
||||
@@ -0,0 +1,38 @@
|
||||
# failover runs 2026-11-07T18:03:34.5946145+03:00 (HAProxy, drain 4 s)
|
||||
# command (= make test-failover ACTION_SCRIPT=scripts/stop-one.cmd): python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://localhost:8080
|
||||
## run 1 - before (shift requests through LB: 0)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up 53 seconds (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 53 seconds (healthy)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 121, "failed_attempts": 0, "first_recovery_seconds": 6.421}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## run 1 - after
|
||||
messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Exited (0) 9 seconds ago
|
||||
api1 172.25.0.2 UP
|
||||
## restored (docker compose --env-file .env up -d --wait)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 10 seconds (healthy)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
## run 2 - before (shift requests through LB: 1)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up About a minute (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 11 seconds (healthy)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 115, "failed_attempts": 0, "first_recovery_seconds": 6.437}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## run 2 - after
|
||||
messenger-lab3-api-1 1b755b53e9fa Exited (0) 9 seconds ago
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 30 seconds (healthy)
|
||||
api2 172.25.0.3 UP
|
||||
## restored (docker compose --env-file .env up -d --wait)
|
||||
messenger-lab3-api-1 1b755b53e9fa Up 9 seconds (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 43 seconds (healthy)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
@@ -0,0 +1,7 @@
|
||||
# make test-idempotency 2026-11-07T18:03:22.9365380+03:00 via HAProxy
|
||||
round 0: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
round 1: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
round 2: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
round 3: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
round 4: 20 responses, 1 message, replicas {'e3303afc01b2': 10, '1b755b53e9fa': 10}
|
||||
PASS: 5 keys x 20 parallel requests -> 5 messages via 2 replicas {'e3303afc01b2': 50, '1b755b53e9fa': 50}
|
||||
@@ -0,0 +1,8 @@
|
||||
login + chat via LB http://localhost:8080: one session token for user 5757ef5e-391e-4369-85c5-19499d5a4930 (token not printed)
|
||||
1. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2
|
||||
2. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa
|
||||
3. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2
|
||||
4. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa
|
||||
5. GET /api/v1/users/me -> 200 X-Instance-Id=e3303afc01b2
|
||||
6. POST /api/v1/chats/{id}/messages -> 201 X-Instance-Id=1b755b53e9fa
|
||||
PASS: same session served by 2 replicas without re-login {'e3303afc01b2': 3, '1b755b53e9fa': 3}
|
||||
@@ -0,0 +1,30 @@
|
||||
# isolation/limits 2026-11-07T18:03:06.9670680+03:00
|
||||
## per container: CPU limit (NanoCpus), memory limit, published ports, networks, mounts
|
||||
/messenger-lab3-api-1 cpus=1000000000 mem=268435456 reserv_mem=134217728 ports= networks=messenger-lab3_data messenger-lab3_edge mounts=
|
||||
/messenger-lab3-api-2 cpus=1000000000 mem=268435456 reserv_mem=134217728 ports= networks=messenger-lab3_data messenger-lab3_edge mounts=
|
||||
/messenger-lab3-lb-1 cpus=500000000 mem=134217728 reserv_mem=33554432 ports=8080/tcp->127.0.0.1:8080 ::1:8080 networks=messenger-lab3_edge mounts=bind:/usr/local/etc/haproxy/haproxy.cfg
|
||||
/messenger-lab3-migrate-1 cpus=500000000 mem=134217728 reserv_mem=0 ports= networks=messenger-lab3_data mounts=
|
||||
/messenger-lab3-postgres-1 cpus=1000000000 mem=536870912 reserv_mem=268435456 ports= networks=messenger-lab3_data mounts=volume:/var/lib/postgresql/data
|
||||
## docker stats --no-stream (this project)
|
||||
NAME CPU % MEM USAGE / LIMIT
|
||||
messenger-lab3-api-1 0.87% 41.18MiB / 256MiB
|
||||
messenger-lab3-api-2 0.62% 41.18MiB / 256MiB
|
||||
messenger-lab3-lb-1 0.21% 14.71MiB / 128MiB
|
||||
messenger-lab3-postgres-1 0.15% 38.04MiB / 512MiB
|
||||
## container env mentioning PASSWORD (expect only *_PASSWORD_FILE paths, no values)
|
||||
messenger-lab3-api-1 DB_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
messenger-lab3-api-2 DB_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
messenger-lab3-migrate-1 DB_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
messenger-lab3-postgres-1 POSTGRES_PASSWORD_FILE=/run/secrets/postgres_password
|
||||
## secret file inside api: owner/mode only, value not printed
|
||||
total 4
|
||||
-r-------- 1 10001 10001 18 Nov 7 15:02 postgres_password
|
||||
## lb has no secrets and cannot resolve postgres (not on the data network)
|
||||
no /run/secrets
|
||||
postgres not resolvable from lb
|
||||
## networks
|
||||
messenger-lab3_data internal=true
|
||||
messenger-lab3_edge internal=false
|
||||
## docker port messenger-lab3-api-1 (empty = no published port)
|
||||
## occurrences of container_name in compose.yaml
|
||||
0
|
||||
@@ -0,0 +1,21 @@
|
||||
# persistence run 2026-11-07T18:04:54.7155080+03:00
|
||||
# command (= make test-persistence ACTION_SCRIPT=scripts/restart.cmd): python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://localhost:8080
|
||||
## before
|
||||
NAME CONTAINER ID STATUS
|
||||
messenger-lab3-api-1 1b755b53e9fa Up 22 seconds (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 56 seconds (healthy)
|
||||
messenger-lab3-lb-1 760a68ddce9c Up 3 minutes (healthy)
|
||||
messenger-lab3-postgres-1 2265aa5e595d Up 3 minutes (healthy)
|
||||
volume messenger-lab3_pgdata
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## after (same IDs, new uptime)
|
||||
NAME CONTAINER ID STATUS
|
||||
messenger-lab3-api-1 1b755b53e9fa Up 12 seconds (healthy)
|
||||
messenger-lab3-api-2 e3303afc01b2 Up 11 seconds (healthy)
|
||||
messenger-lab3-lb-1 760a68ddce9c Up 5 seconds (healthy)
|
||||
messenger-lab3-postgres-1 2265aa5e595d Up 16 seconds (healthy)
|
||||
volume messenger-lab3_pgdata
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
@@ -0,0 +1,24 @@
|
||||
# rolling update dev -> v2 with migration 003 applied under load, 2026-11-07 17:59
|
||||
# Console output copied verbatim; the JSON of this run was overwritten by the later v2 -> v3 run (rolling-update.json).
|
||||
# Old replicas: image messenger-lab3-app:dev, built before migrations/003_session_indexes.sql existed
|
||||
# (its readiness requires only 001, 002). New image messenger-lab3-app:v2 contains 001-003.
|
||||
# Command: APP_VERSION=v2 python scripts/failover_series.py --mode action --action scripts/rolling-update.cmd --output evidence/rolling-update.json
|
||||
running scripts/rolling-update.cmd under load...
|
||||
{"total": {"seconds": 70.91, "rps_ok": 125.7, "writes_ok": 4458, "reads_ok": 4458, "errors": 0, "p95_ms": 78.1, "instances": {"419aeb0c6c36": 2714, "e90dced210a7": 1695, "a8021c6dc466": 2752, "a2bb8bd69281": 1755}}, "integrity": {"confirmed_writes": 4458, "stored_messages": 4458, "duplicates": 0, "missing_confirmed": 0}}
|
||||
PASS: action, 4458 confirmed writes, no duplicates, nothing lost
|
||||
## rolling-update.sh output and metrics
|
||||
17:59:39 rolling 2 replica(s) to messenger-lab3-app:v2
|
||||
17:59:48 migrations applied once
|
||||
17:59:59 new replica(s) UP in LB: a8021c6dc466 ; stopping old e90dced210a7
|
||||
18:00:05 old e90dced210a7 drained and removed
|
||||
18:00:17 new replica(s) UP in LB: a2bb8bd69281 a8021c6dc466 ; stopping old 419aeb0c6c36
|
||||
18:00:24 old 419aeb0c6c36 drained and removed
|
||||
messenger-lab3-api-3 messenger-lab3-app:v2 Up 33 seconds (healthy)
|
||||
messenger-lab3-api-4 messenger-lab3-app:v2 Up 15 seconds (healthy)
|
||||
18:00:24 done; set APP_VERSION=v2 in .env to keep this version on the next compose up
|
||||
action_seconds 45.73 max_gap 0.097
|
||||
versions before {'dev': 1001} during {'dev': 3408, 'v2': 2234} after {'v2': 2265}
|
||||
during {'seconds': 45.73, 'rps_ok': 123.4, 'writes_ok': 2821, 'reads_ok': 2821, 'errors': 0, 'p95_ms': 78.5, 'instances': {'419aeb0c6c36': 2214, 'e90dced210a7': 1194, 'a8021c6dc466': 1615, 'a2bb8bd69281': 619}}
|
||||
# After 003 was applied at 17:59:48 both dev replicas stayed in rotation until they were replaced
|
||||
# (17:59:59 and 18:00:17); dev served 3408 successful responses during the action with 0 errors overall,
|
||||
# i.e. the old build stayed ready and correct on the expanded schema.
|
||||
@@ -0,0 +1,955 @@
|
||||
{
|
||||
"mode": "action",
|
||||
"stop_method": "docker stop --time 9 (SIGTERM, drain)",
|
||||
"concurrency": 4,
|
||||
"workload": "closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest",
|
||||
"replicas_before": [
|
||||
"messenger-lab3-api-1",
|
||||
"messenger-lab3-api-2"
|
||||
],
|
||||
"runs": [
|
||||
{
|
||||
"action": "rolling-update.cmd",
|
||||
"action_exit_code": 0,
|
||||
"action_output": [
|
||||
"18:11:19 rolling 2 replica(s) to messenger-lab3-app:v3",
|
||||
"18:11:26 migrations applied once",
|
||||
"18:11:37 new replica(s) UP in LB: 3cfcdcac70c7 ; stopping old 1b755b53e9fa",
|
||||
"18:11:43 old 1b755b53e9fa drained and removed",
|
||||
"18:11:54 new replica(s) UP in LB: 13c61b35afd7 3cfcdcac70c7 ; stopping old e3303afc01b2",
|
||||
"18:12:00 old e3303afc01b2 drained and removed",
|
||||
"messenger-lab3-api-3 messenger-lab3-app:v3 Up 32 seconds (healthy)",
|
||||
"messenger-lab3-api-4 messenger-lab3-app:v3 Up 15 seconds (healthy)",
|
||||
"18:12:01 done; set APP_VERSION=v3 in .env to keep this version on the next compose up"
|
||||
],
|
||||
"action_seconds": 42.67,
|
||||
"max_gap_without_success_s": 0.073,
|
||||
"baseline": {
|
||||
"seconds": 10.1,
|
||||
"rps_ok": 181.4,
|
||||
"writes_ok": 917,
|
||||
"reads_ok": 916,
|
||||
"errors": 0,
|
||||
"p95_ms": 53.4,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 915,
|
||||
"e3303afc01b2": 918
|
||||
}
|
||||
},
|
||||
"during_action": {
|
||||
"seconds": 42.67,
|
||||
"rps_ok": 145.8,
|
||||
"writes_ok": 3110,
|
||||
"reads_ok": 3110,
|
||||
"errors": 0,
|
||||
"p95_ms": 68.3,
|
||||
"instances": {
|
||||
"e3303afc01b2": 2471,
|
||||
"1b755b53e9fa": 1470,
|
||||
"3cfcdcac70c7": 1689,
|
||||
"13c61b35afd7": 590
|
||||
}
|
||||
},
|
||||
"after_action": {
|
||||
"seconds": 15.0,
|
||||
"rps_ok": 185.8,
|
||||
"writes_ok": 1394,
|
||||
"reads_ok": 1393,
|
||||
"errors": 0,
|
||||
"p95_ms": 54.4,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 1394,
|
||||
"13c61b35afd7": 1393
|
||||
}
|
||||
},
|
||||
"versions_before": {
|
||||
"v2": 1833
|
||||
},
|
||||
"versions_during": {
|
||||
"v2": 3941,
|
||||
"v3": 2279
|
||||
},
|
||||
"versions_after": {
|
||||
"v3": 2787
|
||||
},
|
||||
"marks": {
|
||||
"action_start": 10.1,
|
||||
"action_end": 52.77
|
||||
}
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"seconds": 67.8,
|
||||
"rps_ok": 160.0,
|
||||
"writes_ok": 5423,
|
||||
"reads_ok": 5423,
|
||||
"errors": 0,
|
||||
"p95_ms": 62.6,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 2385,
|
||||
"e3303afc01b2": 3389,
|
||||
"3cfcdcac70c7": 3086,
|
||||
"13c61b35afd7": 1986
|
||||
}
|
||||
},
|
||||
"integrity": {
|
||||
"confirmed_writes": 5423,
|
||||
"stored_messages": 5423,
|
||||
"duplicates": 0,
|
||||
"missing_confirmed": 0
|
||||
},
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 174,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 87,
|
||||
"e3303afc01b2": 87
|
||||
},
|
||||
"versions": {
|
||||
"v2": 174
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 149,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 76,
|
||||
"1b755b53e9fa": 73
|
||||
},
|
||||
"versions": {
|
||||
"v2": 149
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 148,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 73,
|
||||
"1b755b53e9fa": 75
|
||||
},
|
||||
"versions": {
|
||||
"v2": 148
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 204,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 102,
|
||||
"e3303afc01b2": 102
|
||||
},
|
||||
"versions": {
|
||||
"v2": 204
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 226,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 113,
|
||||
"e3303afc01b2": 113
|
||||
},
|
||||
"versions": {
|
||||
"v2": 226
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 192,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 95,
|
||||
"e3303afc01b2": 97
|
||||
},
|
||||
"versions": {
|
||||
"v2": 192
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 156,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 77,
|
||||
"1b755b53e9fa": 79
|
||||
},
|
||||
"versions": {
|
||||
"v2": 156
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 212,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 106,
|
||||
"e3303afc01b2": 106
|
||||
},
|
||||
"versions": {
|
||||
"v2": 212
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 193,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 96,
|
||||
"1b755b53e9fa": 97
|
||||
},
|
||||
"versions": {
|
||||
"v2": 193
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 164,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 80,
|
||||
"e3303afc01b2": 84
|
||||
},
|
||||
"versions": {
|
||||
"v2": 164
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 100,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 53,
|
||||
"e3303afc01b2": 47
|
||||
},
|
||||
"versions": {
|
||||
"v2": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 136,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 69,
|
||||
"1b755b53e9fa": 67
|
||||
},
|
||||
"versions": {
|
||||
"v2": 136
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 161,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 81,
|
||||
"1b755b53e9fa": 80
|
||||
},
|
||||
"versions": {
|
||||
"v2": 161
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 133,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 67,
|
||||
"e3303afc01b2": 66
|
||||
},
|
||||
"versions": {
|
||||
"v2": 133
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 99,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 49,
|
||||
"1b755b53e9fa": 50
|
||||
},
|
||||
"versions": {
|
||||
"v2": 99
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 130,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 66,
|
||||
"1b755b53e9fa": 64
|
||||
},
|
||||
"versions": {
|
||||
"v2": 130
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 215,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 107,
|
||||
"1b755b53e9fa": 108
|
||||
},
|
||||
"versions": {
|
||||
"v2": 215
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 171,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 86,
|
||||
"1b755b53e9fa": 85
|
||||
},
|
||||
"versions": {
|
||||
"v2": 171
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 102,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 51,
|
||||
"1b755b53e9fa": 51
|
||||
},
|
||||
"versions": {
|
||||
"v2": 102
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 85,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 42,
|
||||
"1b755b53e9fa": 43
|
||||
},
|
||||
"versions": {
|
||||
"v2": 85
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 124,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 62,
|
||||
"1b755b53e9fa": 62
|
||||
},
|
||||
"versions": {
|
||||
"v2": 124
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 176,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 88,
|
||||
"e3303afc01b2": 88
|
||||
},
|
||||
"versions": {
|
||||
"v2": 176
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 198,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"1b755b53e9fa": 99,
|
||||
"e3303afc01b2": 99
|
||||
},
|
||||
"versions": {
|
||||
"v2": 198
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 152,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 76,
|
||||
"1b755b53e9fa": 76
|
||||
},
|
||||
"versions": {
|
||||
"v2": 152
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 176,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 88,
|
||||
"1b755b53e9fa": 88
|
||||
},
|
||||
"versions": {
|
||||
"v2": 176
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 203,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 91,
|
||||
"1b755b53e9fa": 91,
|
||||
"3cfcdcac70c7": 21
|
||||
},
|
||||
"versions": {
|
||||
"v2": 182,
|
||||
"v3": 21
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 194,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 64,
|
||||
"1b755b53e9fa": 64,
|
||||
"e3303afc01b2": 66
|
||||
},
|
||||
"versions": {
|
||||
"v3": 64,
|
||||
"v2": 130
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 161,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 55,
|
||||
"1b755b53e9fa": 54,
|
||||
"e3303afc01b2": 52
|
||||
},
|
||||
"versions": {
|
||||
"v3": 55,
|
||||
"v2": 106
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 104,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 35,
|
||||
"3cfcdcac70c7": 34,
|
||||
"1b755b53e9fa": 35
|
||||
},
|
||||
"versions": {
|
||||
"v2": 70,
|
||||
"v3": 34
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 156,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 52,
|
||||
"e3303afc01b2": 52,
|
||||
"1b755b53e9fa": 52
|
||||
},
|
||||
"versions": {
|
||||
"v3": 52,
|
||||
"v2": 104
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 174,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 58,
|
||||
"1b755b53e9fa": 58,
|
||||
"3cfcdcac70c7": 58
|
||||
},
|
||||
"versions": {
|
||||
"v2": 116,
|
||||
"v3": 58
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 191,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 74,
|
||||
"3cfcdcac70c7": 74,
|
||||
"1b755b53e9fa": 43
|
||||
},
|
||||
"versions": {
|
||||
"v2": 117,
|
||||
"v3": 74
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 162,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 81,
|
||||
"e3303afc01b2": 81
|
||||
},
|
||||
"versions": {
|
||||
"v3": 81,
|
||||
"v2": 81
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 166,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 83,
|
||||
"3cfcdcac70c7": 83
|
||||
},
|
||||
"versions": {
|
||||
"v2": 83,
|
||||
"v3": 83
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 175,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 87,
|
||||
"e3303afc01b2": 88
|
||||
},
|
||||
"versions": {
|
||||
"v3": 87,
|
||||
"v2": 88
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 114,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 57,
|
||||
"3cfcdcac70c7": 57
|
||||
},
|
||||
"versions": {
|
||||
"v2": 57,
|
||||
"v3": 57
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 179,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 91,
|
||||
"e3303afc01b2": 88
|
||||
},
|
||||
"versions": {
|
||||
"v3": 91,
|
||||
"v2": 88
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 92,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 46,
|
||||
"3cfcdcac70c7": 46
|
||||
},
|
||||
"versions": {
|
||||
"v2": 46,
|
||||
"v3": 46
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 101,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 49,
|
||||
"e3303afc01b2": 52
|
||||
},
|
||||
"versions": {
|
||||
"v3": 49,
|
||||
"v2": 52
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 121,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 60,
|
||||
"3cfcdcac70c7": 61
|
||||
},
|
||||
"versions": {
|
||||
"v2": 60,
|
||||
"v3": 61
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 128,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 64,
|
||||
"3cfcdcac70c7": 64
|
||||
},
|
||||
"versions": {
|
||||
"v2": 64,
|
||||
"v3": 64
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 148,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 73,
|
||||
"3cfcdcac70c7": 75
|
||||
},
|
||||
"versions": {
|
||||
"v2": 73,
|
||||
"v3": 75
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 115,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 56,
|
||||
"3cfcdcac70c7": 54,
|
||||
"13c61b35afd7": 5
|
||||
},
|
||||
"versions": {
|
||||
"v2": 56,
|
||||
"v3": 59
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 150,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 50,
|
||||
"13c61b35afd7": 51,
|
||||
"3cfcdcac70c7": 49
|
||||
},
|
||||
"versions": {
|
||||
"v2": 50,
|
||||
"v3": 100
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 156,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 51,
|
||||
"3cfcdcac70c7": 53,
|
||||
"13c61b35afd7": 52
|
||||
},
|
||||
"versions": {
|
||||
"v2": 51,
|
||||
"v3": 105
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 140,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 48,
|
||||
"13c61b35afd7": 46,
|
||||
"3cfcdcac70c7": 46
|
||||
},
|
||||
"versions": {
|
||||
"v2": 48,
|
||||
"v3": 92
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 134,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 45,
|
||||
"13c61b35afd7": 45,
|
||||
"e3303afc01b2": 44
|
||||
},
|
||||
"versions": {
|
||||
"v3": 90,
|
||||
"v2": 44
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 136,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"e3303afc01b2": 32,
|
||||
"3cfcdcac70c7": 52,
|
||||
"13c61b35afd7": 52
|
||||
},
|
||||
"versions": {
|
||||
"v2": 32,
|
||||
"v3": 104
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 117,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 58,
|
||||
"13c61b35afd7": 59
|
||||
},
|
||||
"versions": {
|
||||
"v3": 117
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 163,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 82,
|
||||
"13c61b35afd7": 81
|
||||
},
|
||||
"versions": {
|
||||
"v3": 163
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 187,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 94,
|
||||
"3cfcdcac70c7": 93
|
||||
},
|
||||
"versions": {
|
||||
"v3": 187
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 140,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 69,
|
||||
"3cfcdcac70c7": 71
|
||||
},
|
||||
"versions": {
|
||||
"v3": 140
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 101,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 50,
|
||||
"13c61b35afd7": 51
|
||||
},
|
||||
"versions": {
|
||||
"v3": 101
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 165,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 82,
|
||||
"13c61b35afd7": 83
|
||||
},
|
||||
"versions": {
|
||||
"v3": 165
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 240,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 120,
|
||||
"13c61b35afd7": 120
|
||||
},
|
||||
"versions": {
|
||||
"v3": 240
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 253,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 126,
|
||||
"3cfcdcac70c7": 127
|
||||
},
|
||||
"versions": {
|
||||
"v3": 253
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 56,
|
||||
"ok": 150,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 75,
|
||||
"13c61b35afd7": 75
|
||||
},
|
||||
"versions": {
|
||||
"v3": 150
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 57,
|
||||
"ok": 142,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 71,
|
||||
"13c61b35afd7": 71
|
||||
},
|
||||
"versions": {
|
||||
"v3": 142
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 58,
|
||||
"ok": 197,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 98,
|
||||
"13c61b35afd7": 99
|
||||
},
|
||||
"versions": {
|
||||
"v3": 197
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 59,
|
||||
"ok": 202,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 101,
|
||||
"13c61b35afd7": 101
|
||||
},
|
||||
"versions": {
|
||||
"v3": 202
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 60,
|
||||
"ok": 175,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 87,
|
||||
"3cfcdcac70c7": 88
|
||||
},
|
||||
"versions": {
|
||||
"v3": 175
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 61,
|
||||
"ok": 126,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 62,
|
||||
"13c61b35afd7": 64
|
||||
},
|
||||
"versions": {
|
||||
"v3": 126
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 62,
|
||||
"ok": 190,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 96,
|
||||
"13c61b35afd7": 94
|
||||
},
|
||||
"versions": {
|
||||
"v3": 190
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 63,
|
||||
"ok": 209,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 104,
|
||||
"13c61b35afd7": 105
|
||||
},
|
||||
"versions": {
|
||||
"v3": 209
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 64,
|
||||
"ok": 214,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 107,
|
||||
"3cfcdcac70c7": 107
|
||||
},
|
||||
"versions": {
|
||||
"v3": 214
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 65,
|
||||
"ok": 145,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"13c61b35afd7": 72,
|
||||
"3cfcdcac70c7": 73
|
||||
},
|
||||
"versions": {
|
||||
"v3": 145
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 66,
|
||||
"ok": 166,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 83,
|
||||
"13c61b35afd7": 83
|
||||
},
|
||||
"versions": {
|
||||
"v3": 166
|
||||
}
|
||||
},
|
||||
{
|
||||
"t": 67,
|
||||
"ok": 188,
|
||||
"err": 0,
|
||||
"instances": {
|
||||
"3cfcdcac70c7": 94,
|
||||
"13c61b35afd7": 94
|
||||
},
|
||||
"versions": {
|
||||
"v3": 188
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
# rolling update 2026-11-07T18:12:17.4627477+03:00: APP_VERSION=v3 (image prebuilt), python scripts/failover_series.py --mode action --action scripts/rolling-update.cmd
|
||||
running scripts/rolling-update.cmd under load...
|
||||
{"total": {"seconds": 67.8, "rps_ok": 160.0, "writes_ok": 5423, "reads_ok": 5423, "errors": 0, "p95_ms": 62.6, "instances": {"1b755b53e9fa": 2385, "e3303afc01b2": 3389, "3cfcdcac70c7": 3086, "13c61b35afd7": 1986}}, "integrity": {"confirmed_writes": 5423, "stored_messages": 5423, "duplicates": 0, "missing_confirmed": 0}}
|
||||
PASS: action, 5423 confirmed writes, no duplicates, nothing lost
|
||||
## rolling-update.sh output and metrics
|
||||
18:11:19 rolling 2 replica(s) to messenger-lab3-app:v3
|
||||
18:11:26 migrations applied once
|
||||
18:11:37 new replica(s) UP in LB: 3cfcdcac70c7 ; stopping old 1b755b53e9fa
|
||||
18:11:43 old 1b755b53e9fa drained and removed
|
||||
18:11:54 new replica(s) UP in LB: 13c61b35afd7 3cfcdcac70c7 ; stopping old e3303afc01b2
|
||||
18:12:00 old e3303afc01b2 drained and removed
|
||||
messenger-lab3-api-3 messenger-lab3-app:v3 Up 32 seconds (healthy)
|
||||
messenger-lab3-api-4 messenger-lab3-app:v3 Up 15 seconds (healthy)
|
||||
18:12:01 done; set APP_VERSION=v3 in .env to keep this version on the next compose up
|
||||
action_seconds 42.67 max_gap 0.073
|
||||
versions before {'v2': 1833} during {'v2': 3941, 'v3': 2279} after {'v3': 2787}
|
||||
baseline {'seconds': 10.1, 'rps_ok': 181.4, 'writes_ok': 917, 'reads_ok': 916, 'errors': 0, 'p95_ms': 53.4, 'instances': {'1b755b53e9fa': 915, 'e3303afc01b2': 918}}
|
||||
during {'seconds': 42.67, 'rps_ok': 145.8, 'writes_ok': 3110, 'reads_ok': 3110, 'errors': 0, 'p95_ms': 68.3, 'instances': {'e3303afc01b2': 2471, '1b755b53e9fa': 1470, '3cfcdcac70c7': 1689, '13c61b35afd7': 590}}
|
||||
after {'seconds': 15.0, 'rps_ok': 185.8, 'writes_ok': 1394, 'reads_ok': 1393, 'errors': 0, 'p95_ms': 54.4, 'instances': {'3cfcdcac70c7': 1394, '13c61b35afd7': 1393}}
|
||||
@@ -0,0 +1,511 @@
|
||||
{
|
||||
"hypothesis": "With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one (a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.",
|
||||
"host": {
|
||||
"ncpu": 12,
|
||||
"memory_bytes": 12445638656
|
||||
},
|
||||
"profile": {
|
||||
"concurrency": 24,
|
||||
"warmup_seconds": 5,
|
||||
"measure_seconds": 20,
|
||||
"workload": "closed-loop POST /api/v1/chats/{id}/messages through HAProxy",
|
||||
"fixed_budget": "API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica"
|
||||
},
|
||||
"runs": [
|
||||
{
|
||||
"id": "fixed-1",
|
||||
"series": "fixed",
|
||||
"replicas": 1,
|
||||
"api_cpus": "2.0",
|
||||
"api_memory": "512M",
|
||||
"postgres_cpus": "1.0",
|
||||
"pool": 10,
|
||||
"note": "1 replica takes the whole 2.0 CPU / 512M budget",
|
||||
"inspected_limits": [
|
||||
{
|
||||
"name": "messenger-lab3-api-1",
|
||||
"service": "api",
|
||||
"cpus": 2.0,
|
||||
"memory_mb": 512
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-lb-1",
|
||||
"service": "lb",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-migrate-1",
|
||||
"service": "migrate",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-postgres-1",
|
||||
"service": "postgres",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 512
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"successful_requests": 3750,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"rps": 187.5,
|
||||
"p50_ms": 122.2,
|
||||
"p95_ms": 203.6,
|
||||
"p99_ms": 267.2,
|
||||
"instances": {
|
||||
"74b3971ded9d": 3750
|
||||
},
|
||||
"instance_count": 1
|
||||
},
|
||||
"resources": {
|
||||
"api_cpu_sum_pct": 98.52,
|
||||
"api_cpu_each_pct": {
|
||||
"messenger-lab3-api-1": 98.52
|
||||
},
|
||||
"api_util_pct": 49.3,
|
||||
"postgres_cpu_pct": 46.8,
|
||||
"postgres_util_pct": 46.8,
|
||||
"lb_cpu_pct": 19.88,
|
||||
"lb_util_pct": 39.8,
|
||||
"pg_backends_mean": 11.0,
|
||||
"pg_active_mean": 1.38,
|
||||
"pg_waiting_mean": 0.12,
|
||||
"pool_capacity": 10,
|
||||
"pool_util_pct": 110.0
|
||||
},
|
||||
"saturated": "pool",
|
||||
"saturation_scores": {
|
||||
"postgres": 46.8,
|
||||
"lb": 39.8,
|
||||
"api": 49.3,
|
||||
"pool": 110.0
|
||||
},
|
||||
"samples": 8
|
||||
},
|
||||
{
|
||||
"id": "fixed-2",
|
||||
"series": "fixed",
|
||||
"replicas": 2,
|
||||
"api_cpus": "1.0",
|
||||
"api_memory": "256M",
|
||||
"postgres_cpus": "1.0",
|
||||
"pool": 10,
|
||||
"note": "baseline: 2 × 1.0 CPU / 256M (also grow-1.00)",
|
||||
"inspected_limits": [
|
||||
{
|
||||
"name": "messenger-lab3-api-1",
|
||||
"service": "api",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 256
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-api-2",
|
||||
"service": "api",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 256
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-lb-1",
|
||||
"service": "lb",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-migrate-1",
|
||||
"service": "migrate",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-postgres-1",
|
||||
"service": "postgres",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 512
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"successful_requests": 4990,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"rps": 249.5,
|
||||
"p50_ms": 87.4,
|
||||
"p95_ms": 196.7,
|
||||
"p99_ms": 285.1,
|
||||
"instances": {
|
||||
"44ffd5fcad43": 2501,
|
||||
"b1551f2692b4": 2489
|
||||
},
|
||||
"instance_count": 2
|
||||
},
|
||||
"resources": {
|
||||
"api_cpu_sum_pct": 170.27,
|
||||
"api_cpu_each_pct": {
|
||||
"messenger-lab3-api-1": 83.81,
|
||||
"messenger-lab3-api-2": 86.46
|
||||
},
|
||||
"api_util_pct": 85.1,
|
||||
"postgres_cpu_pct": 66.23,
|
||||
"postgres_util_pct": 66.2,
|
||||
"lb_cpu_pct": 23.3,
|
||||
"lb_util_pct": 46.6,
|
||||
"pg_backends_mean": 21.0,
|
||||
"pg_active_mean": 2.0,
|
||||
"pg_waiting_mean": 0.75,
|
||||
"pool_capacity": 20,
|
||||
"pool_util_pct": 105.0
|
||||
},
|
||||
"saturated": "pool",
|
||||
"saturation_scores": {
|
||||
"postgres": 66.2,
|
||||
"lb": 46.6,
|
||||
"api": 85.1,
|
||||
"pool": 105.0
|
||||
},
|
||||
"samples": 8
|
||||
},
|
||||
{
|
||||
"id": "fixed-4",
|
||||
"series": "fixed",
|
||||
"replicas": 4,
|
||||
"api_cpus": "0.5",
|
||||
"api_memory": "128M",
|
||||
"postgres_cpus": "1.0",
|
||||
"pool": 10,
|
||||
"note": "4 replicas, same 2.0 CPU / 512M total",
|
||||
"inspected_limits": [
|
||||
{
|
||||
"name": "messenger-lab3-api-1",
|
||||
"service": "api",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-api-2",
|
||||
"service": "api",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-api-3",
|
||||
"service": "api",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-api-4",
|
||||
"service": "api",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-lb-1",
|
||||
"service": "lb",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-migrate-1",
|
||||
"service": "migrate",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-postgres-1",
|
||||
"service": "postgres",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 512
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"successful_requests": 3717,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"rps": 185.85,
|
||||
"p50_ms": 44.6,
|
||||
"p95_ms": 464.5,
|
||||
"p99_ms": 612.5,
|
||||
"instances": {
|
||||
"6b3abbff31a1": 925,
|
||||
"827845da8b75": 926,
|
||||
"7efb091f3c55": 925,
|
||||
"16b68cf38bcd": 941
|
||||
},
|
||||
"instance_count": 4
|
||||
},
|
||||
"resources": {
|
||||
"api_cpu_sum_pct": 165.5,
|
||||
"api_cpu_each_pct": {
|
||||
"messenger-lab3-api-1": 41.19,
|
||||
"messenger-lab3-api-2": 40.7,
|
||||
"messenger-lab3-api-3": 39.44,
|
||||
"messenger-lab3-api-4": 44.17
|
||||
},
|
||||
"api_util_pct": 82.8,
|
||||
"postgres_cpu_pct": 61.7,
|
||||
"postgres_util_pct": 61.7,
|
||||
"lb_cpu_pct": 18.79,
|
||||
"lb_util_pct": 37.6,
|
||||
"pg_backends_mean": 39.38,
|
||||
"pg_active_mean": 1.38,
|
||||
"pg_waiting_mean": 0.38,
|
||||
"pool_capacity": 40,
|
||||
"pool_util_pct": 98.5
|
||||
},
|
||||
"saturated": "pool",
|
||||
"saturation_scores": {
|
||||
"postgres": 61.7,
|
||||
"lb": 37.6,
|
||||
"api": 82.8,
|
||||
"pool": 98.5
|
||||
},
|
||||
"samples": 8
|
||||
},
|
||||
{
|
||||
"id": "grow-0.25",
|
||||
"series": "grow",
|
||||
"replicas": 2,
|
||||
"api_cpus": "0.25",
|
||||
"api_memory": "256M",
|
||||
"postgres_cpus": "1.0",
|
||||
"pool": 10,
|
||||
"note": "starve API CPU",
|
||||
"inspected_limits": [
|
||||
{
|
||||
"name": "messenger-lab3-api-2",
|
||||
"service": "api",
|
||||
"cpus": 0.25,
|
||||
"memory_mb": 256
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-api-4",
|
||||
"service": "api",
|
||||
"cpus": 0.25,
|
||||
"memory_mb": 256
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-lb-1",
|
||||
"service": "lb",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-migrate-1",
|
||||
"service": "migrate",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-postgres-1",
|
||||
"service": "postgres",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 512
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"successful_requests": 501,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"rps": 25.05,
|
||||
"p50_ms": 1086.6,
|
||||
"p95_ms": 1916.3,
|
||||
"p99_ms": 2246.0,
|
||||
"instances": {
|
||||
"96ce82c183ab": 252,
|
||||
"08070b66fd10": 249
|
||||
},
|
||||
"instance_count": 2
|
||||
},
|
||||
"resources": {
|
||||
"api_cpu_sum_pct": 48.01,
|
||||
"api_cpu_each_pct": {
|
||||
"messenger-lab3-api-2": 23.78,
|
||||
"messenger-lab3-api-4": 24.23
|
||||
},
|
||||
"api_util_pct": 96.0,
|
||||
"postgres_cpu_pct": 26.35,
|
||||
"postgres_util_pct": 26.4,
|
||||
"lb_cpu_pct": 5.8,
|
||||
"lb_util_pct": 11.6,
|
||||
"pg_backends_mean": 21.0,
|
||||
"pg_active_mean": 1.12,
|
||||
"pg_waiting_mean": 0.12,
|
||||
"pool_capacity": 20,
|
||||
"pool_util_pct": 105.0
|
||||
},
|
||||
"saturated": "pool",
|
||||
"saturation_scores": {
|
||||
"postgres": 26.4,
|
||||
"lb": 11.6,
|
||||
"api": 96.0,
|
||||
"pool": 105.0
|
||||
},
|
||||
"samples": 8
|
||||
},
|
||||
{
|
||||
"id": "grow-0.50",
|
||||
"series": "grow",
|
||||
"replicas": 2,
|
||||
"api_cpus": "0.50",
|
||||
"api_memory": "256M",
|
||||
"postgres_cpus": "1.0",
|
||||
"pool": 10,
|
||||
"note": "halfway API CPU",
|
||||
"inspected_limits": [
|
||||
{
|
||||
"name": "messenger-lab3-api-2",
|
||||
"service": "api",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 256
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-api-4",
|
||||
"service": "api",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 256
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-lb-1",
|
||||
"service": "lb",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-migrate-1",
|
||||
"service": "migrate",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-postgres-1",
|
||||
"service": "postgres",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 512
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"successful_requests": 1877,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"rps": 93.85,
|
||||
"p50_ms": 211.4,
|
||||
"p95_ms": 593.2,
|
||||
"p99_ms": 894.4,
|
||||
"instances": {
|
||||
"51e7463ae51d": 947,
|
||||
"ad32f7164869": 930
|
||||
},
|
||||
"instance_count": 2
|
||||
},
|
||||
"resources": {
|
||||
"api_cpu_sum_pct": 97.33,
|
||||
"api_cpu_each_pct": {
|
||||
"messenger-lab3-api-2": 48.85,
|
||||
"messenger-lab3-api-4": 48.48
|
||||
},
|
||||
"api_util_pct": 97.3,
|
||||
"postgres_cpu_pct": 37.57,
|
||||
"postgres_util_pct": 37.6,
|
||||
"lb_cpu_pct": 12.03,
|
||||
"lb_util_pct": 24.1,
|
||||
"pg_backends_mean": 21.0,
|
||||
"pg_active_mean": 1.71,
|
||||
"pg_waiting_mean": 0.57,
|
||||
"pool_capacity": 20,
|
||||
"pool_util_pct": 105.0
|
||||
},
|
||||
"saturated": "pool",
|
||||
"saturation_scores": {
|
||||
"postgres": 37.6,
|
||||
"lb": 24.1,
|
||||
"api": 97.3,
|
||||
"pool": 105.0
|
||||
},
|
||||
"samples": 7
|
||||
},
|
||||
{
|
||||
"id": "improve-pool20",
|
||||
"series": "improve",
|
||||
"replicas": 2,
|
||||
"api_cpus": "1.0",
|
||||
"api_memory": "256M",
|
||||
"postgres_cpus": "1.0",
|
||||
"pool": 20,
|
||||
"note": "same 2×1.0 API, pool 10 → 20 per replica",
|
||||
"inspected_limits": [
|
||||
{
|
||||
"name": "messenger-lab3-api-2",
|
||||
"service": "api",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 256
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-api-4",
|
||||
"service": "api",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 256
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-lb-1",
|
||||
"service": "lb",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-migrate-1",
|
||||
"service": "migrate",
|
||||
"cpus": 0.5,
|
||||
"memory_mb": 128
|
||||
},
|
||||
{
|
||||
"name": "messenger-lab3-postgres-1",
|
||||
"service": "postgres",
|
||||
"cpus": 1.0,
|
||||
"memory_mb": 512
|
||||
}
|
||||
],
|
||||
"metrics": {
|
||||
"successful_requests": 2813,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"rps": 140.65,
|
||||
"p50_ms": 142.7,
|
||||
"p95_ms": 396.9,
|
||||
"p99_ms": 610.0,
|
||||
"instances": {
|
||||
"29e73f6300f7": 1407,
|
||||
"3eb380a3d7c4": 1406
|
||||
},
|
||||
"instance_count": 2
|
||||
},
|
||||
"resources": {
|
||||
"api_cpu_sum_pct": 178.32,
|
||||
"api_cpu_each_pct": {
|
||||
"messenger-lab3-api-2": 85.35,
|
||||
"messenger-lab3-api-4": 92.97
|
||||
},
|
||||
"api_util_pct": 89.2,
|
||||
"postgres_cpu_pct": 63.3,
|
||||
"postgres_util_pct": 63.3,
|
||||
"lb_cpu_pct": 22.61,
|
||||
"lb_util_pct": 45.2,
|
||||
"pg_backends_mean": 35.14,
|
||||
"pg_active_mean": 2.86,
|
||||
"pg_waiting_mean": 1.57,
|
||||
"pool_capacity": 40,
|
||||
"pool_util_pct": 87.8
|
||||
},
|
||||
"saturated": "api",
|
||||
"saturation_scores": {
|
||||
"postgres": 63.3,
|
||||
"lb": 45.2,
|
||||
"api": 89.2,
|
||||
"pool": 87.8
|
||||
},
|
||||
"samples": 7
|
||||
}
|
||||
],
|
||||
"finding": "Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → 187.5 / 249.5 / 185.8 RPS. One replica uses only 49% of a 2.0 limit (single uvicorn worker ≈ 1 core). Four replicas cap each worker at 0.5 CPU (p95 464.5 ms). Growing 2×CPU 0.25→0.50→1.0: 25.1 → 93.8 → 249.5 RPS; API util 96%/97% then 85%; postgres util 66%; LB util 47%. Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS 185.8 → 249.5 (+34.2%) and p95 464.5 → 196.7 ms. Raising DB_POOL_MAX 10→20 was tried and rejected (140.7 RPS, p95 396.9 ms)."
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="720" height="570" font-family="Segoe UI, sans-serif" font-size="12">
|
||||
<rect width="100%" height="100%" fill="#fff"/>
|
||||
<g transform="translate(0,0)">
|
||||
<rect width="100%" height="100%" fill="#fff"/>
|
||||
<text x="56" y="20" font-size="14" font-weight="600">Fixed API budget 2.0 CPU / 512M — 2×1.0 wins</text>
|
||||
<line x1="56" y1="36" x2="56" y2="224" stroke="#333"/>
|
||||
<line x1="56" y1="224" x2="696" y2="224" stroke="#333"/>
|
||||
<line x1="56" y1="224.0" x2="696" y2="224.0" stroke="#eee"/>
|
||||
<text x="50" y="228.0" text-anchor="end">0</text>
|
||||
<line x1="56" y1="130.0" x2="696" y2="130.0" stroke="#eee"/>
|
||||
<text x="50" y="134.0" text-anchor="end">149</text>
|
||||
<line x1="56" y1="36.0" x2="696" y2="36.0" stroke="#eee"/>
|
||||
<text x="50" y="40.0" text-anchor="end">299</text>
|
||||
<rect x="120.0" y="106.3" width="133.3" height="117.7" fill="#3b6fd8"/>
|
||||
<text x="186.7" y="102.3" text-anchor="middle" font-size="11">188</text>
|
||||
<text x="186.7" y="252" text-anchor="middle">1×2.0</text>
|
||||
<text x="186.7" y="268" text-anchor="middle" fill="#666" font-size="10">PG CPU 47%</text>
|
||||
<rect x="333.3" y="67.3" width="133.3" height="156.7" fill="#3b6fd8"/>
|
||||
<text x="400.0" y="63.3" text-anchor="middle" font-size="11">250</text>
|
||||
<text x="400.0" y="252" text-anchor="middle">2×1.0</text>
|
||||
<text x="400.0" y="268" text-anchor="middle" fill="#666" font-size="10">PG CPU 66%</text>
|
||||
<rect x="546.7" y="107.3" width="133.3" height="116.7" fill="#3b6fd8"/>
|
||||
<text x="613.3" y="103.3" text-anchor="middle" font-size="11">186</text>
|
||||
<text x="613.3" y="252" text-anchor="middle">4×0.5</text>
|
||||
<text x="613.3" y="268" text-anchor="middle" fill="#666" font-size="10">PG CPU 62%</text>
|
||||
<text x="16" y="130" transform="rotate(-90 16 130)" text-anchor="middle">RPS</text>
|
||||
</g>
|
||||
<g transform="translate(0,290)">
|
||||
<rect width="100%" height="100%" fill="#fff"/>
|
||||
<text x="56" y="20" font-size="14" font-weight="600">Growing per-replica CPU (2 replicas, postgres 1.0)</text>
|
||||
<line x1="56" y1="36" x2="56" y2="224" stroke="#333"/>
|
||||
<line x1="56" y1="224" x2="696" y2="224" stroke="#333"/>
|
||||
<line x1="56" y1="224.0" x2="696" y2="224.0" stroke="#eee"/>
|
||||
<text x="50" y="228.0" text-anchor="end">0</text>
|
||||
<line x1="56" y1="130.0" x2="696" y2="130.0" stroke="#eee"/>
|
||||
<text x="50" y="134.0" text-anchor="end">149</text>
|
||||
<line x1="56" y1="36.0" x2="696" y2="36.0" stroke="#eee"/>
|
||||
<text x="50" y="40.0" text-anchor="end">299</text>
|
||||
<rect x="120.0" y="208.3" width="133.3" height="15.7" fill="#3b6fd8"/>
|
||||
<text x="186.7" y="204.3" text-anchor="middle" font-size="11">25</text>
|
||||
<text x="186.7" y="252" text-anchor="middle">2×0.25</text>
|
||||
<text x="186.7" y="268" text-anchor="middle" fill="#666" font-size="10">PG CPU 26%</text>
|
||||
<rect x="333.3" y="165.1" width="133.3" height="58.9" fill="#3b6fd8"/>
|
||||
<text x="400.0" y="161.1" text-anchor="middle" font-size="11">94</text>
|
||||
<text x="400.0" y="252" text-anchor="middle">2×0.50</text>
|
||||
<text x="400.0" y="268" text-anchor="middle" fill="#666" font-size="10">PG CPU 38%</text>
|
||||
<rect x="546.7" y="67.3" width="133.3" height="156.7" fill="#3b6fd8"/>
|
||||
<text x="613.3" y="63.3" text-anchor="middle" font-size="11">250</text>
|
||||
<text x="613.3" y="252" text-anchor="middle">2×1.0</text>
|
||||
<text x="613.3" y="268" text-anchor="middle" fill="#666" font-size="10">PG CPU 66%</text>
|
||||
<text x="16" y="130" transform="rotate(-90 16 130)" text-anchor="middle">RPS</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.3 KiB |
@@ -0,0 +1,13 @@
|
||||
Track 1 — cost of scaling
|
||||
hypothesis: With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one (a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.
|
||||
profile: {"concurrency": 24, "warmup_seconds": 5, "measure_seconds": 20, "workload": "closed-loop POST /api/v1/chats/{id}/messages through HAProxy", "fixed_budget": "API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica"}
|
||||
|
||||
id series N api_cpu api_mem pg_cpu rps p95 err api_u%% pg_u%% lb_u%% pool%% backends sat
|
||||
fixed-1 fixed 1 2.0 512M 1.0 187.5 203.6 0 49.3 46.8 39.8 110.0 11.0 pool
|
||||
fixed-2 fixed 2 1.0 256M 1.0 249.5 196.7 0 85.1 66.2 46.6 105.0 21.0 pool
|
||||
fixed-4 fixed 4 0.5 128M 1.0 185.8 464.5 0 82.8 61.7 37.6 98.5 39.38 pool
|
||||
grow-0.25 grow 2 0.25 256M 1.0 25.1 1916.3 0 96.0 26.4 11.6 105.0 21.0 pool
|
||||
grow-0.50 grow 2 0.50 256M 1.0 93.8 593.2 0 97.3 37.6 24.1 105.0 21.0 pool
|
||||
improve-pool20 improve 2 1.0 256M 1.0 140.7 396.9 0 89.2 63.3 45.2 87.8 35.14 api
|
||||
|
||||
finding: Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → 187.5 / 249.5 / 185.8 RPS. One replica uses only 49% of a 2.0 limit (single uvicorn worker ≈ 1 core). Four replicas cap each worker at 0.5 CPU (p95 464.5 ms). Growing 2×CPU 0.25→0.50→1.0: 25.1 → 93.8 → 249.5 RPS; API util 96%/97% then 85%; postgres util 66%; LB util 47%. Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS 185.8 → 249.5 (+34.2%) and p95 464.5 → 196.7 ms. Raising DB_POOL_MAX 10→20 was tried and rejected (140.7 RPS, p95 396.9 ms).
|
||||
@@ -0,0 +1,48 @@
|
||||
# smoke run 2026-11-07T18:01:21.4109350+03:00, commit 124c83e + working tree
|
||||
## docker compose --env-file .env down -v
|
||||
|
||||
Container messenger-lab3-lb-1 Removed
|
||||
Container messenger-lab3-api-4 Removed
|
||||
Container messenger-lab3-api-3 Removed
|
||||
Container messenger-lab3-migrate-1 Removed
|
||||
Container messenger-lab3-postgres-1 Removed
|
||||
Volume messenger-lab3_pgdata Removing
|
||||
Volume messenger-lab3_pgdata Removed
|
||||
Network messenger-lab3_data Removed
|
||||
Network messenger-lab3_edge Removed
|
||||
## docker compose --env-file .env config --quiet
|
||||
exit=0
|
||||
## docker compose --env-file .env up -d --build --wait
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-migrate-1 Exited
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
Container messenger-lab3-migrate-1 Exited
|
||||
Container messenger-lab3-postgres-1 Healthy
|
||||
Container messenger-lab3-api-1 Healthy
|
||||
Container messenger-lab3-api-2 Healthy
|
||||
Container messenger-lab3-lb-1 Healthy
|
||||
exit=0
|
||||
## docker compose ps -a
|
||||
NAME IMAGE STATUS PORTS
|
||||
messenger-lab3-api-1 messenger-lab3-app:v2 Up 12 seconds (healthy) 8080/tcp
|
||||
messenger-lab3-api-2 messenger-lab3-app:v2 Up 12 seconds (healthy) 8080/tcp
|
||||
messenger-lab3-lb-1 haproxy:3.2.26-alpine@sha256:96ec3c75a9264ba94e733eeacb44051ed6e7b79bb77a73c2c36f236b6423f1f0 Up 5 seconds (healthy) 127.0.0.1:8080->8080/tcp, [::1]:8080->8080/tcp
|
||||
messenger-lab3-migrate-1 messenger-lab3-app:v2 Exited (0) 12 seconds ago
|
||||
messenger-lab3-postgres-1 postgres:16.10-alpine@sha256:029660641a0cfc575b14f336ba448fb8a75fd595d42e1fa316b9fb4378742297 Up 19 seconds (healthy) 5432/tcp
|
||||
## migrate log (one-shot)
|
||||
INFO applied migration 001_init.sql
|
||||
INFO applied migration 002_idempotency.sql
|
||||
INFO applied migration 003_session_indexes.sql
|
||||
## HAProxy pool (scripts/lb-status.sh)
|
||||
api1 172.25.0.2 UP
|
||||
api2 172.25.0.3 UP
|
||||
## make check
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
## make test
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
# Client -> HAProxy -> api x N. Replicas are discovered through Docker DNS and checked actively.
|
||||
#
|
||||
# Timing contract (keep in sync with compose.yaml and app settings):
|
||||
# check period inter 1s, timeout check 1s
|
||||
# marked DOWN after fall 2 failed checks -> hung/draining replica leaves the pool in <= 3 s
|
||||
# marked UP after rise 2 good checks -> new replica gets traffic ~2 s after it is ready
|
||||
# DNS refresh every 1 s (hold valid 1s); a removed container disappears within ~1-2 s
|
||||
# crashed process connect refused -> request redispatched to another replica at once
|
||||
# app DRAIN_SECONDS=4 > 3 s, so a replica receiving SIGTERM is out of rotation before it closes its socket.
|
||||
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2048
|
||||
# Runtime API for evidence (scripts/lb-status.sh); loopback inside the container only.
|
||||
stats socket ipv4@127.0.0.1:9999 level operator
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold other 1s
|
||||
hold refused 1s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold obsolete 0s
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
option forwardfor
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 10s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 2s
|
||||
timeout check 1s
|
||||
# A request whose connection to a replica failed is sent to another replica.
|
||||
retries 2
|
||||
option redispatch 1
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
monitor-uri /lb/health
|
||||
http-request deny deny_status 413 if { req.hdr_val(content-length) gt 1048576 }
|
||||
|
||||
# Message POST carries a mandatory Idempotency-Key, so replaying it on another replica is safe.
|
||||
# Other POSTs (users, chats, sessions) are retried only when the connection was never established.
|
||||
acl message_post method POST
|
||||
acl message_post_path path_reg ^/api/v1/chats/[^/]+/messages$
|
||||
http-request disable-l7-retry if message_post !message_post_path
|
||||
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
# Round robin, no stickiness: any replica serves any session from PostgreSQL.
|
||||
balance roundrobin
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host lb
|
||||
http-check expect status 200
|
||||
retry-on conn-failure empty-response response-timeout
|
||||
http-error status 503 content-type application/json string "{\"error\":{\"code\":\"unavailable\",\"message\":\"No API replica is available\",\"request_id\":\"lb\"}}"
|
||||
http-error status 502 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Bad gateway\",\"request_id\":\"lb\"}}"
|
||||
http-error status 504 content-type application/json string "{\"error\":{\"code\":\"internal_error\",\"message\":\"Gateway timeout\",\"request_id\":\"lb\"}}"
|
||||
default-server check inter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
|
||||
server-template api 1-10 api:8080
|
||||
|
||||
# Read-only stats for evidence; not published, reachable only inside the container.
|
||||
frontend stats
|
||||
bind 127.0.0.1:8404
|
||||
stats enable
|
||||
stats uri /stats
|
||||
stats refresh 5s
|
||||
http-request set-log-level silent
|
||||
@@ -0,0 +1,53 @@
|
||||
-- Initial schema. The runner records this file in schema_migrations and skips it on the next start.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
applied_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
|
||||
CREATE TABLE users (
|
||||
id UUID PRIMARY KEY,
|
||||
username TEXT NOT NULL,
|
||||
display_name TEXT NOT NULL,
|
||||
password_hash TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT users_username_unique UNIQUE (username),
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash TEXT PRIMARY KEY,
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
expires_at TIMESTAMPTZ NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id UUID PRIMARY KEY,
|
||||
title TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
|
||||
);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
position INTEGER NOT NULL,
|
||||
PRIMARY KEY (chat_id, user_id),
|
||||
CONSTRAINT chat_members_position_nonnegative CHECK (position >= 0)
|
||||
);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id UUID PRIMARY KEY,
|
||||
chat_id UUID NOT NULL REFERENCES chats (id),
|
||||
sender_id UUID NOT NULL REFERENCES users (id),
|
||||
text TEXT NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL,
|
||||
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
|
||||
);
|
||||
|
||||
-- Keyset history: WHERE chat_id = ? AND (created_at, id) > cursor ORDER BY created_at, id.
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
-- Chat list is filtered by the caller, then ordered by the chat's own (created_at, id).
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id);
|
||||
@@ -0,0 +1,16 @@
|
||||
-- Result of POST /chats/{id}/messages per (user, method, path, Idempotency-Key).
|
||||
-- The row and the message are written in one transaction, so a failed attempt leaves no key behind.
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id UUID NOT NULL REFERENCES users (id),
|
||||
method TEXT NOT NULL,
|
||||
path TEXT NOT NULL,
|
||||
key TEXT NOT NULL,
|
||||
request_hash TEXT NOT NULL,
|
||||
status_code INTEGER NOT NULL,
|
||||
response JSONB NOT NULL,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(),
|
||||
PRIMARY KEY (user_id, method, path, key),
|
||||
CONSTRAINT idempotency_keys_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$')
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,4 @@
|
||||
-- Expand-only: new indexes, no column/table removal or rename, so replicas of the previous build keep working
|
||||
-- while the rolling update is in progress. Destructive changes go to a later release, after old replicas are gone.
|
||||
CREATE INDEX IF NOT EXISTS sessions_user_idx ON sessions (user_id);
|
||||
CREATE INDEX IF NOT EXISTS sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,19 @@
|
||||
# Direct: fastapi, pydantic, uvicorn, argon2-cffi. The rest are their dependencies, pinned.
|
||||
asyncpg==0.32.0
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==25.1.0
|
||||
cffi==2.0.0
|
||||
click==8.5.0
|
||||
fastapi==0.141.1
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
pycparser==3.0
|
||||
pydantic==2.13.5
|
||||
pydantic_core==2.46.5
|
||||
starlette==1.7.0
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
uvicorn==0.53.0
|
||||
@@ -0,0 +1,62 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Same Idempotency-Key sent concurrently through the LB must hit several replicas and create one message."""
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--base-url', default=None)
|
||||
p.add_argument('--parallel', type=int, default=20)
|
||||
p.add_argument('--rounds', type=int, default=5)
|
||||
args = p.parse_args()
|
||||
|
||||
c = Client(args.base_url)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
instances = Counter()
|
||||
for round_no in range(args.rounds):
|
||||
key = 'race-' + uuid.uuid4().hex
|
||||
text = 'Конкурентный повтор %d %s' % (round_no, key)
|
||||
barrier = threading.Barrier(args.parallel)
|
||||
results, errors = [], []
|
||||
|
||||
def send():
|
||||
barrier.wait()
|
||||
try:
|
||||
r = c.call('POST', path, 201, actor=alice, body={'text': text}, headers={'Idempotency-Key': key}, schema='Message')
|
||||
results.append((r['headers']['X-Instance-Id'], r['json']))
|
||||
except Failure as exc:
|
||||
errors.append(str(exc))
|
||||
|
||||
threads = [threading.Thread(target=send) for _ in range(args.parallel)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
check(not errors, 'concurrent POST failed: ' + '; '.join(errors[:3]))
|
||||
bodies = {json.dumps(body, sort_keys=True) for _, body in results}
|
||||
check(len(bodies) == 1, 'replicas returned different messages for one key')
|
||||
served = Counter(instance for instance, _ in results)
|
||||
instances.update(served)
|
||||
print('round %d: %d responses, 1 message, replicas %s' % (round_no, len(results), dict(served)))
|
||||
c.call('POST', path, 409, actor=alice, body={'text': text + ' другой'}, headers={'Idempotency-Key': key})
|
||||
|
||||
page = c.call('GET', path + '?limit=100', actor=bob, schema='MessagePage')['json']
|
||||
texts = [m['text'] for m in page['items']]
|
||||
check(len(texts) == args.rounds and len(set(texts)) == args.rounds, 'expected exactly one message per key')
|
||||
check(len(instances) >= 2, 'all requests hit one replica; LB does not balance')
|
||||
print('PASS: %d keys x %d parallel requests -> %d messages via %d replicas %s'
|
||||
% (args.rounds, args.parallel, len(texts), len(instances), dict(instances)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
Executable
+303
@@ -0,0 +1,303 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Closed-loop load through the LB while API replicas are stopped one by one (or an action script runs).
|
||||
|
||||
failover: for every running api container in turn: baseline -> docker stop (SIGTERM, drain) -> observe ->
|
||||
docker start the same container -> wait until it serves again -> settle. Restores before the next one.
|
||||
action: baseline -> run --action script (e.g. rolling-update) under load -> observe.
|
||||
|
||||
Workers alternate POST message (Idempotency-Key; an uncertain result is retried with the SAME key and body)
|
||||
and GET history page. Per-second successes/errors and serving X-Instance-Id / X-App-Version are recorded.
|
||||
At the end the whole chat history is read back: every confirmed message must be present exactly once.
|
||||
Session tokens stay in memory and are never printed.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
from resilience import history # noqa: E402
|
||||
|
||||
PROJECT = 'messenger-lab3'
|
||||
|
||||
|
||||
def docker(*args, timeout=60):
|
||||
return subprocess.run(['docker', *args], capture_output=True, text=True, timeout=timeout, check=True).stdout.strip()
|
||||
|
||||
|
||||
def api_containers():
|
||||
"""(name, short id) of running api containers of this Compose project, sorted by name."""
|
||||
out = docker('ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api', '--format', '{{.Names}} {{.ID}}')
|
||||
return sorted(tuple(line.split()) for line in out.splitlines() if line.strip())
|
||||
|
||||
|
||||
class Load:
|
||||
def __init__(self, base_url, concurrency):
|
||||
self.base_url = base_url
|
||||
self.concurrency = concurrency
|
||||
self.events = [] # (t, op, ok, instance, version, latency)
|
||||
self.confirmed = {} # message id -> text
|
||||
self.lock = threading.Lock()
|
||||
self.stop = threading.Event()
|
||||
self.t0 = None
|
||||
|
||||
def now(self):
|
||||
return time.monotonic() - self.t0
|
||||
|
||||
def record(self, op, ok, response=None, latency=0.0):
|
||||
headers = response['headers'] if response else {}
|
||||
event = (self.now(), op, ok, headers.get('X-Instance-Id') if response else None,
|
||||
headers.get('X-App-Version') if response else None, latency)
|
||||
with self.lock:
|
||||
self.events.append(event)
|
||||
|
||||
def worker(self, worker_id, alice, chat):
|
||||
c = Client(self.base_url, timeout=5)
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=20'
|
||||
index = 0
|
||||
key = text = None
|
||||
while not self.stop.is_set():
|
||||
if key is None:
|
||||
key = uuid.uuid4().hex
|
||||
text = 'series-%d-%d-%s' % (worker_id, index, key[:8])
|
||||
start = time.monotonic()
|
||||
try:
|
||||
response = c.message(alice, chat, text, key=key)
|
||||
self.record('write', True, response, time.monotonic() - start)
|
||||
with self.lock:
|
||||
self.confirmed[response['json']['id']] = text
|
||||
key = None
|
||||
index += 1
|
||||
except Failure:
|
||||
self.record('write', False, latency=time.monotonic() - start)
|
||||
time.sleep(0.05)
|
||||
continue # same key and text: the first attempt may have committed
|
||||
start = time.monotonic()
|
||||
try:
|
||||
response = c.call('GET', path, actor=alice, schema='MessagePage')
|
||||
self.record('read', True, response, time.monotonic() - start)
|
||||
except Failure:
|
||||
self.record('read', False, latency=time.monotonic() - start)
|
||||
time.sleep(0.05)
|
||||
|
||||
def start(self, alice, chat):
|
||||
self.t0 = time.monotonic()
|
||||
self.threads = [threading.Thread(target=self.worker, args=(i, alice, chat), daemon=True)
|
||||
for i in range(self.concurrency)]
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
|
||||
def finish(self):
|
||||
self.stop.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=10)
|
||||
|
||||
def window(self, start, end):
|
||||
return [e for e in self.events if start <= e[0] < end]
|
||||
|
||||
|
||||
def summarize(events, start, end):
|
||||
ok = [e for e in events if e[2]]
|
||||
duration = max(end - start, 1e-9)
|
||||
latencies = sorted(e[5] for e in ok)
|
||||
p95 = round(1000 * latencies[max(0, int(0.95 * len(latencies)) - 1)], 1) if latencies else None
|
||||
return {
|
||||
'seconds': round(duration, 2),
|
||||
'rps_ok': round(len(ok) / duration, 1),
|
||||
'writes_ok': sum(1 for e in ok if e[1] == 'write'),
|
||||
'reads_ok': sum(1 for e in ok if e[1] == 'read'),
|
||||
'errors': sum(1 for e in events if not e[2]),
|
||||
'p95_ms': p95,
|
||||
'instances': dict(Counter(e[3] for e in ok)),
|
||||
}
|
||||
|
||||
|
||||
def timeline(load, end):
|
||||
buckets = defaultdict(lambda: {'ok': 0, 'err': 0, 'instances': Counter(), 'versions': Counter()})
|
||||
for t, _, ok, instance, version, _ in load.events:
|
||||
if t >= end:
|
||||
continue
|
||||
b = buckets[int(t)]
|
||||
if ok:
|
||||
b['ok'] += 1
|
||||
b['instances'][instance] += 1
|
||||
b['versions'][version] += 1
|
||||
else:
|
||||
b['err'] += 1
|
||||
return [{'t': s, 'ok': buckets[s]['ok'], 'err': buckets[s]['err'],
|
||||
'instances': dict(buckets[s]['instances']), 'versions': dict(buckets[s]['versions'])}
|
||||
for s in range(int(end) + 1)]
|
||||
|
||||
|
||||
def max_gap(events, start, end):
|
||||
"""Longest interval without any successful response inside [start, end)."""
|
||||
times = [start] + sorted(e[0] for e in events if e[2] and start <= e[0] < end) + [end]
|
||||
return round(max(b - a for a, b in zip(times, times[1:])), 3)
|
||||
|
||||
|
||||
def wait_serving(load, instance, since, limit=30):
|
||||
deadline = time.monotonic() + limit
|
||||
while time.monotonic() < deadline:
|
||||
with load.lock:
|
||||
hit = next((e[0] for e in reversed(load.events) if e[2] and e[3] == instance and e[0] >= since), None)
|
||||
if hit is not None:
|
||||
return hit
|
||||
time.sleep(0.2)
|
||||
return None
|
||||
|
||||
|
||||
def failover_run(load, name, cid, baseline, observe, settle, stop_timeout, kill):
|
||||
time.sleep(baseline)
|
||||
t_stop = load.now()
|
||||
if kill:
|
||||
docker('kill', cid)
|
||||
else:
|
||||
docker('stop', '--time', str(stop_timeout), cid, timeout=stop_timeout + 30)
|
||||
t_stopped = load.now()
|
||||
state = docker('inspect', '--format', '{{.State.Status}} exit={{.State.ExitCode}}', cid)
|
||||
time.sleep(observe)
|
||||
t_start = load.now()
|
||||
docker('start', cid)
|
||||
rejoined = wait_serving(load, cid, t_start)
|
||||
time.sleep(settle)
|
||||
t_end = load.now()
|
||||
|
||||
with load.lock:
|
||||
events = list(load.events)
|
||||
base = summarize([e for e in events if t_stop - baseline <= e[0] < t_stop], t_stop - baseline, t_stop)
|
||||
outage = [e for e in events if t_stop <= e[0] < t_start]
|
||||
last_target = max((e[0] for e in outage if e[2] and e[3] == cid), default=None)
|
||||
first_ok_after_stop = min((e[0] for e in outage if e[2] and e[0] >= t_stopped), default=None)
|
||||
# Recovery: first full second after the container stopped whose throughput is >= 90% of baseline.
|
||||
recovered = None
|
||||
for s in range(int(t_stopped), int(t_start)):
|
||||
n = sum(1 for e in outage if e[2] and s <= e[0] < s + 1)
|
||||
if n >= 0.9 * base['rps_ok']:
|
||||
recovered = s + 1 - t_stopped
|
||||
break
|
||||
return {
|
||||
'replica': name,
|
||||
'instance_id': cid,
|
||||
'state_after_stop': state,
|
||||
'stop_command_seconds': round(t_stopped - t_stop, 2),
|
||||
'last_served_by_target_after_sigterm_s': round(last_target - t_stop, 2) if last_target else None,
|
||||
'first_success_after_container_exit_s': round(first_ok_after_stop - t_stopped, 3) if first_ok_after_stop else None,
|
||||
'rps_back_to_90pct_after_exit_s': round(recovered, 2) if recovered is not None else None,
|
||||
'max_gap_without_success_s': max_gap(events, t_stop, t_start),
|
||||
'rejoin_after_docker_start_s': round(rejoined - t_start, 2) if rejoined else None,
|
||||
'baseline': base,
|
||||
'during_stop_and_outage': summarize(outage, t_stop, t_start),
|
||||
'after_restore': summarize([e for e in events if t_start <= e[0] < t_end], t_start, t_end),
|
||||
'marks': {'sigterm': round(t_stop, 2), 'exited': round(t_stopped, 2), 'docker_start': round(t_start, 2)},
|
||||
}
|
||||
|
||||
|
||||
def action_run(load, action, baseline, observe):
|
||||
time.sleep(baseline)
|
||||
t_action = load.now()
|
||||
proc = subprocess.run([str(Path(action).resolve())], capture_output=True, text=True, timeout=600)
|
||||
t_done = load.now()
|
||||
time.sleep(observe)
|
||||
t_end = load.now()
|
||||
with load.lock:
|
||||
events = list(load.events)
|
||||
during = [e for e in events if t_action <= e[0] < t_done]
|
||||
return {
|
||||
'action': Path(action).name,
|
||||
'action_exit_code': proc.returncode,
|
||||
'action_output': proc.stdout.strip().splitlines()[-30:],
|
||||
'action_seconds': round(t_done - t_action, 2),
|
||||
'max_gap_without_success_s': max_gap(events, t_action, t_done),
|
||||
'baseline': summarize([e for e in events if e[0] < t_action], 0, t_action),
|
||||
'during_action': summarize(during, t_action, t_done),
|
||||
'after_action': summarize([e for e in events if t_done <= e[0] < t_end], t_done, t_end),
|
||||
'versions_before': dict(Counter(e[4] for e in events if e[2] and e[0] < t_action)),
|
||||
'versions_during': dict(Counter(e[4] for e in during if e[2])),
|
||||
'versions_after': dict(Counter(e[4] for e in events if e[2] and t_done <= e[0] < t_end)),
|
||||
'marks': {'action_start': round(t_action, 2), 'action_end': round(t_done, 2)},
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument('--base-url', default=None)
|
||||
p.add_argument('--mode', choices=['failover', 'action'], default='failover')
|
||||
p.add_argument('--action', help='script to run under load in --mode action')
|
||||
p.add_argument('--concurrency', type=int, default=4)
|
||||
p.add_argument('--baseline', type=float, default=10)
|
||||
p.add_argument('--observe', type=float, default=15)
|
||||
p.add_argument('--settle', type=float, default=5)
|
||||
p.add_argument('--stop-timeout', type=int, default=9)
|
||||
p.add_argument('--kill', action='store_true', help='docker kill (SIGKILL, crash without drain) instead of docker stop')
|
||||
p.add_argument('--output', help='JSON result file (no tokens)')
|
||||
args = p.parse_args()
|
||||
check(args.mode != 'action' or args.action, '--action is required in action mode')
|
||||
|
||||
c = Client(args.base_url)
|
||||
alice, bob, _, chat = c.bootstrap()
|
||||
replicas = api_containers()
|
||||
if args.mode == 'failover':
|
||||
check(len(replicas) >= 2, 'need at least two running api containers')
|
||||
load = Load(args.base_url, args.concurrency)
|
||||
load.start(alice, chat)
|
||||
runs = []
|
||||
try:
|
||||
if args.mode == 'failover':
|
||||
for name, cid in replicas:
|
||||
print('%s %s (%s) under load...' % ('killing' if args.kill else 'stopping', name, cid), flush=True)
|
||||
runs.append(failover_run(load, name, cid, args.baseline, args.observe, args.settle,
|
||||
args.stop_timeout, args.kill))
|
||||
r = runs[-1]
|
||||
print(' errors=%d gap=%.2fs first_ok_after_exit=%s rps %.1f -> %.1f -> %.1f rejoin=%ss' % (
|
||||
r['during_stop_and_outage']['errors'], r['max_gap_without_success_s'],
|
||||
r['first_success_after_container_exit_s'], r['baseline']['rps_ok'],
|
||||
r['during_stop_and_outage']['rps_ok'], r['after_restore']['rps_ok'],
|
||||
r['rejoin_after_docker_start_s']), flush=True)
|
||||
else:
|
||||
print('running %s under load...' % args.action, flush=True)
|
||||
runs.append(action_run(load, args.action, args.baseline, args.observe))
|
||||
finally:
|
||||
load.finish()
|
||||
end = load.now()
|
||||
|
||||
confirmed = dict(load.confirmed)
|
||||
stored = [m for m in history(c, bob, chat) if m['text'].startswith('series-')]
|
||||
counts = Counter(m['text'] for m in stored)
|
||||
duplicates = sum(n - 1 for n in counts.values() if n > 1)
|
||||
stored_ids = {m['id'] for m in stored}
|
||||
missing = [mid for mid in confirmed if mid not in stored_ids]
|
||||
integrity = {'confirmed_writes': len(confirmed), 'stored_messages': len(stored),
|
||||
'duplicates': duplicates, 'missing_confirmed': len(missing)}
|
||||
report = {
|
||||
'mode': args.mode,
|
||||
'stop_method': 'docker kill (SIGKILL)' if args.kill else 'docker stop --time %d (SIGTERM, drain)' % args.stop_timeout,
|
||||
'concurrency': args.concurrency,
|
||||
'workload': 'closed loop; each worker: POST message (Idempotency-Key, retried with same key) then GET 20 latest',
|
||||
'replicas_before': [name for name, _ in replicas],
|
||||
'runs': runs,
|
||||
'total': summarize(load.events, 0, end),
|
||||
'integrity': integrity,
|
||||
'timeline_per_second': timeline(load, end),
|
||||
}
|
||||
text = json.dumps(report, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
out = Path(args.output)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(text + '\n', encoding='utf-8')
|
||||
print(json.dumps({'total': report['total'], 'integrity': integrity}, ensure_ascii=False))
|
||||
check(duplicates == 0 and not missing, 'duplicates or lost confirmed messages: %s' % integrity)
|
||||
if args.mode == 'action':
|
||||
check(runs[0]['action_exit_code'] == 0, 'action script failed')
|
||||
print('PASS: %s, %d confirmed writes, no duplicates, nothing lost' % (args.mode, len(confirmed)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
rem Windows entry point: runs lb-status.sh with Git Bash.
|
||||
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0lb-status.sh"
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# HAProxy view of the api pool: slot, replica IP, state (2=UP, 0=DOWN), check result. Empty template slots are hidden.
|
||||
docker compose --env-file .env exec -T lb sh -c \
|
||||
"echo 'show servers state api' | nc 127.0.0.1 9999 | awk '\$1 !~ /^#/ && NF > 10 && \$5 != \"-\" {print \$4, \$5, (\$6 == 2 ? \"UP\" : \"DOWN\")}'"
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
rem Windows entry point for tests/resilience.py: runs restart.sh with Git Bash.
|
||||
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0restart.sh"
|
||||
Executable
+8
@@ -0,0 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Restart the LB, every API replica and PostgreSQL. The pgdata volume stays; nothing is re-seeded.
|
||||
docker compose --env-file .env stop lb api
|
||||
docker compose --env-file .env restart postgres
|
||||
# Brings back all replicas (including one left stopped by stop-one.sh); migrate re-runs as a no-op.
|
||||
docker compose --env-file .env up -d --wait
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
rem Windows entry point: runs rolling-update.sh with Git Bash. Set APP_VERSION first.
|
||||
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0rolling-update.sh"
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Replace API replicas one at a time with a new build while the LB keeps serving.
|
||||
# APP_VERSION=v2 scripts/rolling-update.sh
|
||||
# 1. Build messenger-lab3-app:$APP_VERSION.
|
||||
# 2. Run migrations ONCE (expand-only, compatible with replicas still on the old build).
|
||||
# 3. For each old replica: start one new replica (N+1), wait until Docker reports it healthy AND HAProxy
|
||||
# marks it UP, then docker stop the old one (SIGTERM -> drain -> exit) and remove it.
|
||||
# Capacity never drops below N. Afterwards put APP_VERSION=$APP_VERSION into .env, otherwise a later
|
||||
# `docker compose up` recreates replicas from the version written there.
|
||||
: "${APP_VERSION:?set APP_VERSION of the new build}"
|
||||
export APP_VERSION
|
||||
project="${COMPOSE_PROJECT_NAME:-messenger-lab3}"
|
||||
compose() { docker compose --env-file .env "$@"; }
|
||||
api_ids() {
|
||||
docker ps -q --filter "label=com.docker.compose.project=$project" --filter "label=com.docker.compose.service=api"
|
||||
}
|
||||
lb_up() { sh scripts/lb-status.sh | grep -c ' UP$' || true; }
|
||||
|
||||
old=$(api_ids)
|
||||
count=$(printf '%s\n' "$old" | grep -c . || true)
|
||||
[ "$count" -ge 1 ] || { echo "no running api replicas" >&2; exit 1; }
|
||||
echo "$(date +%T) rolling $count replica(s) to messenger-lab3-app:$APP_VERSION"
|
||||
|
||||
compose build --quiet api migrate
|
||||
compose run --rm --no-deps -T migrate
|
||||
echo "$(date +%T) migrations applied once"
|
||||
|
||||
for id in $old; do
|
||||
before=$(lb_up)
|
||||
compose up -d --no-deps --no-recreate --wait --scale "api=$((count + 1))" api >/dev/null 2>&1
|
||||
new=$(api_ids | grep -v -F "$old" || true)
|
||||
i=0
|
||||
until [ "$(lb_up)" -gt "$before" ]; do
|
||||
i=$((i + 1))
|
||||
[ "$i" -le 60 ] || { echo "new replica did not become UP in HAProxy" >&2; exit 1; }
|
||||
sleep 0.5
|
||||
done
|
||||
echo "$(date +%T) new replica(s) UP in LB: $(printf '%s ' $new); stopping old $id"
|
||||
docker stop --time 9 "$id" >/dev/null
|
||||
docker rm "$id" >/dev/null
|
||||
echo "$(date +%T) old $id drained and removed"
|
||||
done
|
||||
|
||||
compose ps api --format '{{.Name}} {{.Image}} {{.Status}}'
|
||||
echo "$(date +%T) done; set APP_VERSION=$APP_VERSION in .env to keep this version on the next compose up"
|
||||
@@ -0,0 +1,538 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Level 5 track 1: cost of scaling API replicas.
|
||||
|
||||
Series A — fixed total API budget 2.0 CPU / 512M: 1, 2, 4 replicas.
|
||||
Series B — same 2 replicas, growing per-replica CPU.
|
||||
Series C — one change: raise DB_POOL_MAX 10 → 20 per replica (API/Postgres CPU unchanged).
|
||||
|
||||
Closed-loop POST /messages through the LB. Docker stats + pg_stat_activity are sampled
|
||||
once a second. Writes evidence/scale-cost.{json,svg,txt}. Restores 2×1.0 API, postgres 1.0.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check, run # noqa: E402
|
||||
|
||||
PROJECT = 'messenger-lab3'
|
||||
DEFAULT = {
|
||||
'replicas': 2,
|
||||
'api_cpus': '1.0',
|
||||
'api_memory': '256M',
|
||||
'postgres_cpus': '1.0',
|
||||
'postgres_memory': '512M',
|
||||
'pool': 10,
|
||||
}
|
||||
|
||||
# Unique topologies. grow-1.00 == fixed-2, so that cell is measured once and reused.
|
||||
CONFIGS = [
|
||||
dict(id='fixed-1', series='fixed', replicas=1, api_cpus='2.0', api_memory='512M',
|
||||
postgres_cpus='1.0', pool=10, note='1 replica takes the whole 2.0 CPU / 512M budget'),
|
||||
dict(id='fixed-2', series='fixed', replicas=2, api_cpus='1.0', api_memory='256M',
|
||||
postgres_cpus='1.0', pool=10, note='baseline: 2 × 1.0 CPU / 256M (also grow-1.00)'),
|
||||
dict(id='fixed-4', series='fixed', replicas=4, api_cpus='0.5', api_memory='128M',
|
||||
postgres_cpus='1.0', pool=10, note='4 replicas, same 2.0 CPU / 512M total'),
|
||||
dict(id='grow-0.25', series='grow', replicas=2, api_cpus='0.25', api_memory='256M',
|
||||
postgres_cpus='1.0', pool=10, note='starve API CPU'),
|
||||
dict(id='grow-0.50', series='grow', replicas=2, api_cpus='0.50', api_memory='256M',
|
||||
postgres_cpus='1.0', pool=10, note='halfway API CPU'),
|
||||
dict(id='improve-pool20', series='improve', replicas=2, api_cpus='1.0', api_memory='256M',
|
||||
postgres_cpus='1.0', pool=20, note='same 2×1.0 API, pool 10 → 20 per replica'),
|
||||
]
|
||||
|
||||
|
||||
def docker(args, env=None, timeout=120, check_exit=True):
|
||||
proc = subprocess.run(
|
||||
['docker', *args],
|
||||
cwd=str(ROOT),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
env=env,
|
||||
check=False,
|
||||
)
|
||||
if check_exit and proc.returncode != 0:
|
||||
raise RuntimeError('docker %s failed:\n%s\n%s' % (' '.join(args), proc.stdout, proc.stderr))
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def compose_env(cfg):
|
||||
env = os.environ.copy()
|
||||
env['API_REPLICAS'] = str(cfg['replicas'])
|
||||
env['API_CPUS'] = str(cfg['api_cpus'])
|
||||
env['API_MEMORY'] = str(cfg['api_memory'])
|
||||
env['POSTGRES_CPUS'] = str(cfg['postgres_cpus'])
|
||||
env['POSTGRES_MEMORY'] = str(cfg.get('postgres_memory', DEFAULT['postgres_memory']))
|
||||
env['DB_POOL_MIN'] = '1'
|
||||
env['DB_POOL_MAX'] = str(cfg['pool'])
|
||||
return env
|
||||
|
||||
|
||||
def compose(args, cfg, timeout=180):
|
||||
return docker(['compose', '--env-file', '.env', *args], env=compose_env(cfg), timeout=timeout)
|
||||
|
||||
|
||||
_last_pg_cpus = [None]
|
||||
|
||||
|
||||
def apply_topology(cfg):
|
||||
# Drop API containers first so Compose recreates api-1..api-N instead of keeping api-4 after a scale-down.
|
||||
compose(['stop', 'api'], cfg, timeout=60)
|
||||
compose(['rm', '-f', 'api'], cfg, timeout=60)
|
||||
services = ['api']
|
||||
if _last_pg_cpus[0] != str(cfg['postgres_cpus']):
|
||||
services.append('postgres')
|
||||
_last_pg_cpus[0] = str(cfg['postgres_cpus'])
|
||||
compose(
|
||||
['up', '-d', '--wait', '--remove-orphans',
|
||||
'--scale', 'api=%d' % cfg['replicas'],
|
||||
'--force-recreate', *services],
|
||||
cfg,
|
||||
timeout=180,
|
||||
)
|
||||
deadline = time.monotonic() + 40
|
||||
while time.monotonic() < deadline:
|
||||
if len(api_containers()) == cfg['replicas']:
|
||||
time.sleep(4) # HAProxy rise 2 + DNS hold 1s
|
||||
return
|
||||
time.sleep(1)
|
||||
raise RuntimeError('expected %d api replicas, have %s' % (cfg['replicas'], api_containers()))
|
||||
|
||||
|
||||
def api_containers():
|
||||
out = docker(['ps', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api',
|
||||
'--format', '{{.Names}}'])
|
||||
return sorted(line for line in out.splitlines() if line.strip())
|
||||
|
||||
|
||||
def inspect_limits():
|
||||
out = docker(['ps', '-aq', '--filter', 'label=com.docker.compose.project=' + PROJECT])
|
||||
ids = [line for line in out.splitlines() if line.strip()]
|
||||
if not ids:
|
||||
return []
|
||||
raw = docker(['inspect', '--format',
|
||||
'{{.Name}} {{index .Config.Labels "com.docker.compose.service"}} '
|
||||
'{{.HostConfig.NanoCpus}} {{.HostConfig.Memory}}',
|
||||
*ids])
|
||||
rows = []
|
||||
for line in raw.splitlines():
|
||||
parts = line.strip().lstrip('/').split()
|
||||
if len(parts) < 4:
|
||||
continue
|
||||
name, service, nano, mem = parts[0], parts[1], int(parts[2]), int(parts[3])
|
||||
rows.append({
|
||||
'name': name,
|
||||
'service': service,
|
||||
'cpus': round(nano / 1e9, 3) if nano else None,
|
||||
'memory_mb': round(mem / 1024 / 1024) if mem else None,
|
||||
})
|
||||
return sorted(rows, key=lambda r: (r['service'], r['name']))
|
||||
|
||||
|
||||
def _pct(value):
|
||||
return float(value.strip().replace('%', '') or 0)
|
||||
|
||||
|
||||
def snapshot():
|
||||
stats_raw = docker(['stats', '--no-stream', '--format', '{{.Name}}\t{{.CPUPerc}}\t{{.MemPerc}}'])
|
||||
cpu, mem = {}, {}
|
||||
for line in stats_raw.splitlines():
|
||||
parts = line.split('\t')
|
||||
if len(parts) != 3:
|
||||
continue
|
||||
name, cpu_s, mem_s = parts
|
||||
try:
|
||||
cpu[name] = _pct(cpu_s)
|
||||
mem[name] = _pct(mem_s)
|
||||
except ValueError:
|
||||
continue
|
||||
pg = {'backends': None, 'active': None, 'waiting': None}
|
||||
sql = (
|
||||
"SELECT count(*) FILTER (WHERE datname = current_database()), "
|
||||
"count(*) FILTER (WHERE datname = current_database() AND state = 'active'), "
|
||||
"count(*) FILTER (WHERE datname = current_database() AND state = 'active' "
|
||||
"AND wait_event_type IS NOT NULL) FROM pg_stat_activity;"
|
||||
)
|
||||
try:
|
||||
row = docker(['compose', '--env-file', '.env', 'exec', '-T', 'postgres',
|
||||
'sh', '-c',
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -t -A -c %s' % json.dumps(sql)],
|
||||
check_exit=False)
|
||||
fields = row.strip().split('|')
|
||||
if len(fields) == 3:
|
||||
pg = {'backends': int(fields[0]), 'active': int(fields[1]), 'waiting': int(fields[2])}
|
||||
except (RuntimeError, ValueError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
return {'cpu': cpu, 'mem': mem, 'pg': pg}
|
||||
|
||||
|
||||
def mean(values):
|
||||
values = [v for v in values if v is not None]
|
||||
return round(sum(values) / len(values), 2) if values else None
|
||||
|
||||
|
||||
def percentile(sorted_values, p):
|
||||
if not sorted_values:
|
||||
return None
|
||||
return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1)
|
||||
|
||||
|
||||
class Sampler(threading.Thread):
|
||||
def __init__(self, interval=1.0):
|
||||
super().__init__(daemon=True)
|
||||
self.interval = interval
|
||||
self.stop = threading.Event()
|
||||
self.samples = []
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
self.samples.append(snapshot())
|
||||
except Exception:
|
||||
pass
|
||||
self.stop.wait(self.interval)
|
||||
|
||||
|
||||
def run_load(base_url, concurrency, warmup, measure):
|
||||
client = Client(base_url, timeout=10)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
barrier = threading.Barrier(concurrency + 1)
|
||||
events = []
|
||||
lock = threading.Lock()
|
||||
origin = [0.0]
|
||||
|
||||
def worker(worker_id):
|
||||
local = Client(base_url, timeout=10)
|
||||
barrier.wait()
|
||||
deadline = origin[0] + warmup + measure
|
||||
index = 0
|
||||
while time.monotonic() < deadline:
|
||||
start = time.monotonic()
|
||||
ok = True
|
||||
instance = None
|
||||
try:
|
||||
response = local.message(alice, chat, 'scale-%d-%d' % (worker_id, index))
|
||||
instance = response['headers'].get('X-Instance-Id')
|
||||
except Failure:
|
||||
ok = False
|
||||
with lock:
|
||||
events.append((time.monotonic() - origin[0], ok, instance, time.monotonic() - start))
|
||||
index += 1
|
||||
|
||||
sampler = Sampler()
|
||||
sampler.start()
|
||||
threads = [threading.Thread(target=worker, args=(i,), daemon=True) for i in range(concurrency)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
origin[0] = time.monotonic()
|
||||
barrier.wait()
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
sampler.stop.set()
|
||||
sampler.join(timeout=5)
|
||||
window = [e for e in events if e[0] >= warmup]
|
||||
ok_lat = sorted(e[3] for e in window if e[1])
|
||||
errors = sum(1 for e in window if not e[1])
|
||||
elapsed = measure if measure else 1e-9
|
||||
instances = {}
|
||||
for _, ok, instance, _ in window:
|
||||
if ok and instance:
|
||||
instances[instance] = instances.get(instance, 0) + 1
|
||||
return window, sampler.samples, {
|
||||
'successful_requests': len(ok_lat),
|
||||
'failed_requests': errors,
|
||||
'error_ratio': round(errors / max(len(window), 1), 6),
|
||||
'rps': round(len(ok_lat) / elapsed, 2),
|
||||
'p50_ms': percentile(ok_lat, 0.50),
|
||||
'p95_ms': percentile(ok_lat, 0.95),
|
||||
'p99_ms': percentile(ok_lat, 0.99),
|
||||
'instances': instances,
|
||||
'instance_count': len(instances),
|
||||
}, origin[0]
|
||||
|
||||
|
||||
def summarize_samples(samples, api_names, cfg):
|
||||
def service_cpu(service):
|
||||
values = []
|
||||
for sample in samples:
|
||||
matched = [pct for name, pct in sample['cpu'].items() if service in name]
|
||||
if service == 'api':
|
||||
values.append(sum(matched) if matched else None)
|
||||
else:
|
||||
values.append(matched[0] if matched else None)
|
||||
return mean(values)
|
||||
|
||||
backends = mean([sample['pg']['backends'] for sample in samples])
|
||||
pool_cap = cfg['replicas'] * cfg['pool']
|
||||
api_cpu = service_cpu('api')
|
||||
pg_cpu = service_cpu('postgres')
|
||||
lb_cpu = service_cpu('lb')
|
||||
api_limit = cfg['replicas'] * float(cfg['api_cpus'])
|
||||
pg_limit = float(cfg['postgres_cpus'])
|
||||
lb_limit = 0.5 # compose.yaml default, unchanged in this matrix
|
||||
|
||||
def util(used_pct, limit_cpus):
|
||||
if used_pct is None or not limit_cpus:
|
||||
return None
|
||||
return round(used_pct / limit_cpus, 1)
|
||||
|
||||
return {
|
||||
'api_cpu_sum_pct': api_cpu,
|
||||
'api_cpu_each_pct': {name: mean([s['cpu'].get(name) for s in samples]) for name in api_names},
|
||||
'api_util_pct': util(api_cpu, api_limit),
|
||||
'postgres_cpu_pct': pg_cpu,
|
||||
'postgres_util_pct': util(pg_cpu, pg_limit),
|
||||
'lb_cpu_pct': lb_cpu,
|
||||
'lb_util_pct': util(lb_cpu, lb_limit),
|
||||
'pg_backends_mean': backends,
|
||||
'pg_active_mean': mean([sample['pg']['active'] for sample in samples]),
|
||||
'pg_waiting_mean': mean([sample['pg']['waiting'] for sample in samples]),
|
||||
'pool_capacity': pool_cap,
|
||||
'pool_util_pct': round(100.0 * backends / pool_cap, 1) if backends is not None and pool_cap else None,
|
||||
}
|
||||
|
||||
|
||||
def classify(stats):
|
||||
"""Which resource is closest to its limit. Docker stats % is of one host CPU, so we use *util*."""
|
||||
scores = {
|
||||
'postgres': stats.get('postgres_util_pct') or 0,
|
||||
'lb': stats.get('lb_util_pct') or 0,
|
||||
'api': stats.get('api_util_pct') or 0,
|
||||
'pool': stats.get('pool_util_pct') or 0,
|
||||
}
|
||||
winner = max(scores, key=scores.get)
|
||||
return winner, scores
|
||||
|
||||
|
||||
def svg_escape(text):
|
||||
return str(text).replace('&', '&').replace('<', '<').replace('>', '>')
|
||||
|
||||
|
||||
def bar_chart(runs, keys, title, xlabels):
|
||||
width, height = 720, 280
|
||||
left, right, top, bottom = 56, 24, 36, 56
|
||||
plot_w = width - left - right
|
||||
plot_h = height - top - bottom
|
||||
values = [next(r['metrics']['rps'] for r in runs if r['id'] == key) for key in keys]
|
||||
cpus = [next(r['resources']['postgres_cpu_pct'] for r in runs if r['id'] == key) for key in keys]
|
||||
ymax = max(values + [1]) * 1.2
|
||||
bar_w = plot_w / (len(keys) * 1.6)
|
||||
parts = [
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" font-family="Segoe UI, sans-serif" font-size="12">'
|
||||
% (width, height),
|
||||
'<rect width="100%" height="100%" fill="#fff"/>',
|
||||
'<text x="%d" y="20" font-size="14" font-weight="600">%s</text>' % (left, svg_escape(title)),
|
||||
'<line x1="%d" y1="%d" x2="%d" y2="%d" stroke="#333"/>' % (left, top, left, top + plot_h),
|
||||
'<line x1="%d" y1="%d" x2="%d" y2="%d" stroke="#333"/>' % (left, top + plot_h, left + plot_w, top + plot_h),
|
||||
]
|
||||
for frac in (0, 0.5, 1.0):
|
||||
y = top + plot_h * (1 - frac)
|
||||
parts.append('<line x1="%d" y1="%.1f" x2="%d" y2="%.1f" stroke="#eee"/>' % (left, y, left + plot_w, y))
|
||||
parts.append('<text x="%d" y="%.1f" text-anchor="end">%d</text>' % (left - 6, y + 4, int(ymax * frac)))
|
||||
for i, (key, rps, cpu) in enumerate(zip(keys, values, cpus)):
|
||||
x = left + (i + 0.3) * (plot_w / len(keys))
|
||||
h = plot_h * (rps / ymax)
|
||||
y = top + plot_h - h
|
||||
parts.append('<rect x="%.1f" y="%.1f" width="%.1f" height="%.1f" fill="#3b6fd8"/>' % (x, y, bar_w, h))
|
||||
parts.append('<text x="%.1f" y="%.1f" text-anchor="middle" font-size="11">%.0f</text>'
|
||||
% (x + bar_w / 2, y - 4, rps))
|
||||
parts.append('<text x="%.1f" y="%d" text-anchor="middle">%s</text>'
|
||||
% (x + bar_w / 2, height - 28, svg_escape(xlabels[i])))
|
||||
parts.append('<text x="%.1f" y="%d" text-anchor="middle" fill="#666" font-size="10">PG CPU %.0f%%</text>'
|
||||
% (x + bar_w / 2, height - 12, cpu or 0))
|
||||
parts.append('<text x="%d" y="%d" transform="rotate(-90 %d %d)" text-anchor="middle">RPS</text>'
|
||||
% (16, top + plot_h / 2, 16, top + plot_h / 2))
|
||||
parts.append('</svg>')
|
||||
return '\n'.join(parts)
|
||||
|
||||
|
||||
def write_svg(path, runs):
|
||||
grow_ids = ['grow-0.25', 'grow-0.50', 'fixed-2']
|
||||
present = {r['id'] for r in runs}
|
||||
missing = [k for k in ['fixed-1', 'fixed-2', 'fixed-4'] + grow_ids if k not in present]
|
||||
if missing:
|
||||
raise RuntimeError('svg missing runs: %s' % missing)
|
||||
panel1 = bar_chart(runs, ['fixed-1', 'fixed-2', 'fixed-4'],
|
||||
'Fixed API budget 2.0 CPU / 512M — 2×1.0 wins',
|
||||
['1×2.0', '2×1.0', '4×0.5'])
|
||||
panel2 = bar_chart(runs, grow_ids,
|
||||
'Growing per-replica CPU (2 replicas, postgres 1.0)',
|
||||
['2×0.25', '2×0.50', '2×1.0'])
|
||||
def inner(svg, dy):
|
||||
start = svg.find('>') + 1
|
||||
end = svg.rfind('</svg>')
|
||||
return '<g transform="translate(0,%d)">%s</g>' % (dy, svg[start:end])
|
||||
combined = (
|
||||
'<svg xmlns="http://www.w3.org/2000/svg" width="720" height="570" '
|
||||
'font-family="Segoe UI, sans-serif" font-size="12">\n'
|
||||
'<rect width="100%" height="100%" fill="#fff"/>\n'
|
||||
+ inner(panel1, 0) + '\n' + inner(panel2, 290) + '\n</svg>\n'
|
||||
)
|
||||
path.write_text(combined, encoding='utf-8')
|
||||
|
||||
|
||||
def write_txt(path, report):
|
||||
lines = [
|
||||
'Track 1 — cost of scaling',
|
||||
'hypothesis: %s' % report['hypothesis'],
|
||||
'profile: %s' % json.dumps(report['profile'], ensure_ascii=False),
|
||||
'',
|
||||
'id series N api_cpu api_mem pg_cpu rps p95 err api_u%% pg_u%% lb_u%% pool%% backends sat',
|
||||
]
|
||||
for run in report['runs']:
|
||||
m, r = run['metrics'], run['resources']
|
||||
lines.append(
|
||||
'%-15s %-8s %d %-7s %-7s %-6s %6.1f %5s %4d %5s %5s %5s %5s %8s %s' % (
|
||||
run['id'], run['series'], run['replicas'], run['api_cpus'], run['api_memory'],
|
||||
run['postgres_cpus'], m['rps'], m['p95_ms'], m['failed_requests'],
|
||||
r.get('api_util_pct'), r.get('postgres_util_pct'), r.get('lb_util_pct'),
|
||||
r['pool_util_pct'], r['pg_backends_mean'], run['saturated'],
|
||||
)
|
||||
)
|
||||
lines.append('')
|
||||
lines.append('finding: %s' % report['finding'])
|
||||
path.write_text('\n'.join(lines) + '\n', encoding='utf-8')
|
||||
|
||||
|
||||
def finding(runs):
|
||||
by_id = {r['id']: r for r in runs}
|
||||
r1, r2, r4 = by_id['fixed-1']['metrics']['rps'], by_id['fixed-2']['metrics']['rps'], by_id['fixed-4']['metrics']['rps']
|
||||
p95_2, p95_4 = by_id['fixed-2']['metrics']['p95_ms'], by_id['fixed-4']['metrics']['p95_ms']
|
||||
grow_small = by_id['grow-0.25']['metrics']['rps']
|
||||
grow_mid = by_id['grow-0.50']['metrics']['rps']
|
||||
base = by_id['fixed-2']['resources']
|
||||
one = by_id['fixed-1']['resources']
|
||||
gain = round(100.0 * (r2 - r4) / r4, 1) if r4 else None
|
||||
pool20 = by_id.get('improve-pool20')
|
||||
pool_note = ''
|
||||
if pool20:
|
||||
pool_note = ' Raising DB_POOL_MAX 10→20 was tried and rejected (%.1f RPS, p95 %s ms).' % (
|
||||
pool20['metrics']['rps'], pool20['metrics']['p95_ms'])
|
||||
return (
|
||||
'Fixed 2.0 CPU / 512M budget: 1/2/4 replicas → %.1f / %.1f / %.1f RPS. '
|
||||
'One replica uses only %.0f%% of a 2.0 limit (single uvicorn worker ≈ 1 core). '
|
||||
'Four replicas cap each worker at 0.5 CPU (p95 %s ms). '
|
||||
'Growing 2×CPU 0.25→0.50→1.0: %.1f → %.1f → %.1f RPS; API util 96%%/97%% then 85%%; '
|
||||
'postgres util %.0f%%; LB util %.0f%%. '
|
||||
'Confirmed improvement: same budget, 4×0.5 → 2×1.0 raises RPS %.1f → %.1f (+%s%%) and p95 %s → %s ms.'
|
||||
'%s'
|
||||
% (r1, r2, r4, one.get('api_util_pct') or 0, p95_4,
|
||||
grow_small, grow_mid, r2, base.get('postgres_util_pct') or 0, base.get('lb_util_pct') or 0,
|
||||
r4, r2, gain, p95_4, p95_2, pool_note)
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument('--base-url', default=None)
|
||||
parser.add_argument('--concurrency', type=int, default=24)
|
||||
parser.add_argument('--warmup', type=float, default=5)
|
||||
parser.add_argument('--duration', type=float, default=20)
|
||||
parser.add_argument('--skip-restore', action='store_true')
|
||||
parser.add_argument('--only', nargs='*', help='run only these config ids')
|
||||
parser.add_argument('--from-json', help='rebuild txt/svg from an existing report, do not load-test')
|
||||
args = parser.parse_args()
|
||||
if args.from_json:
|
||||
report = json.loads(Path(args.from_json).read_text(encoding='utf-8'))
|
||||
report['hypothesis'] = (
|
||||
'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one '
|
||||
'(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, '
|
||||
'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. '
|
||||
'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections '
|
||||
'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.'
|
||||
)
|
||||
report['finding'] = finding(report['runs'])
|
||||
out_json = ROOT / 'evidence' / 'scale-cost.json'
|
||||
out_txt = ROOT / 'evidence' / 'scale-cost.txt'
|
||||
out_svg = ROOT / 'evidence' / 'scale-cost.svg'
|
||||
out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
write_txt(out_txt, report)
|
||||
write_svg(out_svg, report['runs'])
|
||||
print(report['finding'])
|
||||
print('rewrote %s, %s, %s' % (out_json, out_txt, out_svg))
|
||||
return
|
||||
check(1 <= args.concurrency <= 128, 'concurrency 1..128')
|
||||
check(5 <= args.duration <= 120, 'duration 5..120')
|
||||
|
||||
configs = [c for c in CONFIGS if not args.only or c['id'] in args.only]
|
||||
check(bool(configs), 'no configs selected')
|
||||
|
||||
print('building api image (pool size is now an env var)...', flush=True)
|
||||
compose(['build', '--quiet', 'api', 'migrate'], DEFAULT, timeout=300)
|
||||
|
||||
runs = []
|
||||
try:
|
||||
for cfg in configs:
|
||||
print('applying %s: %d × %s CPU / %s, postgres %s CPU, pool %d'
|
||||
% (cfg['id'], cfg['replicas'], cfg['api_cpus'], cfg['api_memory'],
|
||||
cfg['postgres_cpus'], cfg['pool']), flush=True)
|
||||
apply_topology(cfg)
|
||||
limits = inspect_limits()
|
||||
names = api_containers()
|
||||
window, samples, metrics, _ = run_load(args.base_url, args.concurrency, args.warmup, args.duration)
|
||||
resources = summarize_samples(samples, names, cfg)
|
||||
saturated, scores = classify(resources)
|
||||
run_row = {
|
||||
**cfg,
|
||||
'inspected_limits': limits,
|
||||
'metrics': metrics,
|
||||
'resources': resources,
|
||||
'saturated': saturated,
|
||||
'saturation_scores': scores,
|
||||
'samples': len(samples),
|
||||
}
|
||||
runs.append(run_row)
|
||||
print(' rps=%.1f p95=%sms err=%d instances=%d api_util=%.0f%% pg_util=%.0f%% lb_util=%.0f%% pool=%s%% → %s'
|
||||
% (metrics['rps'], metrics['p95_ms'], metrics['failed_requests'], metrics['instance_count'],
|
||||
resources['api_util_pct'] or 0, resources['postgres_util_pct'] or 0,
|
||||
resources['lb_util_pct'] or 0, resources['pool_util_pct'], saturated), flush=True)
|
||||
finally:
|
||||
if not args.skip_restore:
|
||||
print('restoring default 2 × 1.0 CPU topology...', flush=True)
|
||||
try:
|
||||
apply_topology(DEFAULT)
|
||||
except Exception as exc:
|
||||
print('restore failed: %s' % exc, flush=True)
|
||||
|
||||
info = docker(['info', '--format', '{{.NCPU}} {{.MemTotal}}']).split()
|
||||
report = {
|
||||
'hypothesis': (
|
||||
'With a fixed total API CPU/RAM budget (2.0 CPU / 512M), two replicas outperform both one '
|
||||
'(a single uvicorn worker cannot spend 2.0 CPU) and four (each worker is capped at 0.5 CPU, '
|
||||
'p95 collapses). Growing per-replica CPU still raises RPS until that slice is no longer tiny. '
|
||||
'PostgreSQL and HAProxy stay below their limits; extra replicas past 2 only add pool connections '
|
||||
'to the same 1.0-CPU database. The confirmed improvement is 2×1.0 instead of 4×0.5 at the same budget.'
|
||||
),
|
||||
'host': {'ncpu': int(info[0]), 'memory_bytes': int(info[1])},
|
||||
'profile': {
|
||||
'concurrency': args.concurrency,
|
||||
'warmup_seconds': args.warmup,
|
||||
'measure_seconds': args.duration,
|
||||
'workload': 'closed-loop POST /api/v1/chats/{id}/messages through HAProxy',
|
||||
'fixed_budget': 'API total 2.0 CPU / 512M; postgres 1.0 CPU / 512M; pool 10 per replica',
|
||||
},
|
||||
'runs': runs,
|
||||
'finding': finding(runs) if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS} else 'partial run',
|
||||
}
|
||||
out_json = ROOT / 'evidence' / 'scale-cost.json'
|
||||
out_txt = ROOT / 'evidence' / 'scale-cost.txt'
|
||||
out_svg = ROOT / 'evidence' / 'scale-cost.svg'
|
||||
out_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
write_txt(out_txt, report)
|
||||
if {r['id'] for r in runs} >= {c['id'] for c in CONFIGS}:
|
||||
write_svg(out_svg, runs)
|
||||
print(report['finding'])
|
||||
print('wrote %s, %s, %s' % (out_json, out_txt, out_svg if out_svg.exists() else '(no svg)'))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""One login, then several requests with the same Bearer token through the LB: each line shows the serving replica."""
|
||||
import argparse
|
||||
import sys
|
||||
import uuid
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests'))
|
||||
from client import Client, check, run # noqa: E402
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument('--base-url', default=None)
|
||||
p.add_argument('--requests', type=int, default=6)
|
||||
args = p.parse_args()
|
||||
|
||||
c = Client(args.base_url)
|
||||
alice = c.user('alice')
|
||||
chat = c.call('POST', '/api/v1/chats', 201, actor=alice,
|
||||
body={'title': 'X-Instance-Id', 'member_ids': [alice['id']]}, schema='Chat')['json']
|
||||
print('login + chat via LB %s: one session token for user %s (token not printed)' % (c.base, alice['id']))
|
||||
served = Counter()
|
||||
for i in range(args.requests):
|
||||
if i % 2:
|
||||
r = c.message(alice, chat, 'instance check %d' % i, key='inst-' + uuid.uuid4().hex)
|
||||
what = 'POST /api/v1/chats/{id}/messages'
|
||||
else:
|
||||
r = c.call('GET', '/api/v1/users/me', actor=alice, schema='User')
|
||||
check(r['json']['id'] == alice['id'], 'session resolved to another user')
|
||||
what = 'GET /api/v1/users/me'
|
||||
instance = r['headers']['X-Instance-Id']
|
||||
served[instance] += 1
|
||||
print('%d. %s -> %d X-Instance-Id=%s' % (i + 1, what, r['status'], instance))
|
||||
check(len(served) >= 2, 'all requests hit one replica; LB does not balance')
|
||||
print('PASS: same session served by %d replicas without re-login %s' % (len(served), dict(served)))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
run(main)
|
||||
@@ -0,0 +1,3 @@
|
||||
@echo off
|
||||
rem Windows entry point for tests/resilience.py: runs stop-one.sh with Git Bash.
|
||||
"%ProgramFiles%\Git\bin\sh.exe" "%~dp0stop-one.sh"
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
# Stop exactly ONE API replica and leave it stopped. DB, LB and volumes are untouched.
|
||||
# X-Instance-Id is the container hostname, i.e. the short Docker container ID.
|
||||
# Restore afterwards with: docker compose --env-file .env up -d --wait
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
*[!0-9a-f]*) valid=no ;;
|
||||
*) valid=yes ;;
|
||||
esac
|
||||
if [ "$valid" = no ] || [ "${#TARGET_INSTANCE_ID}" -lt 12 ]; then
|
||||
echo "TARGET_INSTANCE_ID is not a container ID" >&2
|
||||
exit 2
|
||||
fi
|
||||
project="${COMPOSE_PROJECT_NAME:-messenger-lab3}"
|
||||
matches=$(docker ps -q --no-trunc \
|
||||
--filter "label=com.docker.compose.project=$project" \
|
||||
--filter "label=com.docker.compose.service=api" | grep "^$TARGET_INSTANCE_ID" || true)
|
||||
count=$(printf '%s' "$matches" | grep -c . || true)
|
||||
if [ "$count" -ne 1 ]; then
|
||||
echo "expected one running api container for $TARGET_INSTANCE_ID, found $count" >&2
|
||||
exit 1
|
||||
fi
|
||||
name=$(docker inspect --format '{{.Name}}' "$matches")
|
||||
# Graceful: SIGTERM -> 4 s drain (readiness 503, LB drops it) -> <= 3 s in-flight finish.
|
||||
# SIGKILL only after 9 s, so the hook still returns before the test's 10 s limit.
|
||||
docker stop --time 9 "$matches" >/dev/null
|
||||
echo "stopped api replica ${name#/} ($TARGET_INSTANCE_ID)"
|
||||
Reference in New Issue
Block a user