This commit is contained in:
2026-10-25 21:46:21 +03:00
parent 5300a1a1e0
commit a7d6487e2d
33 changed files with 5965 additions and 556 deletions
+2
View File
@@ -9,6 +9,8 @@ APP_ENV=lab
POSTGRES_USER=messenger
POSTGRES_DB=messenger
POSTGRES_CPU_LIMIT=1.00
POSTGRES_MEMORY_LIMIT=512M
SESSION_TTL_SECONDS=3600
SESSION_STORE=postgres
+13
View File
@@ -7,6 +7,8 @@ BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
GRADE4_OUTPUT ?= evidence/grade4/results.json
GRADE5_OUTPUT ?= evidence/grade5/results.json
READINESS_OUTPUT ?= evidence/readiness-routing.json
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
@@ -27,8 +29,10 @@ COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
load \
test-persistence \
test-failover \
test-readiness-routing \
rolling-update \
grade4-experiment \
grade5-experiment \
compose-config \
compose-build \
compose-up \
@@ -51,8 +55,10 @@ help:
@echo "load: run synthetic writes against the LB entrypoint"
@echo "test-persistence: run persistence test with ACTION_SCRIPT"
@echo "test-failover: run replica failover test with ACTION_SCRIPT"
@echo "test-readiness-routing: isolate PostgreSQL from one API and verify active LB health checks"
@echo "rolling-update: replace API replicas one by one with surge capacity"
@echo "grade4-experiment: run failovers, idempotency and rolling update with evidence"
@echo "grade5-experiment: measure fixed-budget scaling, saturation and one improvement"
@echo "compose-config: validate resolved Compose configuration"
@echo "compose-build: build application images"
@echo "compose-up: start the stack"
@@ -114,6 +120,9 @@ test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-readiness-routing:
$(PYTHON) scripts/test_readiness_routing.py --env-file "$(ENV_FILE)" --output "$(READINESS_OUTPUT)"
rolling-update:
ENV_FILE="$(ENV_FILE)" ./scripts/rolling-update.sh
@@ -121,6 +130,10 @@ grade4-experiment:
$(PYTHON) scripts/grade4_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE4_OUTPUT)"
$(PYTHON) scripts/render_grade4.py "$(GRADE4_OUTPUT)"
grade5-experiment:
$(PYTHON) scripts/grade5_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE5_OUTPUT)"
$(PYTHON) scripts/render_grade5.py "$(GRADE5_OUTPUT)"
compose-config:
@test -f "$(ENV_FILE)" || \
(echo "$(ENV_FILE) is missing; run: make init-env"; exit 2)
+152 -37
View File
@@ -1,11 +1,11 @@
# Отчёт — лабораторная 3
- Автор, группа: Чурилин, БВТ2301
- Текущий завершённый уровень: оценка «4»
- Итоговая цель: оценка «5»; исследовательский трек выполняется следующим этапом
- Текущий завершённый уровень: оценка «5», трек 1 «Цена масштабирования»
- Итоговая цель: выполнена; уровень «4» и законченный исследовательский трек
- Commit/PR: заполняется автором после локального коммита и публикации
- Использованные источники и библиотеки: преподавательские README/OpenAPI/tests,
документация Go, pgx, PostgreSQL, Docker Compose, Nginx, React, TypeScript,
документация Go, pgx, PostgreSQL, Docker Compose, HAProxy, Nginx, React, TypeScript,
Vite, TanStack Query, React Router, Lucide и Vitest. Для разбора архитектуры,
рефакторинга, реализации и проверок использовался OpenAI Codex; решения
проверены тестами и описанными ниже воспроизводимыми сценариями.
@@ -35,13 +35,16 @@ cp .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
make test
make test-readiness-routing
make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
docker compose --env-file .env up -d --wait api
make test-persistence ACTION_SCRIPT=./scripts/restart.sh
make grade4-experiment
make grade5-experiment
```
По этому же адресу Nginx отдаёт production-сборку React. Переход на
По этому же адресу HAProxy направляет обычные browser-запросы во внутренний
Nginx, который отдаёт production-сборку React. Переход на
`http://localhost:8080` открывает Messenger, а `/api/v1/*` и health endpoints
остаются доступны автоматическим проверкам. Отдельный Node.js на машине для
обычного запуска не нужен: multi-stage image сам выполняет `npm ci`, Vitest,
@@ -84,15 +87,22 @@ PostgreSQL остаётся источником истины для польз
PostgreSQL.
Compose запускает один сервис `api` с `API_REPLICAS=2`. Реплики не имеют host
ports и разделяют PostgreSQL, включая таблицу сессий. Nginx — единственная
публичная точка входа; он использует `least_conn` и Docker DNS resolver с
повторным разрешением имени `api` раз в секунду. Sticky sessions не применяются.
ports и разделяют PostgreSQL, включая таблицу сессий. HAProxy — единственная
публичная точка входа; он использует `leastconn`, Docker DNS resolver и активные
HTTP-проверки `/health/ready` раз в секунду. Реплика исключается после двух
неуспешных readiness-проверок и возвращается после двух успешных. Поэтому живой,
но потерявший PostgreSQL и ставший неготовым процесс перестаёт получать трафик;
Docker healthcheck не является единственным механизмом обнаружения. Sticky
sessions не применяются. Nginx остаётся отдельным внутренним контейнером и
обслуживает только статические файлы React.
Произвольные POST балансировщик автоматически не повторяет. Исключение сделано
только для маршрута отправки сообщения: обязательный `Idempotency-Key` позволяет
Nginx безопасно повторить запрос на другой реплике при 502/503/504 во время
drain. Клиент также сохраняет ключ для повтора после неопределённого результата.
Создание пользователя и чата контрактом не идемпотентно, поэтому автоматический
retry для этих маршрутов запрещён.
HAProxy безопасно повторить запрос на другой реплике при connection failure,
empty response, timeout или 502/503/504 во время обнаружения отказа и drain.
Для этого маршрута тело запроса буферизуется отдельным backend; регистрации
пользователя и создание чата контрактом не идемпотентны, поэтому retry для них
запрещён. Клиент также сохраняет ключ для повтора после неопределённого результата.
Миграции вынесены в одноразовый сервис `migrate`, завершение которого является
предусловием запуска API. Поэтому две реплики не конкурируют за изменение схемы.
@@ -154,7 +164,7 @@ React Router — за маршруты login/register, выбранного ча
`SameSite=Strict`. Флаг `Secure` управляется `SESSION_COOKIE_SECURE`: `false`
нужен только локальному HTTP, на HTTPS должен быть `true`.
Контрактные `/api/v1/*` сознательно остались Bearer-only. Nginx предоставляет
Контрактные `/api/v1/*` сознательно остались Bearer-only. HAProxy предоставляет
отдельный браузерный адаптер `/web-api/v1/*`: берёт token из HttpOnly-cookie,
формирует `Authorization: Bearer ...`, удаляет исходный `Cookie` и проксирует на
обычный `/api/v1/*`. Поэтому преподавательские тесты и внешние API-клиенты не
@@ -173,18 +183,23 @@ cookie.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Воспроизводимая конфигурация и качество Go | `make check && make go-check && make validate PYTHON=../mtusi2026lab2/.venv/bin/python` | форматирование, tests, race detector, vet, Compose config и OpenAPI validation завершились без ошибок | `evidence/lab3-summary.md` |
| Две API-реплики за одним LB | `docker compose --env-file .env up -d --build --wait`; 20 health-запросов через `:8080` | наружу опубликован только Nginx; два разных `X-Instance-Id`, по 10 ответов от каждого | `compose.yaml`, `infrastructure/nginx/nginx.conf`, `evidence/lab3-summary.md` |
| Две API-реплики за одним LB | `docker compose --env-file .env up -d --build --wait`; 20 health-запросов через `:8080` | наружу опубликован только HAProxy; два разных `X-Instance-Id`, по 10 ответов от каждого | `compose.yaml`, `infrastructure/haproxy/haproxy.cfg`, `evidence/lab3-summary.md` |
| Публичный smoke lab3 | `make test` | `tests/smoke.py` завершился `PASS: lab3 public API scenario` | `evidence/lab3-summary.md` |
| Общая сессия без sticky routing | smoke, затем failover и persistence через тот же bearer token | обе реплики обслуживали запросы; сессия работает после остановки API и рестартов | `evidence/lab3-summary.md` |
| Идемпотентность сообщения | `go test ./student_tests -run Idempotency`; 32 конкурентных POST с одним ключом через LB | один Message и ровно одна строка в PostgreSQL; другой payload даёт 409 | `student_tests/idempotency_test.go`, `evidence/lab3-summary.md` |
| Failover каждой API-реплики | два последовательных запуска `make test-failover ACTION_SCRIPT=./scripts/stop-one.sh` с восстановлением между ними | реально остановлены `api-2`, затем `api-1`; получено 142/143 успешных записи, 0 ошибок, восстановление 0.963/0.750 с | `scripts/stop-one.sh`, `evidence/lab3-summary.md` |
| Persistence после рестартов | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS; прежняя сессия и данные доступны после рестарта PostgreSQL, Redis, API и LB | `scripts/restart.sh`, `evidence/lab3-summary.md` |
| Миграции перед API | `docker compose --env-file .env ps -a` и логи `migrate` | миграции 1 и 2 применены; one-shot контейнер завершился с кодом 0; API healthy | `migrations/000002_message_idempotency.up.sql`, `evidence/lab3-summary.md` |
| React frontend и HttpOnly BFF | `docker compose build load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/Dockerfile`, `infrastructure/nginx/nginx.conf` |
| React frontend и HttpOnly BFF | `docker compose build frontend load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/`, `infrastructure/haproxy/` |
| Согласованные health и shutdown | SIGTERM каждой реплики под нагрузкой | API сразу закрывает availability gate, 2 с дренирует запросы и укладывается в 10 с shutdown; `/health/live` остаётся доступен | `internal/transport/http/middleware/availability.go`, `evidence/grade4/` |
| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только Nginx `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 3,464/3,476 с, одна краткая ошибка из 4614 запросов в первом прогоне и 0 из 5815 во втором | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 20 350 успешных запросов, 1 краткий 502 (0,005%), все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только HAProxy `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
| Readiness-aware routing | `make test-readiness-routing` | одной API-реплике заблокирован PostgreSQL: HAProxy исключил её за 2,861 с; 40 health-запросов получили 200, 20 регистраций — 201; после снятия fault реплика вернулась | `scripts/test_readiness_routing.py`, `evidence/readiness-routing.json` |
| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 2,893/2,938 с; во всех фазах 0 ошибочных запросов | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 22 212 успешных запросов, 0 ошибок; все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
| Фиксированный бюджет 1/2/4 API | `make grade5-experiment`; 32 closed-loop клиента | при суммарных 1 CPU, 512 MiB и 16 соединениях получено 578,242 / 529,499 / 353,955 RPS; четыре процесса при том же бюджете дали на 38,79% меньше одной реплики | `evidence/grade5/02-fixed-total.svg`, `measurements.csv` |
| Рост ресурсов 1/2/4 API | тот же workload; по 0,5 CPU, 256 MiB и 8 соединений на реплику | 533,656 / 563,460 / 438,868 RPS; нелинейная серия показывает отсутствие пропорционального выигрыша при общей БД в 1 CPU | `evidence/grade5/03-resource-growth.svg`, `results.json` |
| Saturation point и bottleneck | concurrency 1/2/4/8/16/32, по 10 с после warm-up | первый сигнал насыщения — 8: относительно 4 клиентов RPS вырос только на 8,03%, p95 — в 1,92 раза; PostgreSQL в профиле четырёх реплик достиг 101,91% одного CPU | `evidence/grade5/01-saturation.svg`, `04-bottleneck.svg` |
| Подтверждённое улучшение | четыре API без изменений; PostgreSQL 1 → 2 CPU | RPS 438,868 → 587,574 (+33,88%), ошибок 0, p95 снизился на 14,12%; следующим ограничением стал HAProxy с лимитом 0,25 CPU | `evidence/grade5/05-improvement.svg`, `resources.csv` |
## Эксперименты и отказы
@@ -192,8 +207,8 @@ cookie.
Docker Engine 29.7.2, Compose 5.5.0, 8 логических CPU и 3813,6 MiB RAM,
commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиченными
изменениями текущего уровня. В Compose каждой API-реплике выделено 0,5 CPU и
256 MiB; Nginx — 0,25 CPU/128 MiB; PostgreSQL — 1 CPU/512 MiB; Redis —
0,25 CPU/128 MiB.
256 MiB; HAProxy — 0,25 CPU/128 MiB; frontend Nginx — 0,10 CPU/64 MiB;
PostgreSQL — 1 CPU/512 MiB; Redis — 0,25 CPU/128 MiB.
Команда `make grade4-experiment` создаёт синтетических пользователей и чат,
делает 12 warm-up записей, затем запускает 8 closed-loop клиентов без повторов
@@ -202,44 +217,144 @@ commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиче
следующей, после действия нагрузка продолжается ещё 8 с. Rolling update
временно увеличивает сервис до трёх реплик, ждёт healthcheck и два периода
Docker DNS, заменяет старые контейнеры по одному и возвращает масштаб к двум.
Минимальный подтверждённый размер набора после прогона — 34 808 сообщений.
Минимальный подтверждённый размер набора после прогона — 43 810 сообщений.
| Фаза | Успешно / ошибок | RPS | Доля ошибок | p50 / p95 / p99, мс | Восстановление |
| --- | ---: | ---: | ---: | ---: | ---: |
| baseline | 4017 / 0 | 319,463 | 0% | 16,799 / 55,515 / 93,803 | — |
| остановка реплики 1 | 4613 / 1 | 301,040 | 0,0217% | 17,145 / 69,558 / 89,752 | 3,464 с |
| остановка реплики 2 | 5815 / 0 | 374,449 | 0% | 14,211 / 49,542 / 81,040 | 3,476 с |
| rolling update | 20 350 / 1 | 376,959 | 0,0049% | 15,725 / 48,724 / 79,337 | непрерывная нагрузка |
| baseline | 5762 / 0 | 479,581 | 0% | 9,178 / 56,402 / 79,978 | — |
| остановка реплики 1 | 7853 / 0 | 537,548 | 0% | 8,812 / 50,050 / 66,373 | 2,893 с |
| остановка реплики 2 | 7970 / 0 | 533,154 | 0% | 8,726 / 48,789 / 64,333 | 2,938 с |
| rolling update | 22 212 / 0 | 428,860 | 0% | 10,560 / 58,713 / 83,982 | непрерывная нагрузка |
На baseline средняя выборка одного API-контейнера составила 28,453% CPU и
42,413 MiB RAM, максимум — 36,1% и 66,07 MiB. PostgreSQL оказался самым
нагруженным компонентом: в rolling-фазе среднее 98,172% CPU при лимите одного
ядра, максимум RAM 145,2 MiB. Это не capacity-тест и не доказательство
На baseline средняя выборка одного API-контейнера составила 27,144% CPU и
31,008 MiB RAM, максимум — 30,93% и 66,17 MiB. PostgreSQL оказался самым
нагруженным компонентом: среднее 89,392% CPU при лимите одного ядра, максимум
110,59%; HAProxy использовал в среднем 26,262% CPU. Это не capacity-тест и не доказательство
saturation point, но важный ориентир для исследования уровня 5.
Nginx обновляет Docker DNS раз в 1 с, исключает peer после двух неудач за 2 с,
а connect timeout равен 1 с. При постоянной нагрузке ожидаемое максимальное
обнаружение — около 4 с; измеренные 3,464 и 3,476 с укладываются в требование
10 с. Одна 504 и одна 502 сохранены в отчёте честно: README прямо допускает
краткие ошибки обнаружения. Подтверждённых потерь и дублей нет. Параллельные
32 POST с одним ключом прошли через обе реплики и вернули один Message ID.
HAProxy обновляет Docker DNS раз в 1 с и исключает peer после двух неуспешных
активных `/health/ready`-проверок. Измеренные 2,893 и 2,938 с укладываются в
требование 10 с. Отдельный тест потери доступа одной реплики к PostgreSQL
подтвердил исключение неготового backend за 2,861 с без 5xx: 40 health-запросов
получили 200, а 20 регистраций — 201. Во всех четырёх фазах grade-4 эксперимента
ошибок нет. Параллельные 32 POST с одним ключом прошли через обе реплики и
вернули один Message ID.
Исходные агрегаты, environment/limits, посекундные таблицы и четыре SVG-графика
лежат в `evidence/grade4/`. Percentile рассчитаны только по успешным ответам;
профиль closed-loop имеет coordinated omission. CPU для API агрегируется как
средняя/максимальная выборка отдельного контейнера, а не сумма двух реплик.
## Исследование уровня 5: цена масштабирования
Трек 1 выполнен командой `make grade5-experiment` 23.10.2026 на том же
однохостовом стенде Windows + WSL2: Docker Engine 29.7.2, Compose 5.5.0,
8 логических CPU и 3813,6 MiB RAM. Базовым commit до изменений уровня 5 был
`5300a1a`; измерения ожидаемо зафиксировали dirty working tree, поскольку сам
скрипт и параметризация лимитов ещё не были закоммичены. Все запросы проходили
через единственную публичную точку HAProxy, создавали сообщения с уникальными
`Idempotency-Key` и не повторялись нагрузочным клиентом автоматически.
### Методика и воспроизводимость
Каждая измеряемая фаза длилась 10 секунд. После каждой смены профиля выполнялся
двухсекундный warm-up, затем контейнеры и реально применённые `NanoCpus`, RAM и
`DB_MAX_CONNS` проверялись через `docker inspect`. Скрипт сохраняет посекундные
request buckets и каждую выборку `docker stats`; CPU/RAM API сначала суммируются
по всем репликам в одной временной точке и только затем усредняются. Это устраняет
ошибку, при которой средняя загрузка одного из четырёх контейнеров выдавалась бы
за стоимость всего API-сервиса.
В серии с фиксированным бюджетом сохранялись суммарные 1 CPU, 512 MiB RAM и
16 соединений пула API:
| Реплики | CPU на реплику | RAM на реплику | `DB_MAX_CONNS` на реплику | Суммарный пул |
| ---: | ---: | ---: | ---: | ---: |
| 1 | 1,00 | 512 MiB | 16 | 16 |
| 2 | 0,50 | 256 MiB | 8 | 16 |
| 4 | 0,25 | 128 MiB | 4 | 16 |
В серии роста каждая реплика получала 0,5 CPU, 256 MiB и 8 соединений, поэтому
суммарный API-бюджет менялся от 0,5 CPU/256 MiB/8 соединений до
2 CPU/1024 MiB/32 соединений. Во всех основных профилях PostgreSQL оставался на
1 CPU/512 MiB, HAProxy — 0,25 CPU/128 MiB, frontend Nginx — 0,10 CPU/64 MiB,
Redis — 0,25 CPU/128 MiB. Сравнение
1/2/4 выполнялось при одинаковом concurrency 32. До и после эксперимента
зафиксированы размер набора и базы: 225 626 → 299 524 сообщений и
175 406 103 → 228 793 367 байт. После любого исхода `finally` возвращает стенд к
двум API по 0,5 CPU/256 MiB, пулу 10 на реплику и PostgreSQL с 1 CPU; успешное
восстановление также записано в `results.json`.
### Насыщение и цена реплик
Для поиска saturation point две API-реплики с суммарным 1 CPU и пулом 16
нагружались последовательностью concurrency 1/2/4/8/16/32:
| Concurrency | RPS | p95, мс | PostgreSQL CPU avg/max | Ошибки |
| ---: | ---: | ---: | ---: | ---: |
| 1 | 230,369 | 5,065 | 23,900% / 29,41% | 0 |
| 2 | 320,190 | 7,196 | 54,987% / 62,60% | 0 |
| 4 | 435,822 | 29,304 | 63,093% / 81,80% | 0 |
| 8 | 470,828 | 56,230 | 65,945% / 92,98% | 0 |
| 16 | 472,892 | 73,628 | 65,080% / 88,93% | 0 |
| 32 | 564,012 | 94,785 | 83,825% / 107,62% | 0 |
Заранее зафиксированное правило считает первым saturation signal шаг, где прирост
RPS меньше 10%, а p95 одновременно вырастает не менее чем в 1,5 раза. Это
произошло при переходе 4 → 8: throughput вырос только на 8,03%, а p95 — в
1,92 раза. При 16 клиентах прирост относительно 8 почти отсутствует (+0,44%),
а p95 продолжает расти; при 32 клиентах RPS увеличивается ценой ещё большей
очереди. Поэтому concurrency 8 — первый воспроизводимый saturation signal по
заранее заданному правилу, а не утверждение об абсолютном максимуме системы.
При фиксированном суммарном бюджете 1/2/4 реплики дали соответственно
578,242 / 529,499 / 353,955 RPS. Четыре процесса при том же общем бюджете дали
на 38,79% меньше одной реплики: дробление ресурсов добавило scheduling и pool
overhead, но не увеличило общую ёмкость. В серии роста ресурсов получены
533,656 / 563,460 / 438,868 RPS. Разброс и нестрогая монотонность честно
сохранены: это один прогон на одном Docker-host, где контейнеры конкурируют за
общий scheduler и одну БД. Главный результат здесь не выбор «лучшего» количества
реплик, а отсутствие пропорционального роста: четырёхкратный API-бюджет дал на
17,76% меньше throughput одной реплики в этом конкретном прогоне.
Доминирующее ограничение локализовано в PostgreSQL. В базовом профиле четырёх
реплик API суммарно использовал в среднем 35,690% CPU из доступных 200%, HAProxy —
15,595% из 25%, тогда как PostgreSQL использовал 84,847% в среднем и достигал
101,91% лимита одного ядра. Само по себе это корреляция, поэтому выполнено
контрольное изменение только одного параметра.
### Подтверждённое улучшение
Для тех же четырёх API по 0,5 CPU/256 MiB, того же пула и concurrency 32 лимит
PostgreSQL увеличен с 1 до 2 CPU. RPS вырос с 438,868 до 587,574 — на 33,88%;
ошибок в обеих фазах не было, p95 снизился с 109,446 до 93,991 мс (-14,12%).
Таким образом, снятие именно ограничения CPU БД существенно увеличило throughput
и улучшило tail latency, что подтверждает исходную гипотезу.
После улучшения PostgreSQL использовал в среднем 95,573% из доступных 200%, а
HAProxy — 22,350% при лимите 0,25 CPU и достигал 26,73%. То есть следующее
ограничение ожидаемо сместилось к публичному
балансировщику. Увеличивать API дальше без пересмотра лимита или топологии LB уже
нерационально — это и есть практическая «цена масштабирования».
Полный JSON с окружением, профилями, фактически применёнными лимитами и сырыми
выборками находится в `evidence/grade5/results.json`; агрегаты — в
`measurements.csv`, посекундные запросы — в `timeseries.csv`, ресурсы — в
`resources.csv`. Пять SVG показывают saturation curve, обе серии 1/2/4,
локализацию bottleneck и контрольное улучшение.
## Ограничения и следующие шаги
- Топология однохостовая: Nginx и PostgreSQL остаются single points of failure.
- Топология однохостовая: HAProxy и PostgreSQL остаются single points of failure.
Репликация БД и несколько балансировщиков в требования уровня «4» не входят.
- Redis содержит только кэш, поэтому его потеря допустима. Отдельная
персистентность Redis для корректности API не требуется.
- Истёкшие записи идемпотентности удаляются лениво при повторном обращении к той
же области ключа. Индекс по `expires_at` оставлен для будущей периодической
очистки без изменения схемы.
- Уровень «5» ещё не реализован: сравнение 1/2/4 реплик при фиксированных
ресурсах и поиск saturation point выполняются следующим отдельным этапом.
- Исследование уровня «5» выполняет по одному измеряемому прогону на профиль.
Для production capacity planning нужны несколько повторов, доверительные
интервалы и отдельный генератор нагрузки вне измеряемого Docker-host.
- Rolling-сценарий заменяет бинарник без изменения схемы. Миграции выполняются
однократно до API; для будущей несовместимой схемы потребуется expand/contract,
поэтому одновременная работа разных schema-version сейчас не заявляется.
+27 -4
View File
@@ -2,13 +2,15 @@ services:
load-balancer:
build:
context: .
dockerfile: infrastructure/nginx/Dockerfile
image: mtusi2026lab3-web
dockerfile: infrastructure/haproxy/Dockerfile
image: mtusi2026lab3-load-balancer
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
depends_on:
api:
condition: service_healthy
frontend:
condition: service_healthy
healthcheck:
test:
[
@@ -27,6 +29,27 @@ services:
cpus: "0.25"
memory: 128M
frontend:
build:
context: .
dockerfile: infrastructure/nginx/Dockerfile
image: mtusi2026lab3-web
expose:
- "8080"
healthcheck:
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/ >/dev/null || exit 1"]
interval: 5s
timeout: 3s
retries: 10
start_period: 3s
stop_grace_period: 10s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.10"
memory: 64M
postgres:
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
environment:
@@ -51,8 +74,8 @@ services:
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
cpus: "${POSTGRES_CPU_LIMIT:-1.00}"
memory: ${POSTGRES_MEMORY_LIMIT:-512M}
redis:
image: redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0
+9
View File
@@ -3,3 +3,12 @@
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
## Сохранённые эксперименты
- `grade4/` — failover обеих API-реплик и rolling update под нагрузкой;
- `grade5/` — трек «Цена масштабирования»: saturation sweep, сравнение 1/2/4
реплик с фиксированным и растущим бюджетом, сырые JSON/CSV, пять SVG-графиков
и подтверждённое улучшение PostgreSQL CPU;
- `readiness-routing.json` — результат изоляции PostgreSQL от одной живой
API-реплики: время исключения backend, статусы запросов и восстановление.
+7 -7
View File
@@ -1,12 +1,12 @@
# Lab 3 grade-4 experiment
Generated: 2026-10-23T12:31:12.749687+00:00.
Generated: 2026-10-23T15:23:05.031807+00:00.
## Method
- concurrency: 8;
- baseline duration: 12 s;
- load: closed-loop POST messages through Nginx;
- load: closed-loop POST messages through the public HAProxy gateway;
- failover: both original API containers stopped and restored in turn;
- rolling update: one surge replica, old replicas replaced one by one;
- percentiles cover successful responses; failures are separate.
@@ -15,10 +15,10 @@ Generated: 2026-10-23T12:31:12.749687+00:00.
| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| baseline | 319.463 | 0 (0.000%) | 16.799 | 55.515 | 93.803 | n/a |
| failover-1 | 301.04 | 1 (0.022%) | 17.145 | 69.558 | 89.752 | 3.464 |
| failover-2 | 374.449 | 0 (0.000%) | 14.211 | 49.542 | 81.04 | 3.476 |
| rolling-update | 376.959 | 1 (0.005%) | 15.725 | 48.724 | 79.337 | n/a |
| baseline | 479.581 | 0 (0.000%) | 9.178 | 56.402 | 79.978 | n/a |
| failover-1 | 537.548 | 0 (0.000%) | 8.812 | 50.05 | 66.373 | 2.893 |
| failover-2 | 533.154 | 0 (0.000%) | 8.726 | 48.789 | 64.333 | 2.938 |
| rolling-update | 428.86 | 0 (0.000%) | 10.56 | 58.713 | 83.982 | n/a |
## Parallel idempotency
@@ -36,4 +36,4 @@ Generated: 2026-10-23T12:31:12.749687+00:00.
## Scope
Single-host process failure and rolling replacement only. PostgreSQL and Nginx remain single points of failure.
Single-host process failure and rolling replacement only. PostgreSQL and HAProxy remain single points of failure.
+29 -29
View File
@@ -16,53 +16,53 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
<line x1="80" y1="450.0" x2="1040" y2="450.0" class="grid"/>
<text x="68" y="454.0" text-anchor="end" class="small">0</text>
<line x1="80" y1="378.0" x2="1040" y2="378.0" class="grid"/>
<text x="68" y="382.0" text-anchor="end" class="small">19</text>
<text x="68" y="382.0" text-anchor="end" class="small">17</text>
<line x1="80" y1="306.0" x2="1040" y2="306.0" class="grid"/>
<text x="68" y="310.0" text-anchor="end" class="small">38</text>
<text x="68" y="310.0" text-anchor="end" class="small">34</text>
<line x1="80" y1="234.0" x2="1040" y2="234.0" class="grid"/>
<text x="68" y="238.0" text-anchor="end" class="small">56</text>
<text x="68" y="238.0" text-anchor="end" class="small">50</text>
<line x1="80" y1="162.0" x2="1040" y2="162.0" class="grid"/>
<text x="68" y="166.0" text-anchor="end" class="small">75</text>
<text x="68" y="166.0" text-anchor="end" class="small">67</text>
<line x1="80" y1="90.0" x2="1040" y2="90.0" class="grid"/>
<text x="68" y="94.0" text-anchor="end" class="small">94</text>
<rect x="128.0" y="385.5" width="42.0" height="64.5" rx="7" class="bar"/>
<text x="149.0" y="378.5" text-anchor="middle" class="small">17</text>
<text x="68" y="94.0" text-anchor="end" class="small">84</text>
<rect x="128.0" y="410.7" width="42.0" height="39.3" rx="7" class="bar"/>
<text x="149.0" y="403.7" text-anchor="middle" class="small">9</text>
<text x="149.0" y="470" text-anchor="middle" class="small">p50</text>
<rect x="178.0" y="236.9" width="42.0" height="213.1" rx="7" class="bar2"/>
<text x="199.0" y="229.9" text-anchor="middle" class="small">56</text>
<rect x="178.0" y="208.2" width="42.0" height="241.8" rx="7" class="bar2"/>
<text x="199.0" y="201.2" text-anchor="middle" class="small">56</text>
<text x="199.0" y="470" text-anchor="middle" class="small">p95</text>
<rect x="228.0" y="90.0" width="42.0" height="360.0" rx="7" class="bar2"/>
<text x="249.0" y="83.0" text-anchor="middle" class="small">94</text>
<rect x="228.0" y="107.2" width="42.0" height="342.8" rx="7" class="bar2"/>
<text x="249.0" y="100.2" text-anchor="middle" class="small">80</text>
<text x="249.0" y="470" text-anchor="middle" class="small">p99</text>
<text x="200.0" y="498" text-anchor="middle" class="label">baseline</text>
<rect x="368.0" y="384.2" width="42.0" height="65.8" rx="7" class="bar"/>
<text x="389.0" y="377.2" text-anchor="middle" class="small">17</text>
<rect x="368.0" y="412.2" width="42.0" height="37.8" rx="7" class="bar"/>
<text x="389.0" y="405.2" text-anchor="middle" class="small">9</text>
<text x="389.0" y="470" text-anchor="middle" class="small">p50</text>
<rect x="418.0" y="183.0" width="42.0" height="267.0" rx="7" class="bar2"/>
<text x="439.0" y="176.0" text-anchor="middle" class="small">70</text>
<rect x="418.0" y="235.5" width="42.0" height="214.5" rx="7" class="bar2"/>
<text x="439.0" y="228.5" text-anchor="middle" class="small">50</text>
<text x="439.0" y="470" text-anchor="middle" class="small">p95</text>
<rect x="468.0" y="105.5" width="42.0" height="344.5" rx="7" class="bar2"/>
<text x="489.0" y="98.5" text-anchor="middle" class="small">90</text>
<rect x="468.0" y="165.5" width="42.0" height="284.5" rx="7" class="bar2"/>
<text x="489.0" y="158.5" text-anchor="middle" class="small">66</text>
<text x="489.0" y="470" text-anchor="middle" class="small">p99</text>
<text x="440.0" y="498" text-anchor="middle" class="label">failover-1</text>
<rect x="608.0" y="395.5" width="42.0" height="54.5" rx="7" class="bar"/>
<text x="629.0" y="388.5" text-anchor="middle" class="small">14</text>
<rect x="608.0" y="412.6" width="42.0" height="37.4" rx="7" class="bar"/>
<text x="629.0" y="405.6" text-anchor="middle" class="small">9</text>
<text x="629.0" y="470" text-anchor="middle" class="small">p50</text>
<rect x="658.0" y="259.9" width="42.0" height="190.1" rx="7" class="bar2"/>
<text x="679.0" y="252.9" text-anchor="middle" class="small">50</text>
<rect x="658.0" y="240.9" width="42.0" height="209.1" rx="7" class="bar2"/>
<text x="679.0" y="233.9" text-anchor="middle" class="small">49</text>
<text x="679.0" y="470" text-anchor="middle" class="small">p95</text>
<rect x="708.0" y="139.0" width="42.0" height="311.0" rx="7" class="bar2"/>
<text x="729.0" y="132.0" text-anchor="middle" class="small">81</text>
<rect x="708.0" y="174.2" width="42.0" height="275.8" rx="7" class="bar2"/>
<text x="729.0" y="167.2" text-anchor="middle" class="small">64</text>
<text x="729.0" y="470" text-anchor="middle" class="small">p99</text>
<text x="680.0" y="498" text-anchor="middle" class="label">failover-2</text>
<rect x="848.0" y="389.7" width="42.0" height="60.3" rx="7" class="bar"/>
<text x="869.0" y="382.7" text-anchor="middle" class="small">16</text>
<rect x="848.0" y="404.7" width="42.0" height="45.3" rx="7" class="bar"/>
<text x="869.0" y="397.7" text-anchor="middle" class="small">11</text>
<text x="869.0" y="470" text-anchor="middle" class="small">p50</text>
<rect x="898.0" y="263.0" width="42.0" height="187.0" rx="7" class="bar2"/>
<text x="919.0" y="256.0" text-anchor="middle" class="small">49</text>
<rect x="898.0" y="198.3" width="42.0" height="251.7" rx="7" class="bar2"/>
<text x="919.0" y="191.3" text-anchor="middle" class="small">59</text>
<text x="919.0" y="470" text-anchor="middle" class="small">p95</text>
<rect x="948.0" y="145.5" width="42.0" height="304.5" rx="7" class="bar2"/>
<text x="969.0" y="138.5" text-anchor="middle" class="small">79</text>
<rect x="948.0" y="90.0" width="42.0" height="360.0" rx="7" class="bar2"/>
<text x="969.0" y="83.0" text-anchor="middle" class="small">84</text>
<text x="969.0" y="470" text-anchor="middle" class="small">p99</text>
<text x="920.0" y="498" text-anchor="middle" class="label">rolling-update</text>
<text x="24" y="270.0" transform="rotate(-90 24 270.0)" class="label">Successful latency, ms</text>

Before

Width:  |  Height:  |  Size: 4.4 KiB

After

Width:  |  Height:  |  Size: 4.4 KiB

+30 -24
View File
@@ -14,29 +14,35 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
</style>
<text x="48" y="42" class="title">Grade 4: peak container resources</text>
<text x="80" y="72" class="label">Peak CPU, %</text>
<rect x="103.6" y="270.1" width="57.8" height="179.9" rx="9" class="bar"/>
<text x="132.5" y="262.1" text-anchor="middle" class="small">52.6</text>
<text x="132.5" y="475" text-anchor="middle" class="small">api</text>
<rect x="208.6" y="370.1" width="57.8" height="79.9" rx="9" class="bar"/>
<text x="237.5" y="362.1" text-anchor="middle" class="small">23.4</text>
<text x="237.5" y="475" text-anchor="middle" class="small">load-balancer</text>
<rect x="313.6" y="90.0" width="57.8" height="360.0" rx="9" class="bar"/>
<text x="342.5" y="82.0" text-anchor="middle" class="small">105.2</text>
<text x="342.5" y="475" text-anchor="middle" class="small">postgres</text>
<rect x="418.6" y="429.9" width="57.8" height="20.1" rx="9" class="bar"/>
<text x="447.5" y="421.9" text-anchor="middle" class="small">5.9</text>
<text x="447.5" y="475" text-anchor="middle" class="small">redis</text>
<rect x="98.9" y="288.9" width="46.2" height="161.1" rx="9" class="bar"/>
<text x="122.0" y="280.9" text-anchor="middle" class="small">49.5</text>
<text x="122.0" y="475" text-anchor="middle" class="small">api</text>
<rect x="182.9" y="423.8" width="46.2" height="26.2" rx="9" class="bar"/>
<text x="206.0" y="415.8" text-anchor="middle" class="small">8.1</text>
<text x="206.0" y="475" text-anchor="middle" class="small">frontend</text>
<rect x="266.9" y="354.2" width="46.2" height="95.8" rx="9" class="bar"/>
<text x="290.0" y="346.2" text-anchor="middle" class="small">29.4</text>
<text x="290.0" y="475" text-anchor="middle" class="small">load-balancer</text>
<rect x="350.9" y="90.0" width="46.2" height="360.0" rx="9" class="bar"/>
<text x="374.0" y="82.0" text-anchor="middle" class="small">110.6</text>
<text x="374.0" y="475" text-anchor="middle" class="small">postgres</text>
<rect x="434.9" y="428.6" width="46.2" height="21.4" rx="9" class="bar"/>
<text x="458.0" y="420.6" text-anchor="middle" class="small">6.6</text>
<text x="458.0" y="475" text-anchor="middle" class="small">redis</text>
<text x="600" y="72" class="label">Peak memory, MiB</text>
<rect x="623.6" y="286.2" width="57.8" height="163.8" rx="9" class="bar"/>
<text x="652.5" y="278.2" text-anchor="middle" class="small">66.1</text>
<text x="652.5" y="475" text-anchor="middle" class="small">api</text>
<rect x="728.6" y="416.5" width="57.8" height="33.5" rx="9" class="bar"/>
<text x="757.5" y="408.5" text-anchor="middle" class="small">13.5</text>
<text x="757.5" y="475" text-anchor="middle" class="small">load-balancer</text>
<rect x="833.6" y="90.0" width="57.8" height="360.0" rx="9" class="bar"/>
<text x="862.5" y="82.0" text-anchor="middle" class="small">145.2</text>
<text x="862.5" y="475" text-anchor="middle" class="small">postgres</text>
<rect x="938.6" y="419.6" width="57.8" height="30.4" rx="9" class="bar"/>
<text x="967.5" y="411.6" text-anchor="middle" class="small">12.3</text>
<text x="967.5" y="475" text-anchor="middle" class="small">redis</text>
<rect x="618.9" y="321.9" width="46.2" height="128.1" rx="9" class="bar"/>
<text x="642.0" y="313.9" text-anchor="middle" class="small">66.2</text>
<text x="642.0" y="475" text-anchor="middle" class="small">api</text>
<rect x="702.9" y="426.4" width="46.2" height="23.6" rx="9" class="bar"/>
<text x="726.0" y="418.4" text-anchor="middle" class="small">12.2</text>
<text x="726.0" y="475" text-anchor="middle" class="small">frontend</text>
<rect x="786.9" y="412.0" width="46.2" height="38.0" rx="9" class="bar"/>
<text x="810.0" y="404.0" text-anchor="middle" class="small">19.6</text>
<text x="810.0" y="475" text-anchor="middle" class="small">load-balancer</text>
<rect x="870.9" y="90.0" width="46.2" height="360.0" rx="9" class="bar"/>
<text x="894.0" y="82.0" text-anchor="middle" class="small">185.9</text>
<text x="894.0" y="475" text-anchor="middle" class="small">postgres</text>
<rect x="954.9" y="417.8" width="46.2" height="32.2" rx="9" class="bar"/>
<text x="978.0" y="409.8" text-anchor="middle" class="small">16.6</text>
<text x="978.0" y="475" text-anchor="middle" class="small">redis</text>
</svg>

Before

Width:  |  Height:  |  Size: 2.5 KiB

After

Width:  |  Height:  |  Size: 3.0 KiB

+313 -308
View File
@@ -1,8 +1,8 @@
{
"schema_version": 1,
"environment": {
"utc_started_at": "2026-10-23T12:31:12.749687+00:00",
"git_commit": "34618bd4237bbb8bf6a3a303056fc95d799577f9",
"utc_started_at": "2026-10-23T15:23:05.031807+00:00",
"git_commit": "5300a1a1e0e5497f2d30097bb1542b2743a8aad5",
"working_tree_dirty": true,
"logical_cpu_count": 8,
"host_memory_mib": 3813.6,
@@ -19,6 +19,11 @@
"nano_cpus": 500000000,
"memory_limit_bytes": 268435456
},
{
"service": "frontend",
"nano_cpus": 100000000,
"memory_limit_bytes": 67108864
},
{
"service": "load-balancer",
"nano_cpus": 250000000,
@@ -43,611 +48,611 @@
"action_after_seconds": 3,
"post_action_seconds": 8,
"warmup_requests": 12,
"workload": "closed-loop POST messages via Nginx; no automatic retries"
"workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries"
},
"phases": [
{
"name": "baseline",
"elapsed_seconds": 12.574,
"successful_requests": 4017,
"elapsed_seconds": 12.015,
"successful_requests": 5762,
"failed_requests": 0,
"error_ratio": 0.0,
"errors": {},
"successful_rps": 319.463,
"successful_rps": 479.581,
"successful_latency_ms": {
"p50": 16.799,
"p95": 55.515,
"p99": 93.803
"p50": 9.178,
"p95": 56.402,
"p99": 79.978
},
"instances": [
"0b933a46fccb",
"55aaffcf508a"
"a85567c48d67",
"df8a69558af0"
],
"recovery_seconds": null,
"action": null,
"per_second": {
"0": {
"successful": 118,
"successful": 434,
"failed": 0
},
"1": {
"successful": 308,
"successful": 280,
"failed": 0
},
"2": {
"successful": 386,
"successful": 372,
"failed": 0
},
"3": {
"successful": 378,
"successful": 436,
"failed": 0
},
"4": {
"successful": 344,
"successful": 541,
"failed": 0
},
"5": {
"successful": 322,
"successful": 474,
"failed": 0
},
"6": {
"successful": 316,
"successful": 526,
"failed": 0
},
"7": {
"successful": 384,
"successful": 539,
"failed": 0
},
"8": {
"successful": 380,
"successful": 554,
"failed": 0
},
"9": {
"successful": 402,
"successful": 559,
"failed": 0
},
"10": {
"successful": 377,
"successful": 448,
"failed": 0
},
"11": {
"successful": 288,
"successful": 584,
"failed": 0
},
"12": {
"successful": 14,
"successful": 15,
"failed": 0
}
},
"resources": {
"api": {
"cpu_percent_avg": 28.453,
"cpu_percent_max": 36.1,
"memory_mib_avg": 42.413,
"memory_mib_max": 66.07
"cpu_percent_avg": 27.144,
"cpu_percent_max": 30.93,
"memory_mib_avg": 31.008,
"memory_mib_max": 66.17
},
"frontend": {
"cpu_percent_avg": 1.812,
"cpu_percent_max": 3.76,
"memory_mib_avg": 7.212,
"memory_mib_max": 10.15
},
"load-balancer": {
"cpu_percent_avg": 17.448,
"cpu_percent_max": 20.8,
"memory_mib_avg": 9.214,
"memory_mib_max": 9.559
"cpu_percent_avg": 26.262,
"cpu_percent_max": 27.37,
"memory_mib_avg": 15.645,
"memory_mib_max": 15.97
},
"postgres": {
"cpu_percent_avg": 85.888,
"cpu_percent_max": 105.25,
"memory_mib_avg": 97.975,
"memory_mib_max": 100.8
"cpu_percent_avg": 89.392,
"cpu_percent_max": 110.59,
"memory_mib_avg": 130.125,
"memory_mib_max": 132.2
},
"redis": {
"cpu_percent_avg": 2.552,
"cpu_percent_max": 5.79,
"memory_mib_avg": 8.796,
"memory_mib_max": 11.74
"cpu_percent_avg": 2.488,
"cpu_percent_max": 6.58,
"memory_mib_avg": 15.072,
"memory_mib_max": 15.25
}
}
},
{
"name": "failover-1",
"elapsed_seconds": 15.324,
"successful_requests": 4613,
"failed_requests": 1,
"error_ratio": 0.000217,
"errors": {
"POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 504": 1
},
"successful_rps": 301.04,
"elapsed_seconds": 14.609,
"successful_requests": 7853,
"failed_requests": 0,
"error_ratio": 0.0,
"errors": {},
"successful_rps": 537.548,
"successful_latency_ms": {
"p50": 17.145,
"p95": 69.558,
"p99": 89.752
"p50": 8.812,
"p95": 50.05,
"p99": 66.373
},
"instances": [
"0b933a46fccb",
"55aaffcf508a"
"a85567c48d67",
"df8a69558af0"
],
"recovery_seconds": 3.464,
"recovery_seconds": 2.893,
"action": {
"started_at": 3.006,
"ended_at": 6.463,
"duration_seconds": 3.457,
"started_at": 3.003,
"ended_at": 5.888,
"duration_seconds": 2.885,
"exit_code": 0,
"summary": "stopped API instance 55aaffcf508a"
"summary": "stopped API instance df8a69558af0"
},
"per_second": {
"0": {
"successful": 404,
"successful": 551,
"failed": 0
},
"1": {
"successful": 426,
"successful": 605,
"failed": 0
},
"2": {
"successful": 358,
"successful": 558,
"failed": 0
},
"3": {
"successful": 399,
"successful": 443,
"failed": 0
},
"4": {
"successful": 392,
"successful": 575,
"failed": 0
},
"5": {
"successful": 322,
"successful": 538,
"failed": 0
},
"6": {
"successful": 299,
"successful": 593,
"failed": 0
},
"7": {
"successful": 329,
"failed": 1
"successful": 623,
"failed": 0
},
"8": {
"successful": 313,
"successful": 507,
"failed": 0
},
"9": {
"successful": 229,
"successful": 626,
"failed": 0
},
"10": {
"successful": 155,
"successful": 608,
"failed": 0
},
"11": {
"successful": 187,
"successful": 601,
"failed": 0
},
"12": {
"successful": 299,
"successful": 556,
"failed": 0
},
"13": {
"successful": 317,
"failed": 0
},
"14": {
"successful": 184,
"successful": 469,
"failed": 0
}
},
"resources": {
"api": {
"cpu_percent_avg": 27.62,
"cpu_percent_max": 52.59,
"memory_mib_avg": 11.58,
"memory_mib_max": 12.66
"cpu_percent_avg": 35.916,
"cpu_percent_max": 49.48,
"memory_mib_avg": 11.892,
"memory_mib_max": 13.27
},
"frontend": {
"cpu_percent_avg": 0.762,
"cpu_percent_max": 3.75,
"memory_mib_avg": 6.256,
"memory_mib_max": 6.434
},
"load-balancer": {
"cpu_percent_avg": 18.536,
"cpu_percent_max": 21.25,
"memory_mib_avg": 9.351,
"memory_mib_max": 9.523
"cpu_percent_avg": 22.21,
"cpu_percent_max": 26.89,
"memory_mib_avg": 16.312,
"memory_mib_max": 16.93
},
"postgres": {
"cpu_percent_avg": 81.822,
"cpu_percent_max": 104.67,
"memory_mib_avg": 100.512,
"memory_mib_max": 109.1
"cpu_percent_avg": 73.128,
"cpu_percent_max": 89.46,
"memory_mib_avg": 135.64,
"memory_mib_max": 141.9
},
"redis": {
"cpu_percent_avg": 1.486,
"cpu_percent_max": 5.88,
"memory_mib_avg": 8.04,
"memory_mib_max": 8.25
"cpu_percent_avg": 0.782,
"cpu_percent_max": 3.27,
"memory_mib_avg": 15.258,
"memory_mib_max": 15.73
}
}
},
{
"name": "failover-2",
"elapsed_seconds": 15.529,
"successful_requests": 5815,
"elapsed_seconds": 14.949,
"successful_requests": 7970,
"failed_requests": 0,
"error_ratio": 0.0,
"errors": {},
"successful_rps": 374.449,
"successful_rps": 533.154,
"successful_latency_ms": {
"p50": 14.211,
"p95": 49.542,
"p99": 81.04
"p50": 8.726,
"p95": 48.789,
"p99": 64.333
},
"instances": [
"0b933a46fccb",
"55aaffcf508a"
"a85567c48d67",
"df8a69558af0"
],
"recovery_seconds": 3.476,
"recovery_seconds": 2.938,
"action": {
"started_at": 3.013,
"ended_at": 6.485,
"duration_seconds": 3.472,
"started_at": 3.006,
"ended_at": 5.933,
"duration_seconds": 2.927,
"exit_code": 0,
"summary": "stopped API instance 0b933a46fccb"
"summary": "stopped API instance a85567c48d67"
},
"per_second": {
"0": {
"successful": 396,
"successful": 584,
"failed": 0
},
"1": {
"successful": 439,
"successful": 613,
"failed": 0
},
"2": {
"successful": 412,
"successful": 570,
"failed": 0
},
"3": {
"successful": 355,
"successful": 465,
"failed": 0
},
"4": {
"successful": 378,
"successful": 564,
"failed": 0
},
"5": {
"successful": 440,
"successful": 605,
"failed": 0
},
"6": {
"successful": 362,
"successful": 610,
"failed": 0
},
"7": {
"successful": 419,
"successful": 579,
"failed": 0
},
"8": {
"successful": 385,
"successful": 497,
"failed": 0
},
"9": {
"successful": 372,
"successful": 575,
"failed": 0
},
"10": {
"successful": 420,
"successful": 569,
"failed": 0
},
"11": {
"successful": 423,
"successful": 600,
"failed": 0
},
"12": {
"successful": 432,
"successful": 586,
"failed": 0
},
"13": {
"successful": 375,
"failed": 0
},
"14": {
"successful": 207,
"successful": 553,
"failed": 0
}
},
"resources": {
"api": {
"cpu_percent_avg": 31.411,
"cpu_percent_max": 51.63,
"memory_mib_avg": 10.373,
"memory_mib_max": 11.6
"cpu_percent_avg": 28.073,
"cpu_percent_max": 48.59,
"memory_mib_avg": 10.391,
"memory_mib_max": 13.47
},
"frontend": {
"cpu_percent_avg": 0.696,
"cpu_percent_max": 3.44,
"memory_mib_avg": 6.25,
"memory_mib_max": 6.398
},
"load-balancer": {
"cpu_percent_avg": 15.778,
"cpu_percent_max": 21.79,
"memory_mib_avg": 9.62,
"memory_mib_max": 9.781
"cpu_percent_avg": 25.966,
"cpu_percent_max": 26.65,
"memory_mib_avg": 16.918,
"memory_mib_max": 17.25
},
"postgres": {
"cpu_percent_avg": 76.87,
"cpu_percent_max": 104.51,
"memory_mib_avg": 106.016,
"memory_mib_max": 120.0
"cpu_percent_avg": 72.558,
"cpu_percent_max": 93.6,
"memory_mib_avg": 146.22,
"memory_mib_max": 160.2
},
"redis": {
"cpu_percent_avg": 0.786,
"cpu_percent_max": 2.83,
"memory_mib_avg": 8.844,
"memory_mib_max": 12.28
"cpu_percent_avg": 0.168,
"cpu_percent_max": 0.2,
"memory_mib_avg": 15.646,
"memory_mib_max": 16.64
}
}
},
{
"name": "rolling-update",
"elapsed_seconds": 53.985,
"successful_requests": 20350,
"failed_requests": 1,
"error_ratio": 4.9e-05,
"errors": {
"POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 502": 1
},
"successful_rps": 376.959,
"elapsed_seconds": 51.793,
"successful_requests": 22212,
"failed_requests": 0,
"error_ratio": 0.0,
"errors": {},
"successful_rps": 428.86,
"successful_latency_ms": {
"p50": 15.725,
"p95": 48.724,
"p99": 79.337
"p50": 10.56,
"p95": 58.713,
"p99": 83.982
},
"instances": [
"0b933a46fccb",
"3adbcad9e34b",
"55aaffcf508a",
"ceca3f6f1faf",
"f0a3bedda85a"
"872dbac02cd3",
"a85567c48d67",
"dac03e904506",
"df00f21e6f41",
"df8a69558af0"
],
"recovery_seconds": null,
"action": {
"started_at": 3.016,
"ended_at": 45.961,
"duration_seconds": 42.945,
"started_at": 3.005,
"ended_at": 42.955,
"duration_seconds": 39.95,
"exit_code": 0,
"summary": "rolling update completed with 2 healthy API replicas"
},
"per_second": {
"0": {
"successful": 340,
"successful": 509,
"failed": 0
},
"1": {
"successful": 369,
"successful": 577,
"failed": 0
},
"2": {
"successful": 426,
"successful": 660,
"failed": 0
},
"3": {
"successful": 382,
"successful": 438,
"failed": 0
},
"4": {
"successful": 371,
"successful": 577,
"failed": 0
},
"5": {
"successful": 376,
"successful": 392,
"failed": 0
},
"6": {
"successful": 328,
"successful": 326,
"failed": 0
},
"7": {
"successful": 350,
"successful": 232,
"failed": 0
},
"8": {
"successful": 434,
"successful": 199,
"failed": 0
},
"9": {
"successful": 411,
"successful": 170,
"failed": 0
},
"10": {
"successful": 399,
"successful": 263,
"failed": 0
},
"11": {
"successful": 417,
"successful": 323,
"failed": 0
},
"12": {
"successful": 374,
"successful": 308,
"failed": 0
},
"13": {
"successful": 427,
"failed": 0
},
"14": {
"successful": 414,
"failed": 0
},
"15": {
"successful": 418,
"failed": 0
},
"16": {
"successful": 408,
"failed": 0
},
"17": {
"successful": 352,
"failed": 0
},
"18": {
"successful": 391,
"failed": 0
},
"19": {
"successful": 388,
"failed": 0
},
"20": {
"successful": 413,
"failed": 0
},
"21": {
"successful": 417,
"failed": 0
},
"22": {
"successful": 388,
"failed": 0
},
"23": {
"successful": 396,
"failed": 0
},
"24": {
"successful": 400,
"failed": 0
},
"25": {
"successful": 339,
"failed": 0
},
"26": {
"successful": 225,
"failed": 0
},
"27": {
"successful": 376,
"failed": 0
},
"28": {
"successful": 385,
"failed": 0
},
"29": {
"successful": 396,
"failed": 0
},
"30": {
"successful": 403,
"failed": 0
},
"31": {
"successful": 262,
"failed": 0
},
"32": {
"successful": 428,
"failed": 0
},
"33": {
"successful": 362,
"failed": 0
},
"34": {
"successful": 286,
"14": {
"successful": 443,
"failed": 0
},
"35": {
"successful": 337,
"15": {
"successful": 495,
"failed": 0
},
"36": {
"successful": 328,
"16": {
"successful": 429,
"failed": 0
},
"37": {
"successful": 424,
"17": {
"successful": 370,
"failed": 0
},
"38": {
"successful": 356,
"18": {
"successful": 451,
"failed": 0
},
"39": {
"successful": 386,
"19": {
"successful": 377,
"failed": 0
},
"40": {
"20": {
"successful": 422,
"failed": 0
},
"21": {
"successful": 485,
"failed": 0
},
"22": {
"successful": 473,
"failed": 0
},
"23": {
"successful": 441,
"failed": 0
},
"24": {
"successful": 470,
"failed": 0
},
"25": {
"successful": 487,
"failed": 0
},
"26": {
"successful": 504,
"failed": 0
},
"27": {
"successful": 516,
"failed": 0
},
"28": {
"successful": 441,
"failed": 0
},
"29": {
"successful": 390,
"failed": 0
},
"30": {
"successful": 502,
"failed": 0
},
"31": {
"successful": 450,
"failed": 0
},
"32": {
"successful": 500,
"failed": 0
},
"33": {
"successful": 363,
"failed": 0
},
"34": {
"successful": 470,
"failed": 0
},
"35": {
"successful": 434,
"failed": 0
},
"36": {
"successful": 424,
"failed": 0
},
"37": {
"successful": 373,
"failed": 0
},
"38": {
"successful": 460,
"failed": 0
},
"39": {
"successful": 516,
"failed": 0
},
"40": {
"successful": 375,
"failed": 0
},
"41": {
"successful": 285,
"successful": 492,
"failed": 0
},
"42": {
"successful": 279,
"successful": 477,
"failed": 0
},
"43": {
"successful": 222,
"failed": 1
"successful": 503,
"failed": 0
},
"44": {
"successful": 316,
"successful": 506,
"failed": 0
},
"45": {
"successful": 359,
"successful": 465,
"failed": 0
},
"46": {
"successful": 405,
"successful": 501,
"failed": 0
},
"47": {
"successful": 398,
"successful": 502,
"failed": 0
},
"48": {
"successful": 395,
"successful": 446,
"failed": 0
},
"49": {
"successful": 430,
"successful": 446,
"failed": 0
},
"50": {
"successful": 453,
"failed": 0
},
"51": {
"successful": 409,
"failed": 0
},
"52": {
"successful": 445,
"failed": 0
},
"53": {
"successful": 450,
"successful": 477,
"failed": 0
}
},
"resources": {
"api": {
"cpu_percent_avg": 25.598,
"cpu_percent_max": 37.65,
"memory_mib_avg": 8.956,
"memory_mib_max": 10.97
"cpu_percent_avg": 20.954,
"cpu_percent_max": 31.69,
"memory_mib_avg": 8.855,
"memory_mib_max": 12.74
},
"frontend": {
"cpu_percent_avg": 1.326,
"cpu_percent_max": 8.06,
"memory_mib_avg": 7.43,
"memory_mib_max": 12.17
},
"load-balancer": {
"cpu_percent_avg": 19.935,
"cpu_percent_max": 23.36,
"memory_mib_avg": 9.836,
"memory_mib_max": 13.52
"cpu_percent_avg": 24.581,
"cpu_percent_max": 29.42,
"memory_mib_avg": 17.398,
"memory_mib_max": 19.6
},
"postgres": {
"cpu_percent_avg": 98.172,
"cpu_percent_max": 105.11,
"memory_mib_avg": 127.975,
"memory_mib_max": 145.2
"cpu_percent_avg": 77.881,
"cpu_percent_max": 102.6,
"memory_mib_avg": 170.081,
"memory_mib_max": 185.9
},
"redis": {
"cpu_percent_avg": 0.722,
"cpu_percent_max": 3.94,
"memory_mib_avg": 7.661,
"memory_mib_max": 11.84
"cpu_percent_avg": 0.63,
"cpu_percent_max": 3.84,
"memory_mib_avg": 15.178,
"memory_mib_max": 15.91
}
}
}
@@ -658,18 +663,18 @@
"failed_responses": 0,
"unique_message_ids": 1,
"observed_instances": [
"0b933a46fccb",
"55aaffcf508a"
"a85567c48d67",
"df8a69558af0"
]
},
"dataset": {
"warmup_requests": 12,
"minimum_confirmed_messages": 34808
"minimum_confirmed_messages": 43810
},
"limitations": [
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
"PostgreSQL and Nginx remain single points of failure"
"PostgreSQL and HAProxy remain single points of failure"
]
}
+14 -14
View File
@@ -16,29 +16,29 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
<line x1="80" y1="450.0" x2="1040" y2="450.0" class="grid"/>
<text x="68" y="454.0" text-anchor="end" class="small">0</text>
<line x1="80" y1="378.0" x2="1040" y2="378.0" class="grid"/>
<text x="68" y="382.0" text-anchor="end" class="small">75</text>
<text x="68" y="382.0" text-anchor="end" class="small">108</text>
<line x1="80" y1="306.0" x2="1040" y2="306.0" class="grid"/>
<text x="68" y="310.0" text-anchor="end" class="small">151</text>
<text x="68" y="310.0" text-anchor="end" class="small">215</text>
<line x1="80" y1="234.0" x2="1040" y2="234.0" class="grid"/>
<text x="68" y="238.0" text-anchor="end" class="small">226</text>
<text x="68" y="238.0" text-anchor="end" class="small">323</text>
<line x1="80" y1="162.0" x2="1040" y2="162.0" class="grid"/>
<text x="68" y="166.0" text-anchor="end" class="small">302</text>
<text x="68" y="166.0" text-anchor="end" class="small">430</text>
<line x1="80" y1="90.0" x2="1040" y2="90.0" class="grid"/>
<text x="68" y="94.0" text-anchor="end" class="small">377</text>
<rect x="140.0" y="144.9" width="120.0" height="305.1" rx="10" class="bar"/>
<text x="200.0" y="134.9" text-anchor="middle" class="label">319.5 RPS</text>
<text x="68" y="94.0" text-anchor="end" class="small">538</text>
<rect x="140.0" y="128.8" width="120.0" height="321.2" rx="10" class="bar"/>
<text x="200.0" y="118.8" text-anchor="middle" class="label">479.6 RPS</text>
<text x="200.0" y="478" text-anchor="middle" class="label">baseline</text>
<text x="200.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
<rect x="380.0" y="162.5" width="120.0" height="287.5" rx="10" class="bar"/>
<text x="440.0" y="152.5" text-anchor="middle" class="label">301.0 RPS</text>
<rect x="380.0" y="90.0" width="120.0" height="360.0" rx="10" class="bar"/>
<text x="440.0" y="80.0" text-anchor="middle" class="label">537.5 RPS</text>
<text x="440.0" y="478" text-anchor="middle" class="label">failover-1</text>
<text x="440.0" y="498" text-anchor="middle" class="small">errors 0.02%</text>
<rect x="620.0" y="92.4" width="120.0" height="357.6" rx="10" class="bar"/>
<text x="680.0" y="82.4" text-anchor="middle" class="label">374.4 RPS</text>
<text x="440.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
<rect x="620.0" y="92.9" width="120.0" height="357.1" rx="10" class="bar"/>
<text x="680.0" y="82.9" text-anchor="middle" class="label">533.2 RPS</text>
<text x="680.0" y="478" text-anchor="middle" class="label">failover-2</text>
<text x="680.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
<rect x="860.0" y="90.0" width="120.0" height="360.0" rx="10" class="bar"/>
<text x="920.0" y="80.0" text-anchor="middle" class="label">377.0 RPS</text>
<rect x="860.0" y="162.8" width="120.0" height="287.2" rx="10" class="bar"/>
<text x="920.0" y="152.8" text-anchor="middle" class="label">428.9 RPS</text>
<text x="920.0" y="478" text-anchor="middle" class="label">rolling-update</text>
<text x="920.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
<text x="24" y="270.0" transform="rotate(-90 24 270.0)" class="label">Successful requests per second</text>

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 2.8 KiB

+12 -14
View File
@@ -16,23 +16,21 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
<line x1="70" y1="480.0" x2="1050" y2="480.0" class="grid"/>
<text x="58" y="484.0" text-anchor="end" class="small">0</text>
<line x1="70" y1="402.0" x2="1050" y2="402.0" class="grid"/>
<text x="58" y="406.0" text-anchor="end" class="small">91</text>
<text x="58" y="406.0" text-anchor="end" class="small">132</text>
<line x1="70" y1="324.0" x2="1050" y2="324.0" class="grid"/>
<text x="58" y="328.0" text-anchor="end" class="small">181</text>
<text x="58" y="328.0" text-anchor="end" class="small">264</text>
<line x1="70" y1="246.0" x2="1050" y2="246.0" class="grid"/>
<text x="58" y="250.0" text-anchor="end" class="small">272</text>
<text x="58" y="250.0" text-anchor="end" class="small">396</text>
<line x1="70" y1="168.0" x2="1050" y2="168.0" class="grid"/>
<text x="58" y="172.0" text-anchor="end" class="small">362</text>
<text x="58" y="172.0" text-anchor="end" class="small">528</text>
<line x1="70" y1="90.0" x2="1050" y2="90.0" class="grid"/>
<text x="58" y="94.0" text-anchor="end" class="small">453</text>
<circle cx="277.9" cy="472" r="4" class="error"/>
<circle cx="951.0" cy="472" r="4" class="error"/>
<polyline points="70.0,378.4 79.9,214.8 89.8,147.7 99.7,154.6 109.6,183.8 119.5,202.8 129.4,207.9 139.3,149.4 149.2,152.8 159.1,133.9 169.0,155.4 178.9,232.1 188.8,467.9 208.6,132.2 218.5,113.2 228.4,171.8 238.3,136.5 248.2,142.5 258.1,202.8 268.0,222.6 277.9,196.8 287.8,210.5 297.7,282.8 307.6,346.6 317.5,319.0 327.4,222.6 337.3,207.1 347.2,321.6 367.0,139.1 376.9,102.1 386.8,125.3 396.7,174.4 406.6,154.6 416.5,101.2 426.4,168.3 436.3,119.3 446.2,148.5 456.1,159.7 466.0,118.4 475.9,115.8 485.8,108.1 495.7,157.2 505.6,301.8 525.4,187.3 535.3,162.3 545.2,113.2 555.1,151.1 564.9,160.6 574.8,156.3 584.7,197.6 594.6,178.7 604.5,106.4 614.4,126.2 624.3,136.5 634.2,121.0 644.1,158.0 654.0,112.4 663.9,123.6 673.8,120.1 683.7,128.7 693.6,177.0 703.5,143.4 713.4,146.0 723.3,124.4 733.2,121.0 743.1,146.0 753.0,139.1 762.9,135.6 772.8,188.1 782.7,286.3 792.6,156.3 802.5,148.5 812.4,139.1 822.3,133.0 832.2,254.4 842.1,111.5 852.0,168.3 861.9,233.8 871.8,189.9 881.7,197.6 891.6,115.0 901.5,173.5 911.4,147.7 921.3,116.7 931.2,234.6 941.1,239.8 951.0,288.9 960.9,207.9 970.8,170.9 980.7,131.3 990.6,137.4 1000.5,139.9 1010.4,109.8 1020.3,90.0 1030.2,127.9 1040.1,96.9 1050.0,92.6" class="line"/>
<line x1="238.3" y1="90" x2="238.3" y2="480" class="event"/>
<text x="244.3" y="108" class="small">failover-1 action</text>
<line x1="396.8" y1="90" x2="396.8" y2="480" class="event"/>
<text x="402.8" y="108" class="small">failover-2 action</text>
<line x1="555.2" y1="90" x2="555.2" y2="480" class="event"/>
<text x="561.2" y="108" class="small">rolling-update action</text>
<text x="58" y="94.0" text-anchor="end" class="small">660</text>
<polyline points="70.0,223.5 80.4,314.5 90.9,260.2 101.3,222.4 111.7,160.3 122.1,199.9 132.6,169.2 143.0,161.5 153.4,152.6 163.8,149.7 174.3,215.3 184.7,134.9 195.1,471.1 216.0,154.4 226.4,122.5 236.8,150.3 247.2,218.2 257.7,140.2 268.1,162.1 278.5,129.6 288.9,111.9 299.4,180.4 309.8,110.1 320.2,120.7 330.6,124.9 341.1,151.5 351.5,202.9 372.3,134.9 382.8,117.8 393.2,143.2 403.6,205.2 414.0,146.7 424.5,122.5 434.9,119.5 445.3,137.9 455.7,186.3 466.2,140.2 476.6,143.8 487.0,125.5 497.4,133.7 507.9,153.2 528.7,179.2 539.1,139.0 549.6,90.0 560.0,221.2 570.4,139.0 580.9,248.4 591.3,287.4 601.7,342.9 612.1,362.4 622.6,379.5 633.0,324.6 643.4,289.1 653.8,298.0 664.3,266.1 674.7,218.2 685.1,187.5 695.5,226.5 706.0,261.4 716.4,213.5 726.8,257.2 737.2,230.6 747.7,193.4 758.1,200.5 768.5,219.4 778.9,202.3 789.4,192.2 799.8,182.2 810.2,175.1 820.6,219.4 831.1,249.5 841.5,183.4 851.9,214.1 862.3,184.5 872.8,265.5 883.2,202.3 893.6,223.5 904.0,229.5 914.5,259.6 924.9,208.2 935.3,175.1 945.7,258.4 956.2,189.3 966.6,198.1 977.0,182.8 987.4,181.0 997.9,205.2 1008.3,184.0 1018.7,183.4 1029.1,216.5 1039.6,216.5 1050.0,198.1" class="line"/>
<line x1="247.3" y1="90" x2="247.3" y2="480" class="event"/>
<text x="253.3" y="108" class="small">failover-1 action</text>
<line x1="403.7" y1="90" x2="403.7" y2="480" class="event"/>
<text x="409.7" y="108" class="small">failover-2 action</text>
<line x1="560.1" y1="90" x2="560.1" y2="480" class="event"/>
<text x="566.1" y="108" class="small">rolling-update action</text>
<text x="70" y="512" class="label">One-second buckets; circles mark failed requests</text>
</svg>

Before

Width:  |  Height:  |  Size: 3.2 KiB

After

Width:  |  Height:  |  Size: 3.0 KiB

+63
View File
@@ -0,0 +1,63 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.subtitle { font-size: 14px; fill: #aeb2b8; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.axis { stroke: #747981; stroke-width: 1.5; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
.point { fill: #f4f5f6; }
.point2 { fill: #92979f; }
.bar { fill: #d7d9dc; }
.bar2 { fill: #777c84; }
.bar3 { fill: #4c5057; }
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
</style>
<text x="50" y="44" class="title">Saturation curve</text>
<text x="50" y="70" class="subtitle">Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU</text>
<line x1="90" y1="500.0" x2="1030" y2="500.0" class="grid"/>
<text x="78" y="504.0" text-anchor="end" class="small">0</text>
<line x1="90" y1="422.0" x2="1030" y2="422.0" class="grid"/>
<text x="78" y="426.0" text-anchor="end" class="small">124</text>
<line x1="90" y1="344.0" x2="1030" y2="344.0" class="grid"/>
<text x="78" y="348.0" text-anchor="end" class="small">248</text>
<line x1="90" y1="266.0" x2="1030" y2="266.0" class="grid"/>
<text x="78" y="270.0" text-anchor="end" class="small">372</text>
<line x1="90" y1="188.0" x2="1030" y2="188.0" class="grid"/>
<text x="78" y="192.0" text-anchor="end" class="small">496</text>
<line x1="90" y1="110.0" x2="1030" y2="110.0" class="grid"/>
<text x="78" y="114.0" text-anchor="end" class="small">620</text>
<circle cx="90.0" cy="355.2" r="6" class="point"/>
<circle cx="90.0" cy="481.1" r="6" class="point2"/>
<text x="90.0" y="528" text-anchor="middle" class="label">1</text>
<text x="90.0" y="343.2" text-anchor="middle" class="small">230</text>
<circle cx="278.0" cy="298.7" r="6" class="point"/>
<circle cx="278.0" cy="473.1" r="6" class="point2"/>
<text x="278.0" y="528" text-anchor="middle" class="label">2</text>
<text x="278.0" y="286.7" text-anchor="middle" class="small">320</text>
<circle cx="466.0" cy="226.0" r="6" class="point"/>
<circle cx="466.0" cy="390.4" r="6" class="point2"/>
<text x="466.0" y="528" text-anchor="middle" class="label">4</text>
<text x="466.0" y="214.0" text-anchor="middle" class="small">436</text>
<circle cx="654.0" cy="204.0" r="6" class="point"/>
<circle cx="654.0" cy="289.7" r="6" class="point2"/>
<text x="654.0" y="528" text-anchor="middle" class="label">8</text>
<text x="654.0" y="192.0" text-anchor="middle" class="small">471</text>
<circle cx="842.0" cy="202.7" r="6" class="point"/>
<circle cx="842.0" cy="224.6" r="6" class="point2"/>
<text x="842.0" y="528" text-anchor="middle" class="label">16</text>
<text x="842.0" y="190.7" text-anchor="middle" class="small">473</text>
<circle cx="1030.0" cy="145.5" r="6" class="point"/>
<circle cx="1030.0" cy="145.5" r="6" class="point2"/>
<text x="1030.0" y="528" text-anchor="middle" class="label">32</text>
<text x="1030.0" y="133.5" text-anchor="middle" class="small">564</text>
<polyline points="90.0,355.2 278.0,298.7 466.0,226.0 654.0,204.0 842.0,202.7 1030.0,145.5" class="line"/>
<polyline points="90.0,481.1 278.0,473.1 466.0,390.4 654.0,289.7 842.0,224.6 1030.0,145.5" class="line2"/>
<text x="90" y="562" class="label">Concurrency</text>
<text x="810" y="562" class="label">white: RPS · gray: p95 normalized</text>
<text x="30" y="305.0" transform="rotate(-90 30 305.0)" class="label">Successful requests per second</text>
</svg>

After

Width:  |  Height:  |  Size: 3.6 KiB

+49
View File
@@ -0,0 +1,49 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.subtitle { font-size: 14px; fill: #aeb2b8; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.axis { stroke: #747981; stroke-width: 1.5; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
.point { fill: #f4f5f6; }
.point2 { fill: #92979f; }
.bar { fill: #d7d9dc; }
.bar2 { fill: #777c84; }
.bar3 { fill: #4c5057; }
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
</style>
<text x="50" y="44" class="title">Fixed total API budget</text>
<text x="50" y="70" class="subtitle">1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas</text>
<line x1="90" y1="480.0" x2="1030" y2="480.0" class="grid"/>
<text x="78" y="484.0" text-anchor="end" class="small">0</text>
<line x1="90" y1="406.0" x2="1030" y2="406.0" class="grid"/>
<text x="78" y="410.0" text-anchor="end" class="small">133</text>
<line x1="90" y1="332.0" x2="1030" y2="332.0" class="grid"/>
<text x="78" y="336.0" text-anchor="end" class="small">266</text>
<line x1="90" y1="258.0" x2="1030" y2="258.0" class="grid"/>
<text x="78" y="262.0" text-anchor="end" class="small">399</text>
<line x1="90" y1="184.0" x2="1030" y2="184.0" class="grid"/>
<text x="78" y="188.0" text-anchor="end" class="small">532</text>
<line x1="90" y1="110.0" x2="1030" y2="110.0" class="grid"/>
<text x="78" y="114.0" text-anchor="end" class="small">665</text>
<rect x="171.7" y="158.3" width="150.0" height="321.7" rx="14" class="bar"/>
<text x="246.7" y="146.3" text-anchor="middle" class="label">578.2 RPS</text>
<text x="246.7" y="510" text-anchor="middle" class="label">1 replica(s)</text>
<text x="246.7" y="532" text-anchor="middle" class="small">p95 95.8 ms</text>
<text x="246.7" y="550" text-anchor="middle" class="small">errors 0.000%</text>
<rect x="485.0" y="185.4" width="150.0" height="294.6" rx="14" class="bar"/>
<text x="560.0" y="173.4" text-anchor="middle" class="label">529.5 RPS</text>
<text x="560.0" y="510" text-anchor="middle" class="label">2 replica(s)</text>
<text x="560.0" y="532" text-anchor="middle" class="small">p95 100.1 ms</text>
<text x="560.0" y="550" text-anchor="middle" class="small">errors 0.000%</text>
<rect x="798.3" y="283.1" width="150.0" height="196.9" rx="14" class="bar"/>
<text x="873.3" y="271.1" text-anchor="middle" class="label">354.0 RPS</text>
<text x="873.3" y="510" text-anchor="middle" class="label">4 replica(s)</text>
<text x="873.3" y="532" text-anchor="middle" class="small">p95 139.4 ms</text>
<text x="873.3" y="550" text-anchor="middle" class="small">errors 0.000%</text>
</svg>

After

Width:  |  Height:  |  Size: 2.9 KiB

+49
View File
@@ -0,0 +1,49 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.subtitle { font-size: 14px; fill: #aeb2b8; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.axis { stroke: #747981; stroke-width: 1.5; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
.point { fill: #f4f5f6; }
.point2 { fill: #92979f; }
.bar { fill: #d7d9dc; }
.bar2 { fill: #777c84; }
.bar3 { fill: #4c5057; }
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
</style>
<text x="50" y="44" class="title">Growing API budget</text>
<text x="50" y="70" class="subtitle">Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU</text>
<line x1="90" y1="480.0" x2="1030" y2="480.0" class="grid"/>
<text x="78" y="484.0" text-anchor="end" class="small">0</text>
<line x1="90" y1="406.0" x2="1030" y2="406.0" class="grid"/>
<text x="78" y="410.0" text-anchor="end" class="small">130</text>
<line x1="90" y1="332.0" x2="1030" y2="332.0" class="grid"/>
<text x="78" y="336.0" text-anchor="end" class="small">259</text>
<line x1="90" y1="258.0" x2="1030" y2="258.0" class="grid"/>
<text x="78" y="262.0" text-anchor="end" class="small">389</text>
<line x1="90" y1="184.0" x2="1030" y2="184.0" class="grid"/>
<text x="78" y="188.0" text-anchor="end" class="small">518</text>
<line x1="90" y1="110.0" x2="1030" y2="110.0" class="grid"/>
<text x="78" y="114.0" text-anchor="end" class="small">648</text>
<rect x="171.7" y="175.3" width="150.0" height="304.7" rx="14" class="bar"/>
<text x="246.7" y="163.3" text-anchor="middle" class="label">533.7 RPS</text>
<text x="246.7" y="510" text-anchor="middle" class="label">1 replica(s)</text>
<text x="246.7" y="532" text-anchor="middle" class="small">p95 93.5 ms</text>
<text x="246.7" y="550" text-anchor="middle" class="small">errors 0.000%</text>
<rect x="485.0" y="158.3" width="150.0" height="321.7" rx="14" class="bar"/>
<text x="560.0" y="146.3" text-anchor="middle" class="label">563.5 RPS</text>
<text x="560.0" y="510" text-anchor="middle" class="label">2 replica(s)</text>
<text x="560.0" y="532" text-anchor="middle" class="small">p95 96.4 ms</text>
<text x="560.0" y="550" text-anchor="middle" class="small">errors 0.000%</text>
<rect x="798.3" y="229.4" width="150.0" height="250.6" rx="14" class="bar"/>
<text x="873.3" y="217.4" text-anchor="middle" class="label">438.9 RPS</text>
<text x="873.3" y="510" text-anchor="middle" class="label">4 replica(s)</text>
<text x="873.3" y="532" text-anchor="middle" class="small">p95 109.4 ms</text>
<text x="873.3" y="550" text-anchor="middle" class="small">errors 0.000%</text>
</svg>

After

Width:  |  Height:  |  Size: 2.9 KiB

+83
View File
@@ -0,0 +1,83 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.subtitle { font-size: 14px; fill: #aeb2b8; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.axis { stroke: #747981; stroke-width: 1.5; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
.point { fill: #f4f5f6; }
.point2 { fill: #92979f; }
.bar { fill: #d7d9dc; }
.bar2 { fill: #777c84; }
.bar3 { fill: #4c5057; }
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
</style>
<text x="50" y="44" class="title">Where scaling stops</text>
<text x="50" y="70" class="subtitle">Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis)</text>
<line x1="90" y1="480.0" x2="1030" y2="480.0" class="grid"/>
<text x="78" y="484.0" text-anchor="end" class="small">0</text>
<line x1="90" y1="406.0" x2="1030" y2="406.0" class="grid"/>
<text x="78" y="410.0" text-anchor="end" class="small">20</text>
<line x1="90" y1="332.0" x2="1030" y2="332.0" class="grid"/>
<text x="78" y="336.0" text-anchor="end" class="small">39</text>
<line x1="90" y1="258.0" x2="1030" y2="258.0" class="grid"/>
<text x="78" y="262.0" text-anchor="end" class="small">59</text>
<line x1="90" y1="184.0" x2="1030" y2="184.0" class="grid"/>
<text x="78" y="188.0" text-anchor="end" class="small">79</text>
<line x1="90" y1="110.0" x2="1030" y2="110.0" class="grid"/>
<text x="78" y="114.0" text-anchor="end" class="small">98</text>
<rect x="127.6" y="336.1" width="48.0" height="143.9" rx="7" class="bar"/>
<text x="151.6" y="329.1" text-anchor="middle" class="small">38</text>
<text x="151.6" y="502" text-anchor="middle" class="small">api</text>
<rect x="184.6" y="189.6" width="48.0" height="290.4" rx="7" class="bar2"/>
<text x="208.6" y="182.6" text-anchor="middle" class="small">77</text>
<text x="208.6" y="502" text-anchor="middle" class="small">pos</text>
<rect x="241.6" y="404.6" width="48.0" height="75.4" rx="7" class="bar3"/>
<text x="265.6" y="397.6" text-anchor="middle" class="small">20</text>
<text x="265.6" y="502" text-anchor="middle" class="small">loa</text>
<rect x="298.6" y="477.4" width="48.0" height="2.6" rx="7" class="bar3"/>
<text x="322.6" y="470.4" text-anchor="middle" class="small">1</text>
<text x="322.6" y="502" text-anchor="middle" class="small">fro</text>
<rect x="355.6" y="476.4" width="48.0" height="3.6" rx="7" class="bar3"/>
<text x="379.6" y="469.4" text-anchor="middle" class="small">1</text>
<text x="379.6" y="502" text-anchor="middle" class="small">red</text>
<text x="246.7" y="532" text-anchor="middle" class="label">1 replica(s)</text>
<rect x="440.9" y="304.6" width="48.0" height="175.4" rx="7" class="bar"/>
<text x="464.9" y="297.6" text-anchor="middle" class="small">47</text>
<text x="464.9" y="502" text-anchor="middle" class="small">api</text>
<rect x="497.9" y="158.3" width="48.0" height="321.7" rx="7" class="bar2"/>
<text x="521.9" y="151.3" text-anchor="middle" class="small">86</text>
<text x="521.9" y="502" text-anchor="middle" class="small">pos</text>
<rect x="554.9" y="400.3" width="48.0" height="79.7" rx="7" class="bar3"/>
<text x="578.9" y="393.3" text-anchor="middle" class="small">21</text>
<text x="578.9" y="502" text-anchor="middle" class="small">loa</text>
<rect x="611.9" y="477.1" width="48.0" height="2.9" rx="7" class="bar3"/>
<text x="635.9" y="470.1" text-anchor="middle" class="small">1</text>
<text x="635.9" y="502" text-anchor="middle" class="small">fro</text>
<rect x="668.9" y="476.7" width="48.0" height="3.3" rx="7" class="bar3"/>
<text x="692.9" y="469.7" text-anchor="middle" class="small">1</text>
<text x="692.9" y="502" text-anchor="middle" class="small">red</text>
<text x="560.0" y="532" text-anchor="middle" class="label">2 replica(s)</text>
<rect x="754.3" y="345.8" width="48.0" height="134.2" rx="7" class="bar"/>
<text x="778.3" y="338.8" text-anchor="middle" class="small">36</text>
<text x="778.3" y="502" text-anchor="middle" class="small">api</text>
<rect x="811.3" y="161.1" width="48.0" height="318.9" rx="7" class="bar2"/>
<text x="835.3" y="154.1" text-anchor="middle" class="small">85</text>
<text x="835.3" y="502" text-anchor="middle" class="small">pos</text>
<rect x="868.3" y="421.4" width="48.0" height="58.6" rx="7" class="bar3"/>
<text x="892.3" y="414.4" text-anchor="middle" class="small">16</text>
<text x="892.3" y="502" text-anchor="middle" class="small">loa</text>
<rect x="925.3" y="477.5" width="48.0" height="2.5" rx="7" class="bar3"/>
<text x="949.3" y="470.5" text-anchor="middle" class="small">1</text>
<text x="949.3" y="502" text-anchor="middle" class="small">fro</text>
<rect x="982.3" y="476.7" width="48.0" height="3.3" rx="7" class="bar3"/>
<text x="1006.3" y="469.7" text-anchor="middle" class="small">1</text>
<text x="1006.3" y="502" text-anchor="middle" class="small">red</text>
<text x="873.3" y="532" text-anchor="middle" class="label">4 replica(s)</text>
<text x="30" y="295.0" transform="rotate(-90 30 295.0)" class="label">Average aggregate CPU, %</text>
</svg>

After

Width:  |  Height:  |  Size: 5.2 KiB

+37
View File
@@ -0,0 +1,37 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.subtitle { font-size: 14px; fill: #aeb2b8; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.axis { stroke: #747981; stroke-width: 1.5; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
.point { fill: #f4f5f6; }
.point2 { fill: #92979f; }
.bar { fill: #d7d9dc; }
.bar2 { fill: #777c84; }
.bar3 { fill: #4c5057; }
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
</style>
<text x="50" y="44" class="title">Confirmed improvement</text>
<text x="50" y="70" class="subtitle">Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2</text>
<text x="90" y="98" class="label">Successful RPS</text>
<rect x="125" y="248.4" width="120" height="211.6" rx="14" class="bar2"/>
<text x="185" y="238.4" text-anchor="middle" class="label">438.9</text>
<text x="185" y="488" text-anchor="middle" class="small">PG 1 CPU</text>
<rect x="310" y="176.7" width="120" height="283.3" rx="14" class="bar"/>
<text x="370" y="166.7" text-anchor="middle" class="label">587.6</text>
<text x="370" y="488" text-anchor="middle" class="small">PG 2 CPU</text>
<text x="630" y="98" class="label">p95 latency, ms</text>
<rect x="665" y="176.7" width="120" height="283.3" rx="14" class="bar2"/>
<text x="725" y="166.7" text-anchor="middle" class="label">109.4</text>
<text x="725" y="488" text-anchor="middle" class="small">PG 1 CPU</text>
<rect x="850" y="216.7" width="120" height="243.3" rx="14" class="bar"/>
<text x="910" y="206.7" text-anchor="middle" class="label">94.0</text>
<text x="910" y="488" text-anchor="middle" class="small">PG 2 CPU</text>
<text x="90" y="545" class="label">RPS change: +33.88% · p95 change: -14.12% · confirmed: true</text>
</svg>

After

Width:  |  Height:  |  Size: 2.0 KiB

+69
View File
@@ -0,0 +1,69 @@
# Lab 3 grade-5 scaling experiment
Generated: 2026-10-23T15:23:12.734339+00:00.
## Hypothesis
API replicas stop improving throughput when shared PostgreSQL CPU saturates; adding PostgreSQL CPU should improve the unchanged four-replica profile.
## Reproduction
- workload: closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries;
- phase duration: 10 s;
- comparison concurrency: 32;
- saturation sweep: [1, 2, 4, 8, 16, 32];
- fixed budget: 1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections;
- growth series: 0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica;
- saturation rule: RPS gain below 10% and p95 growth at least 1.5x versus the previous step;
- host: 8 logical CPU, 3813.6 MiB RAM;
- Docker 29.7.2, Compose 5.5.0;
- command: `make grade5-experiment`; warmup runs before every changed profile.
## Measurements
| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
| saturation-c1 | 2 | 1 | 230.369 | 0 (0.000%) | 3.839 | 5.065 | 6.282 | 23.9 | 26.85 |
| saturation-c2 | 2 | 2 | 320.19 | 0 (0.000%) | 5.192 | 7.196 | 15.11 | 54.987 | 48.3 |
| saturation-c4 | 2 | 4 | 435.822 | 0 (0.000%) | 6.171 | 29.304 | 47.665 | 63.093 | 54.453 |
| saturation-c8 | 2 | 8 | 470.828 | 0 (0.000%) | 9.159 | 56.23 | 70.71 | 65.945 | 48.505 |
| saturation-c16 | 2 | 16 | 472.892 | 0 (0.000%) | 19.862 | 73.628 | 98.773 | 65.08 | 38.028 |
| saturation-c32 | 2 | 32 | 564.012 | 0 (0.000%) | 51.089 | 94.785 | 109.138 | 83.825 | 51.965 |
| fixed-r1 | 1 | 32 | 578.242 | 0 (0.000%) | 48.579 | 95.757 | 115.549 | 79.058 | 36.575 |
| fixed-r2 | 2 | 32 | 529.499 | 0 (0.000%) | 53.316 | 100.145 | 129.953 | 81.065 | 44.7 |
| fixed-r4 | 4 | 32 | 353.955 | 0 (0.000%) | 69.354 | 139.408 | 291.312 | 74.677 | 46.25 |
| growth-r1 | 1 | 32 | 533.656 | 0 (0.000%) | 53.378 | 93.541 | 115.58 | 77.252 | 38.273 |
| growth-r2 | 2 | 32 | 563.46 | 0 (0.000%) | 50.516 | 96.428 | 129.46 | 85.597 | 46.653 |
| growth-r4 | 4 | 32 | 438.868 | 0 (0.000%) | 68.331 | 109.446 | 199.616 | 84.847 | 35.69 |
| improved-db-cpu-r4 | 4 | 32 | 587.574 | 0 (0.000%) | 49.466 | 93.991 | 112.002 | 95.573 | 55.2 |
## Conclusion
The first saturation signal appears at concurrency 8: RPS changes by +8.03% while p95 changes by 1.92x versus concurrency 4.
Observed bottleneck classification: `postgres_cpu`.
The PostgreSQL CPU change produced +33.88% RPS and -14.12% p95; confirmed improvement: `true`.
After the improvement the remaining bottleneck classification is `load_balancer_cpu`.
## Graphs
![Saturation](01-saturation.svg)
![Fixed budget](02-fixed-total.svg)
![Growing budget](03-resource-growth.svg)
![Bottleneck](04-bottleneck.svg)
![Improvement](05-improvement.svg)
## Raw data and limits
Raw aggregates are in `measurements.csv`; one-second request buckets are in `timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and applied limits are in `results.json`.
- single Docker host: replicas do not prove host-level availability
- closed-loop workload has coordinated omission
- successful-response latency percentiles are reported separately from failures
- one measured run per profile; host scheduling noise is possible
+14
View File
@@ -0,0 +1,14 @@
phase,kind,replicas,concurrency,elapsed_seconds,successful_requests,failed_requests,error_ratio,successful_rps,p50_ms,p95_ms,p99_ms,api_cpu_avg,postgres_cpu_avg,load_balancer_cpu_avg,frontend_cpu_avg,redis_cpu_avg
saturation-c1,saturation,2,1,10.731,2472,0,0.0,230.369,3.839,5.065,6.282,26.85,23.9,11.662,0.688,0.8
saturation-c2,saturation,2,2,10.915,3495,0,0.0,320.19,5.192,7.196,15.11,48.3,54.987,19.13,0.325,0.193
saturation-c4,saturation,2,4,10.832,4721,0,0.0,435.822,6.171,29.304,47.665,54.453,63.093,20.843,0.95,0.81
saturation-c8,saturation,2,8,10.821,5095,0,0.0,470.828,9.159,56.23,70.71,48.505,65.945,20.867,0.86,0.953
saturation-c16,saturation,2,16,10.876,5143,0,0.0,472.892,19.862,73.628,98.773,38.028,65.08,20.98,1.113,0.172
saturation-c32,saturation,2,32,10.808,6096,0,0.0,564.012,51.089,94.785,109.138,51.965,83.825,21.163,1.837,2.475
fixed-r1,fixed,1,32,10.88,6291,0,0.0,578.242,48.579,95.757,115.549,36.575,79.058,20.732,0.005,1.232
fixed-r2,fixed,2,32,10.905,5774,0,0.0,529.499,53.316,100.145,129.953,44.7,81.065,21.15,0.823,1.093
fixed-r4,fixed,4,32,11.917,4218,0,0.0,353.955,69.354,139.408,291.312,46.25,74.677,21.18,0.637,2.715
growth-r1,growth,1,32,10.951,5844,0,0.0,533.656,53.378,93.541,115.58,38.273,77.252,20.062,0.693,0.958
growth-r2,growth,2,32,10.796,6083,0,0.0,563.46,50.516,96.428,129.46,46.653,85.597,21.192,0.765,0.865
growth-r4,growth,4,32,10.682,4688,0,0.0,438.868,68.331,109.446,199.616,35.69,84.847,15.595,0.667,0.873
improved-db-cpu-r4,improvement,4,32,10.601,6229,0,0.0,587.574,49.466,93.991,112.002,55.2,95.573,22.35,0.83,0.458
1 phase kind replicas concurrency elapsed_seconds successful_requests failed_requests error_ratio successful_rps p50_ms p95_ms p99_ms api_cpu_avg postgres_cpu_avg load_balancer_cpu_avg frontend_cpu_avg redis_cpu_avg
2 saturation-c1 saturation 2 1 10.731 2472 0 0.0 230.369 3.839 5.065 6.282 26.85 23.9 11.662 0.688 0.8
3 saturation-c2 saturation 2 2 10.915 3495 0 0.0 320.19 5.192 7.196 15.11 48.3 54.987 19.13 0.325 0.193
4 saturation-c4 saturation 2 4 10.832 4721 0 0.0 435.822 6.171 29.304 47.665 54.453 63.093 20.843 0.95 0.81
5 saturation-c8 saturation 2 8 10.821 5095 0 0.0 470.828 9.159 56.23 70.71 48.505 65.945 20.867 0.86 0.953
6 saturation-c16 saturation 2 16 10.876 5143 0 0.0 472.892 19.862 73.628 98.773 38.028 65.08 20.98 1.113 0.172
7 saturation-c32 saturation 2 32 10.808 6096 0 0.0 564.012 51.089 94.785 109.138 51.965 83.825 21.163 1.837 2.475
8 fixed-r1 fixed 1 32 10.88 6291 0 0.0 578.242 48.579 95.757 115.549 36.575 79.058 20.732 0.005 1.232
9 fixed-r2 fixed 2 32 10.905 5774 0 0.0 529.499 53.316 100.145 129.953 44.7 81.065 21.15 0.823 1.093
10 fixed-r4 fixed 4 32 11.917 4218 0 0.0 353.955 69.354 139.408 291.312 46.25 74.677 21.18 0.637 2.715
11 growth-r1 growth 1 32 10.951 5844 0 0.0 533.656 53.378 93.541 115.58 38.273 77.252 20.062 0.693 0.958
12 growth-r2 growth 2 32 10.796 6083 0 0.0 563.46 50.516 96.428 129.46 46.653 85.597 21.192 0.765 0.865
13 growth-r4 growth 4 32 10.682 4688 0 0.0 438.868 68.331 109.446 199.616 35.69 84.847 15.595 0.667 0.873
14 improved-db-cpu-r4 improvement 4 32 10.601 6229 0 0.0 587.574 49.466 93.991 112.002 55.2 95.573 22.35 0.83 0.458
+261
View File
@@ -0,0 +1,261 @@
phase,t,service,containers,cpu_percent,memory_mib
saturation-c1,0.0,api,2,26.09,17.976
saturation-c1,0.0,frontend,1,1.61,7.578
saturation-c1,0.0,load-balancer,1,13.0,17.18
saturation-c1,0.0,postgres,1,28.57,169.0
saturation-c1,0.0,redis,1,0.15,15.28
saturation-c1,2.356,api,2,32.65,16.886
saturation-c1,2.356,frontend,1,0.02,7.617
saturation-c1,2.356,load-balancer,1,13.32,17.23
saturation-c1,2.356,postgres,1,28.96,169.5
saturation-c1,2.356,redis,1,2.75,15.4
saturation-c1,5.485,api,2,27.72,17.586
saturation-c1,5.485,frontend,1,0.0,7.383
saturation-c1,5.485,load-balancer,1,13.69,17.41
saturation-c1,5.485,postgres,1,29.41,170.1
saturation-c1,5.485,redis,1,0.15,15.28
saturation-c1,8.605,api,2,20.94,17.41
saturation-c1,8.605,frontend,1,1.12,12.06
saturation-c1,8.605,load-balancer,1,6.64,17.24
saturation-c1,8.605,postgres,1,8.66,170.5
saturation-c1,8.605,redis,1,0.15,15.53
saturation-c2,0.001,api,2,51.0,17.761
saturation-c2,0.001,frontend,1,0.03,7.617
saturation-c2,0.001,load-balancer,1,24.78,17.71
saturation-c2,0.001,postgres,1,57.44,171.2
saturation-c2,0.001,redis,1,0.17,16.03
saturation-c2,2.361,api,2,45.61,17.516
saturation-c2,2.361,frontend,1,0.03,7.367
saturation-c2,2.361,load-balancer,1,24.6,17.41
saturation-c2,2.361,postgres,1,49.85,172.6
saturation-c2,2.361,redis,1,0.18,15.28
saturation-c2,5.543,api,2,51.19,16.258
saturation-c2,5.543,frontend,1,0.02,7.367
saturation-c2,5.543,load-balancer,1,24.27,17.91
saturation-c2,5.543,postgres,1,62.6,173.0
saturation-c2,5.543,redis,1,0.21,15.28
saturation-c2,8.736,api,2,45.4,17.082
saturation-c2,8.736,frontend,1,1.22,8.121
saturation-c2,8.736,load-balancer,1,2.87,17.2
saturation-c2,8.736,postgres,1,50.06,173.8
saturation-c2,8.736,redis,1,0.21,15.78
saturation-c4,0.001,api,2,65.41,17.297
saturation-c4,0.001,frontend,1,0.03,7.367
saturation-c4,0.001,load-balancer,1,25.95,17.71
saturation-c4,0.001,postgres,1,81.8,174.4
saturation-c4,0.001,redis,1,0.19,15.28
saturation-c4,2.372,api,2,60.01,18.188
saturation-c4,2.372,frontend,1,3.75,7.965
saturation-c4,2.372,load-balancer,1,26.04,17.46
saturation-c4,2.372,postgres,1,79.42,175.4
saturation-c4,2.372,redis,1,0.16,15.53
saturation-c4,5.501,api,2,56.53,17.433
saturation-c4,5.501,frontend,1,0.02,7.367
saturation-c4,5.501,load-balancer,1,25.9,21.22
saturation-c4,5.501,postgres,1,73.71,175.7
saturation-c4,5.501,redis,1,0.18,15.28
saturation-c4,8.662,api,2,35.86,16.699
saturation-c4,8.662,frontend,1,0.0,7.367
saturation-c4,8.662,load-balancer,1,5.48,17.53
saturation-c4,8.662,postgres,1,17.44,177.2
saturation-c4,8.662,redis,1,2.71,16.1
saturation-c8,0.006,api,2,48.34,17.403
saturation-c8,0.006,frontend,1,0.0,7.367
saturation-c8,0.006,load-balancer,1,25.42,20.12
saturation-c8,0.006,postgres,1,77.51,185.1
saturation-c8,0.006,redis,1,0.18,15.28
saturation-c8,2.372,api,2,57.14,18.602
saturation-c8,2.372,frontend,1,3.44,7.734
saturation-c8,2.372,load-balancer,1,26.83,17.14
saturation-c8,2.372,postgres,1,92.98,188.5
saturation-c8,2.372,redis,1,3.34,15.28
saturation-c8,5.527,api,2,48.26,18.192
saturation-c8,5.527,frontend,1,0.0,7.363
saturation-c8,5.527,load-balancer,1,25.47,17.28
saturation-c8,5.527,postgres,1,76.05,189.4
saturation-c8,5.527,redis,1,0.15,15.28
saturation-c8,8.685,api,2,40.28,21.813
saturation-c8,8.685,frontend,1,0.0,7.363
saturation-c8,8.685,load-balancer,1,5.75,17.64
saturation-c8,8.685,postgres,1,17.24,190.4
saturation-c8,8.685,redis,1,0.14,15.28
saturation-c16,0.015,api,2,44.39,18.796
saturation-c16,0.015,frontend,1,0.0,7.363
saturation-c16,0.015,load-balancer,1,26.92,17.44
saturation-c16,0.015,postgres,1,82.81,205.9
saturation-c16,0.015,redis,1,0.17,15.28
saturation-c16,2.354,api,2,56.54,17.856
saturation-c16,2.354,frontend,1,0.03,7.375
saturation-c16,2.354,load-balancer,1,26.66,17.69
saturation-c16,2.354,postgres,1,88.93,207.7
saturation-c16,2.354,redis,1,0.19,15.75
saturation-c16,5.525,api,2,41.39,17.73
saturation-c16,5.525,frontend,1,4.4,7.379
saturation-c16,5.525,load-balancer,1,25.76,17.7
saturation-c16,5.525,postgres,1,72.49,208.6
saturation-c16,5.525,redis,1,0.18,15.25
saturation-c16,8.731,api,2,9.79,18.257
saturation-c16,8.731,frontend,1,0.02,7.375
saturation-c16,8.731,load-balancer,1,4.58,17.55
saturation-c16,8.731,postgres,1,16.09,209.7
saturation-c16,8.731,redis,1,0.15,15.33
saturation-c32,0.039,api,2,55.84,18.75
saturation-c32,0.039,frontend,1,6.63,8.125
saturation-c32,0.039,load-balancer,1,25.97,18.0
saturation-c32,0.039,postgres,1,104.2,210.6
saturation-c32,0.039,redis,1,5.85,15.05
saturation-c32,2.29,api,2,56.87,18.933
saturation-c32,2.29,frontend,1,0.0,7.625
saturation-c32,2.29,load-balancer,1,27.03,19.23
saturation-c32,2.29,postgres,1,105.41,212.1
saturation-c32,2.29,redis,1,0.19,15.55
saturation-c32,5.446,api,2,64.13,19.003
saturation-c32,5.446,frontend,1,0.7,7.375
saturation-c32,5.446,load-balancer,1,26.33,18.27
saturation-c32,5.446,postgres,1,107.62,213.1
saturation-c32,5.446,redis,1,3.7,15.27
saturation-c32,8.653,api,2,31.02,19.379
saturation-c32,8.653,frontend,1,0.02,7.625
saturation-c32,8.653,load-balancer,1,5.32,18.36
saturation-c32,8.653,postgres,1,18.07,213.5
saturation-c32,8.653,redis,1,0.16,15.26
fixed-r1,0.034,api,1,45.68,10.61
fixed-r1,0.034,frontend,1,0.02,7.359
fixed-r1,0.034,load-balancer,1,25.97,18.08
fixed-r1,0.034,postgres,1,106.86,214.7
fixed-r1,0.034,redis,1,4.41,15.3
fixed-r1,2.424,api,1,48.23,11.36
fixed-r1,2.424,frontend,1,0.0,7.699
fixed-r1,2.424,load-balancer,1,27.02,22.21
fixed-r1,2.424,postgres,1,96.33,215.7
fixed-r1,2.424,redis,1,0.14,15.44
fixed-r1,5.588,api,1,48.84,12.07
fixed-r1,5.588,frontend,1,0.0,7.359
fixed-r1,5.588,load-balancer,1,26.38,18.43
fixed-r1,5.588,postgres,1,104.71,217.4
fixed-r1,5.588,redis,1,0.19,15.92
fixed-r1,8.743,api,1,3.55,11.93
fixed-r1,8.743,frontend,1,0.0,7.508
fixed-r1,8.743,load-balancer,1,3.56,17.96
fixed-r1,8.743,postgres,1,8.33,218.1
fixed-r1,8.743,redis,1,0.19,16.07
fixed-r2,0.041,api,2,52.24,18.086
fixed-r2,0.041,frontend,1,0.0,7.359
fixed-r2,0.041,load-balancer,1,26.74,18.2
fixed-r2,0.041,postgres,1,100.69,219.4
fixed-r2,0.041,redis,1,3.81,15.18
fixed-r2,2.379,api,2,65.39,18.801
fixed-r2,2.379,frontend,1,1.92,7.859
fixed-r2,2.379,load-balancer,1,26.23,18.61
fixed-r2,2.379,postgres,1,106.33,221.3
fixed-r2,2.379,redis,1,0.16,15.16
fixed-r2,5.564,api,2,54.12,18.379
fixed-r2,5.564,frontend,1,1.37,10.54
fixed-r2,5.564,load-balancer,1,25.33,18.35
fixed-r2,5.564,postgres,1,103.06,222.7
fixed-r2,5.564,redis,1,0.2,14.93
fixed-r2,8.728,api,2,7.05,18.086
fixed-r2,8.728,frontend,1,0.0,7.363
fixed-r2,8.728,load-balancer,1,6.3,17.82
fixed-r2,8.728,postgres,1,14.18,223.3
fixed-r2,8.728,redis,1,0.2,15.43
fixed-r4,0.095,api,4,51.98,27.492
fixed-r4,0.095,frontend,1,0.0,7.363
fixed-r4,0.095,load-balancer,1,27.15,19.06
fixed-r4,0.095,postgres,1,90.1,223.1
fixed-r4,0.095,redis,1,5.94,16.87
fixed-r4,3.244,api,4,64.6,33.144
fixed-r4,3.244,frontend,1,0.02,7.359
fixed-r4,3.244,load-balancer,1,26.34,18.58
fixed-r4,3.244,postgres,1,101.51,225.1
fixed-r4,3.244,redis,1,0.2,16.96
fixed-r4,6.45,api,4,65.35,34.281
fixed-r4,6.45,frontend,1,0.02,7.359
fixed-r4,6.45,load-balancer,1,26.27,18.56
fixed-r4,6.45,postgres,1,106.68,225.7
fixed-r4,6.45,redis,1,4.55,16.7
fixed-r4,9.725,api,4,3.07,32.867
fixed-r4,9.725,frontend,1,2.51,7.762
fixed-r4,9.725,load-balancer,1,4.96,17.89
fixed-r4,9.725,postgres,1,0.42,226.7
fixed-r4,9.725,redis,1,0.17,16.95
growth-r1,0.045,api,1,44.44,9.355
growth-r1,0.045,frontend,1,0.0,7.359
growth-r1,0.045,load-balancer,1,25.9,18.3
growth-r1,0.045,postgres,1,93.36,207.6
growth-r1,0.045,redis,1,3.22,18.55
growth-r1,2.404,api,1,50.32,9.379
growth-r1,2.404,frontend,1,0.0,7.363
growth-r1,2.404,load-balancer,1,25.42,18.36
growth-r1,2.404,postgres,1,96.7,209.4
growth-r1,2.404,redis,1,0.22,18.11
growth-r1,5.579,api,1,50.68,9.66
growth-r1,5.579,frontend,1,0.03,7.613
growth-r1,5.579,load-balancer,1,25.44,18.67
growth-r1,5.579,postgres,1,105.06,210.9
growth-r1,5.579,redis,1,0.22,17.84
growth-r1,8.767,api,1,7.65,9.262
growth-r1,8.767,frontend,1,2.74,8.02
growth-r1,8.767,load-balancer,1,3.49,17.86
growth-r1,8.767,postgres,1,13.89,211.8
growth-r1,8.767,redis,1,0.17,17.93
growth-r2,0.047,api,2,52.25,17.629
growth-r2,0.047,frontend,1,0.02,7.363
growth-r2,0.047,load-balancer,1,24.23,18.59
growth-r2,0.047,postgres,1,100.9,233.3
growth-r2,0.047,redis,1,0.14,17.16
growth-r2,2.292,api,2,62.86,17.606
growth-r2,2.292,frontend,1,0.0,7.355
growth-r2,2.292,load-balancer,1,25.57,18.19
growth-r2,2.292,postgres,1,105.07,235.3
growth-r2,2.292,redis,1,0.16,17.16
growth-r2,5.459,api,2,57.29,18.789
growth-r2,5.459,frontend,1,0.04,7.363
growth-r2,5.459,load-balancer,1,25.89,18.43
growth-r2,5.459,postgres,1,104.58,235.1
growth-r2,5.459,redis,1,0.17,17.73
growth-r2,8.628,api,2,14.21,17.832
growth-r2,8.628,frontend,1,3.0,8.062
growth-r2,8.628,load-balancer,1,9.08,18.0
growth-r2,8.628,postgres,1,31.84,234.3
growth-r2,8.628,redis,1,2.99,22.08
growth-r4,0.043,api,4,25.03,29.672
growth-r4,0.043,frontend,1,0.02,7.316
growth-r4,0.043,load-balancer,1,9.16,18.25
growth-r4,0.043,postgres,1,99.03,269.2
growth-r4,0.043,redis,1,0.15,16.5
growth-r4,2.197,api,4,48.89,31.184
growth-r4,2.197,frontend,1,2.63,7.305
growth-r4,2.197,load-balancer,1,23.31,18.25
growth-r4,2.197,postgres,1,100.3,284.5
growth-r4,2.197,redis,1,2.97,20.3
growth-r4,5.406,api,4,43.96,32.164
growth-r4,5.406,frontend,1,0.0,7.305
growth-r4,5.406,load-balancer,1,19.48,17.79
growth-r4,5.406,postgres,1,101.91,290.1
growth-r4,5.406,redis,1,0.19,16.68
growth-r4,8.538,api,4,24.88,32.855
growth-r4,8.538,frontend,1,0.02,7.551
growth-r4,8.538,load-balancer,1,10.43,17.5
growth-r4,8.538,postgres,1,38.15,279.9
growth-r4,8.538,redis,1,0.18,16.91
improved-db-cpu-r4,0.054,api,4,52.63,30.188
improved-db-cpu-r4,0.054,frontend,1,0.03,7.301
improved-db-cpu-r4,0.054,load-balancer,1,24.59,18.03
improved-db-cpu-r4,0.054,postgres,1,121.18,130.1
improved-db-cpu-r4,0.054,redis,1,1.35,16.63
improved-db-cpu-r4,2.129,api,4,70.26,36.812
improved-db-cpu-r4,2.129,frontend,1,0.0,7.301
improved-db-cpu-r4,2.129,load-balancer,1,25.64,18.99
improved-db-cpu-r4,2.129,postgres,1,106.99,141.4
improved-db-cpu-r4,2.129,redis,1,0.16,16.88
improved-db-cpu-r4,5.271,api,4,65.05,33.652
improved-db-cpu-r4,5.271,frontend,1,3.29,7.43
improved-db-cpu-r4,5.271,load-balancer,1,26.73,18.41
improved-db-cpu-r4,5.271,postgres,1,102.2,145.9
improved-db-cpu-r4,5.271,redis,1,0.14,16.63
improved-db-cpu-r4,8.446,api,4,32.86,34.023
improved-db-cpu-r4,8.446,frontend,1,0.0,7.301
improved-db-cpu-r4,8.446,load-balancer,1,12.44,18.0
improved-db-cpu-r4,8.446,postgres,1,51.92,147.6
improved-db-cpu-r4,8.446,redis,1,0.18,16.88
1 phase t service containers cpu_percent memory_mib
2 saturation-c1 0.0 api 2 26.09 17.976
3 saturation-c1 0.0 frontend 1 1.61 7.578
4 saturation-c1 0.0 load-balancer 1 13.0 17.18
5 saturation-c1 0.0 postgres 1 28.57 169.0
6 saturation-c1 0.0 redis 1 0.15 15.28
7 saturation-c1 2.356 api 2 32.65 16.886
8 saturation-c1 2.356 frontend 1 0.02 7.617
9 saturation-c1 2.356 load-balancer 1 13.32 17.23
10 saturation-c1 2.356 postgres 1 28.96 169.5
11 saturation-c1 2.356 redis 1 2.75 15.4
12 saturation-c1 5.485 api 2 27.72 17.586
13 saturation-c1 5.485 frontend 1 0.0 7.383
14 saturation-c1 5.485 load-balancer 1 13.69 17.41
15 saturation-c1 5.485 postgres 1 29.41 170.1
16 saturation-c1 5.485 redis 1 0.15 15.28
17 saturation-c1 8.605 api 2 20.94 17.41
18 saturation-c1 8.605 frontend 1 1.12 12.06
19 saturation-c1 8.605 load-balancer 1 6.64 17.24
20 saturation-c1 8.605 postgres 1 8.66 170.5
21 saturation-c1 8.605 redis 1 0.15 15.53
22 saturation-c2 0.001 api 2 51.0 17.761
23 saturation-c2 0.001 frontend 1 0.03 7.617
24 saturation-c2 0.001 load-balancer 1 24.78 17.71
25 saturation-c2 0.001 postgres 1 57.44 171.2
26 saturation-c2 0.001 redis 1 0.17 16.03
27 saturation-c2 2.361 api 2 45.61 17.516
28 saturation-c2 2.361 frontend 1 0.03 7.367
29 saturation-c2 2.361 load-balancer 1 24.6 17.41
30 saturation-c2 2.361 postgres 1 49.85 172.6
31 saturation-c2 2.361 redis 1 0.18 15.28
32 saturation-c2 5.543 api 2 51.19 16.258
33 saturation-c2 5.543 frontend 1 0.02 7.367
34 saturation-c2 5.543 load-balancer 1 24.27 17.91
35 saturation-c2 5.543 postgres 1 62.6 173.0
36 saturation-c2 5.543 redis 1 0.21 15.28
37 saturation-c2 8.736 api 2 45.4 17.082
38 saturation-c2 8.736 frontend 1 1.22 8.121
39 saturation-c2 8.736 load-balancer 1 2.87 17.2
40 saturation-c2 8.736 postgres 1 50.06 173.8
41 saturation-c2 8.736 redis 1 0.21 15.78
42 saturation-c4 0.001 api 2 65.41 17.297
43 saturation-c4 0.001 frontend 1 0.03 7.367
44 saturation-c4 0.001 load-balancer 1 25.95 17.71
45 saturation-c4 0.001 postgres 1 81.8 174.4
46 saturation-c4 0.001 redis 1 0.19 15.28
47 saturation-c4 2.372 api 2 60.01 18.188
48 saturation-c4 2.372 frontend 1 3.75 7.965
49 saturation-c4 2.372 load-balancer 1 26.04 17.46
50 saturation-c4 2.372 postgres 1 79.42 175.4
51 saturation-c4 2.372 redis 1 0.16 15.53
52 saturation-c4 5.501 api 2 56.53 17.433
53 saturation-c4 5.501 frontend 1 0.02 7.367
54 saturation-c4 5.501 load-balancer 1 25.9 21.22
55 saturation-c4 5.501 postgres 1 73.71 175.7
56 saturation-c4 5.501 redis 1 0.18 15.28
57 saturation-c4 8.662 api 2 35.86 16.699
58 saturation-c4 8.662 frontend 1 0.0 7.367
59 saturation-c4 8.662 load-balancer 1 5.48 17.53
60 saturation-c4 8.662 postgres 1 17.44 177.2
61 saturation-c4 8.662 redis 1 2.71 16.1
62 saturation-c8 0.006 api 2 48.34 17.403
63 saturation-c8 0.006 frontend 1 0.0 7.367
64 saturation-c8 0.006 load-balancer 1 25.42 20.12
65 saturation-c8 0.006 postgres 1 77.51 185.1
66 saturation-c8 0.006 redis 1 0.18 15.28
67 saturation-c8 2.372 api 2 57.14 18.602
68 saturation-c8 2.372 frontend 1 3.44 7.734
69 saturation-c8 2.372 load-balancer 1 26.83 17.14
70 saturation-c8 2.372 postgres 1 92.98 188.5
71 saturation-c8 2.372 redis 1 3.34 15.28
72 saturation-c8 5.527 api 2 48.26 18.192
73 saturation-c8 5.527 frontend 1 0.0 7.363
74 saturation-c8 5.527 load-balancer 1 25.47 17.28
75 saturation-c8 5.527 postgres 1 76.05 189.4
76 saturation-c8 5.527 redis 1 0.15 15.28
77 saturation-c8 8.685 api 2 40.28 21.813
78 saturation-c8 8.685 frontend 1 0.0 7.363
79 saturation-c8 8.685 load-balancer 1 5.75 17.64
80 saturation-c8 8.685 postgres 1 17.24 190.4
81 saturation-c8 8.685 redis 1 0.14 15.28
82 saturation-c16 0.015 api 2 44.39 18.796
83 saturation-c16 0.015 frontend 1 0.0 7.363
84 saturation-c16 0.015 load-balancer 1 26.92 17.44
85 saturation-c16 0.015 postgres 1 82.81 205.9
86 saturation-c16 0.015 redis 1 0.17 15.28
87 saturation-c16 2.354 api 2 56.54 17.856
88 saturation-c16 2.354 frontend 1 0.03 7.375
89 saturation-c16 2.354 load-balancer 1 26.66 17.69
90 saturation-c16 2.354 postgres 1 88.93 207.7
91 saturation-c16 2.354 redis 1 0.19 15.75
92 saturation-c16 5.525 api 2 41.39 17.73
93 saturation-c16 5.525 frontend 1 4.4 7.379
94 saturation-c16 5.525 load-balancer 1 25.76 17.7
95 saturation-c16 5.525 postgres 1 72.49 208.6
96 saturation-c16 5.525 redis 1 0.18 15.25
97 saturation-c16 8.731 api 2 9.79 18.257
98 saturation-c16 8.731 frontend 1 0.02 7.375
99 saturation-c16 8.731 load-balancer 1 4.58 17.55
100 saturation-c16 8.731 postgres 1 16.09 209.7
101 saturation-c16 8.731 redis 1 0.15 15.33
102 saturation-c32 0.039 api 2 55.84 18.75
103 saturation-c32 0.039 frontend 1 6.63 8.125
104 saturation-c32 0.039 load-balancer 1 25.97 18.0
105 saturation-c32 0.039 postgres 1 104.2 210.6
106 saturation-c32 0.039 redis 1 5.85 15.05
107 saturation-c32 2.29 api 2 56.87 18.933
108 saturation-c32 2.29 frontend 1 0.0 7.625
109 saturation-c32 2.29 load-balancer 1 27.03 19.23
110 saturation-c32 2.29 postgres 1 105.41 212.1
111 saturation-c32 2.29 redis 1 0.19 15.55
112 saturation-c32 5.446 api 2 64.13 19.003
113 saturation-c32 5.446 frontend 1 0.7 7.375
114 saturation-c32 5.446 load-balancer 1 26.33 18.27
115 saturation-c32 5.446 postgres 1 107.62 213.1
116 saturation-c32 5.446 redis 1 3.7 15.27
117 saturation-c32 8.653 api 2 31.02 19.379
118 saturation-c32 8.653 frontend 1 0.02 7.625
119 saturation-c32 8.653 load-balancer 1 5.32 18.36
120 saturation-c32 8.653 postgres 1 18.07 213.5
121 saturation-c32 8.653 redis 1 0.16 15.26
122 fixed-r1 0.034 api 1 45.68 10.61
123 fixed-r1 0.034 frontend 1 0.02 7.359
124 fixed-r1 0.034 load-balancer 1 25.97 18.08
125 fixed-r1 0.034 postgres 1 106.86 214.7
126 fixed-r1 0.034 redis 1 4.41 15.3
127 fixed-r1 2.424 api 1 48.23 11.36
128 fixed-r1 2.424 frontend 1 0.0 7.699
129 fixed-r1 2.424 load-balancer 1 27.02 22.21
130 fixed-r1 2.424 postgres 1 96.33 215.7
131 fixed-r1 2.424 redis 1 0.14 15.44
132 fixed-r1 5.588 api 1 48.84 12.07
133 fixed-r1 5.588 frontend 1 0.0 7.359
134 fixed-r1 5.588 load-balancer 1 26.38 18.43
135 fixed-r1 5.588 postgres 1 104.71 217.4
136 fixed-r1 5.588 redis 1 0.19 15.92
137 fixed-r1 8.743 api 1 3.55 11.93
138 fixed-r1 8.743 frontend 1 0.0 7.508
139 fixed-r1 8.743 load-balancer 1 3.56 17.96
140 fixed-r1 8.743 postgres 1 8.33 218.1
141 fixed-r1 8.743 redis 1 0.19 16.07
142 fixed-r2 0.041 api 2 52.24 18.086
143 fixed-r2 0.041 frontend 1 0.0 7.359
144 fixed-r2 0.041 load-balancer 1 26.74 18.2
145 fixed-r2 0.041 postgres 1 100.69 219.4
146 fixed-r2 0.041 redis 1 3.81 15.18
147 fixed-r2 2.379 api 2 65.39 18.801
148 fixed-r2 2.379 frontend 1 1.92 7.859
149 fixed-r2 2.379 load-balancer 1 26.23 18.61
150 fixed-r2 2.379 postgres 1 106.33 221.3
151 fixed-r2 2.379 redis 1 0.16 15.16
152 fixed-r2 5.564 api 2 54.12 18.379
153 fixed-r2 5.564 frontend 1 1.37 10.54
154 fixed-r2 5.564 load-balancer 1 25.33 18.35
155 fixed-r2 5.564 postgres 1 103.06 222.7
156 fixed-r2 5.564 redis 1 0.2 14.93
157 fixed-r2 8.728 api 2 7.05 18.086
158 fixed-r2 8.728 frontend 1 0.0 7.363
159 fixed-r2 8.728 load-balancer 1 6.3 17.82
160 fixed-r2 8.728 postgres 1 14.18 223.3
161 fixed-r2 8.728 redis 1 0.2 15.43
162 fixed-r4 0.095 api 4 51.98 27.492
163 fixed-r4 0.095 frontend 1 0.0 7.363
164 fixed-r4 0.095 load-balancer 1 27.15 19.06
165 fixed-r4 0.095 postgres 1 90.1 223.1
166 fixed-r4 0.095 redis 1 5.94 16.87
167 fixed-r4 3.244 api 4 64.6 33.144
168 fixed-r4 3.244 frontend 1 0.02 7.359
169 fixed-r4 3.244 load-balancer 1 26.34 18.58
170 fixed-r4 3.244 postgres 1 101.51 225.1
171 fixed-r4 3.244 redis 1 0.2 16.96
172 fixed-r4 6.45 api 4 65.35 34.281
173 fixed-r4 6.45 frontend 1 0.02 7.359
174 fixed-r4 6.45 load-balancer 1 26.27 18.56
175 fixed-r4 6.45 postgres 1 106.68 225.7
176 fixed-r4 6.45 redis 1 4.55 16.7
177 fixed-r4 9.725 api 4 3.07 32.867
178 fixed-r4 9.725 frontend 1 2.51 7.762
179 fixed-r4 9.725 load-balancer 1 4.96 17.89
180 fixed-r4 9.725 postgres 1 0.42 226.7
181 fixed-r4 9.725 redis 1 0.17 16.95
182 growth-r1 0.045 api 1 44.44 9.355
183 growth-r1 0.045 frontend 1 0.0 7.359
184 growth-r1 0.045 load-balancer 1 25.9 18.3
185 growth-r1 0.045 postgres 1 93.36 207.6
186 growth-r1 0.045 redis 1 3.22 18.55
187 growth-r1 2.404 api 1 50.32 9.379
188 growth-r1 2.404 frontend 1 0.0 7.363
189 growth-r1 2.404 load-balancer 1 25.42 18.36
190 growth-r1 2.404 postgres 1 96.7 209.4
191 growth-r1 2.404 redis 1 0.22 18.11
192 growth-r1 5.579 api 1 50.68 9.66
193 growth-r1 5.579 frontend 1 0.03 7.613
194 growth-r1 5.579 load-balancer 1 25.44 18.67
195 growth-r1 5.579 postgres 1 105.06 210.9
196 growth-r1 5.579 redis 1 0.22 17.84
197 growth-r1 8.767 api 1 7.65 9.262
198 growth-r1 8.767 frontend 1 2.74 8.02
199 growth-r1 8.767 load-balancer 1 3.49 17.86
200 growth-r1 8.767 postgres 1 13.89 211.8
201 growth-r1 8.767 redis 1 0.17 17.93
202 growth-r2 0.047 api 2 52.25 17.629
203 growth-r2 0.047 frontend 1 0.02 7.363
204 growth-r2 0.047 load-balancer 1 24.23 18.59
205 growth-r2 0.047 postgres 1 100.9 233.3
206 growth-r2 0.047 redis 1 0.14 17.16
207 growth-r2 2.292 api 2 62.86 17.606
208 growth-r2 2.292 frontend 1 0.0 7.355
209 growth-r2 2.292 load-balancer 1 25.57 18.19
210 growth-r2 2.292 postgres 1 105.07 235.3
211 growth-r2 2.292 redis 1 0.16 17.16
212 growth-r2 5.459 api 2 57.29 18.789
213 growth-r2 5.459 frontend 1 0.04 7.363
214 growth-r2 5.459 load-balancer 1 25.89 18.43
215 growth-r2 5.459 postgres 1 104.58 235.1
216 growth-r2 5.459 redis 1 0.17 17.73
217 growth-r2 8.628 api 2 14.21 17.832
218 growth-r2 8.628 frontend 1 3.0 8.062
219 growth-r2 8.628 load-balancer 1 9.08 18.0
220 growth-r2 8.628 postgres 1 31.84 234.3
221 growth-r2 8.628 redis 1 2.99 22.08
222 growth-r4 0.043 api 4 25.03 29.672
223 growth-r4 0.043 frontend 1 0.02 7.316
224 growth-r4 0.043 load-balancer 1 9.16 18.25
225 growth-r4 0.043 postgres 1 99.03 269.2
226 growth-r4 0.043 redis 1 0.15 16.5
227 growth-r4 2.197 api 4 48.89 31.184
228 growth-r4 2.197 frontend 1 2.63 7.305
229 growth-r4 2.197 load-balancer 1 23.31 18.25
230 growth-r4 2.197 postgres 1 100.3 284.5
231 growth-r4 2.197 redis 1 2.97 20.3
232 growth-r4 5.406 api 4 43.96 32.164
233 growth-r4 5.406 frontend 1 0.0 7.305
234 growth-r4 5.406 load-balancer 1 19.48 17.79
235 growth-r4 5.406 postgres 1 101.91 290.1
236 growth-r4 5.406 redis 1 0.19 16.68
237 growth-r4 8.538 api 4 24.88 32.855
238 growth-r4 8.538 frontend 1 0.02 7.551
239 growth-r4 8.538 load-balancer 1 10.43 17.5
240 growth-r4 8.538 postgres 1 38.15 279.9
241 growth-r4 8.538 redis 1 0.18 16.91
242 improved-db-cpu-r4 0.054 api 4 52.63 30.188
243 improved-db-cpu-r4 0.054 frontend 1 0.03 7.301
244 improved-db-cpu-r4 0.054 load-balancer 1 24.59 18.03
245 improved-db-cpu-r4 0.054 postgres 1 121.18 130.1
246 improved-db-cpu-r4 0.054 redis 1 1.35 16.63
247 improved-db-cpu-r4 2.129 api 4 70.26 36.812
248 improved-db-cpu-r4 2.129 frontend 1 0.0 7.301
249 improved-db-cpu-r4 2.129 load-balancer 1 25.64 18.99
250 improved-db-cpu-r4 2.129 postgres 1 106.99 141.4
251 improved-db-cpu-r4 2.129 redis 1 0.16 16.88
252 improved-db-cpu-r4 5.271 api 4 65.05 33.652
253 improved-db-cpu-r4 5.271 frontend 1 3.29 7.43
254 improved-db-cpu-r4 5.271 load-balancer 1 26.73 18.41
255 improved-db-cpu-r4 5.271 postgres 1 102.2 145.9
256 improved-db-cpu-r4 5.271 redis 1 0.14 16.63
257 improved-db-cpu-r4 8.446 api 4 32.86 34.023
258 improved-db-cpu-r4 8.446 frontend 1 0.0 7.301
259 improved-db-cpu-r4 8.446 load-balancer 1 12.44 18.0
260 improved-db-cpu-r4 8.446 postgres 1 51.92 147.6
261 improved-db-cpu-r4 8.446 redis 1 0.18 16.88
File diff suppressed because it is too large Load Diff
+144
View File
@@ -0,0 +1,144 @@
phase,second,successful,failed
saturation-c1,0,252,0
saturation-c1,1,266,0
saturation-c1,2,257,0
saturation-c1,3,262,0
saturation-c1,4,253,0
saturation-c1,5,249,0
saturation-c1,6,228,0
saturation-c1,7,240,0
saturation-c1,8,219,0
saturation-c1,9,244,0
saturation-c1,10,2,0
saturation-c2,0,444,0
saturation-c2,1,377,0
saturation-c2,2,352,0
saturation-c2,3,297,0
saturation-c2,4,383,0
saturation-c2,5,324,0
saturation-c2,6,275,0
saturation-c2,7,323,0
saturation-c2,8,333,0
saturation-c2,9,381,0
saturation-c2,10,6,0
saturation-c4,0,526,0
saturation-c4,1,453,0
saturation-c4,2,425,0
saturation-c4,3,478,0
saturation-c4,4,510,0
saturation-c4,5,468,0
saturation-c4,6,458,0
saturation-c4,7,428,0
saturation-c4,8,490,0
saturation-c4,9,474,0
saturation-c4,10,11,0
saturation-c8,0,532,0
saturation-c8,1,476,0
saturation-c8,2,530,0
saturation-c8,3,529,0
saturation-c8,4,490,0
saturation-c8,5,527,0
saturation-c8,6,445,0
saturation-c8,7,524,0
saturation-c8,8,507,0
saturation-c8,9,519,0
saturation-c8,10,16,0
saturation-c16,0,349,0
saturation-c16,1,592,0
saturation-c16,2,541,0
saturation-c16,3,491,0
saturation-c16,4,545,0
saturation-c16,5,420,0
saturation-c16,6,499,0
saturation-c16,7,529,0
saturation-c16,8,548,0
saturation-c16,9,601,0
saturation-c16,10,28,0
saturation-c32,0,661,0
saturation-c32,1,604,0
saturation-c32,2,610,0
saturation-c32,3,589,0
saturation-c32,4,571,0
saturation-c32,5,627,0
saturation-c32,6,587,0
saturation-c32,7,619,0
saturation-c32,8,629,0
saturation-c32,9,548,0
saturation-c32,10,51,0
fixed-r1,0,511,0
fixed-r1,1,580,0
fixed-r1,2,659,0
fixed-r1,3,687,0
fixed-r1,4,588,0
fixed-r1,5,675,0
fixed-r1,6,647,0
fixed-r1,7,701,0
fixed-r1,8,690,0
fixed-r1,9,521,0
fixed-r1,10,32,0
fixed-r2,0,452,0
fixed-r2,1,545,0
fixed-r2,2,604,0
fixed-r2,3,652,0
fixed-r2,4,521,0
fixed-r2,5,627,0
fixed-r2,6,571,0
fixed-r2,7,611,0
fixed-r2,8,573,0
fixed-r2,9,586,0
fixed-r2,10,32,0
fixed-r4,0,133,0
fixed-r4,1,276,0
fixed-r4,2,461,0
fixed-r4,3,406,0
fixed-r4,4,503,0
fixed-r4,5,487,0
fixed-r4,6,377,0
fixed-r4,7,427,0
fixed-r4,8,551,0
fixed-r4,9,547,0
fixed-r4,10,50,0
growth-r1,0,527,0
growth-r1,1,596,0
growth-r1,2,604,0
growth-r1,3,616,0
growth-r1,4,516,0
growth-r1,5,508,0
growth-r1,6,606,0
growth-r1,7,620,0
growth-r1,8,628,0
growth-r1,9,591,0
growth-r1,10,32,0
growth-r2,0,461,0
growth-r2,1,662,0
growth-r2,2,670,0
growth-r2,3,655,0
growth-r2,4,640,0
growth-r2,5,495,0
growth-r2,6,635,0
growth-r2,7,626,0
growth-r2,8,598,0
growth-r2,9,609,0
growth-r2,10,32,0
growth-r4,0,106,0
growth-r4,1,514,0
growth-r4,2,587,0
growth-r4,3,506,0
growth-r4,4,423,0
growth-r4,5,409,0
growth-r4,6,459,0
growth-r4,7,462,0
growth-r4,8,618,0
growth-r4,9,549,0
growth-r4,10,55,0
improved-db-cpu-r4,0,492,0
improved-db-cpu-r4,1,583,0
improved-db-cpu-r4,2,629,0
improved-db-cpu-r4,3,681,0
improved-db-cpu-r4,4,667,0
improved-db-cpu-r4,5,632,0
improved-db-cpu-r4,6,585,0
improved-db-cpu-r4,7,677,0
improved-db-cpu-r4,8,602,0
improved-db-cpu-r4,9,629,0
improved-db-cpu-r4,10,52,0
1 phase second successful failed
2 saturation-c1 0 252 0
3 saturation-c1 1 266 0
4 saturation-c1 2 257 0
5 saturation-c1 3 262 0
6 saturation-c1 4 253 0
7 saturation-c1 5 249 0
8 saturation-c1 6 228 0
9 saturation-c1 7 240 0
10 saturation-c1 8 219 0
11 saturation-c1 9 244 0
12 saturation-c1 10 2 0
13 saturation-c2 0 444 0
14 saturation-c2 1 377 0
15 saturation-c2 2 352 0
16 saturation-c2 3 297 0
17 saturation-c2 4 383 0
18 saturation-c2 5 324 0
19 saturation-c2 6 275 0
20 saturation-c2 7 323 0
21 saturation-c2 8 333 0
22 saturation-c2 9 381 0
23 saturation-c2 10 6 0
24 saturation-c4 0 526 0
25 saturation-c4 1 453 0
26 saturation-c4 2 425 0
27 saturation-c4 3 478 0
28 saturation-c4 4 510 0
29 saturation-c4 5 468 0
30 saturation-c4 6 458 0
31 saturation-c4 7 428 0
32 saturation-c4 8 490 0
33 saturation-c4 9 474 0
34 saturation-c4 10 11 0
35 saturation-c8 0 532 0
36 saturation-c8 1 476 0
37 saturation-c8 2 530 0
38 saturation-c8 3 529 0
39 saturation-c8 4 490 0
40 saturation-c8 5 527 0
41 saturation-c8 6 445 0
42 saturation-c8 7 524 0
43 saturation-c8 8 507 0
44 saturation-c8 9 519 0
45 saturation-c8 10 16 0
46 saturation-c16 0 349 0
47 saturation-c16 1 592 0
48 saturation-c16 2 541 0
49 saturation-c16 3 491 0
50 saturation-c16 4 545 0
51 saturation-c16 5 420 0
52 saturation-c16 6 499 0
53 saturation-c16 7 529 0
54 saturation-c16 8 548 0
55 saturation-c16 9 601 0
56 saturation-c16 10 28 0
57 saturation-c32 0 661 0
58 saturation-c32 1 604 0
59 saturation-c32 2 610 0
60 saturation-c32 3 589 0
61 saturation-c32 4 571 0
62 saturation-c32 5 627 0
63 saturation-c32 6 587 0
64 saturation-c32 7 619 0
65 saturation-c32 8 629 0
66 saturation-c32 9 548 0
67 saturation-c32 10 51 0
68 fixed-r1 0 511 0
69 fixed-r1 1 580 0
70 fixed-r1 2 659 0
71 fixed-r1 3 687 0
72 fixed-r1 4 588 0
73 fixed-r1 5 675 0
74 fixed-r1 6 647 0
75 fixed-r1 7 701 0
76 fixed-r1 8 690 0
77 fixed-r1 9 521 0
78 fixed-r1 10 32 0
79 fixed-r2 0 452 0
80 fixed-r2 1 545 0
81 fixed-r2 2 604 0
82 fixed-r2 3 652 0
83 fixed-r2 4 521 0
84 fixed-r2 5 627 0
85 fixed-r2 6 571 0
86 fixed-r2 7 611 0
87 fixed-r2 8 573 0
88 fixed-r2 9 586 0
89 fixed-r2 10 32 0
90 fixed-r4 0 133 0
91 fixed-r4 1 276 0
92 fixed-r4 2 461 0
93 fixed-r4 3 406 0
94 fixed-r4 4 503 0
95 fixed-r4 5 487 0
96 fixed-r4 6 377 0
97 fixed-r4 7 427 0
98 fixed-r4 8 551 0
99 fixed-r4 9 547 0
100 fixed-r4 10 50 0
101 growth-r1 0 527 0
102 growth-r1 1 596 0
103 growth-r1 2 604 0
104 growth-r1 3 616 0
105 growth-r1 4 516 0
106 growth-r1 5 508 0
107 growth-r1 6 606 0
108 growth-r1 7 620 0
109 growth-r1 8 628 0
110 growth-r1 9 591 0
111 growth-r1 10 32 0
112 growth-r2 0 461 0
113 growth-r2 1 662 0
114 growth-r2 2 670 0
115 growth-r2 3 655 0
116 growth-r2 4 640 0
117 growth-r2 5 495 0
118 growth-r2 6 635 0
119 growth-r2 7 626 0
120 growth-r2 8 598 0
121 growth-r2 9 609 0
122 growth-r2 10 32 0
123 growth-r4 0 106 0
124 growth-r4 1 514 0
125 growth-r4 2 587 0
126 growth-r4 3 506 0
127 growth-r4 4 423 0
128 growth-r4 5 409 0
129 growth-r4 6 459 0
130 growth-r4 7 462 0
131 growth-r4 8 618 0
132 growth-r4 9 549 0
133 growth-r4 10 55 0
134 improved-db-cpu-r4 0 492 0
135 improved-db-cpu-r4 1 583 0
136 improved-db-cpu-r4 2 629 0
137 improved-db-cpu-r4 3 681 0
138 improved-db-cpu-r4 4 667 0
139 improved-db-cpu-r4 5 632 0
140 improved-db-cpu-r4 6 585 0
141 improved-db-cpu-r4 7 677 0
142 improved-db-cpu-r4 8 602 0
143 improved-db-cpu-r4 9 629 0
144 improved-db-cpu-r4 10 52 0
+37 -16
View File
@@ -1,6 +1,6 @@
# Lab 3 verification summary
Дата проверки: 2026-10-20.
Дата проверки: 2026-10-23.
Файл содержит обезличенную сводку воспроизводимых проверок. Bearer tokens,
пароли, ключи идемпотентности, UUID пользователей/чатов/сообщений и содержимое
@@ -12,7 +12,8 @@
- Docker Engine 29.7.2;
- Docker Compose 5.5.0;
- две Compose-реплики сервиса `api`;
- единственный опубликованный endpoint: Nginx на `127.0.0.1:8080`;
- единственный опубликованный endpoint: HAProxy на `127.0.0.1:8080`;
- Nginx доступен только внутри Compose network и отдаёт статический React;
- PostgreSQL и Redis доступны только внутри Compose network.
## Статические проверки
@@ -42,6 +43,16 @@ shell syntax for hooks PASS
- преподавательский `python3 tests/smoke.py` завершился строкой
`PASS: lab3 public API scenario`.
## Readiness-aware routing
`make test-readiness-routing` добавил в network namespace одной API-реплики
правило, запрещающее TCP-доступ к PostgreSQL, не останавливая сам процесс. Его
`/health/ready` вернул 503; HAProxy исключил backend за 2,861 с. После этого
40 запросов `/health/live` получили 200, а 20 корректных регистраций — 201;
не готовая реплика не получила ни одного запроса. После удаления fault-правила
она вернулась в ротацию. Машиночитаемый результат сохранён в
`evidence/readiness-routing.json`.
## Failover API
Команда:
@@ -54,30 +65,40 @@ make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
```text
run 1:
stopped container: api-2
observed instances before action: 2
successful writes / failures: 141 / 0
first recovery: 2.668 seconds
run 2, after restoring the desired two replicas:
observed instances before action: 2
successful writes / failures: 142 / 0
first recovery: 0.963 seconds
run 2, after restoring api-2:
stopped container: api-1
observed instances before action: 2
successful writes / failures: 143 / 0
first recovery: 0.750 seconds
first recovery: 2.653 seconds
both results: PASS
```
После первого запуска реплика была восстановлена; один health-запрос сдвинул
очередь LB, чтобы следующий synthetic message выбрал второй target. Hook
сопоставил `TARGET_INSTANCE_ID` с hostname контейнера и в каждом запуске
остановил только выбранную API-реплику. После тестов желаемое число
реплик было восстановлено командой `docker compose up -d --wait api`.
После каждого запуска желаемое число реплик было восстановлено командой
`docker compose up -d --wait api`. Hook сопоставляет `TARGET_INSTANCE_ID` с
hostname контейнера и останавливает только выбранную API-реплику. Отдельный
`make grade4-experiment` целенаправленно остановил обе исходные реплики по
очереди: 7853/0 и 7970/0 успешных/ошибочных запросов, recovery 2,893/2,938 с.
## Persistence и идемпотентность
- `make test-persistence ACTION_SCRIPT=./scripts/restart.sh`: PASS; после
рестарта PostgreSQL, Redis, всех API-реплик и Nginx прежние bearer session и
рестарта PostgreSQL, Redis, всех API-реплик, frontend и HAProxy прежние bearer session и
созданные данные сохранились;
- 32 конкурентных POST сообщения с одинаковыми semantic payload и
`Idempotency-Key` через Nginx вернули один Message ID;
`Idempotency-Key` через HAProxy вернули один Message ID;
- прямой запрос PostgreSQL подтвердил ровно одну строку сообщения;
- повтор с тем же scope/key и другим payload возвращает HTTP 409.
## Негативные и browser-security проверки
- сырой `Authorization: Bearer` с UTF-8 `é` вернул 401, Bearer challenge и
стандартный `Error`; процесс не завершился с panic/500;
- тело JSON из одного байта `FF` вернуло 400 `invalid_request`, а
`error.request_id` совпал с `X-Request-Id`;
- login установил host-only `HttpOnly; SameSite=Strict` cookie;
- cookie без Bearer на контрактном `/api/v1/users/me` дала 401, через
`/web-api/v1/users/me` — 200;
- logout через `/web-api` вернул 204 и инвалидировал серверную сессию;
- статический frontend вернул 200 и `Content-Security-Policy`.
+11
View File
@@ -0,0 +1,11 @@
{
"target_instance": "bd53aadefab0",
"detection_seconds": 2.861,
"health_requests": {
"200": 40
},
"registration_requests": {
"201": 20
},
"recovered": true
}
+7
View File
@@ -0,0 +1,7 @@
FROM haproxy:3.2-alpine@sha256:dfba86f031bd4c9039f10f8cc5de75991d1b6766d5460f0335b980408503baad
COPY infrastructure/haproxy/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
USER haproxy
EXPOSE 8080
+66
View File
@@ -0,0 +1,66 @@
global
log stdout format raw local0
maxconn 2048
defaults
log global
mode http
option httplog
timeout connect 1s
timeout client 30s
timeout server 30s
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 1s
hold other 1s
hold refused 1s
hold nx 1s
hold timeout 1s
hold obsolete 1s
frontend public
bind :8080
acl api_request path_beg /api/ /web-api/ /health/
acl web_api path_beg /web-api/
acl has_session req.cook(mtusi_session) -m found
acl message_create method POST
acl message_path path_reg ^/(api|web-api)/v1/chats/[^/]+/messages$
http-request del-header Authorization if web_api
http-request set-header Authorization "Bearer %[req.cook(mtusi_session)]" if web_api has_session
http-request del-header Cookie if api_request
http-request replace-path ^/web-api/(.*)$ /api/\1 if web_api
use_backend message_backend if message_create message_path
use_backend api_backend if api_request
default_backend frontend_backend
backend api_backend
balance leastconn
option httpchk
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 8 api:8080
backend message_backend
balance leastconn
option httpchk
option redispatch
option http-buffer-request
retries 3
retry-on conn-failure empty-response response-timeout 502 503 504
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
http-check expect status 200
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
server-template api 8 api:8080
backend frontend_backend
option httpchk GET /
http-check expect status 200
server frontend frontend:8080 check inter 2s fall 2 rise 2 resolvers docker init-addr libc,none
+1 -97
View File
@@ -1,24 +1,12 @@
worker_processes auto;
events {
worker_connections 1024;
worker_connections 512;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
resolver 127.0.0.11 valid=1s ipv6=off;
map $cookie_mtusi_session $web_authorization {
default "Bearer $cookie_mtusi_session";
"" "";
}
upstream api_backend {
zone api_backend 64k;
least_conn;
server api:8080 resolve max_fails=2 fail_timeout=2s;
}
server {
listen 8080;
@@ -31,90 +19,6 @@ http {
add_header X-Frame-Options "DENY" always;
add_header Content-Security-Policy "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always;
# Message creation carries an Idempotency-Key. It is therefore safe to
# retry a POST on another replica when the selected backend is draining.
location ~ ^/api/v1/chats/[^/]+/messages$ {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";
proxy_set_header Cookie "";
proxy_connect_timeout 1s;
proxy_send_timeout 20s;
proxy_read_timeout 20s;
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
proxy_next_upstream_tries 3;
}
location ~ ^/web-api/v1/chats/[^/]+/messages$ {
rewrite ^/web-api/(.*)$ /api/$1 break;
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";
proxy_set_header Authorization $web_authorization;
proxy_set_header Cookie "";
proxy_connect_timeout 1s;
proxy_send_timeout 20s;
proxy_read_timeout 20s;
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
proxy_next_upstream_tries 3;
}
location /api/ {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";
proxy_set_header Cookie "";
proxy_connect_timeout 1s;
proxy_send_timeout 20s;
proxy_read_timeout 20s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
location /web-api/ {
rewrite ^/web-api/(.*)$ /api/$1 break;
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";
proxy_set_header Authorization $web_authorization;
proxy_set_header Cookie "";
proxy_connect_timeout 1s;
proxy_send_timeout 20s;
proxy_read_timeout 20s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
location /health/ {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 5s;
proxy_next_upstream error timeout http_502 http_503 http_504;
proxy_next_upstream_tries 2;
}
location /assets/ {
try_files $uri =404;
expires 1y;
+3 -3
View File
@@ -77,7 +77,7 @@ def memory_mib(raw):
def service(name):
for candidate in ("load-balancer", "postgres", "redis", "api"):
for candidate in ("load-balancer", "frontend", "postgres", "redis", "api"):
if f"-{candidate}-" in name:
return candidate
return "other"
@@ -357,7 +357,7 @@ def main():
"action_after_seconds": args.action_after_seconds,
"post_action_seconds": args.post_action_seconds,
"warmup_requests": 12,
"workload": "closed-loop POST messages via Nginx; no automatic retries",
"workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries",
},
"phases": phases,
"idempotency": idem,
@@ -371,7 +371,7 @@ def main():
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
"PostgreSQL and Nginx remain single points of failure",
"PostgreSQL and HAProxy remain single points of failure",
],
}
destination = ROOT / args.output
+577
View File
@@ -0,0 +1,577 @@
#!/usr/bin/env python3
"""Measure the price of horizontal API scaling for lab 3 grade 5."""
import argparse
from collections import Counter, defaultdict
import datetime as dt
import json
import math
import os
from pathlib import Path
import platform
import subprocess
import sys
import threading
import time
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def command(args, env=None, timeout=240, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
env=env,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
detail = (result.stderr or result.stdout).strip().splitlines()
suffix = f": {detail[-1]}" if detail else ""
raise RuntimeError(
f"{' '.join(map(str, args[:4]))} exited {result.returncode}{suffix}"
)
return result
def compose(env_file, *args, env=None, timeout=240, check=True):
return command(
["docker", "compose", "--env-file", env_file, *args],
env=env,
timeout=timeout,
check=check,
)
def inspect(container_id):
raw = command(
["docker", "inspect", "--format", "{{json .}}", container_id]
).stdout
return json.loads(raw)
def service_ids(env_file, service=None):
args = ["ps", "-q"]
if service:
args.append(service)
return [line for line in compose(env_file, *args).stdout.splitlines() if line]
def wait_healthy(env_file, service, expected=1, timeout=120):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
healthy = []
for container_id in service_ids(env_file, service):
state = inspect(container_id)["State"]
if state.get("Health", {}).get("Status") == "healthy":
healthy.append(container_id)
if len(healthy) == expected:
return healthy
time.sleep(0.5)
raise RuntimeError(f"{service}: expected {expected} healthy containers")
def percentile(values, ratio):
if not values:
return None
ordered = sorted(values)
return round(ordered[max(0, math.ceil(ratio * len(ordered)) - 1)], 3)
def memory_mib(raw):
value = raw.split("/", 1)[0].strip()
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
for suffix in ("GiB", "MiB", "KiB", "B"):
if value.endswith(suffix):
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
return 0.0
def profile_environment(profile):
env = dict(os.environ)
env.update({
"API_REPLICAS": str(profile["replicas"]),
"API_CPU_LIMIT": f"{profile['api_cpu']:.2f}",
"API_MEMORY_LIMIT": profile["api_memory"],
"DB_MAX_CONNS": str(profile["db_max_conns"]),
"DB_MIN_CONNS": str(profile["db_min_conns"]),
"POSTGRES_CPU_LIMIT": f"{profile['postgres_cpu']:.2f}",
"POSTGRES_MEMORY_LIMIT": profile["postgres_memory"],
})
return env
def configure_profile(env_file, profile):
env = profile_environment(profile)
compose(
env_file,
"up", "-d", "--no-deps", "postgres",
env=env,
timeout=180,
)
wait_healthy(env_file, "postgres")
compose(
env_file,
"up", "-d", "--no-deps",
"--scale", f"api={profile['replicas']}",
"--force-recreate", "api",
env=env,
timeout=240,
)
api = wait_healthy(env_file, "api", profile["replicas"])
wait_healthy(env_file, "load-balancer")
time.sleep(1.5)
expected_cpu = int(profile["api_cpu"] * 1_000_000_000)
expected_memory = parse_memory_limit(profile["api_memory"])
applied = []
for container_id in api:
data = inspect(container_id)
actual_env = dict(item.split("=", 1) for item in data["Config"]["Env"] if "=" in item)
host = data["HostConfig"]
if host.get("NanoCpus") != expected_cpu or host.get("Memory") != expected_memory:
raise RuntimeError("API container limits do not match the selected profile")
if actual_env.get("DB_MAX_CONNS") != str(profile["db_max_conns"]):
raise RuntimeError("API DB pool size does not match the selected profile")
applied.append({
"instance": data["Config"]["Hostname"],
"cpu_limit": host.get("NanoCpus", 0) / 1_000_000_000,
"memory_limit_mib": round(host.get("Memory", 0) / 1024**2, 1),
"db_max_conns": int(actual_env["DB_MAX_CONNS"]),
})
postgres = inspect(service_ids(env_file, "postgres")[0])
expected_pg_cpu = int(profile["postgres_cpu"] * 1_000_000_000)
if postgres["HostConfig"].get("NanoCpus") != expected_pg_cpu:
raise RuntimeError("PostgreSQL CPU limit does not match the selected profile")
return {
"api": sorted(applied, key=lambda item: item["instance"]),
"postgres_cpu_limit": postgres["HostConfig"].get("NanoCpus", 0) / 1_000_000_000,
"postgres_memory_limit_mib": round(postgres["HostConfig"].get("Memory", 0) / 1024**2, 1),
}
def parse_memory_limit(value):
suffix = value[-1].upper()
multiplier = {"K": 1024, "M": 1024**2, "G": 1024**3}[suffix]
return int(value[:-1]) * multiplier
def resource_targets(env_file):
targets = {}
for container_id in service_ids(env_file):
data = inspect(container_id)
service = data["Config"]["Labels"].get("com.docker.compose.service", "other")
if service in {"api", "postgres", "redis", "load-balancer", "frontend"}:
targets[container_id] = {
"service": service,
"name": data["Name"].lstrip("/"),
}
return targets
def stats_snapshot(targets, elapsed):
if not targets:
return {"t": round(elapsed, 3), "services": {}}
result = command(
["docker", "stats", "--no-stream", "--format", "{{json .}}", *targets],
timeout=15,
check=False,
)
grouped = defaultdict(lambda: {"cpu_percent": 0.0, "memory_mib": 0.0, "containers": 0})
for line in result.stdout.splitlines():
try:
row = json.loads(line)
container_id = row.get("ID", "")
match = next((key for key in targets if key.startswith(container_id)), None)
if not match:
continue
values = grouped[targets[match]["service"]]
values["cpu_percent"] += float(row.get("CPUPerc", "0%").rstrip("%"))
values["memory_mib"] += memory_mib(row.get("MemUsage", "0B / 0B"))
values["containers"] += 1
except (ValueError, json.JSONDecodeError):
continue
return {
"t": round(elapsed, 3),
"services": {
name: {
"cpu_percent": round(values["cpu_percent"], 3),
"memory_mib": round(values["memory_mib"], 3),
"containers": values["containers"],
}
for name, values in sorted(grouped.items())
},
}
def summarize_resources(snapshots):
grouped = defaultdict(lambda: {"cpu": [], "memory": [], "containers": []})
for snapshot in snapshots:
for service, values in snapshot["services"].items():
grouped[service]["cpu"].append(values["cpu_percent"])
grouped[service]["memory"].append(values["memory_mib"])
grouped[service]["containers"].append(values["containers"])
return {
service: {
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
"cpu_percent_max": round(max(values["cpu"]), 3),
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
"memory_mib_max": round(max(values["memory"]), 3),
"containers": max(values["containers"]),
}
for service, values in sorted(grouped.items())
if values["cpu"]
}
def run_phase(name, kind, actor, chat, base_url, duration, concurrency, env_file, profile):
records = []
snapshots = []
lock = threading.Lock()
stop = threading.Event()
barrier = threading.Barrier(concurrency + 1)
targets = resource_targets(env_file)
started = time.monotonic()
def worker(worker_id):
client = Client(base_url, timeout=5)
index = 0
barrier.wait()
while not stop.is_set():
request_started = time.monotonic()
try:
response = client.message(
actor,
chat,
f"grade5-{name}-{worker_id}-{index}",
)
row = {
"t": round(time.monotonic() - started, 3),
"ok": True,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"instance": response["headers"].get("X-Instance-Id", ""),
}
except Failure as exc:
row = {
"t": round(time.monotonic() - started, 3),
"ok": False,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"error": str(exc),
}
with lock:
records.append(row)
index += 1
def sampler():
while not stop.is_set():
snapshots.append(stats_snapshot(targets, time.monotonic() - started))
stop.wait(1)
workers = [
threading.Thread(target=worker, args=(index,), daemon=True)
for index in range(concurrency)
]
for thread in workers:
thread.start()
stats_thread = threading.Thread(target=sampler, daemon=True)
stats_thread.start()
started = time.monotonic()
barrier.wait()
time.sleep(duration)
stop.set()
for thread in workers:
thread.join(10)
stats_thread.join(15)
elapsed = time.monotonic() - started
successes = [row for row in records if row["ok"]]
failures = [row for row in records if not row["ok"]]
latencies = [row["latency_ms"] for row in successes]
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
for row in records:
key = str(max(0, int(row["t"])))
buckets[key]["successful" if row["ok"] else "failed"] += 1
total = len(records)
return {
"name": name,
"kind": kind,
"profile": profile["name"],
"replicas": profile["replicas"],
"concurrency": concurrency,
"elapsed_seconds": round(elapsed, 3),
"successful_requests": len(successes),
"failed_requests": len(failures),
"error_ratio": round(len(failures) / total, 6) if total else None,
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
"successful_rps": round(len(successes) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(latencies, 0.50),
"p95": percentile(latencies, 0.95),
"p99": percentile(latencies, 0.99),
},
"instances": sorted({row["instance"] for row in successes if row.get("instance")}),
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
"resource_samples": snapshots,
"resources": summarize_resources(snapshots),
}
def dataset(env_file):
sql = (
"select count(*)::text || '|' || pg_database_size(current_database())::text "
"from messages"
)
result = compose(
env_file,
"exec", "-T", "postgres", "sh", "-c",
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
"sh", sql,
)
count, size = result.stdout.strip().split("|", 1)
return {"messages": int(count), "database_size_bytes": int(size)}
def host_environment(env_file):
mem_total_kib = 0
with open("/proc/meminfo", encoding="utf-8") as source:
for line in source:
if line.startswith("MemTotal:"):
mem_total_kib = int(line.split()[1])
break
images = compose(env_file, "config", "--images").stdout.splitlines()
return {
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
"logical_cpu_count": os.cpu_count(),
"host_memory_mib": round(mem_total_kib / 1024, 1),
"kernel": platform.release(),
"python": platform.python_version(),
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
"images": images,
}
def analyze(phases):
sweep = sorted(
(phase for phase in phases if phase["kind"] == "saturation"),
key=lambda phase: phase["concurrency"],
)
saturation = None
previous = None
for phase in sweep:
if previous:
gain = (phase["successful_rps"] - previous["successful_rps"]) / max(previous["successful_rps"], 0.001)
latency_growth = phase["successful_latency_ms"]["p95"] / max(previous["successful_latency_ms"]["p95"], 0.001)
if saturation is None and gain < 0.10 and latency_growth >= 1.50:
saturation = {
"concurrency": phase["concurrency"],
"previous_concurrency": previous["concurrency"],
"rps_gain_percent": round(gain * 100, 2),
"p95_growth_ratio": round(latency_growth, 3),
}
previous = phase
baseline = next(phase for phase in phases if phase["name"] == "growth-r4")
improved = next(phase for phase in phases if phase["name"] == "improved-db-cpu-r4")
rps_change = (improved["successful_rps"] - baseline["successful_rps"]) / max(baseline["successful_rps"], 0.001) * 100
p95_change = (improved["successful_latency_ms"]["p95"] - baseline["successful_latency_ms"]["p95"]) / max(baseline["successful_latency_ms"]["p95"], 0.001) * 100
confirmed = (
rps_change >= 5
and improved["error_ratio"] <= baseline["error_ratio"] + 0.001
) or p95_change <= -10
resources = baseline["resources"]
pg = resources.get("postgres", {})
lb = resources.get("load-balancer", {})
bottleneck = "undetermined"
if pg.get("cpu_percent_avg", 0) >= 70 and pg.get("cpu_percent_max", 0) >= 95:
bottleneck = "postgres_cpu"
elif lb.get("cpu_percent_avg", 0) >= 20 and lb.get("cpu_percent_max", 0) >= 24:
bottleneck = "load_balancer_cpu"
improved_resources = improved["resources"]
improved_lb = improved_resources.get("load-balancer", {})
remaining_bottleneck = "undetermined"
if improved_lb.get("cpu_percent_avg", 0) >= 20 and improved_lb.get("cpu_percent_max", 0) >= 24:
remaining_bottleneck = "load_balancer_cpu"
return {
"saturation_point": saturation,
"observed_bottleneck": bottleneck,
"baseline_resource_snapshot": resources,
"improved_resource_snapshot": improved_resources,
"remaining_bottleneck_after_improvement": remaining_bottleneck,
"improvement": {
"change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged",
"baseline_phase": baseline["name"],
"improved_phase": improved["name"],
"rps_change_percent": round(rps_change, 2),
"p95_change_percent": round(p95_change, 2),
"confirmed": confirmed,
},
}
def profiles():
base = {"postgres_cpu": 1.0, "postgres_memory": "512M"}
return {
"fixed-r1": {**base, "name": "fixed-r1", "replicas": 1, "api_cpu": 1.0, "api_memory": "512M", "db_max_conns": 16, "db_min_conns": 2},
"fixed-r2": {**base, "name": "fixed-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"fixed-r4": {**base, "name": "fixed-r4", "replicas": 4, "api_cpu": 0.25, "api_memory": "128M", "db_max_conns": 4, "db_min_conns": 1},
"growth-r1": {**base, "name": "growth-r1", "replicas": 1, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"growth-r2": {**base, "name": "growth-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"growth-r4": {**base, "name": "growth-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
"improved-db-cpu-r4": {**base, "name": "improved-db-cpu-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1, "postgres_cpu": 2.0},
}
def run_warmup(actor, chat, args, profile):
phase = run_phase(
f"warmup-{profile['name']}",
"warmup",
actor,
chat,
args.base_url,
args.warmup_seconds,
min(4, args.comparison_concurrency),
args.env_file,
profile,
)
if phase["failed_requests"]:
raise RuntimeError(f"warmup failed for {profile['name']}")
return phase["successful_requests"]
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--env-file", default=".env")
parser.add_argument("--duration", type=float, default=10)
parser.add_argument("--warmup-seconds", type=float, default=2)
parser.add_argument("--comparison-concurrency", type=int, default=32)
parser.add_argument("--sweep", default="1,2,4,8,16,32")
parser.add_argument("--output", default="evidence/grade5/results.json")
args = parser.parse_args()
sweep = [int(value) for value in args.sweep.split(",")]
if args.duration <= 0 or args.warmup_seconds <= 0:
raise SystemExit("durations must be positive")
if not sweep or min(sweep) < 1 or max(sweep + [args.comparison_concurrency]) > 64:
raise SystemExit("concurrency must be 1..64")
all_profiles = profiles()
restore = {
"name": "restore-default",
"replicas": 2,
"api_cpu": 0.5,
"api_memory": "256M",
"db_max_conns": 10,
"db_min_conns": 2,
"postgres_cpu": 1.0,
"postgres_memory": "512M",
}
phases = []
applied = {}
warmup_requests = 0
restoration = {"attempted": False, "succeeded": False}
environment = host_environment(args.env_file)
start_dataset = dataset(args.env_file)
try:
client = Client(args.base_url, timeout=8)
alice, _, _, chat = client.bootstrap()
saturation_profile = all_profiles["growth-r2"]
applied[saturation_profile["name"]] = configure_profile(args.env_file, saturation_profile)
warmup_requests += run_warmup(alice, chat, args, saturation_profile)
for concurrency in sweep:
print(f"saturation: concurrency={concurrency}", flush=True)
phases.append(run_phase(
f"saturation-c{concurrency}", "saturation", alice, chat,
args.base_url, args.duration, concurrency, args.env_file,
saturation_profile,
))
for profile_name in (
"fixed-r1", "fixed-r2", "fixed-r4",
"growth-r1", "growth-r2", "growth-r4",
"improved-db-cpu-r4",
):
profile = all_profiles[profile_name]
print(f"profile: {profile_name}", flush=True)
applied[profile_name] = configure_profile(args.env_file, profile)
warmup_requests += run_warmup(alice, chat, args, profile)
kind = "improvement" if profile_name.startswith("improved") else profile_name.split("-", 1)[0]
phases.append(run_phase(
profile_name, kind, alice, chat, args.base_url,
args.duration, args.comparison_concurrency, args.env_file,
profile,
))
end_dataset = dataset(args.env_file)
finally:
restoration["attempted"] = True
try:
restoration["applied"] = configure_profile(args.env_file, restore)
restoration["succeeded"] = True
except Exception as exc:
restoration["error"] = f"{type(exc).__name__}: {exc}"
if not restoration["succeeded"]:
raise RuntimeError("experiment completed but default Compose profile was not restored")
report = {
"schema_version": 1,
"environment": environment,
"hypothesis": (
"API replicas stop improving throughput when shared PostgreSQL CPU saturates; "
"adding PostgreSQL CPU should improve the unchanged four-replica profile."
),
"methodology": {
"workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries",
"duration_seconds_per_phase": args.duration,
"warmup_seconds_per_profile": args.warmup_seconds,
"comparison_concurrency": args.comparison_concurrency,
"saturation_concurrency": sweep,
"fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections",
"growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica",
"improvement": "only PostgreSQL CPU changes from 1.00 to 2.00",
"saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step",
"cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer",
},
"profiles": all_profiles,
"applied_limits": applied,
"phases": phases,
"analysis": analyze(phases),
"dataset": {
"before": start_dataset,
"after": end_dataset,
"warmup_successful_requests": warmup_requests,
},
"restoration": restoration,
"limitations": [
"single Docker host: replicas do not prove host-level availability",
"closed-loop workload has coordinated omission",
"successful-response latency percentiles are reported separately from failures",
"one measured run per profile; host scheduling noise is possible",
],
}
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
try:
displayed_output = destination.relative_to(ROOT)
except ValueError:
displayed_output = destination
print(json.dumps({
"output": str(displayed_output),
"analysis": report["analysis"],
"restoration": restoration["succeeded"],
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+2 -2
View File
@@ -148,7 +148,7 @@ def summary(report, output):
"## Method", "",
f"- concurrency: {method['concurrency']};",
f"- baseline duration: {method['baseline_seconds']} s;",
"- load: closed-loop POST messages through Nginx;",
"- load: closed-loop POST messages through the public HAProxy gateway;",
"- failover: both original API containers stopped and restored in turn;",
"- rolling update: one surge replica, old replicas replaced one by one;",
"- percentiles cover successful responses; failures are separate.", "",
@@ -178,7 +178,7 @@ def summary(report, output):
"![Resources](resources.svg)", "",
"## Scope", "",
"Single-host process failure and rolling replacement only. PostgreSQL "
"and Nginx remain single points of failure.", "",
"and HAProxy remain single points of failure.", "",
]
output.write_text("\n".join(lines))
+322
View File
@@ -0,0 +1,322 @@
#!/usr/bin/env python3
"""Render grade-5 scaling evidence as SVG, CSV and Markdown."""
import argparse
import csv
import html
import json
from pathlib import Path
STYLE = """
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.subtitle { font-size: 14px; fill: #aeb2b8; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.axis { stroke: #747981; stroke-width: 1.5; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
.point { fill: #f4f5f6; }
.point2 { fill: #92979f; }
.bar { fill: #d7d9dc; }
.bar2 { fill: #777c84; }
.bar3 { fill: #4c5057; }
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
"""
def svg(title, subtitle, body, width=1120, height=620):
return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>{STYLE}</style>
<text x="50" y="44" class="title">{html.escape(title)}</text>
<text x="50" y="70" class="subtitle">{html.escape(subtitle)}</text>
{body}
</svg>
"""
def grid(maximum, left, top, width, height, suffix=""):
maximum = maximum or 1
parts = []
for step in range(6):
value = maximum * step / 5
y = top + height - height * step / 5
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{left+width}" y2="{y:.1f}" class="grid"/>')
parts.append(f'<text x="{left-12}" y="{y+4:.1f}" text-anchor="end" class="small">{value:.0f}{suffix}</text>')
return parts
def phase_map(report):
return {phase["name"]: phase for phase in report["phases"]}
def saturation_graph(report, output):
phases = sorted(
(phase for phase in report["phases"] if phase["kind"] == "saturation"),
key=lambda phase: phase["concurrency"],
)
left, top, width, height = 90, 110, 940, 390
max_rps = max(phase["successful_rps"] for phase in phases) * 1.1
max_p95 = max(phase["successful_latency_ms"]["p95"] for phase in phases) * 1.1
parts = grid(max_rps, left, top, width, height)
points_rps, points_p95 = [], []
for index, phase in enumerate(phases):
x = left + width * index / max(1, len(phases) - 1)
y_rps = top + height - height * phase["successful_rps"] / max_rps
y_p95 = top + height - height * phase["successful_latency_ms"]["p95"] / max_p95
points_rps.append(f"{x:.1f},{y_rps:.1f}")
points_p95.append(f"{x:.1f},{y_p95:.1f}")
parts.append(f'<circle cx="{x:.1f}" cy="{y_rps:.1f}" r="6" class="point"/>')
parts.append(f'<circle cx="{x:.1f}" cy="{y_p95:.1f}" r="6" class="point2"/>')
parts.append(f'<text x="{x:.1f}" y="{top+height+28}" text-anchor="middle" class="label">{phase["concurrency"]}</text>')
parts.append(f'<text x="{x:.1f}" y="{y_rps-12:.1f}" text-anchor="middle" class="small">{phase["successful_rps"]:.0f}</text>')
parts.append(f'<polyline points="{" ".join(points_rps)}" class="line"/>')
parts.append(f'<polyline points="{" ".join(points_p95)}" class="line2"/>')
parts.append(f'<text x="{left}" y="{top+height+62}" class="label">Concurrency</text>')
parts.append(f'<text x="{left+720}" y="{top+height+62}" class="label">white: RPS · gray: p95 normalized</text>')
parts.append(f'<text x="30" y="{top+height/2}" transform="rotate(-90 30 {top+height/2})" class="label">Successful requests per second</text>')
output.write_text(svg(
"Saturation curve",
"Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU",
"\n".join(parts),
), encoding="utf-8")
def comparison_graph(report, kind, output):
phases = sorted(
(phase for phase in report["phases"] if phase["kind"] == kind),
key=lambda phase: phase["replicas"],
)
left, top, width, height = 90, 110, 940, 370
max_rps = max(phase["successful_rps"] for phase in phases) * 1.15
parts = grid(max_rps, left, top, width, height)
slot = width / len(phases)
for index, phase in enumerate(phases):
bar_width = min(150, slot * 0.52)
x = left + index * slot + (slot - bar_width) / 2
bar_height = height * phase["successful_rps"] / max_rps
y = top + height - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="14" class="bar"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-12:.1f}" text-anchor="middle" class="label">{phase["successful_rps"]:.1f} RPS</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+30}" text-anchor="middle" class="label">{phase["replicas"]} replica(s)</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+52}" text-anchor="middle" class="small">p95 {phase["successful_latency_ms"]["p95"]:.1f} ms</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+70}" text-anchor="middle" class="small">errors {phase["error_ratio"]*100:.3f}%</text>')
if kind == "fixed":
title = "Fixed total API budget"
subtitle = "1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas"
else:
title = "Growing API budget"
subtitle = "Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU"
output.write_text(svg(title, subtitle, "\n".join(parts)), encoding="utf-8")
def bottleneck_graph(report, output):
phases = sorted(
(phase for phase in report["phases"] if phase["kind"] == "growth"),
key=lambda phase: phase["replicas"],
)
services = ("api", "postgres", "load-balancer", "frontend", "redis")
left, top, width, height = 90, 110, 940, 370
maximum = max(
phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
for phase in phases for service in services
) * 1.15 or 1
parts = grid(maximum, left, top, width, height)
slot = width / len(phases)
colors = ("bar", "bar2", "bar3", "bar3", "bar3")
for phase_index, phase in enumerate(phases):
group_left = left + phase_index * slot
bar_width = min(48, slot / 6)
for service_index, service in enumerate(services):
value = phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
x = group_left + slot * 0.12 + service_index * (bar_width + 9)
bar_height = height * value / maximum
y = top + height - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="7" class="{colors[service_index]}"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-7:.1f}" text-anchor="middle" class="small">{value:.0f}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+22}" text-anchor="middle" class="small">{service[:3]}</text>')
parts.append(f'<text x="{group_left+slot/2:.1f}" y="{top+height+52}" text-anchor="middle" class="label">{phase["replicas"]} replica(s)</text>')
parts.append(f'<text x="30" y="{top+height/2}" transform="rotate(-90 30 {top+height/2})" class="label">Average aggregate CPU, %</text>')
output.write_text(svg(
"Where scaling stops",
"Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis)",
"\n".join(parts),
), encoding="utf-8")
def improvement_graph(report, output):
phases = phase_map(report)
baseline = phases["growth-r4"]
improved = phases["improved-db-cpu-r4"]
pair = [baseline, improved]
left, top, panel_width, height = 90, 120, 390, 340
parts = []
panels = [
("Successful RPS", "successful_rps", left),
("p95 latency, ms", "p95", 630),
]
for title, metric, panel_left in panels:
values = [
phase["successful_rps"] if metric == "successful_rps" else phase["successful_latency_ms"]["p95"]
for phase in pair
]
maximum = max(values) * 1.2 or 1
parts.append(f'<text x="{panel_left}" y="{top-22}" class="label">{title}</text>')
for index, (phase, value) in enumerate(zip(pair, values)):
bar_width = 120
x = panel_left + 35 + index * 185
bar_height = height * value / maximum
y = top + height - bar_height
parts.append(f'<rect x="{x}" y="{y:.1f}" width="{bar_width}" height="{bar_height:.1f}" rx="14" class="{"bar" if index else "bar2"}"/>')
parts.append(f'<text x="{x+60}" y="{y-10:.1f}" text-anchor="middle" class="label">{value:.1f}</text>')
parts.append(f'<text x="{x+60}" y="{top+height+28}" text-anchor="middle" class="small">PG {1 if index == 0 else 2} CPU</text>')
analysis = report["analysis"]["improvement"]
parts.append(f'<text x="90" y="545" class="label">RPS change: {analysis["rps_change_percent"]:+.2f}% · p95 change: {analysis["p95_change_percent"]:+.2f}% · confirmed: {str(analysis["confirmed"]).lower()}</text>')
output.write_text(svg(
"Confirmed improvement",
"Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2",
"\n".join(parts),
), encoding="utf-8")
def write_csv(report, destination):
with (destination / "measurements.csv").open("w", newline="", encoding="utf-8") as stream:
fields = [
"phase", "kind", "replicas", "concurrency", "elapsed_seconds",
"successful_requests", "failed_requests", "error_ratio", "successful_rps",
"p50_ms", "p95_ms", "p99_ms", "api_cpu_avg", "postgres_cpu_avg",
"load_balancer_cpu_avg", "frontend_cpu_avg", "redis_cpu_avg",
]
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
for phase in report["phases"]:
writer.writerow({
"phase": phase["name"],
"kind": phase["kind"],
"replicas": phase["replicas"],
"concurrency": phase["concurrency"],
"elapsed_seconds": phase["elapsed_seconds"],
"successful_requests": phase["successful_requests"],
"failed_requests": phase["failed_requests"],
"error_ratio": phase["error_ratio"],
"successful_rps": phase["successful_rps"],
"p50_ms": phase["successful_latency_ms"]["p50"],
"p95_ms": phase["successful_latency_ms"]["p95"],
"p99_ms": phase["successful_latency_ms"]["p99"],
"api_cpu_avg": phase["resources"].get("api", {}).get("cpu_percent_avg"),
"postgres_cpu_avg": phase["resources"].get("postgres", {}).get("cpu_percent_avg"),
"load_balancer_cpu_avg": phase["resources"].get("load-balancer", {}).get("cpu_percent_avg"),
"frontend_cpu_avg": phase["resources"].get("frontend", {}).get("cpu_percent_avg"),
"redis_cpu_avg": phase["resources"].get("redis", {}).get("cpu_percent_avg"),
})
with (destination / "timeseries.csv").open("w", newline="", encoding="utf-8") as stream:
fields = ["phase", "second", "successful", "failed"]
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
for phase in report["phases"]:
for second, counts in phase["per_second"].items():
writer.writerow({"phase": phase["name"], "second": second, **counts})
with (destination / "resources.csv").open("w", newline="", encoding="utf-8") as stream:
fields = ["phase", "t", "service", "containers", "cpu_percent", "memory_mib"]
writer = csv.DictWriter(stream, fieldnames=fields)
writer.writeheader()
for phase in report["phases"]:
for snapshot in phase["resource_samples"]:
for service, values in snapshot["services"].items():
writer.writerow({
"phase": phase["name"], "t": snapshot["t"], "service": service,
**values,
})
def markdown_summary(report, output):
method = report["methodology"]
analysis = report["analysis"]
improvement = analysis["improvement"]
lines = [
"# Lab 3 grade-5 scaling experiment", "",
f"Generated: {report['environment']['utc_started_at']}.", "",
"## Hypothesis", "", report["hypothesis"], "",
"## Reproduction", "",
f"- workload: {method['workload']};",
f"- phase duration: {method['duration_seconds_per_phase']} s;",
f"- comparison concurrency: {method['comparison_concurrency']};",
f"- saturation sweep: {method['saturation_concurrency']};",
f"- fixed budget: {method['fixed_total_budget']};",
f"- growth series: {method['growth_budget']};",
f"- saturation rule: {method['saturation_rule']};",
f"- host: {report['environment']['logical_cpu_count']} logical CPU, {report['environment']['host_memory_mib']} MiB RAM;",
f"- Docker {report['environment']['docker_engine']}, Compose {report['environment']['docker_compose']};",
"- command: `make grade5-experiment`; warmup runs before every changed profile.", "",
"## Measurements", "",
"| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for phase in report["phases"]:
latency = phase["successful_latency_ms"]
resources = phase["resources"]
lines.append(
f"| {phase['name']} | {phase['replicas']} | {phase['concurrency']} | "
f"{phase['successful_rps']} | {phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
f"{latency['p50']} | {latency['p95']} | {latency['p99']} | "
f"{resources.get('postgres', {}).get('cpu_percent_avg', 'n/a')} | "
f"{resources.get('api', {}).get('cpu_percent_avg', 'n/a')} |"
)
saturation = analysis["saturation_point"]
lines += ["", "## Conclusion", ""]
if saturation:
lines.append(
f"The first saturation signal appears at concurrency {saturation['concurrency']}: "
f"RPS changes by {saturation['rps_gain_percent']:+.2f}% while p95 changes by "
f"{saturation['p95_growth_ratio']:.2f}x versus concurrency {saturation['previous_concurrency']}."
)
else:
lines.append("The configured sweep did not locate a saturation point by the declared rule.")
lines += [
"",
f"Observed bottleneck classification: `{analysis['observed_bottleneck']}`.", "",
f"The PostgreSQL CPU change produced {improvement['rps_change_percent']:+.2f}% RPS and "
f"{improvement['p95_change_percent']:+.2f}% p95; confirmed improvement: "
f"`{str(improvement['confirmed']).lower()}`.", "",
f"After the improvement the remaining bottleneck classification is "
f"`{analysis['remaining_bottleneck_after_improvement']}`.", "",
"## Graphs", "",
"![Saturation](01-saturation.svg)", "",
"![Fixed budget](02-fixed-total.svg)", "",
"![Growing budget](03-resource-growth.svg)", "",
"![Bottleneck](04-bottleneck.svg)", "",
"![Improvement](05-improvement.svg)", "",
"## Raw data and limits", "",
"Raw aggregates are in `measurements.csv`; one-second request buckets are in "
"`timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and "
"applied limits are in `results.json`.", "",
]
lines.extend(f"- {item}" for item in report["limitations"])
lines.append("")
output.write_text("\n".join(lines), encoding="utf-8")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("results", nargs="?", default="evidence/grade5/results.json")
args = parser.parse_args()
source = Path(args.results)
report = json.loads(source.read_text(encoding="utf-8"))
destination = source.parent
saturation_graph(report, destination / "01-saturation.svg")
comparison_graph(report, "fixed", destination / "02-fixed-total.svg")
comparison_graph(report, "growth", destination / "03-resource-growth.svg")
bottleneck_graph(report, destination / "04-bottleneck.svg")
improvement_graph(report, destination / "05-improvement.svg")
write_csv(report, destination)
markdown_summary(report, destination / "README.md")
print(f"rendered grade-5 evidence in {destination}")
if __name__ == "__main__":
main()
+5 -1
View File
@@ -19,6 +19,10 @@ compose up -d --no-deps api
compose restart api
compose up -d --wait --no-deps api
# Публичная точка входа перезапускается последней, когда API уже готовы.
# Статический frontend возвращается до публичной точки входа.
compose restart frontend
compose up -d --wait --no-deps frontend
# Публичная точка входа перезапускается последней, когда API и frontend готовы.
compose restart load-balancer
compose up -d --wait --no-deps load-balancer
+198
View File
@@ -0,0 +1,198 @@
#!/usr/bin/env python3
"""Prove that the gateway excludes an API replica which loses readiness."""
import argparse
from collections import Counter
import http.client
import json
import os
from pathlib import Path
import subprocess
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f"
def command(args, timeout=180, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
detail = (result.stderr or result.stdout).strip().splitlines()
suffix = f": {detail[-1]}" if detail else ""
raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}")
return result
def compose(env_file, *args):
return command(["docker", "compose", "--env-file", env_file, *args])
def inspect(container_id):
return json.loads(command([
"docker", "inspect", "--format", "{{json .}}", container_id,
]).stdout)
def call(host, port, method, path, body=None, headers=None, timeout=5):
connection = http.client.HTTPConnection(host, port, timeout=timeout)
connection.request(method, path, body=body, headers=headers or {})
response = connection.getresponse()
payload = response.read()
result = {
"status": response.status,
"instance": response.getheader("X-Instance-Id", ""),
"body": payload,
}
connection.close()
return result
def observe_instances(base_host, base_port, requests=16):
instances = Counter()
statuses = Counter()
for _ in range(requests):
result = call(base_host, base_port, "GET", "/health/live")
statuses[result["status"]] += 1
instances[result["instance"]] += 1
return statuses, instances
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--env-file", default=".env")
parser.add_argument("--host", default="localhost")
parser.add_argument("--port", type=int, default=8080)
parser.add_argument("--output", default="")
args = parser.parse_args()
api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line]
if len(api_ids) != 2:
raise RuntimeError("readiness routing test requires exactly two API replicas")
target_id = api_ids[0]
target = inspect(target_id)
target_instance = target["Config"]["Hostname"]
target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"]
postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip()
postgres = inspect(postgres_id)
postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"]
before_statuses, before_instances = observe_instances(args.host, args.port, 40)
if before_statuses != {200: 40} or len(before_instances) != 2:
raise RuntimeError("both API replicas must serve traffic before fault injection")
helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}"
rule = [
"iptables", "-D", "OUTPUT", "-d", postgres_ip,
"-p", "tcp", "--dport", "5432", "-j", "REJECT",
"--reject-with", "tcp-reset",
]
detection_started = time.monotonic()
try:
script = (
"apk add --no-cache iptables >/dev/null && "
f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 "
"-j REJECT --reject-with tcp-reset && "
"touch /tmp/fault-ready && sleep 180"
)
command([
"docker", "run", "-d", "--name", helper,
"--network", f"container:{target_id}",
"--cap-add", "NET_ADMIN", FAULT_IMAGE,
"sh", "-c", script,
])
deadline = time.monotonic() + 30
while time.monotonic() < deadline:
ready = command(
["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"],
check=False,
)
if ready.returncode == 0:
break
time.sleep(0.2)
else:
raise RuntimeError("fault helper did not become ready")
direct = call(target_ip, 8080, "GET", "/health/ready")
if direct["status"] != 503:
raise RuntimeError(f"target readiness is {direct['status']}, want 503")
deadline = time.monotonic() + 10
while time.monotonic() < deadline:
statuses, instances = observe_instances(args.host, args.port, 16)
if statuses == {200: 16} and target_instance not in instances:
break
time.sleep(0.25)
else:
raise RuntimeError("gateway did not exclude the unready API within 10 seconds")
detection_seconds = time.monotonic() - detection_started
health_statuses, health_instances = observe_instances(args.host, args.port, 40)
registration_statuses = Counter()
registration_instances = Counter()
for _ in range(20):
body = json.dumps({
"username": "readiness_" + uuid.uuid4().hex[:16],
"display_name": "Readiness audit",
"password": "T3st-" + uuid.uuid4().hex,
}).encode()
response = call(
args.host,
args.port,
"POST",
"/api/v1/users",
body=body,
headers={"Content-Type": "application/json"},
timeout=8,
)
registration_statuses[response["status"]] += 1
registration_instances[response["instance"]] += 1
if health_statuses != {200: 40} or target_instance in health_instances:
raise RuntimeError("unready API received liveness traffic through gateway")
if registration_statuses != {201: 20} or target_instance in registration_instances:
raise RuntimeError("unready API received registration traffic through gateway")
finally:
command(["docker", "exec", helper, *rule], check=False)
command(["docker", "rm", "-f", helper], check=False)
deadline = time.monotonic() + 15
while time.monotonic() < deadline:
direct = call(target_ip, 8080, "GET", "/health/ready")
if direct["status"] == 200:
_, instances = observe_instances(args.host, args.port, 24)
if target_instance in instances:
break
time.sleep(0.25)
else:
raise RuntimeError("recovered API did not return to gateway rotation")
result = {
"target_instance": target_instance,
"detection_seconds": round(detection_seconds, 3),
"health_requests": dict(health_statuses),
"registration_requests": dict(registration_statuses),
"recovered": True,
}
rendered = json.dumps(result, ensure_ascii=False, indent=2)
if args.output:
output = Path(args.output)
if not output.is_absolute():
output = ROOT / output
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text(rendered + "\n", encoding="utf-8")
print(rendered)
print("PASS: readiness-aware gateway excluded and restored one API replica")
if __name__ == "__main__":
main()
+5
View File
@@ -209,6 +209,11 @@ func TestSessionAuthenticationLifecycle(t *testing.T) {
authorization: "Basic " + base64.StdEncoding.EncodeToString([]byte(alice.Username+":"+testPassword)),
wantStatus: http.StatusUnauthorized,
},
{
name: "non-ASCII Bearer is rejected without panic",
authorization: "Bearer é",
wantStatus: http.StatusUnauthorized,
},
{
name: "Bearer identity wins over X-User-Id",
authorization: "Bearer " + session.Token,