diff --git a/.env.example b/.env.example
index 459bbb62..677c7bb9 100644
--- a/.env.example
+++ b/.env.example
@@ -9,6 +9,8 @@ APP_ENV=lab
POSTGRES_USER=messenger
POSTGRES_DB=messenger
+POSTGRES_CPU_LIMIT=1.00
+POSTGRES_MEMORY_LIMIT=512M
SESSION_TTL_SECONDS=3600
SESSION_STORE=postgres
diff --git a/Makefile b/Makefile
index 7e286836..1dd07d60 100644
--- a/Makefile
+++ b/Makefile
@@ -7,6 +7,8 @@ BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
GRADE4_OUTPUT ?= evidence/grade4/results.json
+GRADE5_OUTPUT ?= evidence/grade5/results.json
+READINESS_OUTPUT ?= evidence/readiness-routing.json
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
@@ -27,8 +29,10 @@ COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
load \
test-persistence \
test-failover \
+ test-readiness-routing \
rolling-update \
grade4-experiment \
+ grade5-experiment \
compose-config \
compose-build \
compose-up \
@@ -51,8 +55,10 @@ help:
@echo "load: run synthetic writes against the LB entrypoint"
@echo "test-persistence: run persistence test with ACTION_SCRIPT"
@echo "test-failover: run replica failover test with ACTION_SCRIPT"
+ @echo "test-readiness-routing: isolate PostgreSQL from one API and verify active LB health checks"
@echo "rolling-update: replace API replicas one by one with surge capacity"
@echo "grade4-experiment: run failovers, idempotency and rolling update with evidence"
+ @echo "grade5-experiment: measure fixed-budget scaling, saturation and one improvement"
@echo "compose-config: validate resolved Compose configuration"
@echo "compose-build: build application images"
@echo "compose-up: start the stack"
@@ -114,6 +120,9 @@ test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+test-readiness-routing:
+ $(PYTHON) scripts/test_readiness_routing.py --env-file "$(ENV_FILE)" --output "$(READINESS_OUTPUT)"
+
rolling-update:
ENV_FILE="$(ENV_FILE)" ./scripts/rolling-update.sh
@@ -121,6 +130,10 @@ grade4-experiment:
$(PYTHON) scripts/grade4_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE4_OUTPUT)"
$(PYTHON) scripts/render_grade4.py "$(GRADE4_OUTPUT)"
+grade5-experiment:
+ $(PYTHON) scripts/grade5_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE5_OUTPUT)"
+ $(PYTHON) scripts/render_grade5.py "$(GRADE5_OUTPUT)"
+
compose-config:
@test -f "$(ENV_FILE)" || \
(echo "$(ENV_FILE) is missing; run: make init-env"; exit 2)
diff --git a/REPORT.md b/REPORT.md
index 59034639..7ca111bd 100644
--- a/REPORT.md
+++ b/REPORT.md
@@ -1,11 +1,11 @@
# Отчёт — лабораторная 3
- Автор, группа: Чурилин, БВТ2301
-- Текущий завершённый уровень: оценка «4»
-- Итоговая цель: оценка «5»; исследовательский трек выполняется следующим этапом
+- Текущий завершённый уровень: оценка «5», трек 1 «Цена масштабирования»
+- Итоговая цель: выполнена; уровень «4» и законченный исследовательский трек
- Commit/PR: заполняется автором после локального коммита и публикации
- Использованные источники и библиотеки: преподавательские README/OpenAPI/tests,
- документация Go, pgx, PostgreSQL, Docker Compose, Nginx, React, TypeScript,
+ документация Go, pgx, PostgreSQL, Docker Compose, HAProxy, Nginx, React, TypeScript,
Vite, TanStack Query, React Router, Lucide и Vitest. Для разбора архитектуры,
рефакторинга, реализации и проверок использовался OpenAI Codex; решения
проверены тестами и описанными ниже воспроизводимыми сценариями.
@@ -35,13 +35,16 @@ cp .env.example .env
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --wait
make test
+make test-readiness-routing
make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
docker compose --env-file .env up -d --wait api
make test-persistence ACTION_SCRIPT=./scripts/restart.sh
make grade4-experiment
+make grade5-experiment
```
-По этому же адресу Nginx отдаёт production-сборку React. Переход на
+По этому же адресу HAProxy направляет обычные browser-запросы во внутренний
+Nginx, который отдаёт production-сборку React. Переход на
`http://localhost:8080` открывает Messenger, а `/api/v1/*` и health endpoints
остаются доступны автоматическим проверкам. Отдельный Node.js на машине для
обычного запуска не нужен: multi-stage image сам выполняет `npm ci`, Vitest,
@@ -84,15 +87,22 @@ PostgreSQL остаётся источником истины для польз
PostgreSQL.
Compose запускает один сервис `api` с `API_REPLICAS=2`. Реплики не имеют host
-ports и разделяют PostgreSQL, включая таблицу сессий. Nginx — единственная
-публичная точка входа; он использует `least_conn` и Docker DNS resolver с
-повторным разрешением имени `api` раз в секунду. Sticky sessions не применяются.
+ports и разделяют PostgreSQL, включая таблицу сессий. HAProxy — единственная
+публичная точка входа; он использует `leastconn`, Docker DNS resolver и активные
+HTTP-проверки `/health/ready` раз в секунду. Реплика исключается после двух
+неуспешных readiness-проверок и возвращается после двух успешных. Поэтому живой,
+но потерявший PostgreSQL и ставший неготовым процесс перестаёт получать трафик;
+Docker healthcheck не является единственным механизмом обнаружения. Sticky
+sessions не применяются. Nginx остаётся отдельным внутренним контейнером и
+обслуживает только статические файлы React.
+
Произвольные POST балансировщик автоматически не повторяет. Исключение сделано
только для маршрута отправки сообщения: обязательный `Idempotency-Key` позволяет
-Nginx безопасно повторить запрос на другой реплике при 502/503/504 во время
-drain. Клиент также сохраняет ключ для повтора после неопределённого результата.
-Создание пользователя и чата контрактом не идемпотентно, поэтому автоматический
-retry для этих маршрутов запрещён.
+HAProxy безопасно повторить запрос на другой реплике при connection failure,
+empty response, timeout или 502/503/504 во время обнаружения отказа и drain.
+Для этого маршрута тело запроса буферизуется отдельным backend; регистрации
+пользователя и создание чата контрактом не идемпотентны, поэтому retry для них
+запрещён. Клиент также сохраняет ключ для повтора после неопределённого результата.
Миграции вынесены в одноразовый сервис `migrate`, завершение которого является
предусловием запуска API. Поэтому две реплики не конкурируют за изменение схемы.
@@ -154,7 +164,7 @@ React Router — за маршруты login/register, выбранного ча
`SameSite=Strict`. Флаг `Secure` управляется `SESSION_COOKIE_SECURE`: `false`
нужен только локальному HTTP, на HTTPS должен быть `true`.
-Контрактные `/api/v1/*` сознательно остались Bearer-only. Nginx предоставляет
+Контрактные `/api/v1/*` сознательно остались Bearer-only. HAProxy предоставляет
отдельный браузерный адаптер `/web-api/v1/*`: берёт token из HttpOnly-cookie,
формирует `Authorization: Bearer ...`, удаляет исходный `Cookie` и проксирует на
обычный `/api/v1/*`. Поэтому преподавательские тесты и внешние API-клиенты не
@@ -173,18 +183,23 @@ cookie.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Воспроизводимая конфигурация и качество Go | `make check && make go-check && make validate PYTHON=../mtusi2026lab2/.venv/bin/python` | форматирование, tests, race detector, vet, Compose config и OpenAPI validation завершились без ошибок | `evidence/lab3-summary.md` |
-| Две API-реплики за одним LB | `docker compose --env-file .env up -d --build --wait`; 20 health-запросов через `:8080` | наружу опубликован только Nginx; два разных `X-Instance-Id`, по 10 ответов от каждого | `compose.yaml`, `infrastructure/nginx/nginx.conf`, `evidence/lab3-summary.md` |
+| Две API-реплики за одним LB | `docker compose --env-file .env up -d --build --wait`; 20 health-запросов через `:8080` | наружу опубликован только HAProxy; два разных `X-Instance-Id`, по 10 ответов от каждого | `compose.yaml`, `infrastructure/haproxy/haproxy.cfg`, `evidence/lab3-summary.md` |
| Публичный smoke lab3 | `make test` | `tests/smoke.py` завершился `PASS: lab3 public API scenario` | `evidence/lab3-summary.md` |
| Общая сессия без sticky routing | smoke, затем failover и persistence через тот же bearer token | обе реплики обслуживали запросы; сессия работает после остановки API и рестартов | `evidence/lab3-summary.md` |
| Идемпотентность сообщения | `go test ./student_tests -run Idempotency`; 32 конкурентных POST с одним ключом через LB | один Message и ровно одна строка в PostgreSQL; другой payload даёт 409 | `student_tests/idempotency_test.go`, `evidence/lab3-summary.md` |
| Failover каждой API-реплики | два последовательных запуска `make test-failover ACTION_SCRIPT=./scripts/stop-one.sh` с восстановлением между ними | реально остановлены `api-2`, затем `api-1`; получено 142/143 успешных записи, 0 ошибок, восстановление 0.963/0.750 с | `scripts/stop-one.sh`, `evidence/lab3-summary.md` |
| Persistence после рестартов | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS; прежняя сессия и данные доступны после рестарта PostgreSQL, Redis, API и LB | `scripts/restart.sh`, `evidence/lab3-summary.md` |
| Миграции перед API | `docker compose --env-file .env ps -a` и логи `migrate` | миграции 1 и 2 применены; one-shot контейнер завершился с кодом 0; API healthy | `migrations/000002_message_idempotency.up.sql`, `evidence/lab3-summary.md` |
-| React frontend и HttpOnly BFF | `docker compose build load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/Dockerfile`, `infrastructure/nginx/nginx.conf` |
+| React frontend и HttpOnly BFF | `docker compose build frontend load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/`, `infrastructure/haproxy/` |
| Согласованные health и shutdown | SIGTERM каждой реплики под нагрузкой | API сразу закрывает availability gate, 2 с дренирует запросы и укладывается в 10 с shutdown; `/health/live` остаётся доступен | `internal/transport/http/middleware/availability.go`, `evidence/grade4/` |
-| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только Nginx `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
-| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 3,464/3,476 с, одна краткая ошибка из 4614 запросов в первом прогоне и 0 из 5815 во втором | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
-| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 20 350 успешных запросов, 1 краткий 502 (0,005%), все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
+| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только HAProxy `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
+| Readiness-aware routing | `make test-readiness-routing` | одной API-реплике заблокирован PostgreSQL: HAProxy исключил её за 2,861 с; 40 health-запросов получили 200, 20 регистраций — 201; после снятия fault реплика вернулась | `scripts/test_readiness_routing.py`, `evidence/readiness-routing.json` |
+| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 2,893/2,938 с; во всех фазах 0 ошибочных запросов | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
+| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 22 212 успешных запросов, 0 ошибок; все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
+| Фиксированный бюджет 1/2/4 API | `make grade5-experiment`; 32 closed-loop клиента | при суммарных 1 CPU, 512 MiB и 16 соединениях получено 578,242 / 529,499 / 353,955 RPS; четыре процесса при том же бюджете дали на 38,79% меньше одной реплики | `evidence/grade5/02-fixed-total.svg`, `measurements.csv` |
+| Рост ресурсов 1/2/4 API | тот же workload; по 0,5 CPU, 256 MiB и 8 соединений на реплику | 533,656 / 563,460 / 438,868 RPS; нелинейная серия показывает отсутствие пропорционального выигрыша при общей БД в 1 CPU | `evidence/grade5/03-resource-growth.svg`, `results.json` |
+| Saturation point и bottleneck | concurrency 1/2/4/8/16/32, по 10 с после warm-up | первый сигнал насыщения — 8: относительно 4 клиентов RPS вырос только на 8,03%, p95 — в 1,92 раза; PostgreSQL в профиле четырёх реплик достиг 101,91% одного CPU | `evidence/grade5/01-saturation.svg`, `04-bottleneck.svg` |
+| Подтверждённое улучшение | четыре API без изменений; PostgreSQL 1 → 2 CPU | RPS 438,868 → 587,574 (+33,88%), ошибок 0, p95 снизился на 14,12%; следующим ограничением стал HAProxy с лимитом 0,25 CPU | `evidence/grade5/05-improvement.svg`, `resources.csv` |
## Эксперименты и отказы
@@ -192,8 +207,8 @@ cookie.
Docker Engine 29.7.2, Compose 5.5.0, 8 логических CPU и 3813,6 MiB RAM,
commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиченными
изменениями текущего уровня. В Compose каждой API-реплике выделено 0,5 CPU и
-256 MiB; Nginx — 0,25 CPU/128 MiB; PostgreSQL — 1 CPU/512 MiB; Redis —
-0,25 CPU/128 MiB.
+256 MiB; HAProxy — 0,25 CPU/128 MiB; frontend Nginx — 0,10 CPU/64 MiB;
+PostgreSQL — 1 CPU/512 MiB; Redis — 0,25 CPU/128 MiB.
Команда `make grade4-experiment` создаёт синтетических пользователей и чат,
делает 12 warm-up записей, затем запускает 8 closed-loop клиентов без повторов
@@ -202,44 +217,144 @@ commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиче
следующей, после действия нагрузка продолжается ещё 8 с. Rolling update
временно увеличивает сервис до трёх реплик, ждёт healthcheck и два периода
Docker DNS, заменяет старые контейнеры по одному и возвращает масштаб к двум.
-Минимальный подтверждённый размер набора после прогона — 34 808 сообщений.
+Минимальный подтверждённый размер набора после прогона — 43 810 сообщений.
| Фаза | Успешно / ошибок | RPS | Доля ошибок | p50 / p95 / p99, мс | Восстановление |
| --- | ---: | ---: | ---: | ---: | ---: |
-| baseline | 4017 / 0 | 319,463 | 0% | 16,799 / 55,515 / 93,803 | — |
-| остановка реплики 1 | 4613 / 1 | 301,040 | 0,0217% | 17,145 / 69,558 / 89,752 | 3,464 с |
-| остановка реплики 2 | 5815 / 0 | 374,449 | 0% | 14,211 / 49,542 / 81,040 | 3,476 с |
-| rolling update | 20 350 / 1 | 376,959 | 0,0049% | 15,725 / 48,724 / 79,337 | непрерывная нагрузка |
+| baseline | 5762 / 0 | 479,581 | 0% | 9,178 / 56,402 / 79,978 | — |
+| остановка реплики 1 | 7853 / 0 | 537,548 | 0% | 8,812 / 50,050 / 66,373 | 2,893 с |
+| остановка реплики 2 | 7970 / 0 | 533,154 | 0% | 8,726 / 48,789 / 64,333 | 2,938 с |
+| rolling update | 22 212 / 0 | 428,860 | 0% | 10,560 / 58,713 / 83,982 | непрерывная нагрузка |
-На baseline средняя выборка одного API-контейнера составила 28,453% CPU и
-42,413 MiB RAM, максимум — 36,1% и 66,07 MiB. PostgreSQL оказался самым
-нагруженным компонентом: в rolling-фазе среднее 98,172% CPU при лимите одного
-ядра, максимум RAM 145,2 MiB. Это не capacity-тест и не доказательство
+На baseline средняя выборка одного API-контейнера составила 27,144% CPU и
+31,008 MiB RAM, максимум — 30,93% и 66,17 MiB. PostgreSQL оказался самым
+нагруженным компонентом: среднее 89,392% CPU при лимите одного ядра, максимум
+110,59%; HAProxy использовал в среднем 26,262% CPU. Это не capacity-тест и не доказательство
saturation point, но важный ориентир для исследования уровня 5.
-Nginx обновляет Docker DNS раз в 1 с, исключает peer после двух неудач за 2 с,
-а connect timeout равен 1 с. При постоянной нагрузке ожидаемое максимальное
-обнаружение — около 4 с; измеренные 3,464 и 3,476 с укладываются в требование
-10 с. Одна 504 и одна 502 сохранены в отчёте честно: README прямо допускает
-краткие ошибки обнаружения. Подтверждённых потерь и дублей нет. Параллельные
-32 POST с одним ключом прошли через обе реплики и вернули один Message ID.
+HAProxy обновляет Docker DNS раз в 1 с и исключает peer после двух неуспешных
+активных `/health/ready`-проверок. Измеренные 2,893 и 2,938 с укладываются в
+требование 10 с. Отдельный тест потери доступа одной реплики к PostgreSQL
+подтвердил исключение неготового backend за 2,861 с без 5xx: 40 health-запросов
+получили 200, а 20 регистраций — 201. Во всех четырёх фазах grade-4 эксперимента
+ошибок нет. Параллельные 32 POST с одним ключом прошли через обе реплики и
+вернули один Message ID.
Исходные агрегаты, environment/limits, посекундные таблицы и четыре SVG-графика
лежат в `evidence/grade4/`. Percentile рассчитаны только по успешным ответам;
профиль closed-loop имеет coordinated omission. CPU для API агрегируется как
средняя/максимальная выборка отдельного контейнера, а не сумма двух реплик.
+## Исследование уровня 5: цена масштабирования
+
+Трек 1 выполнен командой `make grade5-experiment` 23.10.2026 на том же
+однохостовом стенде Windows + WSL2: Docker Engine 29.7.2, Compose 5.5.0,
+8 логических CPU и 3813,6 MiB RAM. Базовым commit до изменений уровня 5 был
+`5300a1a`; измерения ожидаемо зафиксировали dirty working tree, поскольку сам
+скрипт и параметризация лимитов ещё не были закоммичены. Все запросы проходили
+через единственную публичную точку HAProxy, создавали сообщения с уникальными
+`Idempotency-Key` и не повторялись нагрузочным клиентом автоматически.
+
+### Методика и воспроизводимость
+
+Каждая измеряемая фаза длилась 10 секунд. После каждой смены профиля выполнялся
+двухсекундный warm-up, затем контейнеры и реально применённые `NanoCpus`, RAM и
+`DB_MAX_CONNS` проверялись через `docker inspect`. Скрипт сохраняет посекундные
+request buckets и каждую выборку `docker stats`; CPU/RAM API сначала суммируются
+по всем репликам в одной временной точке и только затем усредняются. Это устраняет
+ошибку, при которой средняя загрузка одного из четырёх контейнеров выдавалась бы
+за стоимость всего API-сервиса.
+
+В серии с фиксированным бюджетом сохранялись суммарные 1 CPU, 512 MiB RAM и
+16 соединений пула API:
+
+| Реплики | CPU на реплику | RAM на реплику | `DB_MAX_CONNS` на реплику | Суммарный пул |
+| ---: | ---: | ---: | ---: | ---: |
+| 1 | 1,00 | 512 MiB | 16 | 16 |
+| 2 | 0,50 | 256 MiB | 8 | 16 |
+| 4 | 0,25 | 128 MiB | 4 | 16 |
+
+В серии роста каждая реплика получала 0,5 CPU, 256 MiB и 8 соединений, поэтому
+суммарный API-бюджет менялся от 0,5 CPU/256 MiB/8 соединений до
+2 CPU/1024 MiB/32 соединений. Во всех основных профилях PostgreSQL оставался на
+1 CPU/512 MiB, HAProxy — 0,25 CPU/128 MiB, frontend Nginx — 0,10 CPU/64 MiB,
+Redis — 0,25 CPU/128 MiB. Сравнение
+1/2/4 выполнялось при одинаковом concurrency 32. До и после эксперимента
+зафиксированы размер набора и базы: 225 626 → 299 524 сообщений и
+175 406 103 → 228 793 367 байт. После любого исхода `finally` возвращает стенд к
+двум API по 0,5 CPU/256 MiB, пулу 10 на реплику и PostgreSQL с 1 CPU; успешное
+восстановление также записано в `results.json`.
+
+### Насыщение и цена реплик
+
+Для поиска saturation point две API-реплики с суммарным 1 CPU и пулом 16
+нагружались последовательностью concurrency 1/2/4/8/16/32:
+
+| Concurrency | RPS | p95, мс | PostgreSQL CPU avg/max | Ошибки |
+| ---: | ---: | ---: | ---: | ---: |
+| 1 | 230,369 | 5,065 | 23,900% / 29,41% | 0 |
+| 2 | 320,190 | 7,196 | 54,987% / 62,60% | 0 |
+| 4 | 435,822 | 29,304 | 63,093% / 81,80% | 0 |
+| 8 | 470,828 | 56,230 | 65,945% / 92,98% | 0 |
+| 16 | 472,892 | 73,628 | 65,080% / 88,93% | 0 |
+| 32 | 564,012 | 94,785 | 83,825% / 107,62% | 0 |
+
+Заранее зафиксированное правило считает первым saturation signal шаг, где прирост
+RPS меньше 10%, а p95 одновременно вырастает не менее чем в 1,5 раза. Это
+произошло при переходе 4 → 8: throughput вырос только на 8,03%, а p95 — в
+1,92 раза. При 16 клиентах прирост относительно 8 почти отсутствует (+0,44%),
+а p95 продолжает расти; при 32 клиентах RPS увеличивается ценой ещё большей
+очереди. Поэтому concurrency 8 — первый воспроизводимый saturation signal по
+заранее заданному правилу, а не утверждение об абсолютном максимуме системы.
+
+При фиксированном суммарном бюджете 1/2/4 реплики дали соответственно
+578,242 / 529,499 / 353,955 RPS. Четыре процесса при том же общем бюджете дали
+на 38,79% меньше одной реплики: дробление ресурсов добавило scheduling и pool
+overhead, но не увеличило общую ёмкость. В серии роста ресурсов получены
+533,656 / 563,460 / 438,868 RPS. Разброс и нестрогая монотонность честно
+сохранены: это один прогон на одном Docker-host, где контейнеры конкурируют за
+общий scheduler и одну БД. Главный результат здесь не выбор «лучшего» количества
+реплик, а отсутствие пропорционального роста: четырёхкратный API-бюджет дал на
+17,76% меньше throughput одной реплики в этом конкретном прогоне.
+
+Доминирующее ограничение локализовано в PostgreSQL. В базовом профиле четырёх
+реплик API суммарно использовал в среднем 35,690% CPU из доступных 200%, HAProxy —
+15,595% из 25%, тогда как PostgreSQL использовал 84,847% в среднем и достигал
+101,91% лимита одного ядра. Само по себе это корреляция, поэтому выполнено
+контрольное изменение только одного параметра.
+
+### Подтверждённое улучшение
+
+Для тех же четырёх API по 0,5 CPU/256 MiB, того же пула и concurrency 32 лимит
+PostgreSQL увеличен с 1 до 2 CPU. RPS вырос с 438,868 до 587,574 — на 33,88%;
+ошибок в обеих фазах не было, p95 снизился с 109,446 до 93,991 мс (-14,12%).
+Таким образом, снятие именно ограничения CPU БД существенно увеличило throughput
+и улучшило tail latency, что подтверждает исходную гипотезу.
+
+После улучшения PostgreSQL использовал в среднем 95,573% из доступных 200%, а
+HAProxy — 22,350% при лимите 0,25 CPU и достигал 26,73%. То есть следующее
+ограничение ожидаемо сместилось к публичному
+балансировщику. Увеличивать API дальше без пересмотра лимита или топологии LB уже
+нерационально — это и есть практическая «цена масштабирования».
+
+Полный JSON с окружением, профилями, фактически применёнными лимитами и сырыми
+выборками находится в `evidence/grade5/results.json`; агрегаты — в
+`measurements.csv`, посекундные запросы — в `timeseries.csv`, ресурсы — в
+`resources.csv`. Пять SVG показывают saturation curve, обе серии 1/2/4,
+локализацию bottleneck и контрольное улучшение.
+
## Ограничения и следующие шаги
-- Топология однохостовая: Nginx и PostgreSQL остаются single points of failure.
+- Топология однохостовая: HAProxy и PostgreSQL остаются single points of failure.
Репликация БД и несколько балансировщиков в требования уровня «4» не входят.
- Redis содержит только кэш, поэтому его потеря допустима. Отдельная
персистентность Redis для корректности API не требуется.
- Истёкшие записи идемпотентности удаляются лениво при повторном обращении к той
же области ключа. Индекс по `expires_at` оставлен для будущей периодической
очистки без изменения схемы.
-- Уровень «5» ещё не реализован: сравнение 1/2/4 реплик при фиксированных
- ресурсах и поиск saturation point выполняются следующим отдельным этапом.
+- Исследование уровня «5» выполняет по одному измеряемому прогону на профиль.
+ Для production capacity planning нужны несколько повторов, доверительные
+ интервалы и отдельный генератор нагрузки вне измеряемого Docker-host.
- Rolling-сценарий заменяет бинарник без изменения схемы. Миграции выполняются
однократно до API; для будущей несовместимой схемы потребуется expand/contract,
поэтому одновременная работа разных schema-version сейчас не заявляется.
diff --git a/compose.yaml b/compose.yaml
index 86e5ae22..2e49ddda 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -2,13 +2,15 @@ services:
load-balancer:
build:
context: .
- dockerfile: infrastructure/nginx/Dockerfile
- image: mtusi2026lab3-web
+ dockerfile: infrastructure/haproxy/Dockerfile
+ image: mtusi2026lab3-load-balancer
ports:
- "127.0.0.1:${APP_PORT:-8080}:8080"
depends_on:
api:
condition: service_healthy
+ frontend:
+ condition: service_healthy
healthcheck:
test:
[
@@ -27,6 +29,27 @@ services:
cpus: "0.25"
memory: 128M
+ frontend:
+ build:
+ context: .
+ dockerfile: infrastructure/nginx/Dockerfile
+ image: mtusi2026lab3-web
+ expose:
+ - "8080"
+ healthcheck:
+ test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/ >/dev/null || exit 1"]
+ interval: 5s
+ timeout: 3s
+ retries: 10
+ start_period: 3s
+ stop_grace_period: 10s
+ restart: unless-stopped
+ deploy:
+ resources:
+ limits:
+ cpus: "0.10"
+ memory: 64M
+
postgres:
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
environment:
@@ -51,8 +74,8 @@ services:
deploy:
resources:
limits:
- cpus: "1.00"
- memory: 512M
+ cpus: "${POSTGRES_CPU_LIMIT:-1.00}"
+ memory: ${POSTGRES_MEMORY_LIMIT:-512M}
redis:
image: redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0
diff --git a/evidence/README.md b/evidence/README.md
index 8df70015..a03474bc 100644
--- a/evidence/README.md
+++ b/evidence/README.md
@@ -3,3 +3,12 @@
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
+
+## Сохранённые эксперименты
+
+- `grade4/` — failover обеих API-реплик и rolling update под нагрузкой;
+- `grade5/` — трек «Цена масштабирования»: saturation sweep, сравнение 1/2/4
+ реплик с фиксированным и растущим бюджетом, сырые JSON/CSV, пять SVG-графиков
+ и подтверждённое улучшение PostgreSQL CPU;
+- `readiness-routing.json` — результат изоляции PostgreSQL от одной живой
+ API-реплики: время исключения backend, статусы запросов и восстановление.
diff --git a/evidence/grade4/README.md b/evidence/grade4/README.md
index 2d9435e3..e42cd53b 100644
--- a/evidence/grade4/README.md
+++ b/evidence/grade4/README.md
@@ -1,12 +1,12 @@
# Lab 3 grade-4 experiment
-Generated: 2026-10-23T12:31:12.749687+00:00.
+Generated: 2026-10-23T15:23:05.031807+00:00.
## Method
- concurrency: 8;
- baseline duration: 12 s;
-- load: closed-loop POST messages through Nginx;
+- load: closed-loop POST messages through the public HAProxy gateway;
- failover: both original API containers stopped and restored in turn;
- rolling update: one surge replica, old replicas replaced one by one;
- percentiles cover successful responses; failures are separate.
@@ -15,10 +15,10 @@ Generated: 2026-10-23T12:31:12.749687+00:00.
| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
-| baseline | 319.463 | 0 (0.000%) | 16.799 | 55.515 | 93.803 | n/a |
-| failover-1 | 301.04 | 1 (0.022%) | 17.145 | 69.558 | 89.752 | 3.464 |
-| failover-2 | 374.449 | 0 (0.000%) | 14.211 | 49.542 | 81.04 | 3.476 |
-| rolling-update | 376.959 | 1 (0.005%) | 15.725 | 48.724 | 79.337 | n/a |
+| baseline | 479.581 | 0 (0.000%) | 9.178 | 56.402 | 79.978 | n/a |
+| failover-1 | 537.548 | 0 (0.000%) | 8.812 | 50.05 | 66.373 | 2.893 |
+| failover-2 | 533.154 | 0 (0.000%) | 8.726 | 48.789 | 64.333 | 2.938 |
+| rolling-update | 428.86 | 0 (0.000%) | 10.56 | 58.713 | 83.982 | n/a |
## Parallel idempotency
@@ -36,4 +36,4 @@ Generated: 2026-10-23T12:31:12.749687+00:00.
## Scope
-Single-host process failure and rolling replacement only. PostgreSQL and Nginx remain single points of failure.
+Single-host process failure and rolling replacement only. PostgreSQL and HAProxy remain single points of failure.
diff --git a/evidence/grade4/latency.svg b/evidence/grade4/latency.svg
index 7c20a4ed..4120856b 100644
--- a/evidence/grade4/latency.svg
+++ b/evidence/grade4/latency.svg
@@ -16,53 +16,53 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
0
-19
+17
-38
+34
-56
+50
-75
+67
-94
-
-17
+84
+
+9
p50
-
-56
+
+56
p95
-
-94
+
+80
p99
baseline
-
-17
+
+9
p50
-
-70
+
+50
p95
-
-90
+
+66
p99
failover-1
-
-14
+
+9
p50
-
-50
+
+49
p95
-
-81
+
+64
p99
failover-2
-
-16
+
+11
p50
-
-49
+
+59
p95
-
-79
+
+84
p99
rolling-update
Successful latency, ms
diff --git a/evidence/grade4/resources.svg b/evidence/grade4/resources.svg
index 2edf10bf..f5bd373d 100644
--- a/evidence/grade4/resources.svg
+++ b/evidence/grade4/resources.svg
@@ -14,29 +14,35 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
Grade 4: peak container resources
Peak CPU, %
-
-52.6
-api
-
-23.4
-load-balancer
-
-105.2
-postgres
-
-5.9
-redis
+
+49.5
+api
+
+8.1
+frontend
+
+29.4
+load-balancer
+
+110.6
+postgres
+
+6.6
+redis
Peak memory, MiB
-
-66.1
-api
-
-13.5
-load-balancer
-
-145.2
-postgres
-
-12.3
-redis
+
+66.2
+api
+
+12.2
+frontend
+
+19.6
+load-balancer
+
+185.9
+postgres
+
+16.6
+redis
diff --git a/evidence/grade4/results.json b/evidence/grade4/results.json
index 739dc0d3..5cd27d9d 100644
--- a/evidence/grade4/results.json
+++ b/evidence/grade4/results.json
@@ -1,8 +1,8 @@
{
"schema_version": 1,
"environment": {
- "utc_started_at": "2026-10-23T12:31:12.749687+00:00",
- "git_commit": "34618bd4237bbb8bf6a3a303056fc95d799577f9",
+ "utc_started_at": "2026-10-23T15:23:05.031807+00:00",
+ "git_commit": "5300a1a1e0e5497f2d30097bb1542b2743a8aad5",
"working_tree_dirty": true,
"logical_cpu_count": 8,
"host_memory_mib": 3813.6,
@@ -19,6 +19,11 @@
"nano_cpus": 500000000,
"memory_limit_bytes": 268435456
},
+ {
+ "service": "frontend",
+ "nano_cpus": 100000000,
+ "memory_limit_bytes": 67108864
+ },
{
"service": "load-balancer",
"nano_cpus": 250000000,
@@ -43,611 +48,611 @@
"action_after_seconds": 3,
"post_action_seconds": 8,
"warmup_requests": 12,
- "workload": "closed-loop POST messages via Nginx; no automatic retries"
+ "workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries"
},
"phases": [
{
"name": "baseline",
- "elapsed_seconds": 12.574,
- "successful_requests": 4017,
+ "elapsed_seconds": 12.015,
+ "successful_requests": 5762,
"failed_requests": 0,
"error_ratio": 0.0,
"errors": {},
- "successful_rps": 319.463,
+ "successful_rps": 479.581,
"successful_latency_ms": {
- "p50": 16.799,
- "p95": 55.515,
- "p99": 93.803
+ "p50": 9.178,
+ "p95": 56.402,
+ "p99": 79.978
},
"instances": [
- "0b933a46fccb",
- "55aaffcf508a"
+ "a85567c48d67",
+ "df8a69558af0"
],
"recovery_seconds": null,
"action": null,
"per_second": {
"0": {
- "successful": 118,
+ "successful": 434,
"failed": 0
},
"1": {
- "successful": 308,
+ "successful": 280,
"failed": 0
},
"2": {
- "successful": 386,
+ "successful": 372,
"failed": 0
},
"3": {
- "successful": 378,
+ "successful": 436,
"failed": 0
},
"4": {
- "successful": 344,
+ "successful": 541,
"failed": 0
},
"5": {
- "successful": 322,
+ "successful": 474,
"failed": 0
},
"6": {
- "successful": 316,
+ "successful": 526,
"failed": 0
},
"7": {
- "successful": 384,
+ "successful": 539,
"failed": 0
},
"8": {
- "successful": 380,
+ "successful": 554,
"failed": 0
},
"9": {
- "successful": 402,
+ "successful": 559,
"failed": 0
},
"10": {
- "successful": 377,
+ "successful": 448,
"failed": 0
},
"11": {
- "successful": 288,
+ "successful": 584,
"failed": 0
},
"12": {
- "successful": 14,
+ "successful": 15,
"failed": 0
}
},
"resources": {
"api": {
- "cpu_percent_avg": 28.453,
- "cpu_percent_max": 36.1,
- "memory_mib_avg": 42.413,
- "memory_mib_max": 66.07
+ "cpu_percent_avg": 27.144,
+ "cpu_percent_max": 30.93,
+ "memory_mib_avg": 31.008,
+ "memory_mib_max": 66.17
+ },
+ "frontend": {
+ "cpu_percent_avg": 1.812,
+ "cpu_percent_max": 3.76,
+ "memory_mib_avg": 7.212,
+ "memory_mib_max": 10.15
},
"load-balancer": {
- "cpu_percent_avg": 17.448,
- "cpu_percent_max": 20.8,
- "memory_mib_avg": 9.214,
- "memory_mib_max": 9.559
+ "cpu_percent_avg": 26.262,
+ "cpu_percent_max": 27.37,
+ "memory_mib_avg": 15.645,
+ "memory_mib_max": 15.97
},
"postgres": {
- "cpu_percent_avg": 85.888,
- "cpu_percent_max": 105.25,
- "memory_mib_avg": 97.975,
- "memory_mib_max": 100.8
+ "cpu_percent_avg": 89.392,
+ "cpu_percent_max": 110.59,
+ "memory_mib_avg": 130.125,
+ "memory_mib_max": 132.2
},
"redis": {
- "cpu_percent_avg": 2.552,
- "cpu_percent_max": 5.79,
- "memory_mib_avg": 8.796,
- "memory_mib_max": 11.74
+ "cpu_percent_avg": 2.488,
+ "cpu_percent_max": 6.58,
+ "memory_mib_avg": 15.072,
+ "memory_mib_max": 15.25
}
}
},
{
"name": "failover-1",
- "elapsed_seconds": 15.324,
- "successful_requests": 4613,
- "failed_requests": 1,
- "error_ratio": 0.000217,
- "errors": {
- "POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 504": 1
- },
- "successful_rps": 301.04,
+ "elapsed_seconds": 14.609,
+ "successful_requests": 7853,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 537.548,
"successful_latency_ms": {
- "p50": 17.145,
- "p95": 69.558,
- "p99": 89.752
+ "p50": 8.812,
+ "p95": 50.05,
+ "p99": 66.373
},
"instances": [
- "0b933a46fccb",
- "55aaffcf508a"
+ "a85567c48d67",
+ "df8a69558af0"
],
- "recovery_seconds": 3.464,
+ "recovery_seconds": 2.893,
"action": {
- "started_at": 3.006,
- "ended_at": 6.463,
- "duration_seconds": 3.457,
+ "started_at": 3.003,
+ "ended_at": 5.888,
+ "duration_seconds": 2.885,
"exit_code": 0,
- "summary": "stopped API instance 55aaffcf508a"
+ "summary": "stopped API instance df8a69558af0"
},
"per_second": {
"0": {
- "successful": 404,
+ "successful": 551,
"failed": 0
},
"1": {
- "successful": 426,
+ "successful": 605,
"failed": 0
},
"2": {
- "successful": 358,
+ "successful": 558,
"failed": 0
},
"3": {
- "successful": 399,
+ "successful": 443,
"failed": 0
},
"4": {
- "successful": 392,
+ "successful": 575,
"failed": 0
},
"5": {
- "successful": 322,
+ "successful": 538,
"failed": 0
},
"6": {
- "successful": 299,
+ "successful": 593,
"failed": 0
},
"7": {
- "successful": 329,
- "failed": 1
+ "successful": 623,
+ "failed": 0
},
"8": {
- "successful": 313,
+ "successful": 507,
"failed": 0
},
"9": {
- "successful": 229,
+ "successful": 626,
"failed": 0
},
"10": {
- "successful": 155,
+ "successful": 608,
"failed": 0
},
"11": {
- "successful": 187,
+ "successful": 601,
"failed": 0
},
"12": {
- "successful": 299,
+ "successful": 556,
"failed": 0
},
"13": {
- "successful": 317,
- "failed": 0
- },
- "14": {
- "successful": 184,
+ "successful": 469,
"failed": 0
}
},
"resources": {
"api": {
- "cpu_percent_avg": 27.62,
- "cpu_percent_max": 52.59,
- "memory_mib_avg": 11.58,
- "memory_mib_max": 12.66
+ "cpu_percent_avg": 35.916,
+ "cpu_percent_max": 49.48,
+ "memory_mib_avg": 11.892,
+ "memory_mib_max": 13.27
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.762,
+ "cpu_percent_max": 3.75,
+ "memory_mib_avg": 6.256,
+ "memory_mib_max": 6.434
},
"load-balancer": {
- "cpu_percent_avg": 18.536,
- "cpu_percent_max": 21.25,
- "memory_mib_avg": 9.351,
- "memory_mib_max": 9.523
+ "cpu_percent_avg": 22.21,
+ "cpu_percent_max": 26.89,
+ "memory_mib_avg": 16.312,
+ "memory_mib_max": 16.93
},
"postgres": {
- "cpu_percent_avg": 81.822,
- "cpu_percent_max": 104.67,
- "memory_mib_avg": 100.512,
- "memory_mib_max": 109.1
+ "cpu_percent_avg": 73.128,
+ "cpu_percent_max": 89.46,
+ "memory_mib_avg": 135.64,
+ "memory_mib_max": 141.9
},
"redis": {
- "cpu_percent_avg": 1.486,
- "cpu_percent_max": 5.88,
- "memory_mib_avg": 8.04,
- "memory_mib_max": 8.25
+ "cpu_percent_avg": 0.782,
+ "cpu_percent_max": 3.27,
+ "memory_mib_avg": 15.258,
+ "memory_mib_max": 15.73
}
}
},
{
"name": "failover-2",
- "elapsed_seconds": 15.529,
- "successful_requests": 5815,
+ "elapsed_seconds": 14.949,
+ "successful_requests": 7970,
"failed_requests": 0,
"error_ratio": 0.0,
"errors": {},
- "successful_rps": 374.449,
+ "successful_rps": 533.154,
"successful_latency_ms": {
- "p50": 14.211,
- "p95": 49.542,
- "p99": 81.04
+ "p50": 8.726,
+ "p95": 48.789,
+ "p99": 64.333
},
"instances": [
- "0b933a46fccb",
- "55aaffcf508a"
+ "a85567c48d67",
+ "df8a69558af0"
],
- "recovery_seconds": 3.476,
+ "recovery_seconds": 2.938,
"action": {
- "started_at": 3.013,
- "ended_at": 6.485,
- "duration_seconds": 3.472,
+ "started_at": 3.006,
+ "ended_at": 5.933,
+ "duration_seconds": 2.927,
"exit_code": 0,
- "summary": "stopped API instance 0b933a46fccb"
+ "summary": "stopped API instance a85567c48d67"
},
"per_second": {
"0": {
- "successful": 396,
+ "successful": 584,
"failed": 0
},
"1": {
- "successful": 439,
+ "successful": 613,
"failed": 0
},
"2": {
- "successful": 412,
+ "successful": 570,
"failed": 0
},
"3": {
- "successful": 355,
+ "successful": 465,
"failed": 0
},
"4": {
- "successful": 378,
+ "successful": 564,
"failed": 0
},
"5": {
- "successful": 440,
+ "successful": 605,
"failed": 0
},
"6": {
- "successful": 362,
+ "successful": 610,
"failed": 0
},
"7": {
- "successful": 419,
+ "successful": 579,
"failed": 0
},
"8": {
- "successful": 385,
+ "successful": 497,
"failed": 0
},
"9": {
- "successful": 372,
+ "successful": 575,
"failed": 0
},
"10": {
- "successful": 420,
+ "successful": 569,
"failed": 0
},
"11": {
- "successful": 423,
+ "successful": 600,
"failed": 0
},
"12": {
- "successful": 432,
+ "successful": 586,
"failed": 0
},
"13": {
- "successful": 375,
- "failed": 0
- },
- "14": {
- "successful": 207,
+ "successful": 553,
"failed": 0
}
},
"resources": {
"api": {
- "cpu_percent_avg": 31.411,
- "cpu_percent_max": 51.63,
- "memory_mib_avg": 10.373,
- "memory_mib_max": 11.6
+ "cpu_percent_avg": 28.073,
+ "cpu_percent_max": 48.59,
+ "memory_mib_avg": 10.391,
+ "memory_mib_max": 13.47
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.696,
+ "cpu_percent_max": 3.44,
+ "memory_mib_avg": 6.25,
+ "memory_mib_max": 6.398
},
"load-balancer": {
- "cpu_percent_avg": 15.778,
- "cpu_percent_max": 21.79,
- "memory_mib_avg": 9.62,
- "memory_mib_max": 9.781
+ "cpu_percent_avg": 25.966,
+ "cpu_percent_max": 26.65,
+ "memory_mib_avg": 16.918,
+ "memory_mib_max": 17.25
},
"postgres": {
- "cpu_percent_avg": 76.87,
- "cpu_percent_max": 104.51,
- "memory_mib_avg": 106.016,
- "memory_mib_max": 120.0
+ "cpu_percent_avg": 72.558,
+ "cpu_percent_max": 93.6,
+ "memory_mib_avg": 146.22,
+ "memory_mib_max": 160.2
},
"redis": {
- "cpu_percent_avg": 0.786,
- "cpu_percent_max": 2.83,
- "memory_mib_avg": 8.844,
- "memory_mib_max": 12.28
+ "cpu_percent_avg": 0.168,
+ "cpu_percent_max": 0.2,
+ "memory_mib_avg": 15.646,
+ "memory_mib_max": 16.64
}
}
},
{
"name": "rolling-update",
- "elapsed_seconds": 53.985,
- "successful_requests": 20350,
- "failed_requests": 1,
- "error_ratio": 4.9e-05,
- "errors": {
- "POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 502": 1
- },
- "successful_rps": 376.959,
+ "elapsed_seconds": 51.793,
+ "successful_requests": 22212,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 428.86,
"successful_latency_ms": {
- "p50": 15.725,
- "p95": 48.724,
- "p99": 79.337
+ "p50": 10.56,
+ "p95": 58.713,
+ "p99": 83.982
},
"instances": [
- "0b933a46fccb",
- "3adbcad9e34b",
- "55aaffcf508a",
- "ceca3f6f1faf",
- "f0a3bedda85a"
+ "872dbac02cd3",
+ "a85567c48d67",
+ "dac03e904506",
+ "df00f21e6f41",
+ "df8a69558af0"
],
"recovery_seconds": null,
"action": {
- "started_at": 3.016,
- "ended_at": 45.961,
- "duration_seconds": 42.945,
+ "started_at": 3.005,
+ "ended_at": 42.955,
+ "duration_seconds": 39.95,
"exit_code": 0,
"summary": "rolling update completed with 2 healthy API replicas"
},
"per_second": {
"0": {
- "successful": 340,
+ "successful": 509,
"failed": 0
},
"1": {
- "successful": 369,
+ "successful": 577,
"failed": 0
},
"2": {
- "successful": 426,
+ "successful": 660,
"failed": 0
},
"3": {
- "successful": 382,
+ "successful": 438,
"failed": 0
},
"4": {
- "successful": 371,
+ "successful": 577,
"failed": 0
},
"5": {
- "successful": 376,
+ "successful": 392,
"failed": 0
},
"6": {
- "successful": 328,
+ "successful": 326,
"failed": 0
},
"7": {
- "successful": 350,
+ "successful": 232,
"failed": 0
},
"8": {
- "successful": 434,
+ "successful": 199,
"failed": 0
},
"9": {
- "successful": 411,
+ "successful": 170,
"failed": 0
},
"10": {
- "successful": 399,
+ "successful": 263,
"failed": 0
},
"11": {
- "successful": 417,
+ "successful": 323,
"failed": 0
},
"12": {
- "successful": 374,
+ "successful": 308,
"failed": 0
},
"13": {
- "successful": 427,
- "failed": 0
- },
- "14": {
- "successful": 414,
- "failed": 0
- },
- "15": {
- "successful": 418,
- "failed": 0
- },
- "16": {
- "successful": 408,
- "failed": 0
- },
- "17": {
- "successful": 352,
- "failed": 0
- },
- "18": {
- "successful": 391,
- "failed": 0
- },
- "19": {
- "successful": 388,
- "failed": 0
- },
- "20": {
- "successful": 413,
- "failed": 0
- },
- "21": {
- "successful": 417,
- "failed": 0
- },
- "22": {
- "successful": 388,
- "failed": 0
- },
- "23": {
- "successful": 396,
- "failed": 0
- },
- "24": {
- "successful": 400,
- "failed": 0
- },
- "25": {
- "successful": 339,
- "failed": 0
- },
- "26": {
- "successful": 225,
- "failed": 0
- },
- "27": {
- "successful": 376,
- "failed": 0
- },
- "28": {
- "successful": 385,
- "failed": 0
- },
- "29": {
- "successful": 396,
- "failed": 0
- },
- "30": {
- "successful": 403,
- "failed": 0
- },
- "31": {
- "successful": 262,
- "failed": 0
- },
- "32": {
- "successful": 428,
- "failed": 0
- },
- "33": {
"successful": 362,
"failed": 0
},
- "34": {
- "successful": 286,
+ "14": {
+ "successful": 443,
"failed": 0
},
- "35": {
- "successful": 337,
+ "15": {
+ "successful": 495,
"failed": 0
},
- "36": {
- "successful": 328,
+ "16": {
+ "successful": 429,
"failed": 0
},
- "37": {
- "successful": 424,
+ "17": {
+ "successful": 370,
"failed": 0
},
- "38": {
- "successful": 356,
+ "18": {
+ "successful": 451,
"failed": 0
},
- "39": {
- "successful": 386,
+ "19": {
+ "successful": 377,
"failed": 0
},
- "40": {
+ "20": {
"successful": 422,
"failed": 0
},
+ "21": {
+ "successful": 485,
+ "failed": 0
+ },
+ "22": {
+ "successful": 473,
+ "failed": 0
+ },
+ "23": {
+ "successful": 441,
+ "failed": 0
+ },
+ "24": {
+ "successful": 470,
+ "failed": 0
+ },
+ "25": {
+ "successful": 487,
+ "failed": 0
+ },
+ "26": {
+ "successful": 504,
+ "failed": 0
+ },
+ "27": {
+ "successful": 516,
+ "failed": 0
+ },
+ "28": {
+ "successful": 441,
+ "failed": 0
+ },
+ "29": {
+ "successful": 390,
+ "failed": 0
+ },
+ "30": {
+ "successful": 502,
+ "failed": 0
+ },
+ "31": {
+ "successful": 450,
+ "failed": 0
+ },
+ "32": {
+ "successful": 500,
+ "failed": 0
+ },
+ "33": {
+ "successful": 363,
+ "failed": 0
+ },
+ "34": {
+ "successful": 470,
+ "failed": 0
+ },
+ "35": {
+ "successful": 434,
+ "failed": 0
+ },
+ "36": {
+ "successful": 424,
+ "failed": 0
+ },
+ "37": {
+ "successful": 373,
+ "failed": 0
+ },
+ "38": {
+ "successful": 460,
+ "failed": 0
+ },
+ "39": {
+ "successful": 516,
+ "failed": 0
+ },
+ "40": {
+ "successful": 375,
+ "failed": 0
+ },
"41": {
- "successful": 285,
+ "successful": 492,
"failed": 0
},
"42": {
- "successful": 279,
+ "successful": 477,
"failed": 0
},
"43": {
- "successful": 222,
- "failed": 1
+ "successful": 503,
+ "failed": 0
},
"44": {
- "successful": 316,
+ "successful": 506,
"failed": 0
},
"45": {
- "successful": 359,
+ "successful": 465,
"failed": 0
},
"46": {
- "successful": 405,
+ "successful": 501,
"failed": 0
},
"47": {
- "successful": 398,
+ "successful": 502,
"failed": 0
},
"48": {
- "successful": 395,
+ "successful": 446,
"failed": 0
},
"49": {
- "successful": 430,
+ "successful": 446,
"failed": 0
},
"50": {
- "successful": 453,
- "failed": 0
- },
- "51": {
- "successful": 409,
- "failed": 0
- },
- "52": {
- "successful": 445,
- "failed": 0
- },
- "53": {
- "successful": 450,
+ "successful": 477,
"failed": 0
}
},
"resources": {
"api": {
- "cpu_percent_avg": 25.598,
- "cpu_percent_max": 37.65,
- "memory_mib_avg": 8.956,
- "memory_mib_max": 10.97
+ "cpu_percent_avg": 20.954,
+ "cpu_percent_max": 31.69,
+ "memory_mib_avg": 8.855,
+ "memory_mib_max": 12.74
+ },
+ "frontend": {
+ "cpu_percent_avg": 1.326,
+ "cpu_percent_max": 8.06,
+ "memory_mib_avg": 7.43,
+ "memory_mib_max": 12.17
},
"load-balancer": {
- "cpu_percent_avg": 19.935,
- "cpu_percent_max": 23.36,
- "memory_mib_avg": 9.836,
- "memory_mib_max": 13.52
+ "cpu_percent_avg": 24.581,
+ "cpu_percent_max": 29.42,
+ "memory_mib_avg": 17.398,
+ "memory_mib_max": 19.6
},
"postgres": {
- "cpu_percent_avg": 98.172,
- "cpu_percent_max": 105.11,
- "memory_mib_avg": 127.975,
- "memory_mib_max": 145.2
+ "cpu_percent_avg": 77.881,
+ "cpu_percent_max": 102.6,
+ "memory_mib_avg": 170.081,
+ "memory_mib_max": 185.9
},
"redis": {
- "cpu_percent_avg": 0.722,
- "cpu_percent_max": 3.94,
- "memory_mib_avg": 7.661,
- "memory_mib_max": 11.84
+ "cpu_percent_avg": 0.63,
+ "cpu_percent_max": 3.84,
+ "memory_mib_avg": 15.178,
+ "memory_mib_max": 15.91
}
}
}
@@ -658,18 +663,18 @@
"failed_responses": 0,
"unique_message_ids": 1,
"observed_instances": [
- "0b933a46fccb",
- "55aaffcf508a"
+ "a85567c48d67",
+ "df8a69558af0"
]
},
"dataset": {
"warmup_requests": 12,
- "minimum_confirmed_messages": 34808
+ "minimum_confirmed_messages": 43810
},
"limitations": [
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
- "PostgreSQL and Nginx remain single points of failure"
+ "PostgreSQL and HAProxy remain single points of failure"
]
}
diff --git a/evidence/grade4/throughput-errors.svg b/evidence/grade4/throughput-errors.svg
index 444752b2..b228e322 100644
--- a/evidence/grade4/throughput-errors.svg
+++ b/evidence/grade4/throughput-errors.svg
@@ -16,29 +16,29 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
0
-75
+108
-151
+215
-226
+323
-302
+430
-377
-
-319.5 RPS
+538
+
+479.6 RPS
baseline
errors 0.00%
-
-301.0 RPS
+
+537.5 RPS
failover-1
-errors 0.02%
-
-374.4 RPS
+errors 0.00%
+
+533.2 RPS
failover-2
errors 0.00%
-
-377.0 RPS
+
+428.9 RPS
rolling-update
errors 0.00%
Successful requests per second
diff --git a/evidence/grade4/timeline.svg b/evidence/grade4/timeline.svg
index b0f1dfa0..884b13f2 100644
--- a/evidence/grade4/timeline.svg
+++ b/evidence/grade4/timeline.svg
@@ -16,23 +16,21 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
0
-91
+132
-181
+264
-272
+396
-362
+528
-453
-
-
-
-
-failover-1 action
-
-failover-2 action
-
-rolling-update action
+660
+
+
+failover-1 action
+
+failover-2 action
+
+rolling-update action
One-second buckets; circles mark failed requests
diff --git a/evidence/grade5/01-saturation.svg b/evidence/grade5/01-saturation.svg
new file mode 100644
index 00000000..7a3ee57a
--- /dev/null
+++ b/evidence/grade5/01-saturation.svg
@@ -0,0 +1,63 @@
+
diff --git a/evidence/grade5/02-fixed-total.svg b/evidence/grade5/02-fixed-total.svg
new file mode 100644
index 00000000..ae3e0f00
--- /dev/null
+++ b/evidence/grade5/02-fixed-total.svg
@@ -0,0 +1,49 @@
+
diff --git a/evidence/grade5/03-resource-growth.svg b/evidence/grade5/03-resource-growth.svg
new file mode 100644
index 00000000..0744eef9
--- /dev/null
+++ b/evidence/grade5/03-resource-growth.svg
@@ -0,0 +1,49 @@
+
diff --git a/evidence/grade5/04-bottleneck.svg b/evidence/grade5/04-bottleneck.svg
new file mode 100644
index 00000000..fa13b0c1
--- /dev/null
+++ b/evidence/grade5/04-bottleneck.svg
@@ -0,0 +1,83 @@
+
diff --git a/evidence/grade5/05-improvement.svg b/evidence/grade5/05-improvement.svg
new file mode 100644
index 00000000..2662e79b
--- /dev/null
+++ b/evidence/grade5/05-improvement.svg
@@ -0,0 +1,37 @@
+
diff --git a/evidence/grade5/README.md b/evidence/grade5/README.md
new file mode 100644
index 00000000..e0532524
--- /dev/null
+++ b/evidence/grade5/README.md
@@ -0,0 +1,69 @@
+# Lab 3 grade-5 scaling experiment
+
+Generated: 2026-10-23T15:23:12.734339+00:00.
+
+## Hypothesis
+
+API replicas stop improving throughput when shared PostgreSQL CPU saturates; adding PostgreSQL CPU should improve the unchanged four-replica profile.
+
+## Reproduction
+
+- workload: closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries;
+- phase duration: 10 s;
+- comparison concurrency: 32;
+- saturation sweep: [1, 2, 4, 8, 16, 32];
+- fixed budget: 1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections;
+- growth series: 0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica;
+- saturation rule: RPS gain below 10% and p95 growth at least 1.5x versus the previous step;
+- host: 8 logical CPU, 3813.6 MiB RAM;
+- Docker 29.7.2, Compose 5.5.0;
+- command: `make grade5-experiment`; warmup runs before every changed profile.
+
+## Measurements
+
+| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
+| saturation-c1 | 2 | 1 | 230.369 | 0 (0.000%) | 3.839 | 5.065 | 6.282 | 23.9 | 26.85 |
+| saturation-c2 | 2 | 2 | 320.19 | 0 (0.000%) | 5.192 | 7.196 | 15.11 | 54.987 | 48.3 |
+| saturation-c4 | 2 | 4 | 435.822 | 0 (0.000%) | 6.171 | 29.304 | 47.665 | 63.093 | 54.453 |
+| saturation-c8 | 2 | 8 | 470.828 | 0 (0.000%) | 9.159 | 56.23 | 70.71 | 65.945 | 48.505 |
+| saturation-c16 | 2 | 16 | 472.892 | 0 (0.000%) | 19.862 | 73.628 | 98.773 | 65.08 | 38.028 |
+| saturation-c32 | 2 | 32 | 564.012 | 0 (0.000%) | 51.089 | 94.785 | 109.138 | 83.825 | 51.965 |
+| fixed-r1 | 1 | 32 | 578.242 | 0 (0.000%) | 48.579 | 95.757 | 115.549 | 79.058 | 36.575 |
+| fixed-r2 | 2 | 32 | 529.499 | 0 (0.000%) | 53.316 | 100.145 | 129.953 | 81.065 | 44.7 |
+| fixed-r4 | 4 | 32 | 353.955 | 0 (0.000%) | 69.354 | 139.408 | 291.312 | 74.677 | 46.25 |
+| growth-r1 | 1 | 32 | 533.656 | 0 (0.000%) | 53.378 | 93.541 | 115.58 | 77.252 | 38.273 |
+| growth-r2 | 2 | 32 | 563.46 | 0 (0.000%) | 50.516 | 96.428 | 129.46 | 85.597 | 46.653 |
+| growth-r4 | 4 | 32 | 438.868 | 0 (0.000%) | 68.331 | 109.446 | 199.616 | 84.847 | 35.69 |
+| improved-db-cpu-r4 | 4 | 32 | 587.574 | 0 (0.000%) | 49.466 | 93.991 | 112.002 | 95.573 | 55.2 |
+
+## Conclusion
+
+The first saturation signal appears at concurrency 8: RPS changes by +8.03% while p95 changes by 1.92x versus concurrency 4.
+
+Observed bottleneck classification: `postgres_cpu`.
+
+The PostgreSQL CPU change produced +33.88% RPS and -14.12% p95; confirmed improvement: `true`.
+
+After the improvement the remaining bottleneck classification is `load_balancer_cpu`.
+
+## Graphs
+
+
+
+
+
+
+
+
+
+
+
+## Raw data and limits
+
+Raw aggregates are in `measurements.csv`; one-second request buckets are in `timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and applied limits are in `results.json`.
+
+- single Docker host: replicas do not prove host-level availability
+- closed-loop workload has coordinated omission
+- successful-response latency percentiles are reported separately from failures
+- one measured run per profile; host scheduling noise is possible
diff --git a/evidence/grade5/measurements.csv b/evidence/grade5/measurements.csv
new file mode 100644
index 00000000..d777c58b
--- /dev/null
+++ b/evidence/grade5/measurements.csv
@@ -0,0 +1,14 @@
+phase,kind,replicas,concurrency,elapsed_seconds,successful_requests,failed_requests,error_ratio,successful_rps,p50_ms,p95_ms,p99_ms,api_cpu_avg,postgres_cpu_avg,load_balancer_cpu_avg,frontend_cpu_avg,redis_cpu_avg
+saturation-c1,saturation,2,1,10.731,2472,0,0.0,230.369,3.839,5.065,6.282,26.85,23.9,11.662,0.688,0.8
+saturation-c2,saturation,2,2,10.915,3495,0,0.0,320.19,5.192,7.196,15.11,48.3,54.987,19.13,0.325,0.193
+saturation-c4,saturation,2,4,10.832,4721,0,0.0,435.822,6.171,29.304,47.665,54.453,63.093,20.843,0.95,0.81
+saturation-c8,saturation,2,8,10.821,5095,0,0.0,470.828,9.159,56.23,70.71,48.505,65.945,20.867,0.86,0.953
+saturation-c16,saturation,2,16,10.876,5143,0,0.0,472.892,19.862,73.628,98.773,38.028,65.08,20.98,1.113,0.172
+saturation-c32,saturation,2,32,10.808,6096,0,0.0,564.012,51.089,94.785,109.138,51.965,83.825,21.163,1.837,2.475
+fixed-r1,fixed,1,32,10.88,6291,0,0.0,578.242,48.579,95.757,115.549,36.575,79.058,20.732,0.005,1.232
+fixed-r2,fixed,2,32,10.905,5774,0,0.0,529.499,53.316,100.145,129.953,44.7,81.065,21.15,0.823,1.093
+fixed-r4,fixed,4,32,11.917,4218,0,0.0,353.955,69.354,139.408,291.312,46.25,74.677,21.18,0.637,2.715
+growth-r1,growth,1,32,10.951,5844,0,0.0,533.656,53.378,93.541,115.58,38.273,77.252,20.062,0.693,0.958
+growth-r2,growth,2,32,10.796,6083,0,0.0,563.46,50.516,96.428,129.46,46.653,85.597,21.192,0.765,0.865
+growth-r4,growth,4,32,10.682,4688,0,0.0,438.868,68.331,109.446,199.616,35.69,84.847,15.595,0.667,0.873
+improved-db-cpu-r4,improvement,4,32,10.601,6229,0,0.0,587.574,49.466,93.991,112.002,55.2,95.573,22.35,0.83,0.458
diff --git a/evidence/grade5/resources.csv b/evidence/grade5/resources.csv
new file mode 100644
index 00000000..b51ffde1
--- /dev/null
+++ b/evidence/grade5/resources.csv
@@ -0,0 +1,261 @@
+phase,t,service,containers,cpu_percent,memory_mib
+saturation-c1,0.0,api,2,26.09,17.976
+saturation-c1,0.0,frontend,1,1.61,7.578
+saturation-c1,0.0,load-balancer,1,13.0,17.18
+saturation-c1,0.0,postgres,1,28.57,169.0
+saturation-c1,0.0,redis,1,0.15,15.28
+saturation-c1,2.356,api,2,32.65,16.886
+saturation-c1,2.356,frontend,1,0.02,7.617
+saturation-c1,2.356,load-balancer,1,13.32,17.23
+saturation-c1,2.356,postgres,1,28.96,169.5
+saturation-c1,2.356,redis,1,2.75,15.4
+saturation-c1,5.485,api,2,27.72,17.586
+saturation-c1,5.485,frontend,1,0.0,7.383
+saturation-c1,5.485,load-balancer,1,13.69,17.41
+saturation-c1,5.485,postgres,1,29.41,170.1
+saturation-c1,5.485,redis,1,0.15,15.28
+saturation-c1,8.605,api,2,20.94,17.41
+saturation-c1,8.605,frontend,1,1.12,12.06
+saturation-c1,8.605,load-balancer,1,6.64,17.24
+saturation-c1,8.605,postgres,1,8.66,170.5
+saturation-c1,8.605,redis,1,0.15,15.53
+saturation-c2,0.001,api,2,51.0,17.761
+saturation-c2,0.001,frontend,1,0.03,7.617
+saturation-c2,0.001,load-balancer,1,24.78,17.71
+saturation-c2,0.001,postgres,1,57.44,171.2
+saturation-c2,0.001,redis,1,0.17,16.03
+saturation-c2,2.361,api,2,45.61,17.516
+saturation-c2,2.361,frontend,1,0.03,7.367
+saturation-c2,2.361,load-balancer,1,24.6,17.41
+saturation-c2,2.361,postgres,1,49.85,172.6
+saturation-c2,2.361,redis,1,0.18,15.28
+saturation-c2,5.543,api,2,51.19,16.258
+saturation-c2,5.543,frontend,1,0.02,7.367
+saturation-c2,5.543,load-balancer,1,24.27,17.91
+saturation-c2,5.543,postgres,1,62.6,173.0
+saturation-c2,5.543,redis,1,0.21,15.28
+saturation-c2,8.736,api,2,45.4,17.082
+saturation-c2,8.736,frontend,1,1.22,8.121
+saturation-c2,8.736,load-balancer,1,2.87,17.2
+saturation-c2,8.736,postgres,1,50.06,173.8
+saturation-c2,8.736,redis,1,0.21,15.78
+saturation-c4,0.001,api,2,65.41,17.297
+saturation-c4,0.001,frontend,1,0.03,7.367
+saturation-c4,0.001,load-balancer,1,25.95,17.71
+saturation-c4,0.001,postgres,1,81.8,174.4
+saturation-c4,0.001,redis,1,0.19,15.28
+saturation-c4,2.372,api,2,60.01,18.188
+saturation-c4,2.372,frontend,1,3.75,7.965
+saturation-c4,2.372,load-balancer,1,26.04,17.46
+saturation-c4,2.372,postgres,1,79.42,175.4
+saturation-c4,2.372,redis,1,0.16,15.53
+saturation-c4,5.501,api,2,56.53,17.433
+saturation-c4,5.501,frontend,1,0.02,7.367
+saturation-c4,5.501,load-balancer,1,25.9,21.22
+saturation-c4,5.501,postgres,1,73.71,175.7
+saturation-c4,5.501,redis,1,0.18,15.28
+saturation-c4,8.662,api,2,35.86,16.699
+saturation-c4,8.662,frontend,1,0.0,7.367
+saturation-c4,8.662,load-balancer,1,5.48,17.53
+saturation-c4,8.662,postgres,1,17.44,177.2
+saturation-c4,8.662,redis,1,2.71,16.1
+saturation-c8,0.006,api,2,48.34,17.403
+saturation-c8,0.006,frontend,1,0.0,7.367
+saturation-c8,0.006,load-balancer,1,25.42,20.12
+saturation-c8,0.006,postgres,1,77.51,185.1
+saturation-c8,0.006,redis,1,0.18,15.28
+saturation-c8,2.372,api,2,57.14,18.602
+saturation-c8,2.372,frontend,1,3.44,7.734
+saturation-c8,2.372,load-balancer,1,26.83,17.14
+saturation-c8,2.372,postgres,1,92.98,188.5
+saturation-c8,2.372,redis,1,3.34,15.28
+saturation-c8,5.527,api,2,48.26,18.192
+saturation-c8,5.527,frontend,1,0.0,7.363
+saturation-c8,5.527,load-balancer,1,25.47,17.28
+saturation-c8,5.527,postgres,1,76.05,189.4
+saturation-c8,5.527,redis,1,0.15,15.28
+saturation-c8,8.685,api,2,40.28,21.813
+saturation-c8,8.685,frontend,1,0.0,7.363
+saturation-c8,8.685,load-balancer,1,5.75,17.64
+saturation-c8,8.685,postgres,1,17.24,190.4
+saturation-c8,8.685,redis,1,0.14,15.28
+saturation-c16,0.015,api,2,44.39,18.796
+saturation-c16,0.015,frontend,1,0.0,7.363
+saturation-c16,0.015,load-balancer,1,26.92,17.44
+saturation-c16,0.015,postgres,1,82.81,205.9
+saturation-c16,0.015,redis,1,0.17,15.28
+saturation-c16,2.354,api,2,56.54,17.856
+saturation-c16,2.354,frontend,1,0.03,7.375
+saturation-c16,2.354,load-balancer,1,26.66,17.69
+saturation-c16,2.354,postgres,1,88.93,207.7
+saturation-c16,2.354,redis,1,0.19,15.75
+saturation-c16,5.525,api,2,41.39,17.73
+saturation-c16,5.525,frontend,1,4.4,7.379
+saturation-c16,5.525,load-balancer,1,25.76,17.7
+saturation-c16,5.525,postgres,1,72.49,208.6
+saturation-c16,5.525,redis,1,0.18,15.25
+saturation-c16,8.731,api,2,9.79,18.257
+saturation-c16,8.731,frontend,1,0.02,7.375
+saturation-c16,8.731,load-balancer,1,4.58,17.55
+saturation-c16,8.731,postgres,1,16.09,209.7
+saturation-c16,8.731,redis,1,0.15,15.33
+saturation-c32,0.039,api,2,55.84,18.75
+saturation-c32,0.039,frontend,1,6.63,8.125
+saturation-c32,0.039,load-balancer,1,25.97,18.0
+saturation-c32,0.039,postgres,1,104.2,210.6
+saturation-c32,0.039,redis,1,5.85,15.05
+saturation-c32,2.29,api,2,56.87,18.933
+saturation-c32,2.29,frontend,1,0.0,7.625
+saturation-c32,2.29,load-balancer,1,27.03,19.23
+saturation-c32,2.29,postgres,1,105.41,212.1
+saturation-c32,2.29,redis,1,0.19,15.55
+saturation-c32,5.446,api,2,64.13,19.003
+saturation-c32,5.446,frontend,1,0.7,7.375
+saturation-c32,5.446,load-balancer,1,26.33,18.27
+saturation-c32,5.446,postgres,1,107.62,213.1
+saturation-c32,5.446,redis,1,3.7,15.27
+saturation-c32,8.653,api,2,31.02,19.379
+saturation-c32,8.653,frontend,1,0.02,7.625
+saturation-c32,8.653,load-balancer,1,5.32,18.36
+saturation-c32,8.653,postgres,1,18.07,213.5
+saturation-c32,8.653,redis,1,0.16,15.26
+fixed-r1,0.034,api,1,45.68,10.61
+fixed-r1,0.034,frontend,1,0.02,7.359
+fixed-r1,0.034,load-balancer,1,25.97,18.08
+fixed-r1,0.034,postgres,1,106.86,214.7
+fixed-r1,0.034,redis,1,4.41,15.3
+fixed-r1,2.424,api,1,48.23,11.36
+fixed-r1,2.424,frontend,1,0.0,7.699
+fixed-r1,2.424,load-balancer,1,27.02,22.21
+fixed-r1,2.424,postgres,1,96.33,215.7
+fixed-r1,2.424,redis,1,0.14,15.44
+fixed-r1,5.588,api,1,48.84,12.07
+fixed-r1,5.588,frontend,1,0.0,7.359
+fixed-r1,5.588,load-balancer,1,26.38,18.43
+fixed-r1,5.588,postgres,1,104.71,217.4
+fixed-r1,5.588,redis,1,0.19,15.92
+fixed-r1,8.743,api,1,3.55,11.93
+fixed-r1,8.743,frontend,1,0.0,7.508
+fixed-r1,8.743,load-balancer,1,3.56,17.96
+fixed-r1,8.743,postgres,1,8.33,218.1
+fixed-r1,8.743,redis,1,0.19,16.07
+fixed-r2,0.041,api,2,52.24,18.086
+fixed-r2,0.041,frontend,1,0.0,7.359
+fixed-r2,0.041,load-balancer,1,26.74,18.2
+fixed-r2,0.041,postgres,1,100.69,219.4
+fixed-r2,0.041,redis,1,3.81,15.18
+fixed-r2,2.379,api,2,65.39,18.801
+fixed-r2,2.379,frontend,1,1.92,7.859
+fixed-r2,2.379,load-balancer,1,26.23,18.61
+fixed-r2,2.379,postgres,1,106.33,221.3
+fixed-r2,2.379,redis,1,0.16,15.16
+fixed-r2,5.564,api,2,54.12,18.379
+fixed-r2,5.564,frontend,1,1.37,10.54
+fixed-r2,5.564,load-balancer,1,25.33,18.35
+fixed-r2,5.564,postgres,1,103.06,222.7
+fixed-r2,5.564,redis,1,0.2,14.93
+fixed-r2,8.728,api,2,7.05,18.086
+fixed-r2,8.728,frontend,1,0.0,7.363
+fixed-r2,8.728,load-balancer,1,6.3,17.82
+fixed-r2,8.728,postgres,1,14.18,223.3
+fixed-r2,8.728,redis,1,0.2,15.43
+fixed-r4,0.095,api,4,51.98,27.492
+fixed-r4,0.095,frontend,1,0.0,7.363
+fixed-r4,0.095,load-balancer,1,27.15,19.06
+fixed-r4,0.095,postgres,1,90.1,223.1
+fixed-r4,0.095,redis,1,5.94,16.87
+fixed-r4,3.244,api,4,64.6,33.144
+fixed-r4,3.244,frontend,1,0.02,7.359
+fixed-r4,3.244,load-balancer,1,26.34,18.58
+fixed-r4,3.244,postgres,1,101.51,225.1
+fixed-r4,3.244,redis,1,0.2,16.96
+fixed-r4,6.45,api,4,65.35,34.281
+fixed-r4,6.45,frontend,1,0.02,7.359
+fixed-r4,6.45,load-balancer,1,26.27,18.56
+fixed-r4,6.45,postgres,1,106.68,225.7
+fixed-r4,6.45,redis,1,4.55,16.7
+fixed-r4,9.725,api,4,3.07,32.867
+fixed-r4,9.725,frontend,1,2.51,7.762
+fixed-r4,9.725,load-balancer,1,4.96,17.89
+fixed-r4,9.725,postgres,1,0.42,226.7
+fixed-r4,9.725,redis,1,0.17,16.95
+growth-r1,0.045,api,1,44.44,9.355
+growth-r1,0.045,frontend,1,0.0,7.359
+growth-r1,0.045,load-balancer,1,25.9,18.3
+growth-r1,0.045,postgres,1,93.36,207.6
+growth-r1,0.045,redis,1,3.22,18.55
+growth-r1,2.404,api,1,50.32,9.379
+growth-r1,2.404,frontend,1,0.0,7.363
+growth-r1,2.404,load-balancer,1,25.42,18.36
+growth-r1,2.404,postgres,1,96.7,209.4
+growth-r1,2.404,redis,1,0.22,18.11
+growth-r1,5.579,api,1,50.68,9.66
+growth-r1,5.579,frontend,1,0.03,7.613
+growth-r1,5.579,load-balancer,1,25.44,18.67
+growth-r1,5.579,postgres,1,105.06,210.9
+growth-r1,5.579,redis,1,0.22,17.84
+growth-r1,8.767,api,1,7.65,9.262
+growth-r1,8.767,frontend,1,2.74,8.02
+growth-r1,8.767,load-balancer,1,3.49,17.86
+growth-r1,8.767,postgres,1,13.89,211.8
+growth-r1,8.767,redis,1,0.17,17.93
+growth-r2,0.047,api,2,52.25,17.629
+growth-r2,0.047,frontend,1,0.02,7.363
+growth-r2,0.047,load-balancer,1,24.23,18.59
+growth-r2,0.047,postgres,1,100.9,233.3
+growth-r2,0.047,redis,1,0.14,17.16
+growth-r2,2.292,api,2,62.86,17.606
+growth-r2,2.292,frontend,1,0.0,7.355
+growth-r2,2.292,load-balancer,1,25.57,18.19
+growth-r2,2.292,postgres,1,105.07,235.3
+growth-r2,2.292,redis,1,0.16,17.16
+growth-r2,5.459,api,2,57.29,18.789
+growth-r2,5.459,frontend,1,0.04,7.363
+growth-r2,5.459,load-balancer,1,25.89,18.43
+growth-r2,5.459,postgres,1,104.58,235.1
+growth-r2,5.459,redis,1,0.17,17.73
+growth-r2,8.628,api,2,14.21,17.832
+growth-r2,8.628,frontend,1,3.0,8.062
+growth-r2,8.628,load-balancer,1,9.08,18.0
+growth-r2,8.628,postgres,1,31.84,234.3
+growth-r2,8.628,redis,1,2.99,22.08
+growth-r4,0.043,api,4,25.03,29.672
+growth-r4,0.043,frontend,1,0.02,7.316
+growth-r4,0.043,load-balancer,1,9.16,18.25
+growth-r4,0.043,postgres,1,99.03,269.2
+growth-r4,0.043,redis,1,0.15,16.5
+growth-r4,2.197,api,4,48.89,31.184
+growth-r4,2.197,frontend,1,2.63,7.305
+growth-r4,2.197,load-balancer,1,23.31,18.25
+growth-r4,2.197,postgres,1,100.3,284.5
+growth-r4,2.197,redis,1,2.97,20.3
+growth-r4,5.406,api,4,43.96,32.164
+growth-r4,5.406,frontend,1,0.0,7.305
+growth-r4,5.406,load-balancer,1,19.48,17.79
+growth-r4,5.406,postgres,1,101.91,290.1
+growth-r4,5.406,redis,1,0.19,16.68
+growth-r4,8.538,api,4,24.88,32.855
+growth-r4,8.538,frontend,1,0.02,7.551
+growth-r4,8.538,load-balancer,1,10.43,17.5
+growth-r4,8.538,postgres,1,38.15,279.9
+growth-r4,8.538,redis,1,0.18,16.91
+improved-db-cpu-r4,0.054,api,4,52.63,30.188
+improved-db-cpu-r4,0.054,frontend,1,0.03,7.301
+improved-db-cpu-r4,0.054,load-balancer,1,24.59,18.03
+improved-db-cpu-r4,0.054,postgres,1,121.18,130.1
+improved-db-cpu-r4,0.054,redis,1,1.35,16.63
+improved-db-cpu-r4,2.129,api,4,70.26,36.812
+improved-db-cpu-r4,2.129,frontend,1,0.0,7.301
+improved-db-cpu-r4,2.129,load-balancer,1,25.64,18.99
+improved-db-cpu-r4,2.129,postgres,1,106.99,141.4
+improved-db-cpu-r4,2.129,redis,1,0.16,16.88
+improved-db-cpu-r4,5.271,api,4,65.05,33.652
+improved-db-cpu-r4,5.271,frontend,1,3.29,7.43
+improved-db-cpu-r4,5.271,load-balancer,1,26.73,18.41
+improved-db-cpu-r4,5.271,postgres,1,102.2,145.9
+improved-db-cpu-r4,5.271,redis,1,0.14,16.63
+improved-db-cpu-r4,8.446,api,4,32.86,34.023
+improved-db-cpu-r4,8.446,frontend,1,0.0,7.301
+improved-db-cpu-r4,8.446,load-balancer,1,12.44,18.0
+improved-db-cpu-r4,8.446,postgres,1,51.92,147.6
+improved-db-cpu-r4,8.446,redis,1,0.18,16.88
diff --git a/evidence/grade5/results.json b/evidence/grade5/results.json
new file mode 100644
index 00000000..7e89b5ef
--- /dev/null
+++ b/evidence/grade5/results.json
@@ -0,0 +1,3354 @@
+{
+ "schema_version": 1,
+ "environment": {
+ "utc_started_at": "2026-10-23T15:23:12.734339+00:00",
+ "git_commit": "5300a1a1e0e5497f2d30097bb1542b2743a8aad5",
+ "working_tree_dirty": true,
+ "logical_cpu_count": 8,
+ "host_memory_mib": 3813.6,
+ "kernel": "6.18.33.2-microsoft-standard-WSL2",
+ "python": "3.14.4",
+ "docker_engine": "29.7.2",
+ "docker_compose": "5.5.0",
+ "images": [
+ "mtusi2026lab3-api",
+ "mtusi2026lab3-load-balancer",
+ "mtusi2026lab3-web",
+ "postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea",
+ "redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0",
+ "migrate/migrate:v4.20.1@sha256:76cc2074cb6642631f34a898ced71e6aeaa6b1a4d78c4daa743275a22e0c5be7"
+ ]
+ },
+ "hypothesis": "API replicas stop improving throughput when shared PostgreSQL CPU saturates; adding PostgreSQL CPU should improve the unchanged four-replica profile.",
+ "methodology": {
+ "workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries",
+ "duration_seconds_per_phase": 10,
+ "warmup_seconds_per_profile": 2,
+ "comparison_concurrency": 32,
+ "saturation_concurrency": [
+ 1,
+ 2,
+ 4,
+ 8,
+ 16,
+ 32
+ ],
+ "fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections",
+ "growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica",
+ "improvement": "only PostgreSQL CPU changes from 1.00 to 2.00",
+ "saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step",
+ "cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer"
+ },
+ "profiles": {
+ "fixed-r1": {
+ "postgres_cpu": 1.0,
+ "postgres_memory": "512M",
+ "name": "fixed-r1",
+ "replicas": 1,
+ "api_cpu": 1.0,
+ "api_memory": "512M",
+ "db_max_conns": 16,
+ "db_min_conns": 2
+ },
+ "fixed-r2": {
+ "postgres_cpu": 1.0,
+ "postgres_memory": "512M",
+ "name": "fixed-r2",
+ "replicas": 2,
+ "api_cpu": 0.5,
+ "api_memory": "256M",
+ "db_max_conns": 8,
+ "db_min_conns": 1
+ },
+ "fixed-r4": {
+ "postgres_cpu": 1.0,
+ "postgres_memory": "512M",
+ "name": "fixed-r4",
+ "replicas": 4,
+ "api_cpu": 0.25,
+ "api_memory": "128M",
+ "db_max_conns": 4,
+ "db_min_conns": 1
+ },
+ "growth-r1": {
+ "postgres_cpu": 1.0,
+ "postgres_memory": "512M",
+ "name": "growth-r1",
+ "replicas": 1,
+ "api_cpu": 0.5,
+ "api_memory": "256M",
+ "db_max_conns": 8,
+ "db_min_conns": 1
+ },
+ "growth-r2": {
+ "postgres_cpu": 1.0,
+ "postgres_memory": "512M",
+ "name": "growth-r2",
+ "replicas": 2,
+ "api_cpu": 0.5,
+ "api_memory": "256M",
+ "db_max_conns": 8,
+ "db_min_conns": 1
+ },
+ "growth-r4": {
+ "postgres_cpu": 1.0,
+ "postgres_memory": "512M",
+ "name": "growth-r4",
+ "replicas": 4,
+ "api_cpu": 0.5,
+ "api_memory": "256M",
+ "db_max_conns": 8,
+ "db_min_conns": 1
+ },
+ "improved-db-cpu-r4": {
+ "postgres_cpu": 2.0,
+ "postgres_memory": "512M",
+ "name": "improved-db-cpu-r4",
+ "replicas": 4,
+ "api_cpu": 0.5,
+ "api_memory": "256M",
+ "db_max_conns": 8,
+ "db_min_conns": 1
+ }
+ },
+ "applied_limits": {
+ "growth-r2": {
+ "api": [
+ {
+ "instance": "3849349fa312",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ },
+ {
+ "instance": "bbcb5c0a1b19",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ }
+ ],
+ "postgres_cpu_limit": 1.0,
+ "postgres_memory_limit_mib": 512.0
+ },
+ "fixed-r1": {
+ "api": [
+ {
+ "instance": "f5b4c0a90be9",
+ "cpu_limit": 1.0,
+ "memory_limit_mib": 512.0,
+ "db_max_conns": 16
+ }
+ ],
+ "postgres_cpu_limit": 1.0,
+ "postgres_memory_limit_mib": 512.0
+ },
+ "fixed-r2": {
+ "api": [
+ {
+ "instance": "71e7fe08b5fe",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ },
+ {
+ "instance": "b072d13bfd31",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ }
+ ],
+ "postgres_cpu_limit": 1.0,
+ "postgres_memory_limit_mib": 512.0
+ },
+ "fixed-r4": {
+ "api": [
+ {
+ "instance": "0cc2174bf528",
+ "cpu_limit": 0.25,
+ "memory_limit_mib": 128.0,
+ "db_max_conns": 4
+ },
+ {
+ "instance": "295ab9e0814c",
+ "cpu_limit": 0.25,
+ "memory_limit_mib": 128.0,
+ "db_max_conns": 4
+ },
+ {
+ "instance": "76432e4c3707",
+ "cpu_limit": 0.25,
+ "memory_limit_mib": 128.0,
+ "db_max_conns": 4
+ },
+ {
+ "instance": "e24ef15f043e",
+ "cpu_limit": 0.25,
+ "memory_limit_mib": 128.0,
+ "db_max_conns": 4
+ }
+ ],
+ "postgres_cpu_limit": 1.0,
+ "postgres_memory_limit_mib": 512.0
+ },
+ "growth-r1": {
+ "api": [
+ {
+ "instance": "15f62705e72d",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ }
+ ],
+ "postgres_cpu_limit": 1.0,
+ "postgres_memory_limit_mib": 512.0
+ },
+ "growth-r4": {
+ "api": [
+ {
+ "instance": "4cdd1d3ba154",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ },
+ {
+ "instance": "5e6a432825a9",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ },
+ {
+ "instance": "98727dbbd98f",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ },
+ {
+ "instance": "9a0b50ef6c5a",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ }
+ ],
+ "postgres_cpu_limit": 1.0,
+ "postgres_memory_limit_mib": 512.0
+ },
+ "improved-db-cpu-r4": {
+ "api": [
+ {
+ "instance": "3078d9394686",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ },
+ {
+ "instance": "950a06115bc5",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ },
+ {
+ "instance": "b04f6abe9ea1",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ },
+ {
+ "instance": "b4c3bdf282a0",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 8
+ }
+ ],
+ "postgres_cpu_limit": 2.0,
+ "postgres_memory_limit_mib": 512.0
+ }
+ },
+ "phases": [
+ {
+ "name": "saturation-c1",
+ "kind": "saturation",
+ "profile": "growth-r2",
+ "replicas": 2,
+ "concurrency": 1,
+ "elapsed_seconds": 10.731,
+ "successful_requests": 2472,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 230.369,
+ "successful_latency_ms": {
+ "p50": 3.839,
+ "p95": 5.065,
+ "p99": 6.282
+ },
+ "instances": [
+ "442f0f041187",
+ "c0e4b7903cdc"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 252,
+ "failed": 0
+ },
+ "1": {
+ "successful": 266,
+ "failed": 0
+ },
+ "2": {
+ "successful": 257,
+ "failed": 0
+ },
+ "3": {
+ "successful": 262,
+ "failed": 0
+ },
+ "4": {
+ "successful": 253,
+ "failed": 0
+ },
+ "5": {
+ "successful": 249,
+ "failed": 0
+ },
+ "6": {
+ "successful": 228,
+ "failed": 0
+ },
+ "7": {
+ "successful": 240,
+ "failed": 0
+ },
+ "8": {
+ "successful": 219,
+ "failed": 0
+ },
+ "9": {
+ "successful": 244,
+ "failed": 0
+ },
+ "10": {
+ "successful": 2,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.0,
+ "services": {
+ "api": {
+ "cpu_percent": 26.09,
+ "memory_mib": 17.976,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 1.61,
+ "memory_mib": 7.578,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 13.0,
+ "memory_mib": 17.18,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 28.57,
+ "memory_mib": 169.0,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.15,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.356,
+ "services": {
+ "api": {
+ "cpu_percent": 32.65,
+ "memory_mib": 16.886,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.617,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 13.32,
+ "memory_mib": 17.23,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 28.96,
+ "memory_mib": 169.5,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 2.75,
+ "memory_mib": 15.4,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.485,
+ "services": {
+ "api": {
+ "cpu_percent": 27.72,
+ "memory_mib": 17.586,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.383,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 13.69,
+ "memory_mib": 17.41,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 29.41,
+ "memory_mib": 170.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.15,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.605,
+ "services": {
+ "api": {
+ "cpu_percent": 20.94,
+ "memory_mib": 17.41,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 1.12,
+ "memory_mib": 12.06,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 6.64,
+ "memory_mib": 17.24,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 8.66,
+ "memory_mib": 170.5,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.15,
+ "memory_mib": 15.53,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 26.85,
+ "cpu_percent_max": 32.65,
+ "memory_mib_avg": 17.465,
+ "memory_mib_max": 17.976,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.688,
+ "cpu_percent_max": 1.61,
+ "memory_mib_avg": 8.659,
+ "memory_mib_max": 12.06,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 11.662,
+ "cpu_percent_max": 13.69,
+ "memory_mib_avg": 17.265,
+ "memory_mib_max": 17.41,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 23.9,
+ "cpu_percent_max": 29.41,
+ "memory_mib_avg": 169.775,
+ "memory_mib_max": 170.5,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.8,
+ "cpu_percent_max": 2.75,
+ "memory_mib_avg": 15.372,
+ "memory_mib_max": 15.53,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "saturation-c2",
+ "kind": "saturation",
+ "profile": "growth-r2",
+ "replicas": 2,
+ "concurrency": 2,
+ "elapsed_seconds": 10.915,
+ "successful_requests": 3495,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 320.19,
+ "successful_latency_ms": {
+ "p50": 5.192,
+ "p95": 7.196,
+ "p99": 15.11
+ },
+ "instances": [
+ "442f0f041187",
+ "c0e4b7903cdc"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 444,
+ "failed": 0
+ },
+ "1": {
+ "successful": 377,
+ "failed": 0
+ },
+ "2": {
+ "successful": 352,
+ "failed": 0
+ },
+ "3": {
+ "successful": 297,
+ "failed": 0
+ },
+ "4": {
+ "successful": 383,
+ "failed": 0
+ },
+ "5": {
+ "successful": 324,
+ "failed": 0
+ },
+ "6": {
+ "successful": 275,
+ "failed": 0
+ },
+ "7": {
+ "successful": 323,
+ "failed": 0
+ },
+ "8": {
+ "successful": 333,
+ "failed": 0
+ },
+ "9": {
+ "successful": 381,
+ "failed": 0
+ },
+ "10": {
+ "successful": 6,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.001,
+ "services": {
+ "api": {
+ "cpu_percent": 51.0,
+ "memory_mib": 17.761,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.03,
+ "memory_mib": 7.617,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 24.78,
+ "memory_mib": 17.71,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 57.44,
+ "memory_mib": 171.2,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.17,
+ "memory_mib": 16.03,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.361,
+ "services": {
+ "api": {
+ "cpu_percent": 45.61,
+ "memory_mib": 17.516,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.03,
+ "memory_mib": 7.367,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 24.6,
+ "memory_mib": 17.41,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 49.85,
+ "memory_mib": 172.6,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.18,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.543,
+ "services": {
+ "api": {
+ "cpu_percent": 51.19,
+ "memory_mib": 16.258,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.367,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 24.27,
+ "memory_mib": 17.91,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 62.6,
+ "memory_mib": 173.0,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.21,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.736,
+ "services": {
+ "api": {
+ "cpu_percent": 45.4,
+ "memory_mib": 17.082,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 1.22,
+ "memory_mib": 8.121,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 2.87,
+ "memory_mib": 17.2,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 50.06,
+ "memory_mib": 173.8,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.21,
+ "memory_mib": 15.78,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 48.3,
+ "cpu_percent_max": 51.19,
+ "memory_mib_avg": 17.154,
+ "memory_mib_max": 17.761,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.325,
+ "cpu_percent_max": 1.22,
+ "memory_mib_avg": 7.618,
+ "memory_mib_max": 8.121,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 19.13,
+ "cpu_percent_max": 24.78,
+ "memory_mib_avg": 17.558,
+ "memory_mib_max": 17.91,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 54.987,
+ "cpu_percent_max": 62.6,
+ "memory_mib_avg": 172.65,
+ "memory_mib_max": 173.8,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.193,
+ "cpu_percent_max": 0.21,
+ "memory_mib_avg": 15.592,
+ "memory_mib_max": 16.03,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "saturation-c4",
+ "kind": "saturation",
+ "profile": "growth-r2",
+ "replicas": 2,
+ "concurrency": 4,
+ "elapsed_seconds": 10.832,
+ "successful_requests": 4721,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 435.822,
+ "successful_latency_ms": {
+ "p50": 6.171,
+ "p95": 29.304,
+ "p99": 47.665
+ },
+ "instances": [
+ "442f0f041187",
+ "c0e4b7903cdc"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 526,
+ "failed": 0
+ },
+ "1": {
+ "successful": 453,
+ "failed": 0
+ },
+ "2": {
+ "successful": 425,
+ "failed": 0
+ },
+ "3": {
+ "successful": 478,
+ "failed": 0
+ },
+ "4": {
+ "successful": 510,
+ "failed": 0
+ },
+ "5": {
+ "successful": 468,
+ "failed": 0
+ },
+ "6": {
+ "successful": 458,
+ "failed": 0
+ },
+ "7": {
+ "successful": 428,
+ "failed": 0
+ },
+ "8": {
+ "successful": 490,
+ "failed": 0
+ },
+ "9": {
+ "successful": 474,
+ "failed": 0
+ },
+ "10": {
+ "successful": 11,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.001,
+ "services": {
+ "api": {
+ "cpu_percent": 65.41,
+ "memory_mib": 17.297,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.03,
+ "memory_mib": 7.367,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.95,
+ "memory_mib": 17.71,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 81.8,
+ "memory_mib": 174.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.19,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.372,
+ "services": {
+ "api": {
+ "cpu_percent": 60.01,
+ "memory_mib": 18.188,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 3.75,
+ "memory_mib": 7.965,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.04,
+ "memory_mib": 17.46,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 79.42,
+ "memory_mib": 175.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.16,
+ "memory_mib": 15.53,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.501,
+ "services": {
+ "api": {
+ "cpu_percent": 56.53,
+ "memory_mib": 17.433,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.367,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.9,
+ "memory_mib": 21.22,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 73.71,
+ "memory_mib": 175.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.18,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.662,
+ "services": {
+ "api": {
+ "cpu_percent": 35.86,
+ "memory_mib": 16.699,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.367,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 5.48,
+ "memory_mib": 17.53,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 17.44,
+ "memory_mib": 177.2,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 2.71,
+ "memory_mib": 16.1,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 54.453,
+ "cpu_percent_max": 65.41,
+ "memory_mib_avg": 17.404,
+ "memory_mib_max": 18.188,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.95,
+ "cpu_percent_max": 3.75,
+ "memory_mib_avg": 7.516,
+ "memory_mib_max": 7.965,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 20.843,
+ "cpu_percent_max": 26.04,
+ "memory_mib_avg": 18.48,
+ "memory_mib_max": 21.22,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 63.093,
+ "cpu_percent_max": 81.8,
+ "memory_mib_avg": 175.675,
+ "memory_mib_max": 177.2,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.81,
+ "cpu_percent_max": 2.71,
+ "memory_mib_avg": 15.547,
+ "memory_mib_max": 16.1,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "saturation-c8",
+ "kind": "saturation",
+ "profile": "growth-r2",
+ "replicas": 2,
+ "concurrency": 8,
+ "elapsed_seconds": 10.821,
+ "successful_requests": 5095,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 470.828,
+ "successful_latency_ms": {
+ "p50": 9.159,
+ "p95": 56.23,
+ "p99": 70.71
+ },
+ "instances": [
+ "442f0f041187",
+ "c0e4b7903cdc"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 532,
+ "failed": 0
+ },
+ "1": {
+ "successful": 476,
+ "failed": 0
+ },
+ "2": {
+ "successful": 530,
+ "failed": 0
+ },
+ "3": {
+ "successful": 529,
+ "failed": 0
+ },
+ "4": {
+ "successful": 490,
+ "failed": 0
+ },
+ "5": {
+ "successful": 527,
+ "failed": 0
+ },
+ "6": {
+ "successful": 445,
+ "failed": 0
+ },
+ "7": {
+ "successful": 524,
+ "failed": 0
+ },
+ "8": {
+ "successful": 507,
+ "failed": 0
+ },
+ "9": {
+ "successful": 519,
+ "failed": 0
+ },
+ "10": {
+ "successful": 16,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.006,
+ "services": {
+ "api": {
+ "cpu_percent": 48.34,
+ "memory_mib": 17.403,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.367,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.42,
+ "memory_mib": 20.12,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 77.51,
+ "memory_mib": 185.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.18,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.372,
+ "services": {
+ "api": {
+ "cpu_percent": 57.14,
+ "memory_mib": 18.602,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 3.44,
+ "memory_mib": 7.734,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.83,
+ "memory_mib": 17.14,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 92.98,
+ "memory_mib": 188.5,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 3.34,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.527,
+ "services": {
+ "api": {
+ "cpu_percent": 48.26,
+ "memory_mib": 18.192,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.363,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.47,
+ "memory_mib": 17.28,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 76.05,
+ "memory_mib": 189.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.15,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.685,
+ "services": {
+ "api": {
+ "cpu_percent": 40.28,
+ "memory_mib": 21.813,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.363,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 5.75,
+ "memory_mib": 17.64,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 17.24,
+ "memory_mib": 190.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.14,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 48.505,
+ "cpu_percent_max": 57.14,
+ "memory_mib_avg": 19.002,
+ "memory_mib_max": 21.813,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.86,
+ "cpu_percent_max": 3.44,
+ "memory_mib_avg": 7.457,
+ "memory_mib_max": 7.734,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 20.867,
+ "cpu_percent_max": 26.83,
+ "memory_mib_avg": 18.045,
+ "memory_mib_max": 20.12,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 65.945,
+ "cpu_percent_max": 92.98,
+ "memory_mib_avg": 188.35,
+ "memory_mib_max": 190.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.953,
+ "cpu_percent_max": 3.34,
+ "memory_mib_avg": 15.28,
+ "memory_mib_max": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "saturation-c16",
+ "kind": "saturation",
+ "profile": "growth-r2",
+ "replicas": 2,
+ "concurrency": 16,
+ "elapsed_seconds": 10.876,
+ "successful_requests": 5143,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 472.892,
+ "successful_latency_ms": {
+ "p50": 19.862,
+ "p95": 73.628,
+ "p99": 98.773
+ },
+ "instances": [
+ "442f0f041187",
+ "c0e4b7903cdc"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 349,
+ "failed": 0
+ },
+ "1": {
+ "successful": 592,
+ "failed": 0
+ },
+ "2": {
+ "successful": 541,
+ "failed": 0
+ },
+ "3": {
+ "successful": 491,
+ "failed": 0
+ },
+ "4": {
+ "successful": 545,
+ "failed": 0
+ },
+ "5": {
+ "successful": 420,
+ "failed": 0
+ },
+ "6": {
+ "successful": 499,
+ "failed": 0
+ },
+ "7": {
+ "successful": 529,
+ "failed": 0
+ },
+ "8": {
+ "successful": 548,
+ "failed": 0
+ },
+ "9": {
+ "successful": 601,
+ "failed": 0
+ },
+ "10": {
+ "successful": 28,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.015,
+ "services": {
+ "api": {
+ "cpu_percent": 44.39,
+ "memory_mib": 18.796,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.363,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.92,
+ "memory_mib": 17.44,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 82.81,
+ "memory_mib": 205.9,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.17,
+ "memory_mib": 15.28,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.354,
+ "services": {
+ "api": {
+ "cpu_percent": 56.54,
+ "memory_mib": 17.856,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.03,
+ "memory_mib": 7.375,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.66,
+ "memory_mib": 17.69,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 88.93,
+ "memory_mib": 207.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.19,
+ "memory_mib": 15.75,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.525,
+ "services": {
+ "api": {
+ "cpu_percent": 41.39,
+ "memory_mib": 17.73,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 4.4,
+ "memory_mib": 7.379,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.76,
+ "memory_mib": 17.7,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 72.49,
+ "memory_mib": 208.6,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.18,
+ "memory_mib": 15.25,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.731,
+ "services": {
+ "api": {
+ "cpu_percent": 9.79,
+ "memory_mib": 18.257,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.375,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 4.58,
+ "memory_mib": 17.55,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 16.09,
+ "memory_mib": 209.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.15,
+ "memory_mib": 15.33,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 38.028,
+ "cpu_percent_max": 56.54,
+ "memory_mib_avg": 18.16,
+ "memory_mib_max": 18.796,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent_avg": 1.113,
+ "cpu_percent_max": 4.4,
+ "memory_mib_avg": 7.373,
+ "memory_mib_max": 7.379,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 20.98,
+ "cpu_percent_max": 26.92,
+ "memory_mib_avg": 17.595,
+ "memory_mib_max": 17.7,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 65.08,
+ "cpu_percent_max": 88.93,
+ "memory_mib_avg": 207.975,
+ "memory_mib_max": 209.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.172,
+ "cpu_percent_max": 0.19,
+ "memory_mib_avg": 15.402,
+ "memory_mib_max": 15.75,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "saturation-c32",
+ "kind": "saturation",
+ "profile": "growth-r2",
+ "replicas": 2,
+ "concurrency": 32,
+ "elapsed_seconds": 10.808,
+ "successful_requests": 6096,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 564.012,
+ "successful_latency_ms": {
+ "p50": 51.089,
+ "p95": 94.785,
+ "p99": 109.138
+ },
+ "instances": [
+ "442f0f041187",
+ "c0e4b7903cdc"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 661,
+ "failed": 0
+ },
+ "1": {
+ "successful": 604,
+ "failed": 0
+ },
+ "2": {
+ "successful": 610,
+ "failed": 0
+ },
+ "3": {
+ "successful": 589,
+ "failed": 0
+ },
+ "4": {
+ "successful": 571,
+ "failed": 0
+ },
+ "5": {
+ "successful": 627,
+ "failed": 0
+ },
+ "6": {
+ "successful": 587,
+ "failed": 0
+ },
+ "7": {
+ "successful": 619,
+ "failed": 0
+ },
+ "8": {
+ "successful": 629,
+ "failed": 0
+ },
+ "9": {
+ "successful": 548,
+ "failed": 0
+ },
+ "10": {
+ "successful": 51,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.039,
+ "services": {
+ "api": {
+ "cpu_percent": 55.84,
+ "memory_mib": 18.75,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 6.63,
+ "memory_mib": 8.125,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.97,
+ "memory_mib": 18.0,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 104.2,
+ "memory_mib": 210.6,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 5.85,
+ "memory_mib": 15.05,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.29,
+ "services": {
+ "api": {
+ "cpu_percent": 56.87,
+ "memory_mib": 18.933,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.625,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 27.03,
+ "memory_mib": 19.23,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 105.41,
+ "memory_mib": 212.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.19,
+ "memory_mib": 15.55,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.446,
+ "services": {
+ "api": {
+ "cpu_percent": 64.13,
+ "memory_mib": 19.003,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.7,
+ "memory_mib": 7.375,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.33,
+ "memory_mib": 18.27,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 107.62,
+ "memory_mib": 213.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 3.7,
+ "memory_mib": 15.27,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.653,
+ "services": {
+ "api": {
+ "cpu_percent": 31.02,
+ "memory_mib": 19.379,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.625,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 5.32,
+ "memory_mib": 18.36,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 18.07,
+ "memory_mib": 213.5,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.16,
+ "memory_mib": 15.26,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 51.965,
+ "cpu_percent_max": 64.13,
+ "memory_mib_avg": 19.016,
+ "memory_mib_max": 19.379,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent_avg": 1.837,
+ "cpu_percent_max": 6.63,
+ "memory_mib_avg": 7.688,
+ "memory_mib_max": 8.125,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 21.163,
+ "cpu_percent_max": 27.03,
+ "memory_mib_avg": 18.465,
+ "memory_mib_max": 19.23,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 83.825,
+ "cpu_percent_max": 107.62,
+ "memory_mib_avg": 212.325,
+ "memory_mib_max": 213.5,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 2.475,
+ "cpu_percent_max": 5.85,
+ "memory_mib_avg": 15.283,
+ "memory_mib_max": 15.55,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "fixed-r1",
+ "kind": "fixed",
+ "profile": "fixed-r1",
+ "replicas": 1,
+ "concurrency": 32,
+ "elapsed_seconds": 10.88,
+ "successful_requests": 6291,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 578.242,
+ "successful_latency_ms": {
+ "p50": 48.579,
+ "p95": 95.757,
+ "p99": 115.549
+ },
+ "instances": [
+ "f5b4c0a90be9"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 511,
+ "failed": 0
+ },
+ "1": {
+ "successful": 580,
+ "failed": 0
+ },
+ "2": {
+ "successful": 659,
+ "failed": 0
+ },
+ "3": {
+ "successful": 687,
+ "failed": 0
+ },
+ "4": {
+ "successful": 588,
+ "failed": 0
+ },
+ "5": {
+ "successful": 675,
+ "failed": 0
+ },
+ "6": {
+ "successful": 647,
+ "failed": 0
+ },
+ "7": {
+ "successful": 701,
+ "failed": 0
+ },
+ "8": {
+ "successful": 690,
+ "failed": 0
+ },
+ "9": {
+ "successful": 521,
+ "failed": 0
+ },
+ "10": {
+ "successful": 32,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.034,
+ "services": {
+ "api": {
+ "cpu_percent": 45.68,
+ "memory_mib": 10.61,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.359,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.97,
+ "memory_mib": 18.08,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 106.86,
+ "memory_mib": 214.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 4.41,
+ "memory_mib": 15.3,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.424,
+ "services": {
+ "api": {
+ "cpu_percent": 48.23,
+ "memory_mib": 11.36,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.699,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 27.02,
+ "memory_mib": 22.21,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 96.33,
+ "memory_mib": 215.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.14,
+ "memory_mib": 15.44,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.588,
+ "services": {
+ "api": {
+ "cpu_percent": 48.84,
+ "memory_mib": 12.07,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.359,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.38,
+ "memory_mib": 18.43,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 104.71,
+ "memory_mib": 217.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.19,
+ "memory_mib": 15.92,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.743,
+ "services": {
+ "api": {
+ "cpu_percent": 3.55,
+ "memory_mib": 11.93,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.508,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 3.56,
+ "memory_mib": 17.96,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 8.33,
+ "memory_mib": 218.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.19,
+ "memory_mib": 16.07,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 36.575,
+ "cpu_percent_max": 48.84,
+ "memory_mib_avg": 11.492,
+ "memory_mib_max": 12.07,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.005,
+ "cpu_percent_max": 0.02,
+ "memory_mib_avg": 7.481,
+ "memory_mib_max": 7.699,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 20.732,
+ "cpu_percent_max": 27.02,
+ "memory_mib_avg": 19.17,
+ "memory_mib_max": 22.21,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 79.058,
+ "cpu_percent_max": 106.86,
+ "memory_mib_avg": 216.475,
+ "memory_mib_max": 218.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 1.232,
+ "cpu_percent_max": 4.41,
+ "memory_mib_avg": 15.683,
+ "memory_mib_max": 16.07,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "fixed-r2",
+ "kind": "fixed",
+ "profile": "fixed-r2",
+ "replicas": 2,
+ "concurrency": 32,
+ "elapsed_seconds": 10.905,
+ "successful_requests": 5774,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 529.499,
+ "successful_latency_ms": {
+ "p50": 53.316,
+ "p95": 100.145,
+ "p99": 129.953
+ },
+ "instances": [
+ "71e7fe08b5fe",
+ "b072d13bfd31"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 452,
+ "failed": 0
+ },
+ "1": {
+ "successful": 545,
+ "failed": 0
+ },
+ "2": {
+ "successful": 604,
+ "failed": 0
+ },
+ "3": {
+ "successful": 652,
+ "failed": 0
+ },
+ "4": {
+ "successful": 521,
+ "failed": 0
+ },
+ "5": {
+ "successful": 627,
+ "failed": 0
+ },
+ "6": {
+ "successful": 571,
+ "failed": 0
+ },
+ "7": {
+ "successful": 611,
+ "failed": 0
+ },
+ "8": {
+ "successful": 573,
+ "failed": 0
+ },
+ "9": {
+ "successful": 586,
+ "failed": 0
+ },
+ "10": {
+ "successful": 32,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.041,
+ "services": {
+ "api": {
+ "cpu_percent": 52.24,
+ "memory_mib": 18.086,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.359,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.74,
+ "memory_mib": 18.2,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 100.69,
+ "memory_mib": 219.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 3.81,
+ "memory_mib": 15.18,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.379,
+ "services": {
+ "api": {
+ "cpu_percent": 65.39,
+ "memory_mib": 18.801,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 1.92,
+ "memory_mib": 7.859,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.23,
+ "memory_mib": 18.61,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 106.33,
+ "memory_mib": 221.3,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.16,
+ "memory_mib": 15.16,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.564,
+ "services": {
+ "api": {
+ "cpu_percent": 54.12,
+ "memory_mib": 18.379,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 1.37,
+ "memory_mib": 10.54,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.33,
+ "memory_mib": 18.35,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 103.06,
+ "memory_mib": 222.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.2,
+ "memory_mib": 14.93,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.728,
+ "services": {
+ "api": {
+ "cpu_percent": 7.05,
+ "memory_mib": 18.086,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.363,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 6.3,
+ "memory_mib": 17.82,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 14.18,
+ "memory_mib": 223.3,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.2,
+ "memory_mib": 15.43,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 44.7,
+ "cpu_percent_max": 65.39,
+ "memory_mib_avg": 18.338,
+ "memory_mib_max": 18.801,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.823,
+ "cpu_percent_max": 1.92,
+ "memory_mib_avg": 8.28,
+ "memory_mib_max": 10.54,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 21.15,
+ "cpu_percent_max": 26.74,
+ "memory_mib_avg": 18.245,
+ "memory_mib_max": 18.61,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 81.065,
+ "cpu_percent_max": 106.33,
+ "memory_mib_avg": 221.675,
+ "memory_mib_max": 223.3,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 1.093,
+ "cpu_percent_max": 3.81,
+ "memory_mib_avg": 15.175,
+ "memory_mib_max": 15.43,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "fixed-r4",
+ "kind": "fixed",
+ "profile": "fixed-r4",
+ "replicas": 4,
+ "concurrency": 32,
+ "elapsed_seconds": 11.917,
+ "successful_requests": 4218,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 353.955,
+ "successful_latency_ms": {
+ "p50": 69.354,
+ "p95": 139.408,
+ "p99": 291.312
+ },
+ "instances": [
+ "0cc2174bf528",
+ "295ab9e0814c",
+ "76432e4c3707",
+ "e24ef15f043e"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 133,
+ "failed": 0
+ },
+ "1": {
+ "successful": 276,
+ "failed": 0
+ },
+ "2": {
+ "successful": 461,
+ "failed": 0
+ },
+ "3": {
+ "successful": 406,
+ "failed": 0
+ },
+ "4": {
+ "successful": 503,
+ "failed": 0
+ },
+ "5": {
+ "successful": 487,
+ "failed": 0
+ },
+ "6": {
+ "successful": 377,
+ "failed": 0
+ },
+ "7": {
+ "successful": 427,
+ "failed": 0
+ },
+ "8": {
+ "successful": 551,
+ "failed": 0
+ },
+ "9": {
+ "successful": 547,
+ "failed": 0
+ },
+ "10": {
+ "successful": 50,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.095,
+ "services": {
+ "api": {
+ "cpu_percent": 51.98,
+ "memory_mib": 27.492,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.363,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 27.15,
+ "memory_mib": 19.06,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 90.1,
+ "memory_mib": 223.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 5.94,
+ "memory_mib": 16.87,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 3.244,
+ "services": {
+ "api": {
+ "cpu_percent": 64.6,
+ "memory_mib": 33.144,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.359,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.34,
+ "memory_mib": 18.58,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 101.51,
+ "memory_mib": 225.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.2,
+ "memory_mib": 16.96,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 6.45,
+ "services": {
+ "api": {
+ "cpu_percent": 65.35,
+ "memory_mib": 34.281,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.359,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.27,
+ "memory_mib": 18.56,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 106.68,
+ "memory_mib": 225.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 4.55,
+ "memory_mib": 16.7,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 9.725,
+ "services": {
+ "api": {
+ "cpu_percent": 3.07,
+ "memory_mib": 32.867,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 2.51,
+ "memory_mib": 7.762,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 4.96,
+ "memory_mib": 17.89,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 0.42,
+ "memory_mib": 226.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.17,
+ "memory_mib": 16.95,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 46.25,
+ "cpu_percent_max": 65.35,
+ "memory_mib_avg": 31.946,
+ "memory_mib_max": 34.281,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.637,
+ "cpu_percent_max": 2.51,
+ "memory_mib_avg": 7.461,
+ "memory_mib_max": 7.762,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 21.18,
+ "cpu_percent_max": 27.15,
+ "memory_mib_avg": 18.523,
+ "memory_mib_max": 19.06,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 74.677,
+ "cpu_percent_max": 106.68,
+ "memory_mib_avg": 225.15,
+ "memory_mib_max": 226.7,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 2.715,
+ "cpu_percent_max": 5.94,
+ "memory_mib_avg": 16.87,
+ "memory_mib_max": 16.96,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "growth-r1",
+ "kind": "growth",
+ "profile": "growth-r1",
+ "replicas": 1,
+ "concurrency": 32,
+ "elapsed_seconds": 10.951,
+ "successful_requests": 5844,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 533.656,
+ "successful_latency_ms": {
+ "p50": 53.378,
+ "p95": 93.541,
+ "p99": 115.58
+ },
+ "instances": [
+ "15f62705e72d"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 527,
+ "failed": 0
+ },
+ "1": {
+ "successful": 596,
+ "failed": 0
+ },
+ "2": {
+ "successful": 604,
+ "failed": 0
+ },
+ "3": {
+ "successful": 616,
+ "failed": 0
+ },
+ "4": {
+ "successful": 516,
+ "failed": 0
+ },
+ "5": {
+ "successful": 508,
+ "failed": 0
+ },
+ "6": {
+ "successful": 606,
+ "failed": 0
+ },
+ "7": {
+ "successful": 620,
+ "failed": 0
+ },
+ "8": {
+ "successful": 628,
+ "failed": 0
+ },
+ "9": {
+ "successful": 591,
+ "failed": 0
+ },
+ "10": {
+ "successful": 32,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.045,
+ "services": {
+ "api": {
+ "cpu_percent": 44.44,
+ "memory_mib": 9.355,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.359,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.9,
+ "memory_mib": 18.3,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 93.36,
+ "memory_mib": 207.6,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 3.22,
+ "memory_mib": 18.55,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.404,
+ "services": {
+ "api": {
+ "cpu_percent": 50.32,
+ "memory_mib": 9.379,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.363,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.42,
+ "memory_mib": 18.36,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 96.7,
+ "memory_mib": 209.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.22,
+ "memory_mib": 18.11,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.579,
+ "services": {
+ "api": {
+ "cpu_percent": 50.68,
+ "memory_mib": 9.66,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent": 0.03,
+ "memory_mib": 7.613,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.44,
+ "memory_mib": 18.67,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 105.06,
+ "memory_mib": 210.9,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.22,
+ "memory_mib": 17.84,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.767,
+ "services": {
+ "api": {
+ "cpu_percent": 7.65,
+ "memory_mib": 9.262,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent": 2.74,
+ "memory_mib": 8.02,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 3.49,
+ "memory_mib": 17.86,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 13.89,
+ "memory_mib": 211.8,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.17,
+ "memory_mib": 17.93,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 38.273,
+ "cpu_percent_max": 50.68,
+ "memory_mib_avg": 9.414,
+ "memory_mib_max": 9.66,
+ "containers": 1
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.693,
+ "cpu_percent_max": 2.74,
+ "memory_mib_avg": 7.589,
+ "memory_mib_max": 8.02,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 20.062,
+ "cpu_percent_max": 25.9,
+ "memory_mib_avg": 18.297,
+ "memory_mib_max": 18.67,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 77.252,
+ "cpu_percent_max": 105.06,
+ "memory_mib_avg": 209.925,
+ "memory_mib_max": 211.8,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.958,
+ "cpu_percent_max": 3.22,
+ "memory_mib_avg": 18.108,
+ "memory_mib_max": 18.55,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "growth-r2",
+ "kind": "growth",
+ "profile": "growth-r2",
+ "replicas": 2,
+ "concurrency": 32,
+ "elapsed_seconds": 10.796,
+ "successful_requests": 6083,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 563.46,
+ "successful_latency_ms": {
+ "p50": 50.516,
+ "p95": 96.428,
+ "p99": 129.46
+ },
+ "instances": [
+ "3849349fa312",
+ "bbcb5c0a1b19"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 461,
+ "failed": 0
+ },
+ "1": {
+ "successful": 662,
+ "failed": 0
+ },
+ "2": {
+ "successful": 670,
+ "failed": 0
+ },
+ "3": {
+ "successful": 655,
+ "failed": 0
+ },
+ "4": {
+ "successful": 640,
+ "failed": 0
+ },
+ "5": {
+ "successful": 495,
+ "failed": 0
+ },
+ "6": {
+ "successful": 635,
+ "failed": 0
+ },
+ "7": {
+ "successful": 626,
+ "failed": 0
+ },
+ "8": {
+ "successful": 598,
+ "failed": 0
+ },
+ "9": {
+ "successful": 609,
+ "failed": 0
+ },
+ "10": {
+ "successful": 32,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.047,
+ "services": {
+ "api": {
+ "cpu_percent": 52.25,
+ "memory_mib": 17.629,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.363,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 24.23,
+ "memory_mib": 18.59,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 100.9,
+ "memory_mib": 233.3,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.14,
+ "memory_mib": 17.16,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.292,
+ "services": {
+ "api": {
+ "cpu_percent": 62.86,
+ "memory_mib": 17.606,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.355,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.57,
+ "memory_mib": 18.19,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 105.07,
+ "memory_mib": 235.3,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.16,
+ "memory_mib": 17.16,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.459,
+ "services": {
+ "api": {
+ "cpu_percent": 57.29,
+ "memory_mib": 18.789,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 0.04,
+ "memory_mib": 7.363,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.89,
+ "memory_mib": 18.43,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 104.58,
+ "memory_mib": 235.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.17,
+ "memory_mib": 17.73,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.628,
+ "services": {
+ "api": {
+ "cpu_percent": 14.21,
+ "memory_mib": 17.832,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent": 3.0,
+ "memory_mib": 8.062,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 9.08,
+ "memory_mib": 18.0,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 31.84,
+ "memory_mib": 234.3,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 2.99,
+ "memory_mib": 22.08,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 46.653,
+ "cpu_percent_max": 62.86,
+ "memory_mib_avg": 17.964,
+ "memory_mib_max": 18.789,
+ "containers": 2
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.765,
+ "cpu_percent_max": 3.0,
+ "memory_mib_avg": 7.536,
+ "memory_mib_max": 8.062,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 21.192,
+ "cpu_percent_max": 25.89,
+ "memory_mib_avg": 18.303,
+ "memory_mib_max": 18.59,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 85.597,
+ "cpu_percent_max": 105.07,
+ "memory_mib_avg": 234.5,
+ "memory_mib_max": 235.3,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.865,
+ "cpu_percent_max": 2.99,
+ "memory_mib_avg": 18.532,
+ "memory_mib_max": 22.08,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "growth-r4",
+ "kind": "growth",
+ "profile": "growth-r4",
+ "replicas": 4,
+ "concurrency": 32,
+ "elapsed_seconds": 10.682,
+ "successful_requests": 4688,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 438.868,
+ "successful_latency_ms": {
+ "p50": 68.331,
+ "p95": 109.446,
+ "p99": 199.616
+ },
+ "instances": [
+ "4cdd1d3ba154",
+ "5e6a432825a9",
+ "98727dbbd98f",
+ "9a0b50ef6c5a"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 106,
+ "failed": 0
+ },
+ "1": {
+ "successful": 514,
+ "failed": 0
+ },
+ "2": {
+ "successful": 587,
+ "failed": 0
+ },
+ "3": {
+ "successful": 506,
+ "failed": 0
+ },
+ "4": {
+ "successful": 423,
+ "failed": 0
+ },
+ "5": {
+ "successful": 409,
+ "failed": 0
+ },
+ "6": {
+ "successful": 459,
+ "failed": 0
+ },
+ "7": {
+ "successful": 462,
+ "failed": 0
+ },
+ "8": {
+ "successful": 618,
+ "failed": 0
+ },
+ "9": {
+ "successful": 549,
+ "failed": 0
+ },
+ "10": {
+ "successful": 55,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.043,
+ "services": {
+ "api": {
+ "cpu_percent": 25.03,
+ "memory_mib": 29.672,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.316,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 9.16,
+ "memory_mib": 18.25,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 99.03,
+ "memory_mib": 269.2,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.15,
+ "memory_mib": 16.5,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.197,
+ "services": {
+ "api": {
+ "cpu_percent": 48.89,
+ "memory_mib": 31.184,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 2.63,
+ "memory_mib": 7.305,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 23.31,
+ "memory_mib": 18.25,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 100.3,
+ "memory_mib": 284.5,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 2.97,
+ "memory_mib": 20.3,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.406,
+ "services": {
+ "api": {
+ "cpu_percent": 43.96,
+ "memory_mib": 32.164,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.305,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 19.48,
+ "memory_mib": 17.79,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 101.91,
+ "memory_mib": 290.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.19,
+ "memory_mib": 16.68,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.538,
+ "services": {
+ "api": {
+ "cpu_percent": 24.88,
+ "memory_mib": 32.855,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.02,
+ "memory_mib": 7.551,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 10.43,
+ "memory_mib": 17.5,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 38.15,
+ "memory_mib": 279.9,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.18,
+ "memory_mib": 16.91,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 35.69,
+ "cpu_percent_max": 48.89,
+ "memory_mib_avg": 31.469,
+ "memory_mib_max": 32.855,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.667,
+ "cpu_percent_max": 2.63,
+ "memory_mib_avg": 7.369,
+ "memory_mib_max": 7.551,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 15.595,
+ "cpu_percent_max": 23.31,
+ "memory_mib_avg": 17.947,
+ "memory_mib_max": 18.25,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 84.847,
+ "cpu_percent_max": 101.91,
+ "memory_mib_avg": 280.925,
+ "memory_mib_max": 290.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.873,
+ "cpu_percent_max": 2.97,
+ "memory_mib_avg": 17.598,
+ "memory_mib_max": 20.3,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "name": "improved-db-cpu-r4",
+ "kind": "improvement",
+ "profile": "improved-db-cpu-r4",
+ "replicas": 4,
+ "concurrency": 32,
+ "elapsed_seconds": 10.601,
+ "successful_requests": 6229,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 587.574,
+ "successful_latency_ms": {
+ "p50": 49.466,
+ "p95": 93.991,
+ "p99": 112.002
+ },
+ "instances": [
+ "3078d9394686",
+ "950a06115bc5",
+ "b04f6abe9ea1",
+ "b4c3bdf282a0"
+ ],
+ "per_second": {
+ "0": {
+ "successful": 492,
+ "failed": 0
+ },
+ "1": {
+ "successful": 583,
+ "failed": 0
+ },
+ "2": {
+ "successful": 629,
+ "failed": 0
+ },
+ "3": {
+ "successful": 681,
+ "failed": 0
+ },
+ "4": {
+ "successful": 667,
+ "failed": 0
+ },
+ "5": {
+ "successful": 632,
+ "failed": 0
+ },
+ "6": {
+ "successful": 585,
+ "failed": 0
+ },
+ "7": {
+ "successful": 677,
+ "failed": 0
+ },
+ "8": {
+ "successful": 602,
+ "failed": 0
+ },
+ "9": {
+ "successful": 629,
+ "failed": 0
+ },
+ "10": {
+ "successful": 52,
+ "failed": 0
+ }
+ },
+ "resource_samples": [
+ {
+ "t": 0.054,
+ "services": {
+ "api": {
+ "cpu_percent": 52.63,
+ "memory_mib": 30.188,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.03,
+ "memory_mib": 7.301,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 24.59,
+ "memory_mib": 18.03,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 121.18,
+ "memory_mib": 130.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 1.35,
+ "memory_mib": 16.63,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 2.129,
+ "services": {
+ "api": {
+ "cpu_percent": 70.26,
+ "memory_mib": 36.812,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.301,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 25.64,
+ "memory_mib": 18.99,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 106.99,
+ "memory_mib": 141.4,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.16,
+ "memory_mib": 16.88,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 5.271,
+ "services": {
+ "api": {
+ "cpu_percent": 65.05,
+ "memory_mib": 33.652,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 3.29,
+ "memory_mib": 7.43,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 26.73,
+ "memory_mib": 18.41,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 102.2,
+ "memory_mib": 145.9,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.14,
+ "memory_mib": 16.63,
+ "containers": 1
+ }
+ }
+ },
+ {
+ "t": 8.446,
+ "services": {
+ "api": {
+ "cpu_percent": 32.86,
+ "memory_mib": 34.023,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent": 0.0,
+ "memory_mib": 7.301,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent": 12.44,
+ "memory_mib": 18.0,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent": 51.92,
+ "memory_mib": 147.6,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent": 0.18,
+ "memory_mib": 16.88,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 55.2,
+ "cpu_percent_max": 70.26,
+ "memory_mib_avg": 33.669,
+ "memory_mib_max": 36.812,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.83,
+ "cpu_percent_max": 3.29,
+ "memory_mib_avg": 7.333,
+ "memory_mib_max": 7.43,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 22.35,
+ "cpu_percent_max": 26.73,
+ "memory_mib_avg": 18.358,
+ "memory_mib_max": 18.99,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 95.573,
+ "cpu_percent_max": 121.18,
+ "memory_mib_avg": 141.25,
+ "memory_mib_max": 147.6,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.458,
+ "cpu_percent_max": 1.35,
+ "memory_mib_avg": 16.755,
+ "memory_mib_max": 16.88,
+ "containers": 1
+ }
+ }
+ }
+ ],
+ "analysis": {
+ "saturation_point": {
+ "concurrency": 8,
+ "previous_concurrency": 4,
+ "rps_gain_percent": 8.03,
+ "p95_growth_ratio": 1.919
+ },
+ "observed_bottleneck": "postgres_cpu",
+ "baseline_resource_snapshot": {
+ "api": {
+ "cpu_percent_avg": 35.69,
+ "cpu_percent_max": 48.89,
+ "memory_mib_avg": 31.469,
+ "memory_mib_max": 32.855,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.667,
+ "cpu_percent_max": 2.63,
+ "memory_mib_avg": 7.369,
+ "memory_mib_max": 7.551,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 15.595,
+ "cpu_percent_max": 23.31,
+ "memory_mib_avg": 17.947,
+ "memory_mib_max": 18.25,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 84.847,
+ "cpu_percent_max": 101.91,
+ "memory_mib_avg": 280.925,
+ "memory_mib_max": 290.1,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.873,
+ "cpu_percent_max": 2.97,
+ "memory_mib_avg": 17.598,
+ "memory_mib_max": 20.3,
+ "containers": 1
+ }
+ },
+ "improved_resource_snapshot": {
+ "api": {
+ "cpu_percent_avg": 55.2,
+ "cpu_percent_max": 70.26,
+ "memory_mib_avg": 33.669,
+ "memory_mib_max": 36.812,
+ "containers": 4
+ },
+ "frontend": {
+ "cpu_percent_avg": 0.83,
+ "cpu_percent_max": 3.29,
+ "memory_mib_avg": 7.333,
+ "memory_mib_max": 7.43,
+ "containers": 1
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 22.35,
+ "cpu_percent_max": 26.73,
+ "memory_mib_avg": 18.358,
+ "memory_mib_max": 18.99,
+ "containers": 1
+ },
+ "postgres": {
+ "cpu_percent_avg": 95.573,
+ "cpu_percent_max": 121.18,
+ "memory_mib_avg": 141.25,
+ "memory_mib_max": 147.6,
+ "containers": 1
+ },
+ "redis": {
+ "cpu_percent_avg": 0.458,
+ "cpu_percent_max": 1.35,
+ "memory_mib_avg": 16.755,
+ "memory_mib_max": 16.88,
+ "containers": 1
+ }
+ },
+ "remaining_bottleneck_after_improvement": "load_balancer_cpu",
+ "improvement": {
+ "change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged",
+ "baseline_phase": "growth-r4",
+ "improved_phase": "improved-db-cpu-r4",
+ "rps_change_percent": 33.88,
+ "p95_change_percent": -14.12,
+ "confirmed": true
+ }
+ },
+ "dataset": {
+ "before": {
+ "messages": 225626,
+ "database_size_bytes": 175406103
+ },
+ "after": {
+ "messages": 299524,
+ "database_size_bytes": 228793367
+ },
+ "warmup_successful_requests": 7749
+ },
+ "restoration": {
+ "attempted": true,
+ "succeeded": true,
+ "applied": {
+ "api": [
+ {
+ "instance": "9c029f1ecdff",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 10
+ },
+ {
+ "instance": "bd53aadefab0",
+ "cpu_limit": 0.5,
+ "memory_limit_mib": 256.0,
+ "db_max_conns": 10
+ }
+ ],
+ "postgres_cpu_limit": 1.0,
+ "postgres_memory_limit_mib": 512.0
+ }
+ },
+ "limitations": [
+ "single Docker host: replicas do not prove host-level availability",
+ "closed-loop workload has coordinated omission",
+ "successful-response latency percentiles are reported separately from failures",
+ "one measured run per profile; host scheduling noise is possible"
+ ]
+}
diff --git a/evidence/grade5/timeseries.csv b/evidence/grade5/timeseries.csv
new file mode 100644
index 00000000..6b425ada
--- /dev/null
+++ b/evidence/grade5/timeseries.csv
@@ -0,0 +1,144 @@
+phase,second,successful,failed
+saturation-c1,0,252,0
+saturation-c1,1,266,0
+saturation-c1,2,257,0
+saturation-c1,3,262,0
+saturation-c1,4,253,0
+saturation-c1,5,249,0
+saturation-c1,6,228,0
+saturation-c1,7,240,0
+saturation-c1,8,219,0
+saturation-c1,9,244,0
+saturation-c1,10,2,0
+saturation-c2,0,444,0
+saturation-c2,1,377,0
+saturation-c2,2,352,0
+saturation-c2,3,297,0
+saturation-c2,4,383,0
+saturation-c2,5,324,0
+saturation-c2,6,275,0
+saturation-c2,7,323,0
+saturation-c2,8,333,0
+saturation-c2,9,381,0
+saturation-c2,10,6,0
+saturation-c4,0,526,0
+saturation-c4,1,453,0
+saturation-c4,2,425,0
+saturation-c4,3,478,0
+saturation-c4,4,510,0
+saturation-c4,5,468,0
+saturation-c4,6,458,0
+saturation-c4,7,428,0
+saturation-c4,8,490,0
+saturation-c4,9,474,0
+saturation-c4,10,11,0
+saturation-c8,0,532,0
+saturation-c8,1,476,0
+saturation-c8,2,530,0
+saturation-c8,3,529,0
+saturation-c8,4,490,0
+saturation-c8,5,527,0
+saturation-c8,6,445,0
+saturation-c8,7,524,0
+saturation-c8,8,507,0
+saturation-c8,9,519,0
+saturation-c8,10,16,0
+saturation-c16,0,349,0
+saturation-c16,1,592,0
+saturation-c16,2,541,0
+saturation-c16,3,491,0
+saturation-c16,4,545,0
+saturation-c16,5,420,0
+saturation-c16,6,499,0
+saturation-c16,7,529,0
+saturation-c16,8,548,0
+saturation-c16,9,601,0
+saturation-c16,10,28,0
+saturation-c32,0,661,0
+saturation-c32,1,604,0
+saturation-c32,2,610,0
+saturation-c32,3,589,0
+saturation-c32,4,571,0
+saturation-c32,5,627,0
+saturation-c32,6,587,0
+saturation-c32,7,619,0
+saturation-c32,8,629,0
+saturation-c32,9,548,0
+saturation-c32,10,51,0
+fixed-r1,0,511,0
+fixed-r1,1,580,0
+fixed-r1,2,659,0
+fixed-r1,3,687,0
+fixed-r1,4,588,0
+fixed-r1,5,675,0
+fixed-r1,6,647,0
+fixed-r1,7,701,0
+fixed-r1,8,690,0
+fixed-r1,9,521,0
+fixed-r1,10,32,0
+fixed-r2,0,452,0
+fixed-r2,1,545,0
+fixed-r2,2,604,0
+fixed-r2,3,652,0
+fixed-r2,4,521,0
+fixed-r2,5,627,0
+fixed-r2,6,571,0
+fixed-r2,7,611,0
+fixed-r2,8,573,0
+fixed-r2,9,586,0
+fixed-r2,10,32,0
+fixed-r4,0,133,0
+fixed-r4,1,276,0
+fixed-r4,2,461,0
+fixed-r4,3,406,0
+fixed-r4,4,503,0
+fixed-r4,5,487,0
+fixed-r4,6,377,0
+fixed-r4,7,427,0
+fixed-r4,8,551,0
+fixed-r4,9,547,0
+fixed-r4,10,50,0
+growth-r1,0,527,0
+growth-r1,1,596,0
+growth-r1,2,604,0
+growth-r1,3,616,0
+growth-r1,4,516,0
+growth-r1,5,508,0
+growth-r1,6,606,0
+growth-r1,7,620,0
+growth-r1,8,628,0
+growth-r1,9,591,0
+growth-r1,10,32,0
+growth-r2,0,461,0
+growth-r2,1,662,0
+growth-r2,2,670,0
+growth-r2,3,655,0
+growth-r2,4,640,0
+growth-r2,5,495,0
+growth-r2,6,635,0
+growth-r2,7,626,0
+growth-r2,8,598,0
+growth-r2,9,609,0
+growth-r2,10,32,0
+growth-r4,0,106,0
+growth-r4,1,514,0
+growth-r4,2,587,0
+growth-r4,3,506,0
+growth-r4,4,423,0
+growth-r4,5,409,0
+growth-r4,6,459,0
+growth-r4,7,462,0
+growth-r4,8,618,0
+growth-r4,9,549,0
+growth-r4,10,55,0
+improved-db-cpu-r4,0,492,0
+improved-db-cpu-r4,1,583,0
+improved-db-cpu-r4,2,629,0
+improved-db-cpu-r4,3,681,0
+improved-db-cpu-r4,4,667,0
+improved-db-cpu-r4,5,632,0
+improved-db-cpu-r4,6,585,0
+improved-db-cpu-r4,7,677,0
+improved-db-cpu-r4,8,602,0
+improved-db-cpu-r4,9,629,0
+improved-db-cpu-r4,10,52,0
diff --git a/evidence/lab3-summary.md b/evidence/lab3-summary.md
index ac792635..8501342e 100644
--- a/evidence/lab3-summary.md
+++ b/evidence/lab3-summary.md
@@ -1,6 +1,6 @@
# Lab 3 verification summary
-Дата проверки: 2026-10-20.
+Дата проверки: 2026-10-23.
Файл содержит обезличенную сводку воспроизводимых проверок. Bearer tokens,
пароли, ключи идемпотентности, UUID пользователей/чатов/сообщений и содержимое
@@ -12,7 +12,8 @@
- Docker Engine 29.7.2;
- Docker Compose 5.5.0;
- две Compose-реплики сервиса `api`;
-- единственный опубликованный endpoint: Nginx на `127.0.0.1:8080`;
+- единственный опубликованный endpoint: HAProxy на `127.0.0.1:8080`;
+- Nginx доступен только внутри Compose network и отдаёт статический React;
- PostgreSQL и Redis доступны только внутри Compose network.
## Статические проверки
@@ -42,6 +43,16 @@ shell syntax for hooks PASS
- преподавательский `python3 tests/smoke.py` завершился строкой
`PASS: lab3 public API scenario`.
+## Readiness-aware routing
+
+`make test-readiness-routing` добавил в network namespace одной API-реплики
+правило, запрещающее TCP-доступ к PostgreSQL, не останавливая сам процесс. Его
+`/health/ready` вернул 503; HAProxy исключил backend за 2,861 с. После этого
+40 запросов `/health/live` получили 200, а 20 корректных регистраций — 201;
+не готовая реплика не получила ни одного запроса. После удаления fault-правила
+она вернулась в ротацию. Машиночитаемый результат сохранён в
+`evidence/readiness-routing.json`.
+
## Failover API
Команда:
@@ -54,30 +65,40 @@ make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
```text
run 1:
- stopped container: api-2
+ observed instances before action: 2
+ successful writes / failures: 141 / 0
+ first recovery: 2.668 seconds
+run 2, after restoring the desired two replicas:
observed instances before action: 2
successful writes / failures: 142 / 0
- first recovery: 0.963 seconds
-run 2, after restoring api-2:
- stopped container: api-1
- observed instances before action: 2
- successful writes / failures: 143 / 0
- first recovery: 0.750 seconds
+ first recovery: 2.653 seconds
both results: PASS
```
-После первого запуска реплика была восстановлена; один health-запрос сдвинул
-очередь LB, чтобы следующий synthetic message выбрал второй target. Hook
-сопоставил `TARGET_INSTANCE_ID` с hostname контейнера и в каждом запуске
-остановил только выбранную API-реплику. После тестов желаемое число
-реплик было восстановлено командой `docker compose up -d --wait api`.
+После каждого запуска желаемое число реплик было восстановлено командой
+`docker compose up -d --wait api`. Hook сопоставляет `TARGET_INSTANCE_ID` с
+hostname контейнера и останавливает только выбранную API-реплику. Отдельный
+`make grade4-experiment` целенаправленно остановил обе исходные реплики по
+очереди: 7853/0 и 7970/0 успешных/ошибочных запросов, recovery 2,893/2,938 с.
## Persistence и идемпотентность
- `make test-persistence ACTION_SCRIPT=./scripts/restart.sh`: PASS; после
- рестарта PostgreSQL, Redis, всех API-реплик и Nginx прежние bearer session и
+ рестарта PostgreSQL, Redis, всех API-реплик, frontend и HAProxy прежние bearer session и
созданные данные сохранились;
- 32 конкурентных POST сообщения с одинаковыми semantic payload и
- `Idempotency-Key` через Nginx вернули один Message ID;
+ `Idempotency-Key` через HAProxy вернули один Message ID;
- прямой запрос PostgreSQL подтвердил ровно одну строку сообщения;
- повтор с тем же scope/key и другим payload возвращает HTTP 409.
+
+## Негативные и browser-security проверки
+
+- сырой `Authorization: Bearer` с UTF-8 `é` вернул 401, Bearer challenge и
+ стандартный `Error`; процесс не завершился с panic/500;
+- тело JSON из одного байта `FF` вернуло 400 `invalid_request`, а
+ `error.request_id` совпал с `X-Request-Id`;
+- login установил host-only `HttpOnly; SameSite=Strict` cookie;
+- cookie без Bearer на контрактном `/api/v1/users/me` дала 401, через
+ `/web-api/v1/users/me` — 200;
+- logout через `/web-api` вернул 204 и инвалидировал серверную сессию;
+- статический frontend вернул 200 и `Content-Security-Policy`.
diff --git a/evidence/readiness-routing.json b/evidence/readiness-routing.json
new file mode 100644
index 00000000..abb6ab7f
--- /dev/null
+++ b/evidence/readiness-routing.json
@@ -0,0 +1,11 @@
+{
+ "target_instance": "bd53aadefab0",
+ "detection_seconds": 2.861,
+ "health_requests": {
+ "200": 40
+ },
+ "registration_requests": {
+ "201": 20
+ },
+ "recovered": true
+}
diff --git a/infrastructure/haproxy/Dockerfile b/infrastructure/haproxy/Dockerfile
new file mode 100644
index 00000000..a2befcec
--- /dev/null
+++ b/infrastructure/haproxy/Dockerfile
@@ -0,0 +1,7 @@
+FROM haproxy:3.2-alpine@sha256:dfba86f031bd4c9039f10f8cc5de75991d1b6766d5460f0335b980408503baad
+
+COPY infrastructure/haproxy/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
+
+USER haproxy
+
+EXPOSE 8080
diff --git a/infrastructure/haproxy/haproxy.cfg b/infrastructure/haproxy/haproxy.cfg
new file mode 100644
index 00000000..3a4f34ab
--- /dev/null
+++ b/infrastructure/haproxy/haproxy.cfg
@@ -0,0 +1,66 @@
+global
+ log stdout format raw local0
+ maxconn 2048
+
+defaults
+ log global
+ mode http
+ option httplog
+ timeout connect 1s
+ timeout client 30s
+ timeout server 30s
+
+resolvers docker
+ nameserver dns 127.0.0.11:53
+ resolve_retries 3
+ timeout resolve 1s
+ timeout retry 1s
+ hold valid 1s
+ hold other 1s
+ hold refused 1s
+ hold nx 1s
+ hold timeout 1s
+ hold obsolete 1s
+
+frontend public
+ bind :8080
+
+ acl api_request path_beg /api/ /web-api/ /health/
+ acl web_api path_beg /web-api/
+ acl has_session req.cook(mtusi_session) -m found
+ acl message_create method POST
+ acl message_path path_reg ^/(api|web-api)/v1/chats/[^/]+/messages$
+
+ http-request del-header Authorization if web_api
+ http-request set-header Authorization "Bearer %[req.cook(mtusi_session)]" if web_api has_session
+ http-request del-header Cookie if api_request
+ http-request replace-path ^/web-api/(.*)$ /api/\1 if web_api
+
+ use_backend message_backend if message_create message_path
+ use_backend api_backend if api_request
+ default_backend frontend_backend
+
+backend api_backend
+ balance leastconn
+ option httpchk
+ http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
+ http-check expect status 200
+ default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
+ server-template api 8 api:8080
+
+backend message_backend
+ balance leastconn
+ option httpchk
+ option redispatch
+ option http-buffer-request
+ retries 3
+ retry-on conn-failure empty-response response-timeout 502 503 504
+ http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
+ http-check expect status 200
+ default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
+ server-template api 8 api:8080
+
+backend frontend_backend
+ option httpchk GET /
+ http-check expect status 200
+ server frontend frontend:8080 check inter 2s fall 2 rise 2 resolvers docker init-addr libc,none
diff --git a/infrastructure/nginx/nginx.conf b/infrastructure/nginx/nginx.conf
index 9132e623..ec07590b 100644
--- a/infrastructure/nginx/nginx.conf
+++ b/infrastructure/nginx/nginx.conf
@@ -1,24 +1,12 @@
worker_processes auto;
events {
- worker_connections 1024;
+ worker_connections 512;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
- resolver 127.0.0.11 valid=1s ipv6=off;
-
- map $cookie_mtusi_session $web_authorization {
- default "Bearer $cookie_mtusi_session";
- "" "";
- }
-
- upstream api_backend {
- zone api_backend 64k;
- least_conn;
- server api:8080 resolve max_fails=2 fail_timeout=2s;
- }
server {
listen 8080;
@@ -31,90 +19,6 @@ http {
add_header X-Frame-Options "DENY" always;
add_header Content-Security-Policy "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always;
- # Message creation carries an Idempotency-Key. It is therefore safe to
- # retry a POST on another replica when the selected backend is draining.
- location ~ ^/api/v1/chats/[^/]+/messages$ {
- proxy_pass http://api_backend;
- proxy_http_version 1.1;
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- proxy_set_header Connection "";
- proxy_set_header Cookie "";
- proxy_connect_timeout 1s;
- proxy_send_timeout 20s;
- proxy_read_timeout 20s;
- proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
- proxy_next_upstream_tries 3;
- }
-
- location ~ ^/web-api/v1/chats/[^/]+/messages$ {
- rewrite ^/web-api/(.*)$ /api/$1 break;
- proxy_pass http://api_backend;
- proxy_http_version 1.1;
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- proxy_set_header Connection "";
- proxy_set_header Authorization $web_authorization;
- proxy_set_header Cookie "";
- proxy_connect_timeout 1s;
- proxy_send_timeout 20s;
- proxy_read_timeout 20s;
- proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
- proxy_next_upstream_tries 3;
- }
-
- location /api/ {
- proxy_pass http://api_backend;
- proxy_http_version 1.1;
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- proxy_set_header Connection "";
- proxy_set_header Cookie "";
- proxy_connect_timeout 1s;
- proxy_send_timeout 20s;
- proxy_read_timeout 20s;
- proxy_next_upstream error timeout http_502 http_503 http_504;
- proxy_next_upstream_tries 2;
- }
-
- location /web-api/ {
- rewrite ^/web-api/(.*)$ /api/$1 break;
- proxy_pass http://api_backend;
- proxy_http_version 1.1;
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- proxy_set_header Connection "";
- proxy_set_header Authorization $web_authorization;
- proxy_set_header Cookie "";
- proxy_connect_timeout 1s;
- proxy_send_timeout 20s;
- proxy_read_timeout 20s;
- proxy_next_upstream error timeout http_502 http_503 http_504;
- proxy_next_upstream_tries 2;
- }
-
- location /health/ {
- proxy_pass http://api_backend;
- proxy_http_version 1.1;
- proxy_set_header Host $host;
- proxy_set_header X-Real-IP $remote_addr;
- proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
- proxy_set_header X-Forwarded-Proto $scheme;
- proxy_set_header Connection "";
- proxy_connect_timeout 1s;
- proxy_read_timeout 5s;
- proxy_next_upstream error timeout http_502 http_503 http_504;
- proxy_next_upstream_tries 2;
- }
-
location /assets/ {
try_files $uri =404;
expires 1y;
diff --git a/scripts/grade4_experiment.py b/scripts/grade4_experiment.py
index 4d938472..c22eb68b 100755
--- a/scripts/grade4_experiment.py
+++ b/scripts/grade4_experiment.py
@@ -77,7 +77,7 @@ def memory_mib(raw):
def service(name):
- for candidate in ("load-balancer", "postgres", "redis", "api"):
+ for candidate in ("load-balancer", "frontend", "postgres", "redis", "api"):
if f"-{candidate}-" in name:
return candidate
return "other"
@@ -357,7 +357,7 @@ def main():
"action_after_seconds": args.action_after_seconds,
"post_action_seconds": args.post_action_seconds,
"warmup_requests": 12,
- "workload": "closed-loop POST messages via Nginx; no automatic retries",
+ "workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries",
},
"phases": phases,
"idempotency": idem,
@@ -371,7 +371,7 @@ def main():
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
- "PostgreSQL and Nginx remain single points of failure",
+ "PostgreSQL and HAProxy remain single points of failure",
],
}
destination = ROOT / args.output
diff --git a/scripts/grade5_experiment.py b/scripts/grade5_experiment.py
new file mode 100755
index 00000000..925d8b69
--- /dev/null
+++ b/scripts/grade5_experiment.py
@@ -0,0 +1,577 @@
+#!/usr/bin/env python3
+"""Measure the price of horizontal API scaling for lab 3 grade 5."""
+
+import argparse
+from collections import Counter, defaultdict
+import datetime as dt
+import json
+import math
+import os
+from pathlib import Path
+import platform
+import subprocess
+import sys
+import threading
+import time
+
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tests"))
+from client import Client, Failure # noqa: E402
+
+
+def command(args, env=None, timeout=240, check=True):
+ result = subprocess.run(
+ [str(value) for value in args],
+ cwd=ROOT,
+ env=env,
+ text=True,
+ capture_output=True,
+ timeout=timeout,
+ check=False,
+ )
+ if check and result.returncode:
+ detail = (result.stderr or result.stdout).strip().splitlines()
+ suffix = f": {detail[-1]}" if detail else ""
+ raise RuntimeError(
+ f"{' '.join(map(str, args[:4]))} exited {result.returncode}{suffix}"
+ )
+ return result
+
+
+def compose(env_file, *args, env=None, timeout=240, check=True):
+ return command(
+ ["docker", "compose", "--env-file", env_file, *args],
+ env=env,
+ timeout=timeout,
+ check=check,
+ )
+
+
+def inspect(container_id):
+ raw = command(
+ ["docker", "inspect", "--format", "{{json .}}", container_id]
+ ).stdout
+ return json.loads(raw)
+
+
+def service_ids(env_file, service=None):
+ args = ["ps", "-q"]
+ if service:
+ args.append(service)
+ return [line for line in compose(env_file, *args).stdout.splitlines() if line]
+
+
+def wait_healthy(env_file, service, expected=1, timeout=120):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ healthy = []
+ for container_id in service_ids(env_file, service):
+ state = inspect(container_id)["State"]
+ if state.get("Health", {}).get("Status") == "healthy":
+ healthy.append(container_id)
+ if len(healthy) == expected:
+ return healthy
+ time.sleep(0.5)
+ raise RuntimeError(f"{service}: expected {expected} healthy containers")
+
+
+def percentile(values, ratio):
+ if not values:
+ return None
+ ordered = sorted(values)
+ return round(ordered[max(0, math.ceil(ratio * len(ordered)) - 1)], 3)
+
+
+def memory_mib(raw):
+ value = raw.split("/", 1)[0].strip()
+ units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
+ for suffix in ("GiB", "MiB", "KiB", "B"):
+ if value.endswith(suffix):
+ return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
+ return 0.0
+
+
+def profile_environment(profile):
+ env = dict(os.environ)
+ env.update({
+ "API_REPLICAS": str(profile["replicas"]),
+ "API_CPU_LIMIT": f"{profile['api_cpu']:.2f}",
+ "API_MEMORY_LIMIT": profile["api_memory"],
+ "DB_MAX_CONNS": str(profile["db_max_conns"]),
+ "DB_MIN_CONNS": str(profile["db_min_conns"]),
+ "POSTGRES_CPU_LIMIT": f"{profile['postgres_cpu']:.2f}",
+ "POSTGRES_MEMORY_LIMIT": profile["postgres_memory"],
+ })
+ return env
+
+
+def configure_profile(env_file, profile):
+ env = profile_environment(profile)
+ compose(
+ env_file,
+ "up", "-d", "--no-deps", "postgres",
+ env=env,
+ timeout=180,
+ )
+ wait_healthy(env_file, "postgres")
+ compose(
+ env_file,
+ "up", "-d", "--no-deps",
+ "--scale", f"api={profile['replicas']}",
+ "--force-recreate", "api",
+ env=env,
+ timeout=240,
+ )
+ api = wait_healthy(env_file, "api", profile["replicas"])
+ wait_healthy(env_file, "load-balancer")
+ time.sleep(1.5)
+
+ expected_cpu = int(profile["api_cpu"] * 1_000_000_000)
+ expected_memory = parse_memory_limit(profile["api_memory"])
+ applied = []
+ for container_id in api:
+ data = inspect(container_id)
+ actual_env = dict(item.split("=", 1) for item in data["Config"]["Env"] if "=" in item)
+ host = data["HostConfig"]
+ if host.get("NanoCpus") != expected_cpu or host.get("Memory") != expected_memory:
+ raise RuntimeError("API container limits do not match the selected profile")
+ if actual_env.get("DB_MAX_CONNS") != str(profile["db_max_conns"]):
+ raise RuntimeError("API DB pool size does not match the selected profile")
+ applied.append({
+ "instance": data["Config"]["Hostname"],
+ "cpu_limit": host.get("NanoCpus", 0) / 1_000_000_000,
+ "memory_limit_mib": round(host.get("Memory", 0) / 1024**2, 1),
+ "db_max_conns": int(actual_env["DB_MAX_CONNS"]),
+ })
+
+ postgres = inspect(service_ids(env_file, "postgres")[0])
+ expected_pg_cpu = int(profile["postgres_cpu"] * 1_000_000_000)
+ if postgres["HostConfig"].get("NanoCpus") != expected_pg_cpu:
+ raise RuntimeError("PostgreSQL CPU limit does not match the selected profile")
+ return {
+ "api": sorted(applied, key=lambda item: item["instance"]),
+ "postgres_cpu_limit": postgres["HostConfig"].get("NanoCpus", 0) / 1_000_000_000,
+ "postgres_memory_limit_mib": round(postgres["HostConfig"].get("Memory", 0) / 1024**2, 1),
+ }
+
+
+def parse_memory_limit(value):
+ suffix = value[-1].upper()
+ multiplier = {"K": 1024, "M": 1024**2, "G": 1024**3}[suffix]
+ return int(value[:-1]) * multiplier
+
+
+def resource_targets(env_file):
+ targets = {}
+ for container_id in service_ids(env_file):
+ data = inspect(container_id)
+ service = data["Config"]["Labels"].get("com.docker.compose.service", "other")
+ if service in {"api", "postgres", "redis", "load-balancer", "frontend"}:
+ targets[container_id] = {
+ "service": service,
+ "name": data["Name"].lstrip("/"),
+ }
+ return targets
+
+
+def stats_snapshot(targets, elapsed):
+ if not targets:
+ return {"t": round(elapsed, 3), "services": {}}
+ result = command(
+ ["docker", "stats", "--no-stream", "--format", "{{json .}}", *targets],
+ timeout=15,
+ check=False,
+ )
+ grouped = defaultdict(lambda: {"cpu_percent": 0.0, "memory_mib": 0.0, "containers": 0})
+ for line in result.stdout.splitlines():
+ try:
+ row = json.loads(line)
+ container_id = row.get("ID", "")
+ match = next((key for key in targets if key.startswith(container_id)), None)
+ if not match:
+ continue
+ values = grouped[targets[match]["service"]]
+ values["cpu_percent"] += float(row.get("CPUPerc", "0%").rstrip("%"))
+ values["memory_mib"] += memory_mib(row.get("MemUsage", "0B / 0B"))
+ values["containers"] += 1
+ except (ValueError, json.JSONDecodeError):
+ continue
+ return {
+ "t": round(elapsed, 3),
+ "services": {
+ name: {
+ "cpu_percent": round(values["cpu_percent"], 3),
+ "memory_mib": round(values["memory_mib"], 3),
+ "containers": values["containers"],
+ }
+ for name, values in sorted(grouped.items())
+ },
+ }
+
+
+def summarize_resources(snapshots):
+ grouped = defaultdict(lambda: {"cpu": [], "memory": [], "containers": []})
+ for snapshot in snapshots:
+ for service, values in snapshot["services"].items():
+ grouped[service]["cpu"].append(values["cpu_percent"])
+ grouped[service]["memory"].append(values["memory_mib"])
+ grouped[service]["containers"].append(values["containers"])
+ return {
+ service: {
+ "cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
+ "cpu_percent_max": round(max(values["cpu"]), 3),
+ "memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
+ "memory_mib_max": round(max(values["memory"]), 3),
+ "containers": max(values["containers"]),
+ }
+ for service, values in sorted(grouped.items())
+ if values["cpu"]
+ }
+
+
+def run_phase(name, kind, actor, chat, base_url, duration, concurrency, env_file, profile):
+ records = []
+ snapshots = []
+ lock = threading.Lock()
+ stop = threading.Event()
+ barrier = threading.Barrier(concurrency + 1)
+ targets = resource_targets(env_file)
+ started = time.monotonic()
+
+ def worker(worker_id):
+ client = Client(base_url, timeout=5)
+ index = 0
+ barrier.wait()
+ while not stop.is_set():
+ request_started = time.monotonic()
+ try:
+ response = client.message(
+ actor,
+ chat,
+ f"grade5-{name}-{worker_id}-{index}",
+ )
+ row = {
+ "t": round(time.monotonic() - started, 3),
+ "ok": True,
+ "latency_ms": round((time.monotonic() - request_started) * 1000, 3),
+ "instance": response["headers"].get("X-Instance-Id", ""),
+ }
+ except Failure as exc:
+ row = {
+ "t": round(time.monotonic() - started, 3),
+ "ok": False,
+ "latency_ms": round((time.monotonic() - request_started) * 1000, 3),
+ "error": str(exc),
+ }
+ with lock:
+ records.append(row)
+ index += 1
+
+ def sampler():
+ while not stop.is_set():
+ snapshots.append(stats_snapshot(targets, time.monotonic() - started))
+ stop.wait(1)
+
+ workers = [
+ threading.Thread(target=worker, args=(index,), daemon=True)
+ for index in range(concurrency)
+ ]
+ for thread in workers:
+ thread.start()
+ stats_thread = threading.Thread(target=sampler, daemon=True)
+ stats_thread.start()
+ started = time.monotonic()
+ barrier.wait()
+ time.sleep(duration)
+ stop.set()
+ for thread in workers:
+ thread.join(10)
+ stats_thread.join(15)
+
+ elapsed = time.monotonic() - started
+ successes = [row for row in records if row["ok"]]
+ failures = [row for row in records if not row["ok"]]
+ latencies = [row["latency_ms"] for row in successes]
+ buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
+ for row in records:
+ key = str(max(0, int(row["t"])))
+ buckets[key]["successful" if row["ok"] else "failed"] += 1
+ total = len(records)
+ return {
+ "name": name,
+ "kind": kind,
+ "profile": profile["name"],
+ "replicas": profile["replicas"],
+ "concurrency": concurrency,
+ "elapsed_seconds": round(elapsed, 3),
+ "successful_requests": len(successes),
+ "failed_requests": len(failures),
+ "error_ratio": round(len(failures) / total, 6) if total else None,
+ "errors": dict(sorted(Counter(row["error"] for row in failures).items())),
+ "successful_rps": round(len(successes) / elapsed, 3),
+ "successful_latency_ms": {
+ "p50": percentile(latencies, 0.50),
+ "p95": percentile(latencies, 0.95),
+ "p99": percentile(latencies, 0.99),
+ },
+ "instances": sorted({row["instance"] for row in successes if row.get("instance")}),
+ "per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
+ "resource_samples": snapshots,
+ "resources": summarize_resources(snapshots),
+ }
+
+
+def dataset(env_file):
+ sql = (
+ "select count(*)::text || '|' || pg_database_size(current_database())::text "
+ "from messages"
+ )
+ result = compose(
+ env_file,
+ "exec", "-T", "postgres", "sh", "-c",
+ 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
+ "sh", sql,
+ )
+ count, size = result.stdout.strip().split("|", 1)
+ return {"messages": int(count), "database_size_bytes": int(size)}
+
+
+def host_environment(env_file):
+ mem_total_kib = 0
+ with open("/proc/meminfo", encoding="utf-8") as source:
+ for line in source:
+ if line.startswith("MemTotal:"):
+ mem_total_kib = int(line.split()[1])
+ break
+ images = compose(env_file, "config", "--images").stdout.splitlines()
+ return {
+ "utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
+ "git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
+ "working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
+ "logical_cpu_count": os.cpu_count(),
+ "host_memory_mib": round(mem_total_kib / 1024, 1),
+ "kernel": platform.release(),
+ "python": platform.python_version(),
+ "docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
+ "docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
+ "images": images,
+ }
+
+
+def analyze(phases):
+ sweep = sorted(
+ (phase for phase in phases if phase["kind"] == "saturation"),
+ key=lambda phase: phase["concurrency"],
+ )
+ saturation = None
+ previous = None
+ for phase in sweep:
+ if previous:
+ gain = (phase["successful_rps"] - previous["successful_rps"]) / max(previous["successful_rps"], 0.001)
+ latency_growth = phase["successful_latency_ms"]["p95"] / max(previous["successful_latency_ms"]["p95"], 0.001)
+ if saturation is None and gain < 0.10 and latency_growth >= 1.50:
+ saturation = {
+ "concurrency": phase["concurrency"],
+ "previous_concurrency": previous["concurrency"],
+ "rps_gain_percent": round(gain * 100, 2),
+ "p95_growth_ratio": round(latency_growth, 3),
+ }
+ previous = phase
+
+ baseline = next(phase for phase in phases if phase["name"] == "growth-r4")
+ improved = next(phase for phase in phases if phase["name"] == "improved-db-cpu-r4")
+ rps_change = (improved["successful_rps"] - baseline["successful_rps"]) / max(baseline["successful_rps"], 0.001) * 100
+ p95_change = (improved["successful_latency_ms"]["p95"] - baseline["successful_latency_ms"]["p95"]) / max(baseline["successful_latency_ms"]["p95"], 0.001) * 100
+ confirmed = (
+ rps_change >= 5
+ and improved["error_ratio"] <= baseline["error_ratio"] + 0.001
+ ) or p95_change <= -10
+
+ resources = baseline["resources"]
+ pg = resources.get("postgres", {})
+ lb = resources.get("load-balancer", {})
+ bottleneck = "undetermined"
+ if pg.get("cpu_percent_avg", 0) >= 70 and pg.get("cpu_percent_max", 0) >= 95:
+ bottleneck = "postgres_cpu"
+ elif lb.get("cpu_percent_avg", 0) >= 20 and lb.get("cpu_percent_max", 0) >= 24:
+ bottleneck = "load_balancer_cpu"
+ improved_resources = improved["resources"]
+ improved_lb = improved_resources.get("load-balancer", {})
+ remaining_bottleneck = "undetermined"
+ if improved_lb.get("cpu_percent_avg", 0) >= 20 and improved_lb.get("cpu_percent_max", 0) >= 24:
+ remaining_bottleneck = "load_balancer_cpu"
+ return {
+ "saturation_point": saturation,
+ "observed_bottleneck": bottleneck,
+ "baseline_resource_snapshot": resources,
+ "improved_resource_snapshot": improved_resources,
+ "remaining_bottleneck_after_improvement": remaining_bottleneck,
+ "improvement": {
+ "change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged",
+ "baseline_phase": baseline["name"],
+ "improved_phase": improved["name"],
+ "rps_change_percent": round(rps_change, 2),
+ "p95_change_percent": round(p95_change, 2),
+ "confirmed": confirmed,
+ },
+ }
+
+
+def profiles():
+ base = {"postgres_cpu": 1.0, "postgres_memory": "512M"}
+ return {
+ "fixed-r1": {**base, "name": "fixed-r1", "replicas": 1, "api_cpu": 1.0, "api_memory": "512M", "db_max_conns": 16, "db_min_conns": 2},
+ "fixed-r2": {**base, "name": "fixed-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
+ "fixed-r4": {**base, "name": "fixed-r4", "replicas": 4, "api_cpu": 0.25, "api_memory": "128M", "db_max_conns": 4, "db_min_conns": 1},
+ "growth-r1": {**base, "name": "growth-r1", "replicas": 1, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
+ "growth-r2": {**base, "name": "growth-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
+ "growth-r4": {**base, "name": "growth-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
+ "improved-db-cpu-r4": {**base, "name": "improved-db-cpu-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1, "postgres_cpu": 2.0},
+ }
+
+
+def run_warmup(actor, chat, args, profile):
+ phase = run_phase(
+ f"warmup-{profile['name']}",
+ "warmup",
+ actor,
+ chat,
+ args.base_url,
+ args.warmup_seconds,
+ min(4, args.comparison_concurrency),
+ args.env_file,
+ profile,
+ )
+ if phase["failed_requests"]:
+ raise RuntimeError(f"warmup failed for {profile['name']}")
+ return phase["successful_requests"]
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--base-url", default="http://localhost:8080")
+ parser.add_argument("--env-file", default=".env")
+ parser.add_argument("--duration", type=float, default=10)
+ parser.add_argument("--warmup-seconds", type=float, default=2)
+ parser.add_argument("--comparison-concurrency", type=int, default=32)
+ parser.add_argument("--sweep", default="1,2,4,8,16,32")
+ parser.add_argument("--output", default="evidence/grade5/results.json")
+ args = parser.parse_args()
+ sweep = [int(value) for value in args.sweep.split(",")]
+ if args.duration <= 0 or args.warmup_seconds <= 0:
+ raise SystemExit("durations must be positive")
+ if not sweep or min(sweep) < 1 or max(sweep + [args.comparison_concurrency]) > 64:
+ raise SystemExit("concurrency must be 1..64")
+
+ all_profiles = profiles()
+ restore = {
+ "name": "restore-default",
+ "replicas": 2,
+ "api_cpu": 0.5,
+ "api_memory": "256M",
+ "db_max_conns": 10,
+ "db_min_conns": 2,
+ "postgres_cpu": 1.0,
+ "postgres_memory": "512M",
+ }
+ phases = []
+ applied = {}
+ warmup_requests = 0
+ restoration = {"attempted": False, "succeeded": False}
+ environment = host_environment(args.env_file)
+ start_dataset = dataset(args.env_file)
+ try:
+ client = Client(args.base_url, timeout=8)
+ alice, _, _, chat = client.bootstrap()
+
+ saturation_profile = all_profiles["growth-r2"]
+ applied[saturation_profile["name"]] = configure_profile(args.env_file, saturation_profile)
+ warmup_requests += run_warmup(alice, chat, args, saturation_profile)
+ for concurrency in sweep:
+ print(f"saturation: concurrency={concurrency}", flush=True)
+ phases.append(run_phase(
+ f"saturation-c{concurrency}", "saturation", alice, chat,
+ args.base_url, args.duration, concurrency, args.env_file,
+ saturation_profile,
+ ))
+
+ for profile_name in (
+ "fixed-r1", "fixed-r2", "fixed-r4",
+ "growth-r1", "growth-r2", "growth-r4",
+ "improved-db-cpu-r4",
+ ):
+ profile = all_profiles[profile_name]
+ print(f"profile: {profile_name}", flush=True)
+ applied[profile_name] = configure_profile(args.env_file, profile)
+ warmup_requests += run_warmup(alice, chat, args, profile)
+ kind = "improvement" if profile_name.startswith("improved") else profile_name.split("-", 1)[0]
+ phases.append(run_phase(
+ profile_name, kind, alice, chat, args.base_url,
+ args.duration, args.comparison_concurrency, args.env_file,
+ profile,
+ ))
+ end_dataset = dataset(args.env_file)
+ finally:
+ restoration["attempted"] = True
+ try:
+ restoration["applied"] = configure_profile(args.env_file, restore)
+ restoration["succeeded"] = True
+ except Exception as exc:
+ restoration["error"] = f"{type(exc).__name__}: {exc}"
+
+ if not restoration["succeeded"]:
+ raise RuntimeError("experiment completed but default Compose profile was not restored")
+
+ report = {
+ "schema_version": 1,
+ "environment": environment,
+ "hypothesis": (
+ "API replicas stop improving throughput when shared PostgreSQL CPU saturates; "
+ "adding PostgreSQL CPU should improve the unchanged four-replica profile."
+ ),
+ "methodology": {
+ "workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries",
+ "duration_seconds_per_phase": args.duration,
+ "warmup_seconds_per_profile": args.warmup_seconds,
+ "comparison_concurrency": args.comparison_concurrency,
+ "saturation_concurrency": sweep,
+ "fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections",
+ "growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica",
+ "improvement": "only PostgreSQL CPU changes from 1.00 to 2.00",
+ "saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step",
+ "cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer",
+ },
+ "profiles": all_profiles,
+ "applied_limits": applied,
+ "phases": phases,
+ "analysis": analyze(phases),
+ "dataset": {
+ "before": start_dataset,
+ "after": end_dataset,
+ "warmup_successful_requests": warmup_requests,
+ },
+ "restoration": restoration,
+ "limitations": [
+ "single Docker host: replicas do not prove host-level availability",
+ "closed-loop workload has coordinated omission",
+ "successful-response latency percentiles are reported separately from failures",
+ "one measured run per profile; host scheduling noise is possible",
+ ],
+ }
+ destination = ROOT / args.output
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+ try:
+ displayed_output = destination.relative_to(ROOT)
+ except ValueError:
+ displayed_output = destination
+ print(json.dumps({
+ "output": str(displayed_output),
+ "analysis": report["analysis"],
+ "restoration": restoration["succeeded"],
+ }, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/render_grade4.py b/scripts/render_grade4.py
index 9c62886d..e510ffdf 100755
--- a/scripts/render_grade4.py
+++ b/scripts/render_grade4.py
@@ -148,7 +148,7 @@ def summary(report, output):
"## Method", "",
f"- concurrency: {method['concurrency']};",
f"- baseline duration: {method['baseline_seconds']} s;",
- "- load: closed-loop POST messages through Nginx;",
+ "- load: closed-loop POST messages through the public HAProxy gateway;",
"- failover: both original API containers stopped and restored in turn;",
"- rolling update: one surge replica, old replicas replaced one by one;",
"- percentiles cover successful responses; failures are separate.", "",
@@ -178,7 +178,7 @@ def summary(report, output):
"", "",
"## Scope", "",
"Single-host process failure and rolling replacement only. PostgreSQL "
- "and Nginx remain single points of failure.", "",
+ "and HAProxy remain single points of failure.", "",
]
output.write_text("\n".join(lines))
diff --git a/scripts/render_grade5.py b/scripts/render_grade5.py
new file mode 100755
index 00000000..60014c30
--- /dev/null
+++ b/scripts/render_grade5.py
@@ -0,0 +1,322 @@
+#!/usr/bin/env python3
+"""Render grade-5 scaling evidence as SVG, CSV and Markdown."""
+
+import argparse
+import csv
+import html
+import json
+from pathlib import Path
+
+
+STYLE = """
+text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
+.title { font-size: 26px; font-weight: 650; }
+.subtitle { font-size: 14px; fill: #aeb2b8; }
+.label { font-size: 14px; fill: #c8cbd0; }
+.small { font-size: 12px; fill: #a2a5aa; }
+.grid { stroke: #34373d; stroke-width: 1; }
+.axis { stroke: #747981; stroke-width: 1.5; }
+.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
+.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
+.point { fill: #f4f5f6; }
+.point2 { fill: #92979f; }
+.bar { fill: #d7d9dc; }
+.bar2 { fill: #777c84; }
+.bar3 { fill: #4c5057; }
+.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
+"""
+
+
+def svg(title, subtitle, body, width=1120, height=620):
+ return f"""
+"""
+
+
+def grid(maximum, left, top, width, height, suffix=""):
+ maximum = maximum or 1
+ parts = []
+ for step in range(6):
+ value = maximum * step / 5
+ y = top + height - height * step / 5
+ parts.append(f'')
+ parts.append(f'{value:.0f}{suffix}')
+ return parts
+
+
+def phase_map(report):
+ return {phase["name"]: phase for phase in report["phases"]}
+
+
+def saturation_graph(report, output):
+ phases = sorted(
+ (phase for phase in report["phases"] if phase["kind"] == "saturation"),
+ key=lambda phase: phase["concurrency"],
+ )
+ left, top, width, height = 90, 110, 940, 390
+ max_rps = max(phase["successful_rps"] for phase in phases) * 1.1
+ max_p95 = max(phase["successful_latency_ms"]["p95"] for phase in phases) * 1.1
+ parts = grid(max_rps, left, top, width, height)
+ points_rps, points_p95 = [], []
+ for index, phase in enumerate(phases):
+ x = left + width * index / max(1, len(phases) - 1)
+ y_rps = top + height - height * phase["successful_rps"] / max_rps
+ y_p95 = top + height - height * phase["successful_latency_ms"]["p95"] / max_p95
+ points_rps.append(f"{x:.1f},{y_rps:.1f}")
+ points_p95.append(f"{x:.1f},{y_p95:.1f}")
+ parts.append(f'')
+ parts.append(f'')
+ parts.append(f'{phase["concurrency"]}')
+ parts.append(f'{phase["successful_rps"]:.0f}')
+ parts.append(f'')
+ parts.append(f'')
+ parts.append(f'Concurrency')
+ parts.append(f'white: RPS · gray: p95 normalized')
+ parts.append(f'Successful requests per second')
+ output.write_text(svg(
+ "Saturation curve",
+ "Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU",
+ "\n".join(parts),
+ ), encoding="utf-8")
+
+
+def comparison_graph(report, kind, output):
+ phases = sorted(
+ (phase for phase in report["phases"] if phase["kind"] == kind),
+ key=lambda phase: phase["replicas"],
+ )
+ left, top, width, height = 90, 110, 940, 370
+ max_rps = max(phase["successful_rps"] for phase in phases) * 1.15
+ parts = grid(max_rps, left, top, width, height)
+ slot = width / len(phases)
+ for index, phase in enumerate(phases):
+ bar_width = min(150, slot * 0.52)
+ x = left + index * slot + (slot - bar_width) / 2
+ bar_height = height * phase["successful_rps"] / max_rps
+ y = top + height - bar_height
+ parts.append(f'')
+ parts.append(f'{phase["successful_rps"]:.1f} RPS')
+ parts.append(f'{phase["replicas"]} replica(s)')
+ parts.append(f'p95 {phase["successful_latency_ms"]["p95"]:.1f} ms')
+ parts.append(f'errors {phase["error_ratio"]*100:.3f}%')
+ if kind == "fixed":
+ title = "Fixed total API budget"
+ subtitle = "1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas"
+ else:
+ title = "Growing API budget"
+ subtitle = "Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU"
+ output.write_text(svg(title, subtitle, "\n".join(parts)), encoding="utf-8")
+
+
+def bottleneck_graph(report, output):
+ phases = sorted(
+ (phase for phase in report["phases"] if phase["kind"] == "growth"),
+ key=lambda phase: phase["replicas"],
+ )
+ services = ("api", "postgres", "load-balancer", "frontend", "redis")
+ left, top, width, height = 90, 110, 940, 370
+ maximum = max(
+ phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
+ for phase in phases for service in services
+ ) * 1.15 or 1
+ parts = grid(maximum, left, top, width, height)
+ slot = width / len(phases)
+ colors = ("bar", "bar2", "bar3", "bar3", "bar3")
+ for phase_index, phase in enumerate(phases):
+ group_left = left + phase_index * slot
+ bar_width = min(48, slot / 6)
+ for service_index, service in enumerate(services):
+ value = phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
+ x = group_left + slot * 0.12 + service_index * (bar_width + 9)
+ bar_height = height * value / maximum
+ y = top + height - bar_height
+ parts.append(f'')
+ parts.append(f'{value:.0f}')
+ parts.append(f'{service[:3]}')
+ parts.append(f'{phase["replicas"]} replica(s)')
+ parts.append(f'Average aggregate CPU, %')
+ output.write_text(svg(
+ "Where scaling stops",
+ "Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis)",
+ "\n".join(parts),
+ ), encoding="utf-8")
+
+
+def improvement_graph(report, output):
+ phases = phase_map(report)
+ baseline = phases["growth-r4"]
+ improved = phases["improved-db-cpu-r4"]
+ pair = [baseline, improved]
+ left, top, panel_width, height = 90, 120, 390, 340
+ parts = []
+ panels = [
+ ("Successful RPS", "successful_rps", left),
+ ("p95 latency, ms", "p95", 630),
+ ]
+ for title, metric, panel_left in panels:
+ values = [
+ phase["successful_rps"] if metric == "successful_rps" else phase["successful_latency_ms"]["p95"]
+ for phase in pair
+ ]
+ maximum = max(values) * 1.2 or 1
+ parts.append(f'{title}')
+ for index, (phase, value) in enumerate(zip(pair, values)):
+ bar_width = 120
+ x = panel_left + 35 + index * 185
+ bar_height = height * value / maximum
+ y = top + height - bar_height
+ parts.append(f'')
+ parts.append(f'{value:.1f}')
+ parts.append(f'PG {1 if index == 0 else 2} CPU')
+ analysis = report["analysis"]["improvement"]
+ parts.append(f'RPS change: {analysis["rps_change_percent"]:+.2f}% · p95 change: {analysis["p95_change_percent"]:+.2f}% · confirmed: {str(analysis["confirmed"]).lower()}')
+ output.write_text(svg(
+ "Confirmed improvement",
+ "Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2",
+ "\n".join(parts),
+ ), encoding="utf-8")
+
+
+def write_csv(report, destination):
+ with (destination / "measurements.csv").open("w", newline="", encoding="utf-8") as stream:
+ fields = [
+ "phase", "kind", "replicas", "concurrency", "elapsed_seconds",
+ "successful_requests", "failed_requests", "error_ratio", "successful_rps",
+ "p50_ms", "p95_ms", "p99_ms", "api_cpu_avg", "postgres_cpu_avg",
+ "load_balancer_cpu_avg", "frontend_cpu_avg", "redis_cpu_avg",
+ ]
+ writer = csv.DictWriter(stream, fieldnames=fields)
+ writer.writeheader()
+ for phase in report["phases"]:
+ writer.writerow({
+ "phase": phase["name"],
+ "kind": phase["kind"],
+ "replicas": phase["replicas"],
+ "concurrency": phase["concurrency"],
+ "elapsed_seconds": phase["elapsed_seconds"],
+ "successful_requests": phase["successful_requests"],
+ "failed_requests": phase["failed_requests"],
+ "error_ratio": phase["error_ratio"],
+ "successful_rps": phase["successful_rps"],
+ "p50_ms": phase["successful_latency_ms"]["p50"],
+ "p95_ms": phase["successful_latency_ms"]["p95"],
+ "p99_ms": phase["successful_latency_ms"]["p99"],
+ "api_cpu_avg": phase["resources"].get("api", {}).get("cpu_percent_avg"),
+ "postgres_cpu_avg": phase["resources"].get("postgres", {}).get("cpu_percent_avg"),
+ "load_balancer_cpu_avg": phase["resources"].get("load-balancer", {}).get("cpu_percent_avg"),
+ "frontend_cpu_avg": phase["resources"].get("frontend", {}).get("cpu_percent_avg"),
+ "redis_cpu_avg": phase["resources"].get("redis", {}).get("cpu_percent_avg"),
+ })
+
+ with (destination / "timeseries.csv").open("w", newline="", encoding="utf-8") as stream:
+ fields = ["phase", "second", "successful", "failed"]
+ writer = csv.DictWriter(stream, fieldnames=fields)
+ writer.writeheader()
+ for phase in report["phases"]:
+ for second, counts in phase["per_second"].items():
+ writer.writerow({"phase": phase["name"], "second": second, **counts})
+
+ with (destination / "resources.csv").open("w", newline="", encoding="utf-8") as stream:
+ fields = ["phase", "t", "service", "containers", "cpu_percent", "memory_mib"]
+ writer = csv.DictWriter(stream, fieldnames=fields)
+ writer.writeheader()
+ for phase in report["phases"]:
+ for snapshot in phase["resource_samples"]:
+ for service, values in snapshot["services"].items():
+ writer.writerow({
+ "phase": phase["name"], "t": snapshot["t"], "service": service,
+ **values,
+ })
+
+
+def markdown_summary(report, output):
+ method = report["methodology"]
+ analysis = report["analysis"]
+ improvement = analysis["improvement"]
+ lines = [
+ "# Lab 3 grade-5 scaling experiment", "",
+ f"Generated: {report['environment']['utc_started_at']}.", "",
+ "## Hypothesis", "", report["hypothesis"], "",
+ "## Reproduction", "",
+ f"- workload: {method['workload']};",
+ f"- phase duration: {method['duration_seconds_per_phase']} s;",
+ f"- comparison concurrency: {method['comparison_concurrency']};",
+ f"- saturation sweep: {method['saturation_concurrency']};",
+ f"- fixed budget: {method['fixed_total_budget']};",
+ f"- growth series: {method['growth_budget']};",
+ f"- saturation rule: {method['saturation_rule']};",
+ f"- host: {report['environment']['logical_cpu_count']} logical CPU, {report['environment']['host_memory_mib']} MiB RAM;",
+ f"- Docker {report['environment']['docker_engine']}, Compose {report['environment']['docker_compose']};",
+ "- command: `make grade5-experiment`; warmup runs before every changed profile.", "",
+ "## Measurements", "",
+ "| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |",
+ "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+ ]
+ for phase in report["phases"]:
+ latency = phase["successful_latency_ms"]
+ resources = phase["resources"]
+ lines.append(
+ f"| {phase['name']} | {phase['replicas']} | {phase['concurrency']} | "
+ f"{phase['successful_rps']} | {phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
+ f"{latency['p50']} | {latency['p95']} | {latency['p99']} | "
+ f"{resources.get('postgres', {}).get('cpu_percent_avg', 'n/a')} | "
+ f"{resources.get('api', {}).get('cpu_percent_avg', 'n/a')} |"
+ )
+ saturation = analysis["saturation_point"]
+ lines += ["", "## Conclusion", ""]
+ if saturation:
+ lines.append(
+ f"The first saturation signal appears at concurrency {saturation['concurrency']}: "
+ f"RPS changes by {saturation['rps_gain_percent']:+.2f}% while p95 changes by "
+ f"{saturation['p95_growth_ratio']:.2f}x versus concurrency {saturation['previous_concurrency']}."
+ )
+ else:
+ lines.append("The configured sweep did not locate a saturation point by the declared rule.")
+ lines += [
+ "",
+ f"Observed bottleneck classification: `{analysis['observed_bottleneck']}`.", "",
+ f"The PostgreSQL CPU change produced {improvement['rps_change_percent']:+.2f}% RPS and "
+ f"{improvement['p95_change_percent']:+.2f}% p95; confirmed improvement: "
+ f"`{str(improvement['confirmed']).lower()}`.", "",
+ f"After the improvement the remaining bottleneck classification is "
+ f"`{analysis['remaining_bottleneck_after_improvement']}`.", "",
+ "## Graphs", "",
+ "", "",
+ "", "",
+ "", "",
+ "", "",
+ "", "",
+ "## Raw data and limits", "",
+ "Raw aggregates are in `measurements.csv`; one-second request buckets are in "
+ "`timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and "
+ "applied limits are in `results.json`.", "",
+ ]
+ lines.extend(f"- {item}" for item in report["limitations"])
+ lines.append("")
+ output.write_text("\n".join(lines), encoding="utf-8")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("results", nargs="?", default="evidence/grade5/results.json")
+ args = parser.parse_args()
+ source = Path(args.results)
+ report = json.loads(source.read_text(encoding="utf-8"))
+ destination = source.parent
+ saturation_graph(report, destination / "01-saturation.svg")
+ comparison_graph(report, "fixed", destination / "02-fixed-total.svg")
+ comparison_graph(report, "growth", destination / "03-resource-growth.svg")
+ bottleneck_graph(report, destination / "04-bottleneck.svg")
+ improvement_graph(report, destination / "05-improvement.svg")
+ write_csv(report, destination)
+ markdown_summary(report, destination / "README.md")
+ print(f"rendered grade-5 evidence in {destination}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/restart.sh b/scripts/restart.sh
index 80ea344d..fd04a052 100755
--- a/scripts/restart.sh
+++ b/scripts/restart.sh
@@ -19,6 +19,10 @@ compose up -d --no-deps api
compose restart api
compose up -d --wait --no-deps api
-# Публичная точка входа перезапускается последней, когда API уже готовы.
+# Статический frontend возвращается до публичной точки входа.
+compose restart frontend
+compose up -d --wait --no-deps frontend
+
+# Публичная точка входа перезапускается последней, когда API и frontend готовы.
compose restart load-balancer
compose up -d --wait --no-deps load-balancer
diff --git a/scripts/test_readiness_routing.py b/scripts/test_readiness_routing.py
new file mode 100755
index 00000000..257b3246
--- /dev/null
+++ b/scripts/test_readiness_routing.py
@@ -0,0 +1,198 @@
+#!/usr/bin/env python3
+"""Prove that the gateway excludes an API replica which loses readiness."""
+
+import argparse
+from collections import Counter
+import http.client
+import json
+import os
+from pathlib import Path
+import subprocess
+import time
+import uuid
+
+
+ROOT = Path(__file__).resolve().parents[1]
+FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f"
+
+
+def command(args, timeout=180, check=True):
+ result = subprocess.run(
+ [str(value) for value in args],
+ cwd=ROOT,
+ text=True,
+ capture_output=True,
+ timeout=timeout,
+ check=False,
+ )
+ if check and result.returncode:
+ detail = (result.stderr or result.stdout).strip().splitlines()
+ suffix = f": {detail[-1]}" if detail else ""
+ raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}")
+ return result
+
+
+def compose(env_file, *args):
+ return command(["docker", "compose", "--env-file", env_file, *args])
+
+
+def inspect(container_id):
+ return json.loads(command([
+ "docker", "inspect", "--format", "{{json .}}", container_id,
+ ]).stdout)
+
+
+def call(host, port, method, path, body=None, headers=None, timeout=5):
+ connection = http.client.HTTPConnection(host, port, timeout=timeout)
+ connection.request(method, path, body=body, headers=headers or {})
+ response = connection.getresponse()
+ payload = response.read()
+ result = {
+ "status": response.status,
+ "instance": response.getheader("X-Instance-Id", ""),
+ "body": payload,
+ }
+ connection.close()
+ return result
+
+
+def observe_instances(base_host, base_port, requests=16):
+ instances = Counter()
+ statuses = Counter()
+ for _ in range(requests):
+ result = call(base_host, base_port, "GET", "/health/live")
+ statuses[result["status"]] += 1
+ instances[result["instance"]] += 1
+ return statuses, instances
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--env-file", default=".env")
+ parser.add_argument("--host", default="localhost")
+ parser.add_argument("--port", type=int, default=8080)
+ parser.add_argument("--output", default="")
+ args = parser.parse_args()
+
+ api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line]
+ if len(api_ids) != 2:
+ raise RuntimeError("readiness routing test requires exactly two API replicas")
+ target_id = api_ids[0]
+ target = inspect(target_id)
+ target_instance = target["Config"]["Hostname"]
+ target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"]
+
+ postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip()
+ postgres = inspect(postgres_id)
+ postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"]
+
+ before_statuses, before_instances = observe_instances(args.host, args.port, 40)
+ if before_statuses != {200: 40} or len(before_instances) != 2:
+ raise RuntimeError("both API replicas must serve traffic before fault injection")
+
+ helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}"
+ rule = [
+ "iptables", "-D", "OUTPUT", "-d", postgres_ip,
+ "-p", "tcp", "--dport", "5432", "-j", "REJECT",
+ "--reject-with", "tcp-reset",
+ ]
+ detection_started = time.monotonic()
+ try:
+ script = (
+ "apk add --no-cache iptables >/dev/null && "
+ f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 "
+ "-j REJECT --reject-with tcp-reset && "
+ "touch /tmp/fault-ready && sleep 180"
+ )
+ command([
+ "docker", "run", "-d", "--name", helper,
+ "--network", f"container:{target_id}",
+ "--cap-add", "NET_ADMIN", FAULT_IMAGE,
+ "sh", "-c", script,
+ ])
+ deadline = time.monotonic() + 30
+ while time.monotonic() < deadline:
+ ready = command(
+ ["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"],
+ check=False,
+ )
+ if ready.returncode == 0:
+ break
+ time.sleep(0.2)
+ else:
+ raise RuntimeError("fault helper did not become ready")
+
+ direct = call(target_ip, 8080, "GET", "/health/ready")
+ if direct["status"] != 503:
+ raise RuntimeError(f"target readiness is {direct['status']}, want 503")
+
+ deadline = time.monotonic() + 10
+ while time.monotonic() < deadline:
+ statuses, instances = observe_instances(args.host, args.port, 16)
+ if statuses == {200: 16} and target_instance not in instances:
+ break
+ time.sleep(0.25)
+ else:
+ raise RuntimeError("gateway did not exclude the unready API within 10 seconds")
+ detection_seconds = time.monotonic() - detection_started
+
+ health_statuses, health_instances = observe_instances(args.host, args.port, 40)
+ registration_statuses = Counter()
+ registration_instances = Counter()
+ for _ in range(20):
+ body = json.dumps({
+ "username": "readiness_" + uuid.uuid4().hex[:16],
+ "display_name": "Readiness audit",
+ "password": "T3st-" + uuid.uuid4().hex,
+ }).encode()
+ response = call(
+ args.host,
+ args.port,
+ "POST",
+ "/api/v1/users",
+ body=body,
+ headers={"Content-Type": "application/json"},
+ timeout=8,
+ )
+ registration_statuses[response["status"]] += 1
+ registration_instances[response["instance"]] += 1
+
+ if health_statuses != {200: 40} or target_instance in health_instances:
+ raise RuntimeError("unready API received liveness traffic through gateway")
+ if registration_statuses != {201: 20} or target_instance in registration_instances:
+ raise RuntimeError("unready API received registration traffic through gateway")
+ finally:
+ command(["docker", "exec", helper, *rule], check=False)
+ command(["docker", "rm", "-f", helper], check=False)
+
+ deadline = time.monotonic() + 15
+ while time.monotonic() < deadline:
+ direct = call(target_ip, 8080, "GET", "/health/ready")
+ if direct["status"] == 200:
+ _, instances = observe_instances(args.host, args.port, 24)
+ if target_instance in instances:
+ break
+ time.sleep(0.25)
+ else:
+ raise RuntimeError("recovered API did not return to gateway rotation")
+
+ result = {
+ "target_instance": target_instance,
+ "detection_seconds": round(detection_seconds, 3),
+ "health_requests": dict(health_statuses),
+ "registration_requests": dict(registration_statuses),
+ "recovered": True,
+ }
+ rendered = json.dumps(result, ensure_ascii=False, indent=2)
+ if args.output:
+ output = Path(args.output)
+ if not output.is_absolute():
+ output = ROOT / output
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_text(rendered + "\n", encoding="utf-8")
+ print(rendered)
+ print("PASS: readiness-aware gateway excluded and restored one API replica")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/student_tests/auth_test.go b/student_tests/auth_test.go
index 6b5e6e31..08e43ff6 100644
--- a/student_tests/auth_test.go
+++ b/student_tests/auth_test.go
@@ -209,6 +209,11 @@ func TestSessionAuthenticationLifecycle(t *testing.T) {
authorization: "Basic " + base64.StdEncoding.EncodeToString([]byte(alice.Username+":"+testPassword)),
wantStatus: http.StatusUnauthorized,
},
+ {
+ name: "non-ASCII Bearer is rejected without panic",
+ authorization: "Bearer é",
+ wantStatus: http.StatusUnauthorized,
+ },
{
name: "Bearer identity wins over X-User-Id",
authorization: "Bearer " + session.Token,