From a7d6487e2dd7a40810f06bc60a833f3dec4a7137 Mon Sep 17 00:00:00 2001 From: d1frend Date: Sun, 25 Oct 2026 21:46:21 +0300 Subject: [PATCH] =?UTF-8?q?=D0=BD=D0=B0=205?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 2 + Makefile | 13 + REPORT.md | 189 +- compose.yaml | 31 +- evidence/README.md | 9 + evidence/grade4/README.md | 14 +- evidence/grade4/latency.svg | 58 +- evidence/grade4/resources.svg | 54 +- evidence/grade4/results.json | 621 ++--- evidence/grade4/throughput-errors.svg | 28 +- evidence/grade4/timeline.svg | 26 +- evidence/grade5/01-saturation.svg | 63 + evidence/grade5/02-fixed-total.svg | 49 + evidence/grade5/03-resource-growth.svg | 49 + evidence/grade5/04-bottleneck.svg | 83 + evidence/grade5/05-improvement.svg | 37 + evidence/grade5/README.md | 69 + evidence/grade5/measurements.csv | 14 + evidence/grade5/resources.csv | 261 ++ evidence/grade5/results.json | 3354 ++++++++++++++++++++++++ evidence/grade5/timeseries.csv | 144 + evidence/lab3-summary.md | 53 +- evidence/readiness-routing.json | 11 + infrastructure/haproxy/Dockerfile | 7 + infrastructure/haproxy/haproxy.cfg | 66 + infrastructure/nginx/nginx.conf | 98 +- scripts/grade4_experiment.py | 6 +- scripts/grade5_experiment.py | 577 ++++ scripts/render_grade4.py | 4 +- scripts/render_grade5.py | 322 +++ scripts/restart.sh | 6 +- scripts/test_readiness_routing.py | 198 ++ student_tests/auth_test.go | 5 + 33 files changed, 5965 insertions(+), 556 deletions(-) create mode 100644 evidence/grade5/01-saturation.svg create mode 100644 evidence/grade5/02-fixed-total.svg create mode 100644 evidence/grade5/03-resource-growth.svg create mode 100644 evidence/grade5/04-bottleneck.svg create mode 100644 evidence/grade5/05-improvement.svg create mode 100644 evidence/grade5/README.md create mode 100644 evidence/grade5/measurements.csv create mode 100644 evidence/grade5/resources.csv create mode 100644 evidence/grade5/results.json create mode 100644 evidence/grade5/timeseries.csv create mode 100644 evidence/readiness-routing.json create mode 100644 infrastructure/haproxy/Dockerfile create mode 100644 infrastructure/haproxy/haproxy.cfg create mode 100755 scripts/grade5_experiment.py create mode 100755 scripts/render_grade5.py create mode 100755 scripts/test_readiness_routing.py diff --git a/.env.example b/.env.example index 459bbb62..677c7bb9 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,8 @@ APP_ENV=lab POSTGRES_USER=messenger POSTGRES_DB=messenger +POSTGRES_CPU_LIMIT=1.00 +POSTGRES_MEMORY_LIMIT=512M SESSION_TTL_SECONDS=3600 SESSION_STORE=postgres diff --git a/Makefile b/Makefile index 7e286836..1dd07d60 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,8 @@ BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= GRADE4_OUTPUT ?= evidence/grade4/results.json +GRADE5_OUTPUT ?= evidence/grade5/results.json +READINESS_OUTPUT ?= evidence/readiness-routing.json TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)" @@ -27,8 +29,10 @@ COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)" load \ test-persistence \ test-failover \ + test-readiness-routing \ rolling-update \ grade4-experiment \ + grade5-experiment \ compose-config \ compose-build \ compose-up \ @@ -51,8 +55,10 @@ help: @echo "load: run synthetic writes against the LB entrypoint" @echo "test-persistence: run persistence test with ACTION_SCRIPT" @echo "test-failover: run replica failover test with ACTION_SCRIPT" + @echo "test-readiness-routing: isolate PostgreSQL from one API and verify active LB health checks" @echo "rolling-update: replace API replicas one by one with surge capacity" @echo "grade4-experiment: run failovers, idempotency and rolling update with evidence" + @echo "grade5-experiment: measure fixed-budget scaling, saturation and one improvement" @echo "compose-config: validate resolved Compose configuration" @echo "compose-build: build application images" @echo "compose-up: start the stack" @@ -114,6 +120,9 @@ test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +test-readiness-routing: + $(PYTHON) scripts/test_readiness_routing.py --env-file "$(ENV_FILE)" --output "$(READINESS_OUTPUT)" + rolling-update: ENV_FILE="$(ENV_FILE)" ./scripts/rolling-update.sh @@ -121,6 +130,10 @@ grade4-experiment: $(PYTHON) scripts/grade4_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE4_OUTPUT)" $(PYTHON) scripts/render_grade4.py "$(GRADE4_OUTPUT)" +grade5-experiment: + $(PYTHON) scripts/grade5_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE5_OUTPUT)" + $(PYTHON) scripts/render_grade5.py "$(GRADE5_OUTPUT)" + compose-config: @test -f "$(ENV_FILE)" || \ (echo "$(ENV_FILE) is missing; run: make init-env"; exit 2) diff --git a/REPORT.md b/REPORT.md index 59034639..7ca111bd 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,11 +1,11 @@ # Отчёт — лабораторная 3 - Автор, группа: Чурилин, БВТ2301 -- Текущий завершённый уровень: оценка «4» -- Итоговая цель: оценка «5»; исследовательский трек выполняется следующим этапом +- Текущий завершённый уровень: оценка «5», трек 1 «Цена масштабирования» +- Итоговая цель: выполнена; уровень «4» и законченный исследовательский трек - Commit/PR: заполняется автором после локального коммита и публикации - Использованные источники и библиотеки: преподавательские README/OpenAPI/tests, - документация Go, pgx, PostgreSQL, Docker Compose, Nginx, React, TypeScript, + документация Go, pgx, PostgreSQL, Docker Compose, HAProxy, Nginx, React, TypeScript, Vite, TanStack Query, React Router, Lucide и Vitest. Для разбора архитектуры, рефакторинга, реализации и проверок использовался OpenAI Codex; решения проверены тестами и описанными ниже воспроизводимыми сценариями. @@ -35,13 +35,16 @@ cp .env.example .env docker compose --env-file .env config --quiet docker compose --env-file .env up -d --build --wait make test +make test-readiness-routing make test-failover ACTION_SCRIPT=./scripts/stop-one.sh docker compose --env-file .env up -d --wait api make test-persistence ACTION_SCRIPT=./scripts/restart.sh make grade4-experiment +make grade5-experiment ``` -По этому же адресу Nginx отдаёт production-сборку React. Переход на +По этому же адресу HAProxy направляет обычные browser-запросы во внутренний +Nginx, который отдаёт production-сборку React. Переход на `http://localhost:8080` открывает Messenger, а `/api/v1/*` и health endpoints остаются доступны автоматическим проверкам. Отдельный Node.js на машине для обычного запуска не нужен: multi-stage image сам выполняет `npm ci`, Vitest, @@ -84,15 +87,22 @@ PostgreSQL остаётся источником истины для польз PostgreSQL. Compose запускает один сервис `api` с `API_REPLICAS=2`. Реплики не имеют host -ports и разделяют PostgreSQL, включая таблицу сессий. Nginx — единственная -публичная точка входа; он использует `least_conn` и Docker DNS resolver с -повторным разрешением имени `api` раз в секунду. Sticky sessions не применяются. +ports и разделяют PostgreSQL, включая таблицу сессий. HAProxy — единственная +публичная точка входа; он использует `leastconn`, Docker DNS resolver и активные +HTTP-проверки `/health/ready` раз в секунду. Реплика исключается после двух +неуспешных readiness-проверок и возвращается после двух успешных. Поэтому живой, +но потерявший PostgreSQL и ставший неготовым процесс перестаёт получать трафик; +Docker healthcheck не является единственным механизмом обнаружения. Sticky +sessions не применяются. Nginx остаётся отдельным внутренним контейнером и +обслуживает только статические файлы React. + Произвольные POST балансировщик автоматически не повторяет. Исключение сделано только для маршрута отправки сообщения: обязательный `Idempotency-Key` позволяет -Nginx безопасно повторить запрос на другой реплике при 502/503/504 во время -drain. Клиент также сохраняет ключ для повтора после неопределённого результата. -Создание пользователя и чата контрактом не идемпотентно, поэтому автоматический -retry для этих маршрутов запрещён. +HAProxy безопасно повторить запрос на другой реплике при connection failure, +empty response, timeout или 502/503/504 во время обнаружения отказа и drain. +Для этого маршрута тело запроса буферизуется отдельным backend; регистрации +пользователя и создание чата контрактом не идемпотентны, поэтому retry для них +запрещён. Клиент также сохраняет ключ для повтора после неопределённого результата. Миграции вынесены в одноразовый сервис `migrate`, завершение которого является предусловием запуска API. Поэтому две реплики не конкурируют за изменение схемы. @@ -154,7 +164,7 @@ React Router — за маршруты login/register, выбранного ча `SameSite=Strict`. Флаг `Secure` управляется `SESSION_COOKIE_SECURE`: `false` нужен только локальному HTTP, на HTTPS должен быть `true`. -Контрактные `/api/v1/*` сознательно остались Bearer-only. Nginx предоставляет +Контрактные `/api/v1/*` сознательно остались Bearer-only. HAProxy предоставляет отдельный браузерный адаптер `/web-api/v1/*`: берёт token из HttpOnly-cookie, формирует `Authorization: Bearer ...`, удаляет исходный `Cookie` и проксирует на обычный `/api/v1/*`. Поэтому преподавательские тесты и внешние API-клиенты не @@ -173,18 +183,23 @@ cookie. | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | | Воспроизводимая конфигурация и качество Go | `make check && make go-check && make validate PYTHON=../mtusi2026lab2/.venv/bin/python` | форматирование, tests, race detector, vet, Compose config и OpenAPI validation завершились без ошибок | `evidence/lab3-summary.md` | -| Две API-реплики за одним LB | `docker compose --env-file .env up -d --build --wait`; 20 health-запросов через `:8080` | наружу опубликован только Nginx; два разных `X-Instance-Id`, по 10 ответов от каждого | `compose.yaml`, `infrastructure/nginx/nginx.conf`, `evidence/lab3-summary.md` | +| Две API-реплики за одним LB | `docker compose --env-file .env up -d --build --wait`; 20 health-запросов через `:8080` | наружу опубликован только HAProxy; два разных `X-Instance-Id`, по 10 ответов от каждого | `compose.yaml`, `infrastructure/haproxy/haproxy.cfg`, `evidence/lab3-summary.md` | | Публичный smoke lab3 | `make test` | `tests/smoke.py` завершился `PASS: lab3 public API scenario` | `evidence/lab3-summary.md` | | Общая сессия без sticky routing | smoke, затем failover и persistence через тот же bearer token | обе реплики обслуживали запросы; сессия работает после остановки API и рестартов | `evidence/lab3-summary.md` | | Идемпотентность сообщения | `go test ./student_tests -run Idempotency`; 32 конкурентных POST с одним ключом через LB | один Message и ровно одна строка в PostgreSQL; другой payload даёт 409 | `student_tests/idempotency_test.go`, `evidence/lab3-summary.md` | | Failover каждой API-реплики | два последовательных запуска `make test-failover ACTION_SCRIPT=./scripts/stop-one.sh` с восстановлением между ними | реально остановлены `api-2`, затем `api-1`; получено 142/143 успешных записи, 0 ошибок, восстановление 0.963/0.750 с | `scripts/stop-one.sh`, `evidence/lab3-summary.md` | | Persistence после рестартов | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS; прежняя сессия и данные доступны после рестарта PostgreSQL, Redis, API и LB | `scripts/restart.sh`, `evidence/lab3-summary.md` | | Миграции перед API | `docker compose --env-file .env ps -a` и логи `migrate` | миграции 1 и 2 применены; one-shot контейнер завершился с кодом 0; API healthy | `migrations/000002_message_idempotency.up.sql`, `evidence/lab3-summary.md` | -| React frontend и HttpOnly BFF | `docker compose build load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/Dockerfile`, `infrastructure/nginx/nginx.conf` | +| React frontend и HttpOnly BFF | `docker compose build frontend load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/`, `infrastructure/haproxy/` | | Согласованные health и shutdown | SIGTERM каждой реплики под нагрузкой | API сразу закрывает availability gate, 2 с дренирует запросы и укладывается в 10 с shutdown; `/health/live` остаётся доступен | `internal/transport/http/middleware/availability.go`, `evidence/grade4/` | -| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только Nginx `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` | -| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 3,464/3,476 с, одна краткая ошибка из 4614 запросов в первом прогоне и 0 из 5815 во втором | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` | -| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 20 350 успешных запросов, 1 краткий 502 (0,005%), все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` | +| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только HAProxy `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` | +| Readiness-aware routing | `make test-readiness-routing` | одной API-реплике заблокирован PostgreSQL: HAProxy исключил её за 2,861 с; 40 health-запросов получили 200, 20 регистраций — 201; после снятия fault реплика вернулась | `scripts/test_readiness_routing.py`, `evidence/readiness-routing.json` | +| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 2,893/2,938 с; во всех фазах 0 ошибочных запросов | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` | +| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 22 212 успешных запросов, 0 ошибок; все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` | +| Фиксированный бюджет 1/2/4 API | `make grade5-experiment`; 32 closed-loop клиента | при суммарных 1 CPU, 512 MiB и 16 соединениях получено 578,242 / 529,499 / 353,955 RPS; четыре процесса при том же бюджете дали на 38,79% меньше одной реплики | `evidence/grade5/02-fixed-total.svg`, `measurements.csv` | +| Рост ресурсов 1/2/4 API | тот же workload; по 0,5 CPU, 256 MiB и 8 соединений на реплику | 533,656 / 563,460 / 438,868 RPS; нелинейная серия показывает отсутствие пропорционального выигрыша при общей БД в 1 CPU | `evidence/grade5/03-resource-growth.svg`, `results.json` | +| Saturation point и bottleneck | concurrency 1/2/4/8/16/32, по 10 с после warm-up | первый сигнал насыщения — 8: относительно 4 клиентов RPS вырос только на 8,03%, p95 — в 1,92 раза; PostgreSQL в профиле четырёх реплик достиг 101,91% одного CPU | `evidence/grade5/01-saturation.svg`, `04-bottleneck.svg` | +| Подтверждённое улучшение | четыре API без изменений; PostgreSQL 1 → 2 CPU | RPS 438,868 → 587,574 (+33,88%), ошибок 0, p95 снизился на 14,12%; следующим ограничением стал HAProxy с лимитом 0,25 CPU | `evidence/grade5/05-improvement.svg`, `resources.csv` | ## Эксперименты и отказы @@ -192,8 +207,8 @@ cookie. Docker Engine 29.7.2, Compose 5.5.0, 8 логических CPU и 3813,6 MiB RAM, commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиченными изменениями текущего уровня. В Compose каждой API-реплике выделено 0,5 CPU и -256 MiB; Nginx — 0,25 CPU/128 MiB; PostgreSQL — 1 CPU/512 MiB; Redis — -0,25 CPU/128 MiB. +256 MiB; HAProxy — 0,25 CPU/128 MiB; frontend Nginx — 0,10 CPU/64 MiB; +PostgreSQL — 1 CPU/512 MiB; Redis — 0,25 CPU/128 MiB. Команда `make grade4-experiment` создаёт синтетических пользователей и чат, делает 12 warm-up записей, затем запускает 8 closed-loop клиентов без повторов @@ -202,44 +217,144 @@ commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиче следующей, после действия нагрузка продолжается ещё 8 с. Rolling update временно увеличивает сервис до трёх реплик, ждёт healthcheck и два периода Docker DNS, заменяет старые контейнеры по одному и возвращает масштаб к двум. -Минимальный подтверждённый размер набора после прогона — 34 808 сообщений. +Минимальный подтверждённый размер набора после прогона — 43 810 сообщений. | Фаза | Успешно / ошибок | RPS | Доля ошибок | p50 / p95 / p99, мс | Восстановление | | --- | ---: | ---: | ---: | ---: | ---: | -| baseline | 4017 / 0 | 319,463 | 0% | 16,799 / 55,515 / 93,803 | — | -| остановка реплики 1 | 4613 / 1 | 301,040 | 0,0217% | 17,145 / 69,558 / 89,752 | 3,464 с | -| остановка реплики 2 | 5815 / 0 | 374,449 | 0% | 14,211 / 49,542 / 81,040 | 3,476 с | -| rolling update | 20 350 / 1 | 376,959 | 0,0049% | 15,725 / 48,724 / 79,337 | непрерывная нагрузка | +| baseline | 5762 / 0 | 479,581 | 0% | 9,178 / 56,402 / 79,978 | — | +| остановка реплики 1 | 7853 / 0 | 537,548 | 0% | 8,812 / 50,050 / 66,373 | 2,893 с | +| остановка реплики 2 | 7970 / 0 | 533,154 | 0% | 8,726 / 48,789 / 64,333 | 2,938 с | +| rolling update | 22 212 / 0 | 428,860 | 0% | 10,560 / 58,713 / 83,982 | непрерывная нагрузка | -На baseline средняя выборка одного API-контейнера составила 28,453% CPU и -42,413 MiB RAM, максимум — 36,1% и 66,07 MiB. PostgreSQL оказался самым -нагруженным компонентом: в rolling-фазе среднее 98,172% CPU при лимите одного -ядра, максимум RAM 145,2 MiB. Это не capacity-тест и не доказательство +На baseline средняя выборка одного API-контейнера составила 27,144% CPU и +31,008 MiB RAM, максимум — 30,93% и 66,17 MiB. PostgreSQL оказался самым +нагруженным компонентом: среднее 89,392% CPU при лимите одного ядра, максимум +110,59%; HAProxy использовал в среднем 26,262% CPU. Это не capacity-тест и не доказательство saturation point, но важный ориентир для исследования уровня 5. -Nginx обновляет Docker DNS раз в 1 с, исключает peer после двух неудач за 2 с, -а connect timeout равен 1 с. При постоянной нагрузке ожидаемое максимальное -обнаружение — около 4 с; измеренные 3,464 и 3,476 с укладываются в требование -10 с. Одна 504 и одна 502 сохранены в отчёте честно: README прямо допускает -краткие ошибки обнаружения. Подтверждённых потерь и дублей нет. Параллельные -32 POST с одним ключом прошли через обе реплики и вернули один Message ID. +HAProxy обновляет Docker DNS раз в 1 с и исключает peer после двух неуспешных +активных `/health/ready`-проверок. Измеренные 2,893 и 2,938 с укладываются в +требование 10 с. Отдельный тест потери доступа одной реплики к PostgreSQL +подтвердил исключение неготового backend за 2,861 с без 5xx: 40 health-запросов +получили 200, а 20 регистраций — 201. Во всех четырёх фазах grade-4 эксперимента +ошибок нет. Параллельные 32 POST с одним ключом прошли через обе реплики и +вернули один Message ID. Исходные агрегаты, environment/limits, посекундные таблицы и четыре SVG-графика лежат в `evidence/grade4/`. Percentile рассчитаны только по успешным ответам; профиль closed-loop имеет coordinated omission. CPU для API агрегируется как средняя/максимальная выборка отдельного контейнера, а не сумма двух реплик. +## Исследование уровня 5: цена масштабирования + +Трек 1 выполнен командой `make grade5-experiment` 23.10.2026 на том же +однохостовом стенде Windows + WSL2: Docker Engine 29.7.2, Compose 5.5.0, +8 логических CPU и 3813,6 MiB RAM. Базовым commit до изменений уровня 5 был +`5300a1a`; измерения ожидаемо зафиксировали dirty working tree, поскольку сам +скрипт и параметризация лимитов ещё не были закоммичены. Все запросы проходили +через единственную публичную точку HAProxy, создавали сообщения с уникальными +`Idempotency-Key` и не повторялись нагрузочным клиентом автоматически. + +### Методика и воспроизводимость + +Каждая измеряемая фаза длилась 10 секунд. После каждой смены профиля выполнялся +двухсекундный warm-up, затем контейнеры и реально применённые `NanoCpus`, RAM и +`DB_MAX_CONNS` проверялись через `docker inspect`. Скрипт сохраняет посекундные +request buckets и каждую выборку `docker stats`; CPU/RAM API сначала суммируются +по всем репликам в одной временной точке и только затем усредняются. Это устраняет +ошибку, при которой средняя загрузка одного из четырёх контейнеров выдавалась бы +за стоимость всего API-сервиса. + +В серии с фиксированным бюджетом сохранялись суммарные 1 CPU, 512 MiB RAM и +16 соединений пула API: + +| Реплики | CPU на реплику | RAM на реплику | `DB_MAX_CONNS` на реплику | Суммарный пул | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 1,00 | 512 MiB | 16 | 16 | +| 2 | 0,50 | 256 MiB | 8 | 16 | +| 4 | 0,25 | 128 MiB | 4 | 16 | + +В серии роста каждая реплика получала 0,5 CPU, 256 MiB и 8 соединений, поэтому +суммарный API-бюджет менялся от 0,5 CPU/256 MiB/8 соединений до +2 CPU/1024 MiB/32 соединений. Во всех основных профилях PostgreSQL оставался на +1 CPU/512 MiB, HAProxy — 0,25 CPU/128 MiB, frontend Nginx — 0,10 CPU/64 MiB, +Redis — 0,25 CPU/128 MiB. Сравнение +1/2/4 выполнялось при одинаковом concurrency 32. До и после эксперимента +зафиксированы размер набора и базы: 225 626 → 299 524 сообщений и +175 406 103 → 228 793 367 байт. После любого исхода `finally` возвращает стенд к +двум API по 0,5 CPU/256 MiB, пулу 10 на реплику и PostgreSQL с 1 CPU; успешное +восстановление также записано в `results.json`. + +### Насыщение и цена реплик + +Для поиска saturation point две API-реплики с суммарным 1 CPU и пулом 16 +нагружались последовательностью concurrency 1/2/4/8/16/32: + +| Concurrency | RPS | p95, мс | PostgreSQL CPU avg/max | Ошибки | +| ---: | ---: | ---: | ---: | ---: | +| 1 | 230,369 | 5,065 | 23,900% / 29,41% | 0 | +| 2 | 320,190 | 7,196 | 54,987% / 62,60% | 0 | +| 4 | 435,822 | 29,304 | 63,093% / 81,80% | 0 | +| 8 | 470,828 | 56,230 | 65,945% / 92,98% | 0 | +| 16 | 472,892 | 73,628 | 65,080% / 88,93% | 0 | +| 32 | 564,012 | 94,785 | 83,825% / 107,62% | 0 | + +Заранее зафиксированное правило считает первым saturation signal шаг, где прирост +RPS меньше 10%, а p95 одновременно вырастает не менее чем в 1,5 раза. Это +произошло при переходе 4 → 8: throughput вырос только на 8,03%, а p95 — в +1,92 раза. При 16 клиентах прирост относительно 8 почти отсутствует (+0,44%), +а p95 продолжает расти; при 32 клиентах RPS увеличивается ценой ещё большей +очереди. Поэтому concurrency 8 — первый воспроизводимый saturation signal по +заранее заданному правилу, а не утверждение об абсолютном максимуме системы. + +При фиксированном суммарном бюджете 1/2/4 реплики дали соответственно +578,242 / 529,499 / 353,955 RPS. Четыре процесса при том же общем бюджете дали +на 38,79% меньше одной реплики: дробление ресурсов добавило scheduling и pool +overhead, но не увеличило общую ёмкость. В серии роста ресурсов получены +533,656 / 563,460 / 438,868 RPS. Разброс и нестрогая монотонность честно +сохранены: это один прогон на одном Docker-host, где контейнеры конкурируют за +общий scheduler и одну БД. Главный результат здесь не выбор «лучшего» количества +реплик, а отсутствие пропорционального роста: четырёхкратный API-бюджет дал на +17,76% меньше throughput одной реплики в этом конкретном прогоне. + +Доминирующее ограничение локализовано в PostgreSQL. В базовом профиле четырёх +реплик API суммарно использовал в среднем 35,690% CPU из доступных 200%, HAProxy — +15,595% из 25%, тогда как PostgreSQL использовал 84,847% в среднем и достигал +101,91% лимита одного ядра. Само по себе это корреляция, поэтому выполнено +контрольное изменение только одного параметра. + +### Подтверждённое улучшение + +Для тех же четырёх API по 0,5 CPU/256 MiB, того же пула и concurrency 32 лимит +PostgreSQL увеличен с 1 до 2 CPU. RPS вырос с 438,868 до 587,574 — на 33,88%; +ошибок в обеих фазах не было, p95 снизился с 109,446 до 93,991 мс (-14,12%). +Таким образом, снятие именно ограничения CPU БД существенно увеличило throughput +и улучшило tail latency, что подтверждает исходную гипотезу. + +После улучшения PostgreSQL использовал в среднем 95,573% из доступных 200%, а +HAProxy — 22,350% при лимите 0,25 CPU и достигал 26,73%. То есть следующее +ограничение ожидаемо сместилось к публичному +балансировщику. Увеличивать API дальше без пересмотра лимита или топологии LB уже +нерационально — это и есть практическая «цена масштабирования». + +Полный JSON с окружением, профилями, фактически применёнными лимитами и сырыми +выборками находится в `evidence/grade5/results.json`; агрегаты — в +`measurements.csv`, посекундные запросы — в `timeseries.csv`, ресурсы — в +`resources.csv`. Пять SVG показывают saturation curve, обе серии 1/2/4, +локализацию bottleneck и контрольное улучшение. + ## Ограничения и следующие шаги -- Топология однохостовая: Nginx и PostgreSQL остаются single points of failure. +- Топология однохостовая: HAProxy и PostgreSQL остаются single points of failure. Репликация БД и несколько балансировщиков в требования уровня «4» не входят. - Redis содержит только кэш, поэтому его потеря допустима. Отдельная персистентность Redis для корректности API не требуется. - Истёкшие записи идемпотентности удаляются лениво при повторном обращении к той же области ключа. Индекс по `expires_at` оставлен для будущей периодической очистки без изменения схемы. -- Уровень «5» ещё не реализован: сравнение 1/2/4 реплик при фиксированных - ресурсах и поиск saturation point выполняются следующим отдельным этапом. +- Исследование уровня «5» выполняет по одному измеряемому прогону на профиль. + Для production capacity planning нужны несколько повторов, доверительные + интервалы и отдельный генератор нагрузки вне измеряемого Docker-host. - Rolling-сценарий заменяет бинарник без изменения схемы. Миграции выполняются однократно до API; для будущей несовместимой схемы потребуется expand/contract, поэтому одновременная работа разных schema-version сейчас не заявляется. diff --git a/compose.yaml b/compose.yaml index 86e5ae22..2e49ddda 100644 --- a/compose.yaml +++ b/compose.yaml @@ -2,13 +2,15 @@ services: load-balancer: build: context: . - dockerfile: infrastructure/nginx/Dockerfile - image: mtusi2026lab3-web + dockerfile: infrastructure/haproxy/Dockerfile + image: mtusi2026lab3-load-balancer ports: - "127.0.0.1:${APP_PORT:-8080}:8080" depends_on: api: condition: service_healthy + frontend: + condition: service_healthy healthcheck: test: [ @@ -27,6 +29,27 @@ services: cpus: "0.25" memory: 128M + frontend: + build: + context: . + dockerfile: infrastructure/nginx/Dockerfile + image: mtusi2026lab3-web + expose: + - "8080" + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/ >/dev/null || exit 1"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 3s + stop_grace_period: 10s + restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.10" + memory: 64M + postgres: image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea environment: @@ -51,8 +74,8 @@ services: deploy: resources: limits: - cpus: "1.00" - memory: 512M + cpus: "${POSTGRES_CPU_LIMIT:-1.00}" + memory: ${POSTGRES_MEMORY_LIMIT:-512M} redis: image: redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0 diff --git a/evidence/README.md b/evidence/README.md index 8df70015..a03474bc 100644 --- a/evidence/README.md +++ b/evidence/README.md @@ -3,3 +3,12 @@ Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки. Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно. + +## Сохранённые эксперименты + +- `grade4/` — failover обеих API-реплик и rolling update под нагрузкой; +- `grade5/` — трек «Цена масштабирования»: saturation sweep, сравнение 1/2/4 + реплик с фиксированным и растущим бюджетом, сырые JSON/CSV, пять SVG-графиков + и подтверждённое улучшение PostgreSQL CPU; +- `readiness-routing.json` — результат изоляции PostgreSQL от одной живой + API-реплики: время исключения backend, статусы запросов и восстановление. diff --git a/evidence/grade4/README.md b/evidence/grade4/README.md index 2d9435e3..e42cd53b 100644 --- a/evidence/grade4/README.md +++ b/evidence/grade4/README.md @@ -1,12 +1,12 @@ # Lab 3 grade-4 experiment -Generated: 2026-10-23T12:31:12.749687+00:00. +Generated: 2026-10-23T15:23:05.031807+00:00. ## Method - concurrency: 8; - baseline duration: 12 s; -- load: closed-loop POST messages through Nginx; +- load: closed-loop POST messages through the public HAProxy gateway; - failover: both original API containers stopped and restored in turn; - rolling update: one surge replica, old replicas replaced one by one; - percentiles cover successful responses; failures are separate. @@ -15,10 +15,10 @@ Generated: 2026-10-23T12:31:12.749687+00:00. | Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s | | --- | ---: | ---: | ---: | ---: | ---: | ---: | -| baseline | 319.463 | 0 (0.000%) | 16.799 | 55.515 | 93.803 | n/a | -| failover-1 | 301.04 | 1 (0.022%) | 17.145 | 69.558 | 89.752 | 3.464 | -| failover-2 | 374.449 | 0 (0.000%) | 14.211 | 49.542 | 81.04 | 3.476 | -| rolling-update | 376.959 | 1 (0.005%) | 15.725 | 48.724 | 79.337 | n/a | +| baseline | 479.581 | 0 (0.000%) | 9.178 | 56.402 | 79.978 | n/a | +| failover-1 | 537.548 | 0 (0.000%) | 8.812 | 50.05 | 66.373 | 2.893 | +| failover-2 | 533.154 | 0 (0.000%) | 8.726 | 48.789 | 64.333 | 2.938 | +| rolling-update | 428.86 | 0 (0.000%) | 10.56 | 58.713 | 83.982 | n/a | ## Parallel idempotency @@ -36,4 +36,4 @@ Generated: 2026-10-23T12:31:12.749687+00:00. ## Scope -Single-host process failure and rolling replacement only. PostgreSQL and Nginx remain single points of failure. +Single-host process failure and rolling replacement only. PostgreSQL and HAProxy remain single points of failure. diff --git a/evidence/grade4/latency.svg b/evidence/grade4/latency.svg index 7c20a4ed..4120856b 100644 --- a/evidence/grade4/latency.svg +++ b/evidence/grade4/latency.svg @@ -16,53 +16,53 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; } 0 -19 +17 -38 +34 -56 +50 -75 +67 -94 - -17 +84 + +9 p50 - -56 + +56 p95 - -94 + +80 p99 baseline - -17 + +9 p50 - -70 + +50 p95 - -90 + +66 p99 failover-1 - -14 + +9 p50 - -50 + +49 p95 - -81 + +64 p99 failover-2 - -16 + +11 p50 - -49 + +59 p95 - -79 + +84 p99 rolling-update Successful latency, ms diff --git a/evidence/grade4/resources.svg b/evidence/grade4/resources.svg index 2edf10bf..f5bd373d 100644 --- a/evidence/grade4/resources.svg +++ b/evidence/grade4/resources.svg @@ -14,29 +14,35 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; } Grade 4: peak container resources Peak CPU, % - -52.6 -api - -23.4 -load-balancer - -105.2 -postgres - -5.9 -redis + +49.5 +api + +8.1 +frontend + +29.4 +load-balancer + +110.6 +postgres + +6.6 +redis Peak memory, MiB - -66.1 -api - -13.5 -load-balancer - -145.2 -postgres - -12.3 -redis + +66.2 +api + +12.2 +frontend + +19.6 +load-balancer + +185.9 +postgres + +16.6 +redis diff --git a/evidence/grade4/results.json b/evidence/grade4/results.json index 739dc0d3..5cd27d9d 100644 --- a/evidence/grade4/results.json +++ b/evidence/grade4/results.json @@ -1,8 +1,8 @@ { "schema_version": 1, "environment": { - "utc_started_at": "2026-10-23T12:31:12.749687+00:00", - "git_commit": "34618bd4237bbb8bf6a3a303056fc95d799577f9", + "utc_started_at": "2026-10-23T15:23:05.031807+00:00", + "git_commit": "5300a1a1e0e5497f2d30097bb1542b2743a8aad5", "working_tree_dirty": true, "logical_cpu_count": 8, "host_memory_mib": 3813.6, @@ -19,6 +19,11 @@ "nano_cpus": 500000000, "memory_limit_bytes": 268435456 }, + { + "service": "frontend", + "nano_cpus": 100000000, + "memory_limit_bytes": 67108864 + }, { "service": "load-balancer", "nano_cpus": 250000000, @@ -43,611 +48,611 @@ "action_after_seconds": 3, "post_action_seconds": 8, "warmup_requests": 12, - "workload": "closed-loop POST messages via Nginx; no automatic retries" + "workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries" }, "phases": [ { "name": "baseline", - "elapsed_seconds": 12.574, - "successful_requests": 4017, + "elapsed_seconds": 12.015, + "successful_requests": 5762, "failed_requests": 0, "error_ratio": 0.0, "errors": {}, - "successful_rps": 319.463, + "successful_rps": 479.581, "successful_latency_ms": { - "p50": 16.799, - "p95": 55.515, - "p99": 93.803 + "p50": 9.178, + "p95": 56.402, + "p99": 79.978 }, "instances": [ - "0b933a46fccb", - "55aaffcf508a" + "a85567c48d67", + "df8a69558af0" ], "recovery_seconds": null, "action": null, "per_second": { "0": { - "successful": 118, + "successful": 434, "failed": 0 }, "1": { - "successful": 308, + "successful": 280, "failed": 0 }, "2": { - "successful": 386, + "successful": 372, "failed": 0 }, "3": { - "successful": 378, + "successful": 436, "failed": 0 }, "4": { - "successful": 344, + "successful": 541, "failed": 0 }, "5": { - "successful": 322, + "successful": 474, "failed": 0 }, "6": { - "successful": 316, + "successful": 526, "failed": 0 }, "7": { - "successful": 384, + "successful": 539, "failed": 0 }, "8": { - "successful": 380, + "successful": 554, "failed": 0 }, "9": { - "successful": 402, + "successful": 559, "failed": 0 }, "10": { - "successful": 377, + "successful": 448, "failed": 0 }, "11": { - "successful": 288, + "successful": 584, "failed": 0 }, "12": { - "successful": 14, + "successful": 15, "failed": 0 } }, "resources": { "api": { - "cpu_percent_avg": 28.453, - "cpu_percent_max": 36.1, - "memory_mib_avg": 42.413, - "memory_mib_max": 66.07 + "cpu_percent_avg": 27.144, + "cpu_percent_max": 30.93, + "memory_mib_avg": 31.008, + "memory_mib_max": 66.17 + }, + "frontend": { + "cpu_percent_avg": 1.812, + "cpu_percent_max": 3.76, + "memory_mib_avg": 7.212, + "memory_mib_max": 10.15 }, "load-balancer": { - "cpu_percent_avg": 17.448, - "cpu_percent_max": 20.8, - "memory_mib_avg": 9.214, - "memory_mib_max": 9.559 + "cpu_percent_avg": 26.262, + "cpu_percent_max": 27.37, + "memory_mib_avg": 15.645, + "memory_mib_max": 15.97 }, "postgres": { - "cpu_percent_avg": 85.888, - "cpu_percent_max": 105.25, - "memory_mib_avg": 97.975, - "memory_mib_max": 100.8 + "cpu_percent_avg": 89.392, + "cpu_percent_max": 110.59, + "memory_mib_avg": 130.125, + "memory_mib_max": 132.2 }, "redis": { - "cpu_percent_avg": 2.552, - "cpu_percent_max": 5.79, - "memory_mib_avg": 8.796, - "memory_mib_max": 11.74 + "cpu_percent_avg": 2.488, + "cpu_percent_max": 6.58, + "memory_mib_avg": 15.072, + "memory_mib_max": 15.25 } } }, { "name": "failover-1", - "elapsed_seconds": 15.324, - "successful_requests": 4613, - "failed_requests": 1, - "error_ratio": 0.000217, - "errors": { - "POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 504": 1 - }, - "successful_rps": 301.04, + "elapsed_seconds": 14.609, + "successful_requests": 7853, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 537.548, "successful_latency_ms": { - "p50": 17.145, - "p95": 69.558, - "p99": 89.752 + "p50": 8.812, + "p95": 50.05, + "p99": 66.373 }, "instances": [ - "0b933a46fccb", - "55aaffcf508a" + "a85567c48d67", + "df8a69558af0" ], - "recovery_seconds": 3.464, + "recovery_seconds": 2.893, "action": { - "started_at": 3.006, - "ended_at": 6.463, - "duration_seconds": 3.457, + "started_at": 3.003, + "ended_at": 5.888, + "duration_seconds": 2.885, "exit_code": 0, - "summary": "stopped API instance 55aaffcf508a" + "summary": "stopped API instance df8a69558af0" }, "per_second": { "0": { - "successful": 404, + "successful": 551, "failed": 0 }, "1": { - "successful": 426, + "successful": 605, "failed": 0 }, "2": { - "successful": 358, + "successful": 558, "failed": 0 }, "3": { - "successful": 399, + "successful": 443, "failed": 0 }, "4": { - "successful": 392, + "successful": 575, "failed": 0 }, "5": { - "successful": 322, + "successful": 538, "failed": 0 }, "6": { - "successful": 299, + "successful": 593, "failed": 0 }, "7": { - "successful": 329, - "failed": 1 + "successful": 623, + "failed": 0 }, "8": { - "successful": 313, + "successful": 507, "failed": 0 }, "9": { - "successful": 229, + "successful": 626, "failed": 0 }, "10": { - "successful": 155, + "successful": 608, "failed": 0 }, "11": { - "successful": 187, + "successful": 601, "failed": 0 }, "12": { - "successful": 299, + "successful": 556, "failed": 0 }, "13": { - "successful": 317, - "failed": 0 - }, - "14": { - "successful": 184, + "successful": 469, "failed": 0 } }, "resources": { "api": { - "cpu_percent_avg": 27.62, - "cpu_percent_max": 52.59, - "memory_mib_avg": 11.58, - "memory_mib_max": 12.66 + "cpu_percent_avg": 35.916, + "cpu_percent_max": 49.48, + "memory_mib_avg": 11.892, + "memory_mib_max": 13.27 + }, + "frontend": { + "cpu_percent_avg": 0.762, + "cpu_percent_max": 3.75, + "memory_mib_avg": 6.256, + "memory_mib_max": 6.434 }, "load-balancer": { - "cpu_percent_avg": 18.536, - "cpu_percent_max": 21.25, - "memory_mib_avg": 9.351, - "memory_mib_max": 9.523 + "cpu_percent_avg": 22.21, + "cpu_percent_max": 26.89, + "memory_mib_avg": 16.312, + "memory_mib_max": 16.93 }, "postgres": { - "cpu_percent_avg": 81.822, - "cpu_percent_max": 104.67, - "memory_mib_avg": 100.512, - "memory_mib_max": 109.1 + "cpu_percent_avg": 73.128, + "cpu_percent_max": 89.46, + "memory_mib_avg": 135.64, + "memory_mib_max": 141.9 }, "redis": { - "cpu_percent_avg": 1.486, - "cpu_percent_max": 5.88, - "memory_mib_avg": 8.04, - "memory_mib_max": 8.25 + "cpu_percent_avg": 0.782, + "cpu_percent_max": 3.27, + "memory_mib_avg": 15.258, + "memory_mib_max": 15.73 } } }, { "name": "failover-2", - "elapsed_seconds": 15.529, - "successful_requests": 5815, + "elapsed_seconds": 14.949, + "successful_requests": 7970, "failed_requests": 0, "error_ratio": 0.0, "errors": {}, - "successful_rps": 374.449, + "successful_rps": 533.154, "successful_latency_ms": { - "p50": 14.211, - "p95": 49.542, - "p99": 81.04 + "p50": 8.726, + "p95": 48.789, + "p99": 64.333 }, "instances": [ - "0b933a46fccb", - "55aaffcf508a" + "a85567c48d67", + "df8a69558af0" ], - "recovery_seconds": 3.476, + "recovery_seconds": 2.938, "action": { - "started_at": 3.013, - "ended_at": 6.485, - "duration_seconds": 3.472, + "started_at": 3.006, + "ended_at": 5.933, + "duration_seconds": 2.927, "exit_code": 0, - "summary": "stopped API instance 0b933a46fccb" + "summary": "stopped API instance a85567c48d67" }, "per_second": { "0": { - "successful": 396, + "successful": 584, "failed": 0 }, "1": { - "successful": 439, + "successful": 613, "failed": 0 }, "2": { - "successful": 412, + "successful": 570, "failed": 0 }, "3": { - "successful": 355, + "successful": 465, "failed": 0 }, "4": { - "successful": 378, + "successful": 564, "failed": 0 }, "5": { - "successful": 440, + "successful": 605, "failed": 0 }, "6": { - "successful": 362, + "successful": 610, "failed": 0 }, "7": { - "successful": 419, + "successful": 579, "failed": 0 }, "8": { - "successful": 385, + "successful": 497, "failed": 0 }, "9": { - "successful": 372, + "successful": 575, "failed": 0 }, "10": { - "successful": 420, + "successful": 569, "failed": 0 }, "11": { - "successful": 423, + "successful": 600, "failed": 0 }, "12": { - "successful": 432, + "successful": 586, "failed": 0 }, "13": { - "successful": 375, - "failed": 0 - }, - "14": { - "successful": 207, + "successful": 553, "failed": 0 } }, "resources": { "api": { - "cpu_percent_avg": 31.411, - "cpu_percent_max": 51.63, - "memory_mib_avg": 10.373, - "memory_mib_max": 11.6 + "cpu_percent_avg": 28.073, + "cpu_percent_max": 48.59, + "memory_mib_avg": 10.391, + "memory_mib_max": 13.47 + }, + "frontend": { + "cpu_percent_avg": 0.696, + "cpu_percent_max": 3.44, + "memory_mib_avg": 6.25, + "memory_mib_max": 6.398 }, "load-balancer": { - "cpu_percent_avg": 15.778, - "cpu_percent_max": 21.79, - "memory_mib_avg": 9.62, - "memory_mib_max": 9.781 + "cpu_percent_avg": 25.966, + "cpu_percent_max": 26.65, + "memory_mib_avg": 16.918, + "memory_mib_max": 17.25 }, "postgres": { - "cpu_percent_avg": 76.87, - "cpu_percent_max": 104.51, - "memory_mib_avg": 106.016, - "memory_mib_max": 120.0 + "cpu_percent_avg": 72.558, + "cpu_percent_max": 93.6, + "memory_mib_avg": 146.22, + "memory_mib_max": 160.2 }, "redis": { - "cpu_percent_avg": 0.786, - "cpu_percent_max": 2.83, - "memory_mib_avg": 8.844, - "memory_mib_max": 12.28 + "cpu_percent_avg": 0.168, + "cpu_percent_max": 0.2, + "memory_mib_avg": 15.646, + "memory_mib_max": 16.64 } } }, { "name": "rolling-update", - "elapsed_seconds": 53.985, - "successful_requests": 20350, - "failed_requests": 1, - "error_ratio": 4.9e-05, - "errors": { - "POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 502": 1 - }, - "successful_rps": 376.959, + "elapsed_seconds": 51.793, + "successful_requests": 22212, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 428.86, "successful_latency_ms": { - "p50": 15.725, - "p95": 48.724, - "p99": 79.337 + "p50": 10.56, + "p95": 58.713, + "p99": 83.982 }, "instances": [ - "0b933a46fccb", - "3adbcad9e34b", - "55aaffcf508a", - "ceca3f6f1faf", - "f0a3bedda85a" + "872dbac02cd3", + "a85567c48d67", + "dac03e904506", + "df00f21e6f41", + "df8a69558af0" ], "recovery_seconds": null, "action": { - "started_at": 3.016, - "ended_at": 45.961, - "duration_seconds": 42.945, + "started_at": 3.005, + "ended_at": 42.955, + "duration_seconds": 39.95, "exit_code": 0, "summary": "rolling update completed with 2 healthy API replicas" }, "per_second": { "0": { - "successful": 340, + "successful": 509, "failed": 0 }, "1": { - "successful": 369, + "successful": 577, "failed": 0 }, "2": { - "successful": 426, + "successful": 660, "failed": 0 }, "3": { - "successful": 382, + "successful": 438, "failed": 0 }, "4": { - "successful": 371, + "successful": 577, "failed": 0 }, "5": { - "successful": 376, + "successful": 392, "failed": 0 }, "6": { - "successful": 328, + "successful": 326, "failed": 0 }, "7": { - "successful": 350, + "successful": 232, "failed": 0 }, "8": { - "successful": 434, + "successful": 199, "failed": 0 }, "9": { - "successful": 411, + "successful": 170, "failed": 0 }, "10": { - "successful": 399, + "successful": 263, "failed": 0 }, "11": { - "successful": 417, + "successful": 323, "failed": 0 }, "12": { - "successful": 374, + "successful": 308, "failed": 0 }, "13": { - "successful": 427, - "failed": 0 - }, - "14": { - "successful": 414, - "failed": 0 - }, - "15": { - "successful": 418, - "failed": 0 - }, - "16": { - "successful": 408, - "failed": 0 - }, - "17": { - "successful": 352, - "failed": 0 - }, - "18": { - "successful": 391, - "failed": 0 - }, - "19": { - "successful": 388, - "failed": 0 - }, - "20": { - "successful": 413, - "failed": 0 - }, - "21": { - "successful": 417, - "failed": 0 - }, - "22": { - "successful": 388, - "failed": 0 - }, - "23": { - "successful": 396, - "failed": 0 - }, - "24": { - "successful": 400, - "failed": 0 - }, - "25": { - "successful": 339, - "failed": 0 - }, - "26": { - "successful": 225, - "failed": 0 - }, - "27": { - "successful": 376, - "failed": 0 - }, - "28": { - "successful": 385, - "failed": 0 - }, - "29": { - "successful": 396, - "failed": 0 - }, - "30": { - "successful": 403, - "failed": 0 - }, - "31": { - "successful": 262, - "failed": 0 - }, - "32": { - "successful": 428, - "failed": 0 - }, - "33": { "successful": 362, "failed": 0 }, - "34": { - "successful": 286, + "14": { + "successful": 443, "failed": 0 }, - "35": { - "successful": 337, + "15": { + "successful": 495, "failed": 0 }, - "36": { - "successful": 328, + "16": { + "successful": 429, "failed": 0 }, - "37": { - "successful": 424, + "17": { + "successful": 370, "failed": 0 }, - "38": { - "successful": 356, + "18": { + "successful": 451, "failed": 0 }, - "39": { - "successful": 386, + "19": { + "successful": 377, "failed": 0 }, - "40": { + "20": { "successful": 422, "failed": 0 }, + "21": { + "successful": 485, + "failed": 0 + }, + "22": { + "successful": 473, + "failed": 0 + }, + "23": { + "successful": 441, + "failed": 0 + }, + "24": { + "successful": 470, + "failed": 0 + }, + "25": { + "successful": 487, + "failed": 0 + }, + "26": { + "successful": 504, + "failed": 0 + }, + "27": { + "successful": 516, + "failed": 0 + }, + "28": { + "successful": 441, + "failed": 0 + }, + "29": { + "successful": 390, + "failed": 0 + }, + "30": { + "successful": 502, + "failed": 0 + }, + "31": { + "successful": 450, + "failed": 0 + }, + "32": { + "successful": 500, + "failed": 0 + }, + "33": { + "successful": 363, + "failed": 0 + }, + "34": { + "successful": 470, + "failed": 0 + }, + "35": { + "successful": 434, + "failed": 0 + }, + "36": { + "successful": 424, + "failed": 0 + }, + "37": { + "successful": 373, + "failed": 0 + }, + "38": { + "successful": 460, + "failed": 0 + }, + "39": { + "successful": 516, + "failed": 0 + }, + "40": { + "successful": 375, + "failed": 0 + }, "41": { - "successful": 285, + "successful": 492, "failed": 0 }, "42": { - "successful": 279, + "successful": 477, "failed": 0 }, "43": { - "successful": 222, - "failed": 1 + "successful": 503, + "failed": 0 }, "44": { - "successful": 316, + "successful": 506, "failed": 0 }, "45": { - "successful": 359, + "successful": 465, "failed": 0 }, "46": { - "successful": 405, + "successful": 501, "failed": 0 }, "47": { - "successful": 398, + "successful": 502, "failed": 0 }, "48": { - "successful": 395, + "successful": 446, "failed": 0 }, "49": { - "successful": 430, + "successful": 446, "failed": 0 }, "50": { - "successful": 453, - "failed": 0 - }, - "51": { - "successful": 409, - "failed": 0 - }, - "52": { - "successful": 445, - "failed": 0 - }, - "53": { - "successful": 450, + "successful": 477, "failed": 0 } }, "resources": { "api": { - "cpu_percent_avg": 25.598, - "cpu_percent_max": 37.65, - "memory_mib_avg": 8.956, - "memory_mib_max": 10.97 + "cpu_percent_avg": 20.954, + "cpu_percent_max": 31.69, + "memory_mib_avg": 8.855, + "memory_mib_max": 12.74 + }, + "frontend": { + "cpu_percent_avg": 1.326, + "cpu_percent_max": 8.06, + "memory_mib_avg": 7.43, + "memory_mib_max": 12.17 }, "load-balancer": { - "cpu_percent_avg": 19.935, - "cpu_percent_max": 23.36, - "memory_mib_avg": 9.836, - "memory_mib_max": 13.52 + "cpu_percent_avg": 24.581, + "cpu_percent_max": 29.42, + "memory_mib_avg": 17.398, + "memory_mib_max": 19.6 }, "postgres": { - "cpu_percent_avg": 98.172, - "cpu_percent_max": 105.11, - "memory_mib_avg": 127.975, - "memory_mib_max": 145.2 + "cpu_percent_avg": 77.881, + "cpu_percent_max": 102.6, + "memory_mib_avg": 170.081, + "memory_mib_max": 185.9 }, "redis": { - "cpu_percent_avg": 0.722, - "cpu_percent_max": 3.94, - "memory_mib_avg": 7.661, - "memory_mib_max": 11.84 + "cpu_percent_avg": 0.63, + "cpu_percent_max": 3.84, + "memory_mib_avg": 15.178, + "memory_mib_max": 15.91 } } } @@ -658,18 +663,18 @@ "failed_responses": 0, "unique_message_ids": 1, "observed_instances": [ - "0b933a46fccb", - "55aaffcf508a" + "a85567c48d67", + "df8a69558af0" ] }, "dataset": { "warmup_requests": 12, - "minimum_confirmed_messages": 34808 + "minimum_confirmed_messages": 43810 }, "limitations": [ "single Docker host", "closed-loop coordinated omission", "successful-response latency percentiles only", - "PostgreSQL and Nginx remain single points of failure" + "PostgreSQL and HAProxy remain single points of failure" ] } diff --git a/evidence/grade4/throughput-errors.svg b/evidence/grade4/throughput-errors.svg index 444752b2..b228e322 100644 --- a/evidence/grade4/throughput-errors.svg +++ b/evidence/grade4/throughput-errors.svg @@ -16,29 +16,29 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; } 0 -75 +108 -151 +215 -226 +323 -302 +430 -377 - -319.5 RPS +538 + +479.6 RPS baseline errors 0.00% - -301.0 RPS + +537.5 RPS failover-1 -errors 0.02% - -374.4 RPS +errors 0.00% + +533.2 RPS failover-2 errors 0.00% - -377.0 RPS + +428.9 RPS rolling-update errors 0.00% Successful requests per second diff --git a/evidence/grade4/timeline.svg b/evidence/grade4/timeline.svg index b0f1dfa0..884b13f2 100644 --- a/evidence/grade4/timeline.svg +++ b/evidence/grade4/timeline.svg @@ -16,23 +16,21 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; } 0 -91 +132 -181 +264 -272 +396 -362 +528 -453 - - - - -failover-1 action - -failover-2 action - -rolling-update action +660 + + +failover-1 action + +failover-2 action + +rolling-update action One-second buckets; circles mark failed requests diff --git a/evidence/grade5/01-saturation.svg b/evidence/grade5/01-saturation.svg new file mode 100644 index 00000000..7a3ee57a --- /dev/null +++ b/evidence/grade5/01-saturation.svg @@ -0,0 +1,63 @@ + + + +Saturation curve +Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU + +0 + +124 + +248 + +372 + +496 + +620 + + +1 +230 + + +2 +320 + + +4 +436 + + +8 +471 + + +16 +473 + + +32 +564 + + +Concurrency +white: RPS · gray: p95 normalized +Successful requests per second + diff --git a/evidence/grade5/02-fixed-total.svg b/evidence/grade5/02-fixed-total.svg new file mode 100644 index 00000000..ae3e0f00 --- /dev/null +++ b/evidence/grade5/02-fixed-total.svg @@ -0,0 +1,49 @@ + + + +Fixed total API budget +1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas + +0 + +133 + +266 + +399 + +532 + +665 + +578.2 RPS +1 replica(s) +p95 95.8 ms +errors 0.000% + +529.5 RPS +2 replica(s) +p95 100.1 ms +errors 0.000% + +354.0 RPS +4 replica(s) +p95 139.4 ms +errors 0.000% + diff --git a/evidence/grade5/03-resource-growth.svg b/evidence/grade5/03-resource-growth.svg new file mode 100644 index 00000000..0744eef9 --- /dev/null +++ b/evidence/grade5/03-resource-growth.svg @@ -0,0 +1,49 @@ + + + +Growing API budget +Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU + +0 + +130 + +259 + +389 + +518 + +648 + +533.7 RPS +1 replica(s) +p95 93.5 ms +errors 0.000% + +563.5 RPS +2 replica(s) +p95 96.4 ms +errors 0.000% + +438.9 RPS +4 replica(s) +p95 109.4 ms +errors 0.000% + diff --git a/evidence/grade5/04-bottleneck.svg b/evidence/grade5/04-bottleneck.svg new file mode 100644 index 00000000..fa13b0c1 --- /dev/null +++ b/evidence/grade5/04-bottleneck.svg @@ -0,0 +1,83 @@ + + + +Where scaling stops +Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis) + +0 + +20 + +39 + +59 + +79 + +98 + +38 +api + +77 +pos + +20 +loa + +1 +fro + +1 +red +1 replica(s) + +47 +api + +86 +pos + +21 +loa + +1 +fro + +1 +red +2 replica(s) + +36 +api + +85 +pos + +16 +loa + +1 +fro + +1 +red +4 replica(s) +Average aggregate CPU, % + diff --git a/evidence/grade5/05-improvement.svg b/evidence/grade5/05-improvement.svg new file mode 100644 index 00000000..2662e79b --- /dev/null +++ b/evidence/grade5/05-improvement.svg @@ -0,0 +1,37 @@ + + + +Confirmed improvement +Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2 +Successful RPS + +438.9 +PG 1 CPU + +587.6 +PG 2 CPU +p95 latency, ms + +109.4 +PG 1 CPU + +94.0 +PG 2 CPU +RPS change: +33.88% · p95 change: -14.12% · confirmed: true + diff --git a/evidence/grade5/README.md b/evidence/grade5/README.md new file mode 100644 index 00000000..e0532524 --- /dev/null +++ b/evidence/grade5/README.md @@ -0,0 +1,69 @@ +# Lab 3 grade-5 scaling experiment + +Generated: 2026-10-23T15:23:12.734339+00:00. + +## Hypothesis + +API replicas stop improving throughput when shared PostgreSQL CPU saturates; adding PostgreSQL CPU should improve the unchanged four-replica profile. + +## Reproduction + +- workload: closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries; +- phase duration: 10 s; +- comparison concurrency: 32; +- saturation sweep: [1, 2, 4, 8, 16, 32]; +- fixed budget: 1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections; +- growth series: 0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica; +- saturation rule: RPS gain below 10% and p95 growth at least 1.5x versus the previous step; +- host: 8 logical CPU, 3813.6 MiB RAM; +- Docker 29.7.2, Compose 5.5.0; +- command: `make grade5-experiment`; warmup runs before every changed profile. + +## Measurements + +| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| saturation-c1 | 2 | 1 | 230.369 | 0 (0.000%) | 3.839 | 5.065 | 6.282 | 23.9 | 26.85 | +| saturation-c2 | 2 | 2 | 320.19 | 0 (0.000%) | 5.192 | 7.196 | 15.11 | 54.987 | 48.3 | +| saturation-c4 | 2 | 4 | 435.822 | 0 (0.000%) | 6.171 | 29.304 | 47.665 | 63.093 | 54.453 | +| saturation-c8 | 2 | 8 | 470.828 | 0 (0.000%) | 9.159 | 56.23 | 70.71 | 65.945 | 48.505 | +| saturation-c16 | 2 | 16 | 472.892 | 0 (0.000%) | 19.862 | 73.628 | 98.773 | 65.08 | 38.028 | +| saturation-c32 | 2 | 32 | 564.012 | 0 (0.000%) | 51.089 | 94.785 | 109.138 | 83.825 | 51.965 | +| fixed-r1 | 1 | 32 | 578.242 | 0 (0.000%) | 48.579 | 95.757 | 115.549 | 79.058 | 36.575 | +| fixed-r2 | 2 | 32 | 529.499 | 0 (0.000%) | 53.316 | 100.145 | 129.953 | 81.065 | 44.7 | +| fixed-r4 | 4 | 32 | 353.955 | 0 (0.000%) | 69.354 | 139.408 | 291.312 | 74.677 | 46.25 | +| growth-r1 | 1 | 32 | 533.656 | 0 (0.000%) | 53.378 | 93.541 | 115.58 | 77.252 | 38.273 | +| growth-r2 | 2 | 32 | 563.46 | 0 (0.000%) | 50.516 | 96.428 | 129.46 | 85.597 | 46.653 | +| growth-r4 | 4 | 32 | 438.868 | 0 (0.000%) | 68.331 | 109.446 | 199.616 | 84.847 | 35.69 | +| improved-db-cpu-r4 | 4 | 32 | 587.574 | 0 (0.000%) | 49.466 | 93.991 | 112.002 | 95.573 | 55.2 | + +## Conclusion + +The first saturation signal appears at concurrency 8: RPS changes by +8.03% while p95 changes by 1.92x versus concurrency 4. + +Observed bottleneck classification: `postgres_cpu`. + +The PostgreSQL CPU change produced +33.88% RPS and -14.12% p95; confirmed improvement: `true`. + +After the improvement the remaining bottleneck classification is `load_balancer_cpu`. + +## Graphs + +![Saturation](01-saturation.svg) + +![Fixed budget](02-fixed-total.svg) + +![Growing budget](03-resource-growth.svg) + +![Bottleneck](04-bottleneck.svg) + +![Improvement](05-improvement.svg) + +## Raw data and limits + +Raw aggregates are in `measurements.csv`; one-second request buckets are in `timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and applied limits are in `results.json`. + +- single Docker host: replicas do not prove host-level availability +- closed-loop workload has coordinated omission +- successful-response latency percentiles are reported separately from failures +- one measured run per profile; host scheduling noise is possible diff --git a/evidence/grade5/measurements.csv b/evidence/grade5/measurements.csv new file mode 100644 index 00000000..d777c58b --- /dev/null +++ b/evidence/grade5/measurements.csv @@ -0,0 +1,14 @@ +phase,kind,replicas,concurrency,elapsed_seconds,successful_requests,failed_requests,error_ratio,successful_rps,p50_ms,p95_ms,p99_ms,api_cpu_avg,postgres_cpu_avg,load_balancer_cpu_avg,frontend_cpu_avg,redis_cpu_avg +saturation-c1,saturation,2,1,10.731,2472,0,0.0,230.369,3.839,5.065,6.282,26.85,23.9,11.662,0.688,0.8 +saturation-c2,saturation,2,2,10.915,3495,0,0.0,320.19,5.192,7.196,15.11,48.3,54.987,19.13,0.325,0.193 +saturation-c4,saturation,2,4,10.832,4721,0,0.0,435.822,6.171,29.304,47.665,54.453,63.093,20.843,0.95,0.81 +saturation-c8,saturation,2,8,10.821,5095,0,0.0,470.828,9.159,56.23,70.71,48.505,65.945,20.867,0.86,0.953 +saturation-c16,saturation,2,16,10.876,5143,0,0.0,472.892,19.862,73.628,98.773,38.028,65.08,20.98,1.113,0.172 +saturation-c32,saturation,2,32,10.808,6096,0,0.0,564.012,51.089,94.785,109.138,51.965,83.825,21.163,1.837,2.475 +fixed-r1,fixed,1,32,10.88,6291,0,0.0,578.242,48.579,95.757,115.549,36.575,79.058,20.732,0.005,1.232 +fixed-r2,fixed,2,32,10.905,5774,0,0.0,529.499,53.316,100.145,129.953,44.7,81.065,21.15,0.823,1.093 +fixed-r4,fixed,4,32,11.917,4218,0,0.0,353.955,69.354,139.408,291.312,46.25,74.677,21.18,0.637,2.715 +growth-r1,growth,1,32,10.951,5844,0,0.0,533.656,53.378,93.541,115.58,38.273,77.252,20.062,0.693,0.958 +growth-r2,growth,2,32,10.796,6083,0,0.0,563.46,50.516,96.428,129.46,46.653,85.597,21.192,0.765,0.865 +growth-r4,growth,4,32,10.682,4688,0,0.0,438.868,68.331,109.446,199.616,35.69,84.847,15.595,0.667,0.873 +improved-db-cpu-r4,improvement,4,32,10.601,6229,0,0.0,587.574,49.466,93.991,112.002,55.2,95.573,22.35,0.83,0.458 diff --git a/evidence/grade5/resources.csv b/evidence/grade5/resources.csv new file mode 100644 index 00000000..b51ffde1 --- /dev/null +++ b/evidence/grade5/resources.csv @@ -0,0 +1,261 @@ +phase,t,service,containers,cpu_percent,memory_mib +saturation-c1,0.0,api,2,26.09,17.976 +saturation-c1,0.0,frontend,1,1.61,7.578 +saturation-c1,0.0,load-balancer,1,13.0,17.18 +saturation-c1,0.0,postgres,1,28.57,169.0 +saturation-c1,0.0,redis,1,0.15,15.28 +saturation-c1,2.356,api,2,32.65,16.886 +saturation-c1,2.356,frontend,1,0.02,7.617 +saturation-c1,2.356,load-balancer,1,13.32,17.23 +saturation-c1,2.356,postgres,1,28.96,169.5 +saturation-c1,2.356,redis,1,2.75,15.4 +saturation-c1,5.485,api,2,27.72,17.586 +saturation-c1,5.485,frontend,1,0.0,7.383 +saturation-c1,5.485,load-balancer,1,13.69,17.41 +saturation-c1,5.485,postgres,1,29.41,170.1 +saturation-c1,5.485,redis,1,0.15,15.28 +saturation-c1,8.605,api,2,20.94,17.41 +saturation-c1,8.605,frontend,1,1.12,12.06 +saturation-c1,8.605,load-balancer,1,6.64,17.24 +saturation-c1,8.605,postgres,1,8.66,170.5 +saturation-c1,8.605,redis,1,0.15,15.53 +saturation-c2,0.001,api,2,51.0,17.761 +saturation-c2,0.001,frontend,1,0.03,7.617 +saturation-c2,0.001,load-balancer,1,24.78,17.71 +saturation-c2,0.001,postgres,1,57.44,171.2 +saturation-c2,0.001,redis,1,0.17,16.03 +saturation-c2,2.361,api,2,45.61,17.516 +saturation-c2,2.361,frontend,1,0.03,7.367 +saturation-c2,2.361,load-balancer,1,24.6,17.41 +saturation-c2,2.361,postgres,1,49.85,172.6 +saturation-c2,2.361,redis,1,0.18,15.28 +saturation-c2,5.543,api,2,51.19,16.258 +saturation-c2,5.543,frontend,1,0.02,7.367 +saturation-c2,5.543,load-balancer,1,24.27,17.91 +saturation-c2,5.543,postgres,1,62.6,173.0 +saturation-c2,5.543,redis,1,0.21,15.28 +saturation-c2,8.736,api,2,45.4,17.082 +saturation-c2,8.736,frontend,1,1.22,8.121 +saturation-c2,8.736,load-balancer,1,2.87,17.2 +saturation-c2,8.736,postgres,1,50.06,173.8 +saturation-c2,8.736,redis,1,0.21,15.78 +saturation-c4,0.001,api,2,65.41,17.297 +saturation-c4,0.001,frontend,1,0.03,7.367 +saturation-c4,0.001,load-balancer,1,25.95,17.71 +saturation-c4,0.001,postgres,1,81.8,174.4 +saturation-c4,0.001,redis,1,0.19,15.28 +saturation-c4,2.372,api,2,60.01,18.188 +saturation-c4,2.372,frontend,1,3.75,7.965 +saturation-c4,2.372,load-balancer,1,26.04,17.46 +saturation-c4,2.372,postgres,1,79.42,175.4 +saturation-c4,2.372,redis,1,0.16,15.53 +saturation-c4,5.501,api,2,56.53,17.433 +saturation-c4,5.501,frontend,1,0.02,7.367 +saturation-c4,5.501,load-balancer,1,25.9,21.22 +saturation-c4,5.501,postgres,1,73.71,175.7 +saturation-c4,5.501,redis,1,0.18,15.28 +saturation-c4,8.662,api,2,35.86,16.699 +saturation-c4,8.662,frontend,1,0.0,7.367 +saturation-c4,8.662,load-balancer,1,5.48,17.53 +saturation-c4,8.662,postgres,1,17.44,177.2 +saturation-c4,8.662,redis,1,2.71,16.1 +saturation-c8,0.006,api,2,48.34,17.403 +saturation-c8,0.006,frontend,1,0.0,7.367 +saturation-c8,0.006,load-balancer,1,25.42,20.12 +saturation-c8,0.006,postgres,1,77.51,185.1 +saturation-c8,0.006,redis,1,0.18,15.28 +saturation-c8,2.372,api,2,57.14,18.602 +saturation-c8,2.372,frontend,1,3.44,7.734 +saturation-c8,2.372,load-balancer,1,26.83,17.14 +saturation-c8,2.372,postgres,1,92.98,188.5 +saturation-c8,2.372,redis,1,3.34,15.28 +saturation-c8,5.527,api,2,48.26,18.192 +saturation-c8,5.527,frontend,1,0.0,7.363 +saturation-c8,5.527,load-balancer,1,25.47,17.28 +saturation-c8,5.527,postgres,1,76.05,189.4 +saturation-c8,5.527,redis,1,0.15,15.28 +saturation-c8,8.685,api,2,40.28,21.813 +saturation-c8,8.685,frontend,1,0.0,7.363 +saturation-c8,8.685,load-balancer,1,5.75,17.64 +saturation-c8,8.685,postgres,1,17.24,190.4 +saturation-c8,8.685,redis,1,0.14,15.28 +saturation-c16,0.015,api,2,44.39,18.796 +saturation-c16,0.015,frontend,1,0.0,7.363 +saturation-c16,0.015,load-balancer,1,26.92,17.44 +saturation-c16,0.015,postgres,1,82.81,205.9 +saturation-c16,0.015,redis,1,0.17,15.28 +saturation-c16,2.354,api,2,56.54,17.856 +saturation-c16,2.354,frontend,1,0.03,7.375 +saturation-c16,2.354,load-balancer,1,26.66,17.69 +saturation-c16,2.354,postgres,1,88.93,207.7 +saturation-c16,2.354,redis,1,0.19,15.75 +saturation-c16,5.525,api,2,41.39,17.73 +saturation-c16,5.525,frontend,1,4.4,7.379 +saturation-c16,5.525,load-balancer,1,25.76,17.7 +saturation-c16,5.525,postgres,1,72.49,208.6 +saturation-c16,5.525,redis,1,0.18,15.25 +saturation-c16,8.731,api,2,9.79,18.257 +saturation-c16,8.731,frontend,1,0.02,7.375 +saturation-c16,8.731,load-balancer,1,4.58,17.55 +saturation-c16,8.731,postgres,1,16.09,209.7 +saturation-c16,8.731,redis,1,0.15,15.33 +saturation-c32,0.039,api,2,55.84,18.75 +saturation-c32,0.039,frontend,1,6.63,8.125 +saturation-c32,0.039,load-balancer,1,25.97,18.0 +saturation-c32,0.039,postgres,1,104.2,210.6 +saturation-c32,0.039,redis,1,5.85,15.05 +saturation-c32,2.29,api,2,56.87,18.933 +saturation-c32,2.29,frontend,1,0.0,7.625 +saturation-c32,2.29,load-balancer,1,27.03,19.23 +saturation-c32,2.29,postgres,1,105.41,212.1 +saturation-c32,2.29,redis,1,0.19,15.55 +saturation-c32,5.446,api,2,64.13,19.003 +saturation-c32,5.446,frontend,1,0.7,7.375 +saturation-c32,5.446,load-balancer,1,26.33,18.27 +saturation-c32,5.446,postgres,1,107.62,213.1 +saturation-c32,5.446,redis,1,3.7,15.27 +saturation-c32,8.653,api,2,31.02,19.379 +saturation-c32,8.653,frontend,1,0.02,7.625 +saturation-c32,8.653,load-balancer,1,5.32,18.36 +saturation-c32,8.653,postgres,1,18.07,213.5 +saturation-c32,8.653,redis,1,0.16,15.26 +fixed-r1,0.034,api,1,45.68,10.61 +fixed-r1,0.034,frontend,1,0.02,7.359 +fixed-r1,0.034,load-balancer,1,25.97,18.08 +fixed-r1,0.034,postgres,1,106.86,214.7 +fixed-r1,0.034,redis,1,4.41,15.3 +fixed-r1,2.424,api,1,48.23,11.36 +fixed-r1,2.424,frontend,1,0.0,7.699 +fixed-r1,2.424,load-balancer,1,27.02,22.21 +fixed-r1,2.424,postgres,1,96.33,215.7 +fixed-r1,2.424,redis,1,0.14,15.44 +fixed-r1,5.588,api,1,48.84,12.07 +fixed-r1,5.588,frontend,1,0.0,7.359 +fixed-r1,5.588,load-balancer,1,26.38,18.43 +fixed-r1,5.588,postgres,1,104.71,217.4 +fixed-r1,5.588,redis,1,0.19,15.92 +fixed-r1,8.743,api,1,3.55,11.93 +fixed-r1,8.743,frontend,1,0.0,7.508 +fixed-r1,8.743,load-balancer,1,3.56,17.96 +fixed-r1,8.743,postgres,1,8.33,218.1 +fixed-r1,8.743,redis,1,0.19,16.07 +fixed-r2,0.041,api,2,52.24,18.086 +fixed-r2,0.041,frontend,1,0.0,7.359 +fixed-r2,0.041,load-balancer,1,26.74,18.2 +fixed-r2,0.041,postgres,1,100.69,219.4 +fixed-r2,0.041,redis,1,3.81,15.18 +fixed-r2,2.379,api,2,65.39,18.801 +fixed-r2,2.379,frontend,1,1.92,7.859 +fixed-r2,2.379,load-balancer,1,26.23,18.61 +fixed-r2,2.379,postgres,1,106.33,221.3 +fixed-r2,2.379,redis,1,0.16,15.16 +fixed-r2,5.564,api,2,54.12,18.379 +fixed-r2,5.564,frontend,1,1.37,10.54 +fixed-r2,5.564,load-balancer,1,25.33,18.35 +fixed-r2,5.564,postgres,1,103.06,222.7 +fixed-r2,5.564,redis,1,0.2,14.93 +fixed-r2,8.728,api,2,7.05,18.086 +fixed-r2,8.728,frontend,1,0.0,7.363 +fixed-r2,8.728,load-balancer,1,6.3,17.82 +fixed-r2,8.728,postgres,1,14.18,223.3 +fixed-r2,8.728,redis,1,0.2,15.43 +fixed-r4,0.095,api,4,51.98,27.492 +fixed-r4,0.095,frontend,1,0.0,7.363 +fixed-r4,0.095,load-balancer,1,27.15,19.06 +fixed-r4,0.095,postgres,1,90.1,223.1 +fixed-r4,0.095,redis,1,5.94,16.87 +fixed-r4,3.244,api,4,64.6,33.144 +fixed-r4,3.244,frontend,1,0.02,7.359 +fixed-r4,3.244,load-balancer,1,26.34,18.58 +fixed-r4,3.244,postgres,1,101.51,225.1 +fixed-r4,3.244,redis,1,0.2,16.96 +fixed-r4,6.45,api,4,65.35,34.281 +fixed-r4,6.45,frontend,1,0.02,7.359 +fixed-r4,6.45,load-balancer,1,26.27,18.56 +fixed-r4,6.45,postgres,1,106.68,225.7 +fixed-r4,6.45,redis,1,4.55,16.7 +fixed-r4,9.725,api,4,3.07,32.867 +fixed-r4,9.725,frontend,1,2.51,7.762 +fixed-r4,9.725,load-balancer,1,4.96,17.89 +fixed-r4,9.725,postgres,1,0.42,226.7 +fixed-r4,9.725,redis,1,0.17,16.95 +growth-r1,0.045,api,1,44.44,9.355 +growth-r1,0.045,frontend,1,0.0,7.359 +growth-r1,0.045,load-balancer,1,25.9,18.3 +growth-r1,0.045,postgres,1,93.36,207.6 +growth-r1,0.045,redis,1,3.22,18.55 +growth-r1,2.404,api,1,50.32,9.379 +growth-r1,2.404,frontend,1,0.0,7.363 +growth-r1,2.404,load-balancer,1,25.42,18.36 +growth-r1,2.404,postgres,1,96.7,209.4 +growth-r1,2.404,redis,1,0.22,18.11 +growth-r1,5.579,api,1,50.68,9.66 +growth-r1,5.579,frontend,1,0.03,7.613 +growth-r1,5.579,load-balancer,1,25.44,18.67 +growth-r1,5.579,postgres,1,105.06,210.9 +growth-r1,5.579,redis,1,0.22,17.84 +growth-r1,8.767,api,1,7.65,9.262 +growth-r1,8.767,frontend,1,2.74,8.02 +growth-r1,8.767,load-balancer,1,3.49,17.86 +growth-r1,8.767,postgres,1,13.89,211.8 +growth-r1,8.767,redis,1,0.17,17.93 +growth-r2,0.047,api,2,52.25,17.629 +growth-r2,0.047,frontend,1,0.02,7.363 +growth-r2,0.047,load-balancer,1,24.23,18.59 +growth-r2,0.047,postgres,1,100.9,233.3 +growth-r2,0.047,redis,1,0.14,17.16 +growth-r2,2.292,api,2,62.86,17.606 +growth-r2,2.292,frontend,1,0.0,7.355 +growth-r2,2.292,load-balancer,1,25.57,18.19 +growth-r2,2.292,postgres,1,105.07,235.3 +growth-r2,2.292,redis,1,0.16,17.16 +growth-r2,5.459,api,2,57.29,18.789 +growth-r2,5.459,frontend,1,0.04,7.363 +growth-r2,5.459,load-balancer,1,25.89,18.43 +growth-r2,5.459,postgres,1,104.58,235.1 +growth-r2,5.459,redis,1,0.17,17.73 +growth-r2,8.628,api,2,14.21,17.832 +growth-r2,8.628,frontend,1,3.0,8.062 +growth-r2,8.628,load-balancer,1,9.08,18.0 +growth-r2,8.628,postgres,1,31.84,234.3 +growth-r2,8.628,redis,1,2.99,22.08 +growth-r4,0.043,api,4,25.03,29.672 +growth-r4,0.043,frontend,1,0.02,7.316 +growth-r4,0.043,load-balancer,1,9.16,18.25 +growth-r4,0.043,postgres,1,99.03,269.2 +growth-r4,0.043,redis,1,0.15,16.5 +growth-r4,2.197,api,4,48.89,31.184 +growth-r4,2.197,frontend,1,2.63,7.305 +growth-r4,2.197,load-balancer,1,23.31,18.25 +growth-r4,2.197,postgres,1,100.3,284.5 +growth-r4,2.197,redis,1,2.97,20.3 +growth-r4,5.406,api,4,43.96,32.164 +growth-r4,5.406,frontend,1,0.0,7.305 +growth-r4,5.406,load-balancer,1,19.48,17.79 +growth-r4,5.406,postgres,1,101.91,290.1 +growth-r4,5.406,redis,1,0.19,16.68 +growth-r4,8.538,api,4,24.88,32.855 +growth-r4,8.538,frontend,1,0.02,7.551 +growth-r4,8.538,load-balancer,1,10.43,17.5 +growth-r4,8.538,postgres,1,38.15,279.9 +growth-r4,8.538,redis,1,0.18,16.91 +improved-db-cpu-r4,0.054,api,4,52.63,30.188 +improved-db-cpu-r4,0.054,frontend,1,0.03,7.301 +improved-db-cpu-r4,0.054,load-balancer,1,24.59,18.03 +improved-db-cpu-r4,0.054,postgres,1,121.18,130.1 +improved-db-cpu-r4,0.054,redis,1,1.35,16.63 +improved-db-cpu-r4,2.129,api,4,70.26,36.812 +improved-db-cpu-r4,2.129,frontend,1,0.0,7.301 +improved-db-cpu-r4,2.129,load-balancer,1,25.64,18.99 +improved-db-cpu-r4,2.129,postgres,1,106.99,141.4 +improved-db-cpu-r4,2.129,redis,1,0.16,16.88 +improved-db-cpu-r4,5.271,api,4,65.05,33.652 +improved-db-cpu-r4,5.271,frontend,1,3.29,7.43 +improved-db-cpu-r4,5.271,load-balancer,1,26.73,18.41 +improved-db-cpu-r4,5.271,postgres,1,102.2,145.9 +improved-db-cpu-r4,5.271,redis,1,0.14,16.63 +improved-db-cpu-r4,8.446,api,4,32.86,34.023 +improved-db-cpu-r4,8.446,frontend,1,0.0,7.301 +improved-db-cpu-r4,8.446,load-balancer,1,12.44,18.0 +improved-db-cpu-r4,8.446,postgres,1,51.92,147.6 +improved-db-cpu-r4,8.446,redis,1,0.18,16.88 diff --git a/evidence/grade5/results.json b/evidence/grade5/results.json new file mode 100644 index 00000000..7e89b5ef --- /dev/null +++ b/evidence/grade5/results.json @@ -0,0 +1,3354 @@ +{ + "schema_version": 1, + "environment": { + "utc_started_at": "2026-10-23T15:23:12.734339+00:00", + "git_commit": "5300a1a1e0e5497f2d30097bb1542b2743a8aad5", + "working_tree_dirty": true, + "logical_cpu_count": 8, + "host_memory_mib": 3813.6, + "kernel": "6.18.33.2-microsoft-standard-WSL2", + "python": "3.14.4", + "docker_engine": "29.7.2", + "docker_compose": "5.5.0", + "images": [ + "mtusi2026lab3-api", + "mtusi2026lab3-load-balancer", + "mtusi2026lab3-web", + "postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea", + "redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0", + "migrate/migrate:v4.20.1@sha256:76cc2074cb6642631f34a898ced71e6aeaa6b1a4d78c4daa743275a22e0c5be7" + ] + }, + "hypothesis": "API replicas stop improving throughput when shared PostgreSQL CPU saturates; adding PostgreSQL CPU should improve the unchanged four-replica profile.", + "methodology": { + "workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries", + "duration_seconds_per_phase": 10, + "warmup_seconds_per_profile": 2, + "comparison_concurrency": 32, + "saturation_concurrency": [ + 1, + 2, + 4, + 8, + 16, + 32 + ], + "fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections", + "growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica", + "improvement": "only PostgreSQL CPU changes from 1.00 to 2.00", + "saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step", + "cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer" + }, + "profiles": { + "fixed-r1": { + "postgres_cpu": 1.0, + "postgres_memory": "512M", + "name": "fixed-r1", + "replicas": 1, + "api_cpu": 1.0, + "api_memory": "512M", + "db_max_conns": 16, + "db_min_conns": 2 + }, + "fixed-r2": { + "postgres_cpu": 1.0, + "postgres_memory": "512M", + "name": "fixed-r2", + "replicas": 2, + "api_cpu": 0.5, + "api_memory": "256M", + "db_max_conns": 8, + "db_min_conns": 1 + }, + "fixed-r4": { + "postgres_cpu": 1.0, + "postgres_memory": "512M", + "name": "fixed-r4", + "replicas": 4, + "api_cpu": 0.25, + "api_memory": "128M", + "db_max_conns": 4, + "db_min_conns": 1 + }, + "growth-r1": { + "postgres_cpu": 1.0, + "postgres_memory": "512M", + "name": "growth-r1", + "replicas": 1, + "api_cpu": 0.5, + "api_memory": "256M", + "db_max_conns": 8, + "db_min_conns": 1 + }, + "growth-r2": { + "postgres_cpu": 1.0, + "postgres_memory": "512M", + "name": "growth-r2", + "replicas": 2, + "api_cpu": 0.5, + "api_memory": "256M", + "db_max_conns": 8, + "db_min_conns": 1 + }, + "growth-r4": { + "postgres_cpu": 1.0, + "postgres_memory": "512M", + "name": "growth-r4", + "replicas": 4, + "api_cpu": 0.5, + "api_memory": "256M", + "db_max_conns": 8, + "db_min_conns": 1 + }, + "improved-db-cpu-r4": { + "postgres_cpu": 2.0, + "postgres_memory": "512M", + "name": "improved-db-cpu-r4", + "replicas": 4, + "api_cpu": 0.5, + "api_memory": "256M", + "db_max_conns": 8, + "db_min_conns": 1 + } + }, + "applied_limits": { + "growth-r2": { + "api": [ + { + "instance": "3849349fa312", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + }, + { + "instance": "bbcb5c0a1b19", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + } + ], + "postgres_cpu_limit": 1.0, + "postgres_memory_limit_mib": 512.0 + }, + "fixed-r1": { + "api": [ + { + "instance": "f5b4c0a90be9", + "cpu_limit": 1.0, + "memory_limit_mib": 512.0, + "db_max_conns": 16 + } + ], + "postgres_cpu_limit": 1.0, + "postgres_memory_limit_mib": 512.0 + }, + "fixed-r2": { + "api": [ + { + "instance": "71e7fe08b5fe", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + }, + { + "instance": "b072d13bfd31", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + } + ], + "postgres_cpu_limit": 1.0, + "postgres_memory_limit_mib": 512.0 + }, + "fixed-r4": { + "api": [ + { + "instance": "0cc2174bf528", + "cpu_limit": 0.25, + "memory_limit_mib": 128.0, + "db_max_conns": 4 + }, + { + "instance": "295ab9e0814c", + "cpu_limit": 0.25, + "memory_limit_mib": 128.0, + "db_max_conns": 4 + }, + { + "instance": "76432e4c3707", + "cpu_limit": 0.25, + "memory_limit_mib": 128.0, + "db_max_conns": 4 + }, + { + "instance": "e24ef15f043e", + "cpu_limit": 0.25, + "memory_limit_mib": 128.0, + "db_max_conns": 4 + } + ], + "postgres_cpu_limit": 1.0, + "postgres_memory_limit_mib": 512.0 + }, + "growth-r1": { + "api": [ + { + "instance": "15f62705e72d", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + } + ], + "postgres_cpu_limit": 1.0, + "postgres_memory_limit_mib": 512.0 + }, + "growth-r4": { + "api": [ + { + "instance": "4cdd1d3ba154", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + }, + { + "instance": "5e6a432825a9", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + }, + { + "instance": "98727dbbd98f", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + }, + { + "instance": "9a0b50ef6c5a", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + } + ], + "postgres_cpu_limit": 1.0, + "postgres_memory_limit_mib": 512.0 + }, + "improved-db-cpu-r4": { + "api": [ + { + "instance": "3078d9394686", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + }, + { + "instance": "950a06115bc5", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + }, + { + "instance": "b04f6abe9ea1", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + }, + { + "instance": "b4c3bdf282a0", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 8 + } + ], + "postgres_cpu_limit": 2.0, + "postgres_memory_limit_mib": 512.0 + } + }, + "phases": [ + { + "name": "saturation-c1", + "kind": "saturation", + "profile": "growth-r2", + "replicas": 2, + "concurrency": 1, + "elapsed_seconds": 10.731, + "successful_requests": 2472, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 230.369, + "successful_latency_ms": { + "p50": 3.839, + "p95": 5.065, + "p99": 6.282 + }, + "instances": [ + "442f0f041187", + "c0e4b7903cdc" + ], + "per_second": { + "0": { + "successful": 252, + "failed": 0 + }, + "1": { + "successful": 266, + "failed": 0 + }, + "2": { + "successful": 257, + "failed": 0 + }, + "3": { + "successful": 262, + "failed": 0 + }, + "4": { + "successful": 253, + "failed": 0 + }, + "5": { + "successful": 249, + "failed": 0 + }, + "6": { + "successful": 228, + "failed": 0 + }, + "7": { + "successful": 240, + "failed": 0 + }, + "8": { + "successful": 219, + "failed": 0 + }, + "9": { + "successful": 244, + "failed": 0 + }, + "10": { + "successful": 2, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.0, + "services": { + "api": { + "cpu_percent": 26.09, + "memory_mib": 17.976, + "containers": 2 + }, + "frontend": { + "cpu_percent": 1.61, + "memory_mib": 7.578, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 13.0, + "memory_mib": 17.18, + "containers": 1 + }, + "postgres": { + "cpu_percent": 28.57, + "memory_mib": 169.0, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.15, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 2.356, + "services": { + "api": { + "cpu_percent": 32.65, + "memory_mib": 16.886, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.617, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 13.32, + "memory_mib": 17.23, + "containers": 1 + }, + "postgres": { + "cpu_percent": 28.96, + "memory_mib": 169.5, + "containers": 1 + }, + "redis": { + "cpu_percent": 2.75, + "memory_mib": 15.4, + "containers": 1 + } + } + }, + { + "t": 5.485, + "services": { + "api": { + "cpu_percent": 27.72, + "memory_mib": 17.586, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.383, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 13.69, + "memory_mib": 17.41, + "containers": 1 + }, + "postgres": { + "cpu_percent": 29.41, + "memory_mib": 170.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.15, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 8.605, + "services": { + "api": { + "cpu_percent": 20.94, + "memory_mib": 17.41, + "containers": 2 + }, + "frontend": { + "cpu_percent": 1.12, + "memory_mib": 12.06, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 6.64, + "memory_mib": 17.24, + "containers": 1 + }, + "postgres": { + "cpu_percent": 8.66, + "memory_mib": 170.5, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.15, + "memory_mib": 15.53, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 26.85, + "cpu_percent_max": 32.65, + "memory_mib_avg": 17.465, + "memory_mib_max": 17.976, + "containers": 2 + }, + "frontend": { + "cpu_percent_avg": 0.688, + "cpu_percent_max": 1.61, + "memory_mib_avg": 8.659, + "memory_mib_max": 12.06, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 11.662, + "cpu_percent_max": 13.69, + "memory_mib_avg": 17.265, + "memory_mib_max": 17.41, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 23.9, + "cpu_percent_max": 29.41, + "memory_mib_avg": 169.775, + "memory_mib_max": 170.5, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.8, + "cpu_percent_max": 2.75, + "memory_mib_avg": 15.372, + "memory_mib_max": 15.53, + "containers": 1 + } + } + }, + { + "name": "saturation-c2", + "kind": "saturation", + "profile": "growth-r2", + "replicas": 2, + "concurrency": 2, + "elapsed_seconds": 10.915, + "successful_requests": 3495, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 320.19, + "successful_latency_ms": { + "p50": 5.192, + "p95": 7.196, + "p99": 15.11 + }, + "instances": [ + "442f0f041187", + "c0e4b7903cdc" + ], + "per_second": { + "0": { + "successful": 444, + "failed": 0 + }, + "1": { + "successful": 377, + "failed": 0 + }, + "2": { + "successful": 352, + "failed": 0 + }, + "3": { + "successful": 297, + "failed": 0 + }, + "4": { + "successful": 383, + "failed": 0 + }, + "5": { + "successful": 324, + "failed": 0 + }, + "6": { + "successful": 275, + "failed": 0 + }, + "7": { + "successful": 323, + "failed": 0 + }, + "8": { + "successful": 333, + "failed": 0 + }, + "9": { + "successful": 381, + "failed": 0 + }, + "10": { + "successful": 6, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.001, + "services": { + "api": { + "cpu_percent": 51.0, + "memory_mib": 17.761, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.03, + "memory_mib": 7.617, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 24.78, + "memory_mib": 17.71, + "containers": 1 + }, + "postgres": { + "cpu_percent": 57.44, + "memory_mib": 171.2, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.17, + "memory_mib": 16.03, + "containers": 1 + } + } + }, + { + "t": 2.361, + "services": { + "api": { + "cpu_percent": 45.61, + "memory_mib": 17.516, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.03, + "memory_mib": 7.367, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 24.6, + "memory_mib": 17.41, + "containers": 1 + }, + "postgres": { + "cpu_percent": 49.85, + "memory_mib": 172.6, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.18, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 5.543, + "services": { + "api": { + "cpu_percent": 51.19, + "memory_mib": 16.258, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.367, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 24.27, + "memory_mib": 17.91, + "containers": 1 + }, + "postgres": { + "cpu_percent": 62.6, + "memory_mib": 173.0, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.21, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 8.736, + "services": { + "api": { + "cpu_percent": 45.4, + "memory_mib": 17.082, + "containers": 2 + }, + "frontend": { + "cpu_percent": 1.22, + "memory_mib": 8.121, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 2.87, + "memory_mib": 17.2, + "containers": 1 + }, + "postgres": { + "cpu_percent": 50.06, + "memory_mib": 173.8, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.21, + "memory_mib": 15.78, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 48.3, + "cpu_percent_max": 51.19, + "memory_mib_avg": 17.154, + "memory_mib_max": 17.761, + "containers": 2 + }, + "frontend": { + "cpu_percent_avg": 0.325, + "cpu_percent_max": 1.22, + "memory_mib_avg": 7.618, + "memory_mib_max": 8.121, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 19.13, + "cpu_percent_max": 24.78, + "memory_mib_avg": 17.558, + "memory_mib_max": 17.91, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 54.987, + "cpu_percent_max": 62.6, + "memory_mib_avg": 172.65, + "memory_mib_max": 173.8, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.193, + "cpu_percent_max": 0.21, + "memory_mib_avg": 15.592, + "memory_mib_max": 16.03, + "containers": 1 + } + } + }, + { + "name": "saturation-c4", + "kind": "saturation", + "profile": "growth-r2", + "replicas": 2, + "concurrency": 4, + "elapsed_seconds": 10.832, + "successful_requests": 4721, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 435.822, + "successful_latency_ms": { + "p50": 6.171, + "p95": 29.304, + "p99": 47.665 + }, + "instances": [ + "442f0f041187", + "c0e4b7903cdc" + ], + "per_second": { + "0": { + "successful": 526, + "failed": 0 + }, + "1": { + "successful": 453, + "failed": 0 + }, + "2": { + "successful": 425, + "failed": 0 + }, + "3": { + "successful": 478, + "failed": 0 + }, + "4": { + "successful": 510, + "failed": 0 + }, + "5": { + "successful": 468, + "failed": 0 + }, + "6": { + "successful": 458, + "failed": 0 + }, + "7": { + "successful": 428, + "failed": 0 + }, + "8": { + "successful": 490, + "failed": 0 + }, + "9": { + "successful": 474, + "failed": 0 + }, + "10": { + "successful": 11, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.001, + "services": { + "api": { + "cpu_percent": 65.41, + "memory_mib": 17.297, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.03, + "memory_mib": 7.367, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.95, + "memory_mib": 17.71, + "containers": 1 + }, + "postgres": { + "cpu_percent": 81.8, + "memory_mib": 174.4, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.19, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 2.372, + "services": { + "api": { + "cpu_percent": 60.01, + "memory_mib": 18.188, + "containers": 2 + }, + "frontend": { + "cpu_percent": 3.75, + "memory_mib": 7.965, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.04, + "memory_mib": 17.46, + "containers": 1 + }, + "postgres": { + "cpu_percent": 79.42, + "memory_mib": 175.4, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.16, + "memory_mib": 15.53, + "containers": 1 + } + } + }, + { + "t": 5.501, + "services": { + "api": { + "cpu_percent": 56.53, + "memory_mib": 17.433, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.367, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.9, + "memory_mib": 21.22, + "containers": 1 + }, + "postgres": { + "cpu_percent": 73.71, + "memory_mib": 175.7, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.18, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 8.662, + "services": { + "api": { + "cpu_percent": 35.86, + "memory_mib": 16.699, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.367, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 5.48, + "memory_mib": 17.53, + "containers": 1 + }, + "postgres": { + "cpu_percent": 17.44, + "memory_mib": 177.2, + "containers": 1 + }, + "redis": { + "cpu_percent": 2.71, + "memory_mib": 16.1, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 54.453, + "cpu_percent_max": 65.41, + "memory_mib_avg": 17.404, + "memory_mib_max": 18.188, + "containers": 2 + }, + "frontend": { + "cpu_percent_avg": 0.95, + "cpu_percent_max": 3.75, + "memory_mib_avg": 7.516, + "memory_mib_max": 7.965, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 20.843, + "cpu_percent_max": 26.04, + "memory_mib_avg": 18.48, + "memory_mib_max": 21.22, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 63.093, + "cpu_percent_max": 81.8, + "memory_mib_avg": 175.675, + "memory_mib_max": 177.2, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.81, + "cpu_percent_max": 2.71, + "memory_mib_avg": 15.547, + "memory_mib_max": 16.1, + "containers": 1 + } + } + }, + { + "name": "saturation-c8", + "kind": "saturation", + "profile": "growth-r2", + "replicas": 2, + "concurrency": 8, + "elapsed_seconds": 10.821, + "successful_requests": 5095, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 470.828, + "successful_latency_ms": { + "p50": 9.159, + "p95": 56.23, + "p99": 70.71 + }, + "instances": [ + "442f0f041187", + "c0e4b7903cdc" + ], + "per_second": { + "0": { + "successful": 532, + "failed": 0 + }, + "1": { + "successful": 476, + "failed": 0 + }, + "2": { + "successful": 530, + "failed": 0 + }, + "3": { + "successful": 529, + "failed": 0 + }, + "4": { + "successful": 490, + "failed": 0 + }, + "5": { + "successful": 527, + "failed": 0 + }, + "6": { + "successful": 445, + "failed": 0 + }, + "7": { + "successful": 524, + "failed": 0 + }, + "8": { + "successful": 507, + "failed": 0 + }, + "9": { + "successful": 519, + "failed": 0 + }, + "10": { + "successful": 16, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.006, + "services": { + "api": { + "cpu_percent": 48.34, + "memory_mib": 17.403, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.367, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.42, + "memory_mib": 20.12, + "containers": 1 + }, + "postgres": { + "cpu_percent": 77.51, + "memory_mib": 185.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.18, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 2.372, + "services": { + "api": { + "cpu_percent": 57.14, + "memory_mib": 18.602, + "containers": 2 + }, + "frontend": { + "cpu_percent": 3.44, + "memory_mib": 7.734, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.83, + "memory_mib": 17.14, + "containers": 1 + }, + "postgres": { + "cpu_percent": 92.98, + "memory_mib": 188.5, + "containers": 1 + }, + "redis": { + "cpu_percent": 3.34, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 5.527, + "services": { + "api": { + "cpu_percent": 48.26, + "memory_mib": 18.192, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.363, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.47, + "memory_mib": 17.28, + "containers": 1 + }, + "postgres": { + "cpu_percent": 76.05, + "memory_mib": 189.4, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.15, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 8.685, + "services": { + "api": { + "cpu_percent": 40.28, + "memory_mib": 21.813, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.363, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 5.75, + "memory_mib": 17.64, + "containers": 1 + }, + "postgres": { + "cpu_percent": 17.24, + "memory_mib": 190.4, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.14, + "memory_mib": 15.28, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 48.505, + "cpu_percent_max": 57.14, + "memory_mib_avg": 19.002, + "memory_mib_max": 21.813, + "containers": 2 + }, + "frontend": { + "cpu_percent_avg": 0.86, + "cpu_percent_max": 3.44, + "memory_mib_avg": 7.457, + "memory_mib_max": 7.734, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 20.867, + "cpu_percent_max": 26.83, + "memory_mib_avg": 18.045, + "memory_mib_max": 20.12, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 65.945, + "cpu_percent_max": 92.98, + "memory_mib_avg": 188.35, + "memory_mib_max": 190.4, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.953, + "cpu_percent_max": 3.34, + "memory_mib_avg": 15.28, + "memory_mib_max": 15.28, + "containers": 1 + } + } + }, + { + "name": "saturation-c16", + "kind": "saturation", + "profile": "growth-r2", + "replicas": 2, + "concurrency": 16, + "elapsed_seconds": 10.876, + "successful_requests": 5143, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 472.892, + "successful_latency_ms": { + "p50": 19.862, + "p95": 73.628, + "p99": 98.773 + }, + "instances": [ + "442f0f041187", + "c0e4b7903cdc" + ], + "per_second": { + "0": { + "successful": 349, + "failed": 0 + }, + "1": { + "successful": 592, + "failed": 0 + }, + "2": { + "successful": 541, + "failed": 0 + }, + "3": { + "successful": 491, + "failed": 0 + }, + "4": { + "successful": 545, + "failed": 0 + }, + "5": { + "successful": 420, + "failed": 0 + }, + "6": { + "successful": 499, + "failed": 0 + }, + "7": { + "successful": 529, + "failed": 0 + }, + "8": { + "successful": 548, + "failed": 0 + }, + "9": { + "successful": 601, + "failed": 0 + }, + "10": { + "successful": 28, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.015, + "services": { + "api": { + "cpu_percent": 44.39, + "memory_mib": 18.796, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.363, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.92, + "memory_mib": 17.44, + "containers": 1 + }, + "postgres": { + "cpu_percent": 82.81, + "memory_mib": 205.9, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.17, + "memory_mib": 15.28, + "containers": 1 + } + } + }, + { + "t": 2.354, + "services": { + "api": { + "cpu_percent": 56.54, + "memory_mib": 17.856, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.03, + "memory_mib": 7.375, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.66, + "memory_mib": 17.69, + "containers": 1 + }, + "postgres": { + "cpu_percent": 88.93, + "memory_mib": 207.7, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.19, + "memory_mib": 15.75, + "containers": 1 + } + } + }, + { + "t": 5.525, + "services": { + "api": { + "cpu_percent": 41.39, + "memory_mib": 17.73, + "containers": 2 + }, + "frontend": { + "cpu_percent": 4.4, + "memory_mib": 7.379, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.76, + "memory_mib": 17.7, + "containers": 1 + }, + "postgres": { + "cpu_percent": 72.49, + "memory_mib": 208.6, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.18, + "memory_mib": 15.25, + "containers": 1 + } + } + }, + { + "t": 8.731, + "services": { + "api": { + "cpu_percent": 9.79, + "memory_mib": 18.257, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.375, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 4.58, + "memory_mib": 17.55, + "containers": 1 + }, + "postgres": { + "cpu_percent": 16.09, + "memory_mib": 209.7, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.15, + "memory_mib": 15.33, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 38.028, + "cpu_percent_max": 56.54, + "memory_mib_avg": 18.16, + "memory_mib_max": 18.796, + "containers": 2 + }, + "frontend": { + "cpu_percent_avg": 1.113, + "cpu_percent_max": 4.4, + "memory_mib_avg": 7.373, + "memory_mib_max": 7.379, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 20.98, + "cpu_percent_max": 26.92, + "memory_mib_avg": 17.595, + "memory_mib_max": 17.7, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 65.08, + "cpu_percent_max": 88.93, + "memory_mib_avg": 207.975, + "memory_mib_max": 209.7, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.172, + "cpu_percent_max": 0.19, + "memory_mib_avg": 15.402, + "memory_mib_max": 15.75, + "containers": 1 + } + } + }, + { + "name": "saturation-c32", + "kind": "saturation", + "profile": "growth-r2", + "replicas": 2, + "concurrency": 32, + "elapsed_seconds": 10.808, + "successful_requests": 6096, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 564.012, + "successful_latency_ms": { + "p50": 51.089, + "p95": 94.785, + "p99": 109.138 + }, + "instances": [ + "442f0f041187", + "c0e4b7903cdc" + ], + "per_second": { + "0": { + "successful": 661, + "failed": 0 + }, + "1": { + "successful": 604, + "failed": 0 + }, + "2": { + "successful": 610, + "failed": 0 + }, + "3": { + "successful": 589, + "failed": 0 + }, + "4": { + "successful": 571, + "failed": 0 + }, + "5": { + "successful": 627, + "failed": 0 + }, + "6": { + "successful": 587, + "failed": 0 + }, + "7": { + "successful": 619, + "failed": 0 + }, + "8": { + "successful": 629, + "failed": 0 + }, + "9": { + "successful": 548, + "failed": 0 + }, + "10": { + "successful": 51, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.039, + "services": { + "api": { + "cpu_percent": 55.84, + "memory_mib": 18.75, + "containers": 2 + }, + "frontend": { + "cpu_percent": 6.63, + "memory_mib": 8.125, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.97, + "memory_mib": 18.0, + "containers": 1 + }, + "postgres": { + "cpu_percent": 104.2, + "memory_mib": 210.6, + "containers": 1 + }, + "redis": { + "cpu_percent": 5.85, + "memory_mib": 15.05, + "containers": 1 + } + } + }, + { + "t": 2.29, + "services": { + "api": { + "cpu_percent": 56.87, + "memory_mib": 18.933, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.625, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 27.03, + "memory_mib": 19.23, + "containers": 1 + }, + "postgres": { + "cpu_percent": 105.41, + "memory_mib": 212.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.19, + "memory_mib": 15.55, + "containers": 1 + } + } + }, + { + "t": 5.446, + "services": { + "api": { + "cpu_percent": 64.13, + "memory_mib": 19.003, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.7, + "memory_mib": 7.375, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.33, + "memory_mib": 18.27, + "containers": 1 + }, + "postgres": { + "cpu_percent": 107.62, + "memory_mib": 213.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 3.7, + "memory_mib": 15.27, + "containers": 1 + } + } + }, + { + "t": 8.653, + "services": { + "api": { + "cpu_percent": 31.02, + "memory_mib": 19.379, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.625, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 5.32, + "memory_mib": 18.36, + "containers": 1 + }, + "postgres": { + "cpu_percent": 18.07, + "memory_mib": 213.5, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.16, + "memory_mib": 15.26, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 51.965, + "cpu_percent_max": 64.13, + "memory_mib_avg": 19.016, + "memory_mib_max": 19.379, + "containers": 2 + }, + "frontend": { + "cpu_percent_avg": 1.837, + "cpu_percent_max": 6.63, + "memory_mib_avg": 7.688, + "memory_mib_max": 8.125, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 21.163, + "cpu_percent_max": 27.03, + "memory_mib_avg": 18.465, + "memory_mib_max": 19.23, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 83.825, + "cpu_percent_max": 107.62, + "memory_mib_avg": 212.325, + "memory_mib_max": 213.5, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 2.475, + "cpu_percent_max": 5.85, + "memory_mib_avg": 15.283, + "memory_mib_max": 15.55, + "containers": 1 + } + } + }, + { + "name": "fixed-r1", + "kind": "fixed", + "profile": "fixed-r1", + "replicas": 1, + "concurrency": 32, + "elapsed_seconds": 10.88, + "successful_requests": 6291, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 578.242, + "successful_latency_ms": { + "p50": 48.579, + "p95": 95.757, + "p99": 115.549 + }, + "instances": [ + "f5b4c0a90be9" + ], + "per_second": { + "0": { + "successful": 511, + "failed": 0 + }, + "1": { + "successful": 580, + "failed": 0 + }, + "2": { + "successful": 659, + "failed": 0 + }, + "3": { + "successful": 687, + "failed": 0 + }, + "4": { + "successful": 588, + "failed": 0 + }, + "5": { + "successful": 675, + "failed": 0 + }, + "6": { + "successful": 647, + "failed": 0 + }, + "7": { + "successful": 701, + "failed": 0 + }, + "8": { + "successful": 690, + "failed": 0 + }, + "9": { + "successful": 521, + "failed": 0 + }, + "10": { + "successful": 32, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.034, + "services": { + "api": { + "cpu_percent": 45.68, + "memory_mib": 10.61, + "containers": 1 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.359, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.97, + "memory_mib": 18.08, + "containers": 1 + }, + "postgres": { + "cpu_percent": 106.86, + "memory_mib": 214.7, + "containers": 1 + }, + "redis": { + "cpu_percent": 4.41, + "memory_mib": 15.3, + "containers": 1 + } + } + }, + { + "t": 2.424, + "services": { + "api": { + "cpu_percent": 48.23, + "memory_mib": 11.36, + "containers": 1 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.699, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 27.02, + "memory_mib": 22.21, + "containers": 1 + }, + "postgres": { + "cpu_percent": 96.33, + "memory_mib": 215.7, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.14, + "memory_mib": 15.44, + "containers": 1 + } + } + }, + { + "t": 5.588, + "services": { + "api": { + "cpu_percent": 48.84, + "memory_mib": 12.07, + "containers": 1 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.359, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.38, + "memory_mib": 18.43, + "containers": 1 + }, + "postgres": { + "cpu_percent": 104.71, + "memory_mib": 217.4, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.19, + "memory_mib": 15.92, + "containers": 1 + } + } + }, + { + "t": 8.743, + "services": { + "api": { + "cpu_percent": 3.55, + "memory_mib": 11.93, + "containers": 1 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.508, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 3.56, + "memory_mib": 17.96, + "containers": 1 + }, + "postgres": { + "cpu_percent": 8.33, + "memory_mib": 218.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.19, + "memory_mib": 16.07, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 36.575, + "cpu_percent_max": 48.84, + "memory_mib_avg": 11.492, + "memory_mib_max": 12.07, + "containers": 1 + }, + "frontend": { + "cpu_percent_avg": 0.005, + "cpu_percent_max": 0.02, + "memory_mib_avg": 7.481, + "memory_mib_max": 7.699, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 20.732, + "cpu_percent_max": 27.02, + "memory_mib_avg": 19.17, + "memory_mib_max": 22.21, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 79.058, + "cpu_percent_max": 106.86, + "memory_mib_avg": 216.475, + "memory_mib_max": 218.1, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 1.232, + "cpu_percent_max": 4.41, + "memory_mib_avg": 15.683, + "memory_mib_max": 16.07, + "containers": 1 + } + } + }, + { + "name": "fixed-r2", + "kind": "fixed", + "profile": "fixed-r2", + "replicas": 2, + "concurrency": 32, + "elapsed_seconds": 10.905, + "successful_requests": 5774, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 529.499, + "successful_latency_ms": { + "p50": 53.316, + "p95": 100.145, + "p99": 129.953 + }, + "instances": [ + "71e7fe08b5fe", + "b072d13bfd31" + ], + "per_second": { + "0": { + "successful": 452, + "failed": 0 + }, + "1": { + "successful": 545, + "failed": 0 + }, + "2": { + "successful": 604, + "failed": 0 + }, + "3": { + "successful": 652, + "failed": 0 + }, + "4": { + "successful": 521, + "failed": 0 + }, + "5": { + "successful": 627, + "failed": 0 + }, + "6": { + "successful": 571, + "failed": 0 + }, + "7": { + "successful": 611, + "failed": 0 + }, + "8": { + "successful": 573, + "failed": 0 + }, + "9": { + "successful": 586, + "failed": 0 + }, + "10": { + "successful": 32, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.041, + "services": { + "api": { + "cpu_percent": 52.24, + "memory_mib": 18.086, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.359, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.74, + "memory_mib": 18.2, + "containers": 1 + }, + "postgres": { + "cpu_percent": 100.69, + "memory_mib": 219.4, + "containers": 1 + }, + "redis": { + "cpu_percent": 3.81, + "memory_mib": 15.18, + "containers": 1 + } + } + }, + { + "t": 2.379, + "services": { + "api": { + "cpu_percent": 65.39, + "memory_mib": 18.801, + "containers": 2 + }, + "frontend": { + "cpu_percent": 1.92, + "memory_mib": 7.859, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.23, + "memory_mib": 18.61, + "containers": 1 + }, + "postgres": { + "cpu_percent": 106.33, + "memory_mib": 221.3, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.16, + "memory_mib": 15.16, + "containers": 1 + } + } + }, + { + "t": 5.564, + "services": { + "api": { + "cpu_percent": 54.12, + "memory_mib": 18.379, + "containers": 2 + }, + "frontend": { + "cpu_percent": 1.37, + "memory_mib": 10.54, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.33, + "memory_mib": 18.35, + "containers": 1 + }, + "postgres": { + "cpu_percent": 103.06, + "memory_mib": 222.7, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.2, + "memory_mib": 14.93, + "containers": 1 + } + } + }, + { + "t": 8.728, + "services": { + "api": { + "cpu_percent": 7.05, + "memory_mib": 18.086, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.363, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 6.3, + "memory_mib": 17.82, + "containers": 1 + }, + "postgres": { + "cpu_percent": 14.18, + "memory_mib": 223.3, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.2, + "memory_mib": 15.43, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 44.7, + "cpu_percent_max": 65.39, + "memory_mib_avg": 18.338, + "memory_mib_max": 18.801, + "containers": 2 + }, + "frontend": { + "cpu_percent_avg": 0.823, + "cpu_percent_max": 1.92, + "memory_mib_avg": 8.28, + "memory_mib_max": 10.54, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 21.15, + "cpu_percent_max": 26.74, + "memory_mib_avg": 18.245, + "memory_mib_max": 18.61, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 81.065, + "cpu_percent_max": 106.33, + "memory_mib_avg": 221.675, + "memory_mib_max": 223.3, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 1.093, + "cpu_percent_max": 3.81, + "memory_mib_avg": 15.175, + "memory_mib_max": 15.43, + "containers": 1 + } + } + }, + { + "name": "fixed-r4", + "kind": "fixed", + "profile": "fixed-r4", + "replicas": 4, + "concurrency": 32, + "elapsed_seconds": 11.917, + "successful_requests": 4218, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 353.955, + "successful_latency_ms": { + "p50": 69.354, + "p95": 139.408, + "p99": 291.312 + }, + "instances": [ + "0cc2174bf528", + "295ab9e0814c", + "76432e4c3707", + "e24ef15f043e" + ], + "per_second": { + "0": { + "successful": 133, + "failed": 0 + }, + "1": { + "successful": 276, + "failed": 0 + }, + "2": { + "successful": 461, + "failed": 0 + }, + "3": { + "successful": 406, + "failed": 0 + }, + "4": { + "successful": 503, + "failed": 0 + }, + "5": { + "successful": 487, + "failed": 0 + }, + "6": { + "successful": 377, + "failed": 0 + }, + "7": { + "successful": 427, + "failed": 0 + }, + "8": { + "successful": 551, + "failed": 0 + }, + "9": { + "successful": 547, + "failed": 0 + }, + "10": { + "successful": 50, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.095, + "services": { + "api": { + "cpu_percent": 51.98, + "memory_mib": 27.492, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.363, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 27.15, + "memory_mib": 19.06, + "containers": 1 + }, + "postgres": { + "cpu_percent": 90.1, + "memory_mib": 223.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 5.94, + "memory_mib": 16.87, + "containers": 1 + } + } + }, + { + "t": 3.244, + "services": { + "api": { + "cpu_percent": 64.6, + "memory_mib": 33.144, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.359, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.34, + "memory_mib": 18.58, + "containers": 1 + }, + "postgres": { + "cpu_percent": 101.51, + "memory_mib": 225.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.2, + "memory_mib": 16.96, + "containers": 1 + } + } + }, + { + "t": 6.45, + "services": { + "api": { + "cpu_percent": 65.35, + "memory_mib": 34.281, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.359, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.27, + "memory_mib": 18.56, + "containers": 1 + }, + "postgres": { + "cpu_percent": 106.68, + "memory_mib": 225.7, + "containers": 1 + }, + "redis": { + "cpu_percent": 4.55, + "memory_mib": 16.7, + "containers": 1 + } + } + }, + { + "t": 9.725, + "services": { + "api": { + "cpu_percent": 3.07, + "memory_mib": 32.867, + "containers": 4 + }, + "frontend": { + "cpu_percent": 2.51, + "memory_mib": 7.762, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 4.96, + "memory_mib": 17.89, + "containers": 1 + }, + "postgres": { + "cpu_percent": 0.42, + "memory_mib": 226.7, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.17, + "memory_mib": 16.95, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 46.25, + "cpu_percent_max": 65.35, + "memory_mib_avg": 31.946, + "memory_mib_max": 34.281, + "containers": 4 + }, + "frontend": { + "cpu_percent_avg": 0.637, + "cpu_percent_max": 2.51, + "memory_mib_avg": 7.461, + "memory_mib_max": 7.762, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 21.18, + "cpu_percent_max": 27.15, + "memory_mib_avg": 18.523, + "memory_mib_max": 19.06, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 74.677, + "cpu_percent_max": 106.68, + "memory_mib_avg": 225.15, + "memory_mib_max": 226.7, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 2.715, + "cpu_percent_max": 5.94, + "memory_mib_avg": 16.87, + "memory_mib_max": 16.96, + "containers": 1 + } + } + }, + { + "name": "growth-r1", + "kind": "growth", + "profile": "growth-r1", + "replicas": 1, + "concurrency": 32, + "elapsed_seconds": 10.951, + "successful_requests": 5844, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 533.656, + "successful_latency_ms": { + "p50": 53.378, + "p95": 93.541, + "p99": 115.58 + }, + "instances": [ + "15f62705e72d" + ], + "per_second": { + "0": { + "successful": 527, + "failed": 0 + }, + "1": { + "successful": 596, + "failed": 0 + }, + "2": { + "successful": 604, + "failed": 0 + }, + "3": { + "successful": 616, + "failed": 0 + }, + "4": { + "successful": 516, + "failed": 0 + }, + "5": { + "successful": 508, + "failed": 0 + }, + "6": { + "successful": 606, + "failed": 0 + }, + "7": { + "successful": 620, + "failed": 0 + }, + "8": { + "successful": 628, + "failed": 0 + }, + "9": { + "successful": 591, + "failed": 0 + }, + "10": { + "successful": 32, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.045, + "services": { + "api": { + "cpu_percent": 44.44, + "memory_mib": 9.355, + "containers": 1 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.359, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.9, + "memory_mib": 18.3, + "containers": 1 + }, + "postgres": { + "cpu_percent": 93.36, + "memory_mib": 207.6, + "containers": 1 + }, + "redis": { + "cpu_percent": 3.22, + "memory_mib": 18.55, + "containers": 1 + } + } + }, + { + "t": 2.404, + "services": { + "api": { + "cpu_percent": 50.32, + "memory_mib": 9.379, + "containers": 1 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.363, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.42, + "memory_mib": 18.36, + "containers": 1 + }, + "postgres": { + "cpu_percent": 96.7, + "memory_mib": 209.4, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.22, + "memory_mib": 18.11, + "containers": 1 + } + } + }, + { + "t": 5.579, + "services": { + "api": { + "cpu_percent": 50.68, + "memory_mib": 9.66, + "containers": 1 + }, + "frontend": { + "cpu_percent": 0.03, + "memory_mib": 7.613, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.44, + "memory_mib": 18.67, + "containers": 1 + }, + "postgres": { + "cpu_percent": 105.06, + "memory_mib": 210.9, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.22, + "memory_mib": 17.84, + "containers": 1 + } + } + }, + { + "t": 8.767, + "services": { + "api": { + "cpu_percent": 7.65, + "memory_mib": 9.262, + "containers": 1 + }, + "frontend": { + "cpu_percent": 2.74, + "memory_mib": 8.02, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 3.49, + "memory_mib": 17.86, + "containers": 1 + }, + "postgres": { + "cpu_percent": 13.89, + "memory_mib": 211.8, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.17, + "memory_mib": 17.93, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 38.273, + "cpu_percent_max": 50.68, + "memory_mib_avg": 9.414, + "memory_mib_max": 9.66, + "containers": 1 + }, + "frontend": { + "cpu_percent_avg": 0.693, + "cpu_percent_max": 2.74, + "memory_mib_avg": 7.589, + "memory_mib_max": 8.02, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 20.062, + "cpu_percent_max": 25.9, + "memory_mib_avg": 18.297, + "memory_mib_max": 18.67, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 77.252, + "cpu_percent_max": 105.06, + "memory_mib_avg": 209.925, + "memory_mib_max": 211.8, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.958, + "cpu_percent_max": 3.22, + "memory_mib_avg": 18.108, + "memory_mib_max": 18.55, + "containers": 1 + } + } + }, + { + "name": "growth-r2", + "kind": "growth", + "profile": "growth-r2", + "replicas": 2, + "concurrency": 32, + "elapsed_seconds": 10.796, + "successful_requests": 6083, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 563.46, + "successful_latency_ms": { + "p50": 50.516, + "p95": 96.428, + "p99": 129.46 + }, + "instances": [ + "3849349fa312", + "bbcb5c0a1b19" + ], + "per_second": { + "0": { + "successful": 461, + "failed": 0 + }, + "1": { + "successful": 662, + "failed": 0 + }, + "2": { + "successful": 670, + "failed": 0 + }, + "3": { + "successful": 655, + "failed": 0 + }, + "4": { + "successful": 640, + "failed": 0 + }, + "5": { + "successful": 495, + "failed": 0 + }, + "6": { + "successful": 635, + "failed": 0 + }, + "7": { + "successful": 626, + "failed": 0 + }, + "8": { + "successful": 598, + "failed": 0 + }, + "9": { + "successful": 609, + "failed": 0 + }, + "10": { + "successful": 32, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.047, + "services": { + "api": { + "cpu_percent": 52.25, + "memory_mib": 17.629, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.363, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 24.23, + "memory_mib": 18.59, + "containers": 1 + }, + "postgres": { + "cpu_percent": 100.9, + "memory_mib": 233.3, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.14, + "memory_mib": 17.16, + "containers": 1 + } + } + }, + { + "t": 2.292, + "services": { + "api": { + "cpu_percent": 62.86, + "memory_mib": 17.606, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.355, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.57, + "memory_mib": 18.19, + "containers": 1 + }, + "postgres": { + "cpu_percent": 105.07, + "memory_mib": 235.3, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.16, + "memory_mib": 17.16, + "containers": 1 + } + } + }, + { + "t": 5.459, + "services": { + "api": { + "cpu_percent": 57.29, + "memory_mib": 18.789, + "containers": 2 + }, + "frontend": { + "cpu_percent": 0.04, + "memory_mib": 7.363, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.89, + "memory_mib": 18.43, + "containers": 1 + }, + "postgres": { + "cpu_percent": 104.58, + "memory_mib": 235.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.17, + "memory_mib": 17.73, + "containers": 1 + } + } + }, + { + "t": 8.628, + "services": { + "api": { + "cpu_percent": 14.21, + "memory_mib": 17.832, + "containers": 2 + }, + "frontend": { + "cpu_percent": 3.0, + "memory_mib": 8.062, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 9.08, + "memory_mib": 18.0, + "containers": 1 + }, + "postgres": { + "cpu_percent": 31.84, + "memory_mib": 234.3, + "containers": 1 + }, + "redis": { + "cpu_percent": 2.99, + "memory_mib": 22.08, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 46.653, + "cpu_percent_max": 62.86, + "memory_mib_avg": 17.964, + "memory_mib_max": 18.789, + "containers": 2 + }, + "frontend": { + "cpu_percent_avg": 0.765, + "cpu_percent_max": 3.0, + "memory_mib_avg": 7.536, + "memory_mib_max": 8.062, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 21.192, + "cpu_percent_max": 25.89, + "memory_mib_avg": 18.303, + "memory_mib_max": 18.59, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 85.597, + "cpu_percent_max": 105.07, + "memory_mib_avg": 234.5, + "memory_mib_max": 235.3, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.865, + "cpu_percent_max": 2.99, + "memory_mib_avg": 18.532, + "memory_mib_max": 22.08, + "containers": 1 + } + } + }, + { + "name": "growth-r4", + "kind": "growth", + "profile": "growth-r4", + "replicas": 4, + "concurrency": 32, + "elapsed_seconds": 10.682, + "successful_requests": 4688, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 438.868, + "successful_latency_ms": { + "p50": 68.331, + "p95": 109.446, + "p99": 199.616 + }, + "instances": [ + "4cdd1d3ba154", + "5e6a432825a9", + "98727dbbd98f", + "9a0b50ef6c5a" + ], + "per_second": { + "0": { + "successful": 106, + "failed": 0 + }, + "1": { + "successful": 514, + "failed": 0 + }, + "2": { + "successful": 587, + "failed": 0 + }, + "3": { + "successful": 506, + "failed": 0 + }, + "4": { + "successful": 423, + "failed": 0 + }, + "5": { + "successful": 409, + "failed": 0 + }, + "6": { + "successful": 459, + "failed": 0 + }, + "7": { + "successful": 462, + "failed": 0 + }, + "8": { + "successful": 618, + "failed": 0 + }, + "9": { + "successful": 549, + "failed": 0 + }, + "10": { + "successful": 55, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.043, + "services": { + "api": { + "cpu_percent": 25.03, + "memory_mib": 29.672, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.316, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 9.16, + "memory_mib": 18.25, + "containers": 1 + }, + "postgres": { + "cpu_percent": 99.03, + "memory_mib": 269.2, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.15, + "memory_mib": 16.5, + "containers": 1 + } + } + }, + { + "t": 2.197, + "services": { + "api": { + "cpu_percent": 48.89, + "memory_mib": 31.184, + "containers": 4 + }, + "frontend": { + "cpu_percent": 2.63, + "memory_mib": 7.305, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 23.31, + "memory_mib": 18.25, + "containers": 1 + }, + "postgres": { + "cpu_percent": 100.3, + "memory_mib": 284.5, + "containers": 1 + }, + "redis": { + "cpu_percent": 2.97, + "memory_mib": 20.3, + "containers": 1 + } + } + }, + { + "t": 5.406, + "services": { + "api": { + "cpu_percent": 43.96, + "memory_mib": 32.164, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.305, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 19.48, + "memory_mib": 17.79, + "containers": 1 + }, + "postgres": { + "cpu_percent": 101.91, + "memory_mib": 290.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.19, + "memory_mib": 16.68, + "containers": 1 + } + } + }, + { + "t": 8.538, + "services": { + "api": { + "cpu_percent": 24.88, + "memory_mib": 32.855, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.02, + "memory_mib": 7.551, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 10.43, + "memory_mib": 17.5, + "containers": 1 + }, + "postgres": { + "cpu_percent": 38.15, + "memory_mib": 279.9, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.18, + "memory_mib": 16.91, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 35.69, + "cpu_percent_max": 48.89, + "memory_mib_avg": 31.469, + "memory_mib_max": 32.855, + "containers": 4 + }, + "frontend": { + "cpu_percent_avg": 0.667, + "cpu_percent_max": 2.63, + "memory_mib_avg": 7.369, + "memory_mib_max": 7.551, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 15.595, + "cpu_percent_max": 23.31, + "memory_mib_avg": 17.947, + "memory_mib_max": 18.25, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 84.847, + "cpu_percent_max": 101.91, + "memory_mib_avg": 280.925, + "memory_mib_max": 290.1, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.873, + "cpu_percent_max": 2.97, + "memory_mib_avg": 17.598, + "memory_mib_max": 20.3, + "containers": 1 + } + } + }, + { + "name": "improved-db-cpu-r4", + "kind": "improvement", + "profile": "improved-db-cpu-r4", + "replicas": 4, + "concurrency": 32, + "elapsed_seconds": 10.601, + "successful_requests": 6229, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 587.574, + "successful_latency_ms": { + "p50": 49.466, + "p95": 93.991, + "p99": 112.002 + }, + "instances": [ + "3078d9394686", + "950a06115bc5", + "b04f6abe9ea1", + "b4c3bdf282a0" + ], + "per_second": { + "0": { + "successful": 492, + "failed": 0 + }, + "1": { + "successful": 583, + "failed": 0 + }, + "2": { + "successful": 629, + "failed": 0 + }, + "3": { + "successful": 681, + "failed": 0 + }, + "4": { + "successful": 667, + "failed": 0 + }, + "5": { + "successful": 632, + "failed": 0 + }, + "6": { + "successful": 585, + "failed": 0 + }, + "7": { + "successful": 677, + "failed": 0 + }, + "8": { + "successful": 602, + "failed": 0 + }, + "9": { + "successful": 629, + "failed": 0 + }, + "10": { + "successful": 52, + "failed": 0 + } + }, + "resource_samples": [ + { + "t": 0.054, + "services": { + "api": { + "cpu_percent": 52.63, + "memory_mib": 30.188, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.03, + "memory_mib": 7.301, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 24.59, + "memory_mib": 18.03, + "containers": 1 + }, + "postgres": { + "cpu_percent": 121.18, + "memory_mib": 130.1, + "containers": 1 + }, + "redis": { + "cpu_percent": 1.35, + "memory_mib": 16.63, + "containers": 1 + } + } + }, + { + "t": 2.129, + "services": { + "api": { + "cpu_percent": 70.26, + "memory_mib": 36.812, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.301, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 25.64, + "memory_mib": 18.99, + "containers": 1 + }, + "postgres": { + "cpu_percent": 106.99, + "memory_mib": 141.4, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.16, + "memory_mib": 16.88, + "containers": 1 + } + } + }, + { + "t": 5.271, + "services": { + "api": { + "cpu_percent": 65.05, + "memory_mib": 33.652, + "containers": 4 + }, + "frontend": { + "cpu_percent": 3.29, + "memory_mib": 7.43, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 26.73, + "memory_mib": 18.41, + "containers": 1 + }, + "postgres": { + "cpu_percent": 102.2, + "memory_mib": 145.9, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.14, + "memory_mib": 16.63, + "containers": 1 + } + } + }, + { + "t": 8.446, + "services": { + "api": { + "cpu_percent": 32.86, + "memory_mib": 34.023, + "containers": 4 + }, + "frontend": { + "cpu_percent": 0.0, + "memory_mib": 7.301, + "containers": 1 + }, + "load-balancer": { + "cpu_percent": 12.44, + "memory_mib": 18.0, + "containers": 1 + }, + "postgres": { + "cpu_percent": 51.92, + "memory_mib": 147.6, + "containers": 1 + }, + "redis": { + "cpu_percent": 0.18, + "memory_mib": 16.88, + "containers": 1 + } + } + } + ], + "resources": { + "api": { + "cpu_percent_avg": 55.2, + "cpu_percent_max": 70.26, + "memory_mib_avg": 33.669, + "memory_mib_max": 36.812, + "containers": 4 + }, + "frontend": { + "cpu_percent_avg": 0.83, + "cpu_percent_max": 3.29, + "memory_mib_avg": 7.333, + "memory_mib_max": 7.43, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 22.35, + "cpu_percent_max": 26.73, + "memory_mib_avg": 18.358, + "memory_mib_max": 18.99, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 95.573, + "cpu_percent_max": 121.18, + "memory_mib_avg": 141.25, + "memory_mib_max": 147.6, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.458, + "cpu_percent_max": 1.35, + "memory_mib_avg": 16.755, + "memory_mib_max": 16.88, + "containers": 1 + } + } + } + ], + "analysis": { + "saturation_point": { + "concurrency": 8, + "previous_concurrency": 4, + "rps_gain_percent": 8.03, + "p95_growth_ratio": 1.919 + }, + "observed_bottleneck": "postgres_cpu", + "baseline_resource_snapshot": { + "api": { + "cpu_percent_avg": 35.69, + "cpu_percent_max": 48.89, + "memory_mib_avg": 31.469, + "memory_mib_max": 32.855, + "containers": 4 + }, + "frontend": { + "cpu_percent_avg": 0.667, + "cpu_percent_max": 2.63, + "memory_mib_avg": 7.369, + "memory_mib_max": 7.551, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 15.595, + "cpu_percent_max": 23.31, + "memory_mib_avg": 17.947, + "memory_mib_max": 18.25, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 84.847, + "cpu_percent_max": 101.91, + "memory_mib_avg": 280.925, + "memory_mib_max": 290.1, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.873, + "cpu_percent_max": 2.97, + "memory_mib_avg": 17.598, + "memory_mib_max": 20.3, + "containers": 1 + } + }, + "improved_resource_snapshot": { + "api": { + "cpu_percent_avg": 55.2, + "cpu_percent_max": 70.26, + "memory_mib_avg": 33.669, + "memory_mib_max": 36.812, + "containers": 4 + }, + "frontend": { + "cpu_percent_avg": 0.83, + "cpu_percent_max": 3.29, + "memory_mib_avg": 7.333, + "memory_mib_max": 7.43, + "containers": 1 + }, + "load-balancer": { + "cpu_percent_avg": 22.35, + "cpu_percent_max": 26.73, + "memory_mib_avg": 18.358, + "memory_mib_max": 18.99, + "containers": 1 + }, + "postgres": { + "cpu_percent_avg": 95.573, + "cpu_percent_max": 121.18, + "memory_mib_avg": 141.25, + "memory_mib_max": 147.6, + "containers": 1 + }, + "redis": { + "cpu_percent_avg": 0.458, + "cpu_percent_max": 1.35, + "memory_mib_avg": 16.755, + "memory_mib_max": 16.88, + "containers": 1 + } + }, + "remaining_bottleneck_after_improvement": "load_balancer_cpu", + "improvement": { + "change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged", + "baseline_phase": "growth-r4", + "improved_phase": "improved-db-cpu-r4", + "rps_change_percent": 33.88, + "p95_change_percent": -14.12, + "confirmed": true + } + }, + "dataset": { + "before": { + "messages": 225626, + "database_size_bytes": 175406103 + }, + "after": { + "messages": 299524, + "database_size_bytes": 228793367 + }, + "warmup_successful_requests": 7749 + }, + "restoration": { + "attempted": true, + "succeeded": true, + "applied": { + "api": [ + { + "instance": "9c029f1ecdff", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 10 + }, + { + "instance": "bd53aadefab0", + "cpu_limit": 0.5, + "memory_limit_mib": 256.0, + "db_max_conns": 10 + } + ], + "postgres_cpu_limit": 1.0, + "postgres_memory_limit_mib": 512.0 + } + }, + "limitations": [ + "single Docker host: replicas do not prove host-level availability", + "closed-loop workload has coordinated omission", + "successful-response latency percentiles are reported separately from failures", + "one measured run per profile; host scheduling noise is possible" + ] +} diff --git a/evidence/grade5/timeseries.csv b/evidence/grade5/timeseries.csv new file mode 100644 index 00000000..6b425ada --- /dev/null +++ b/evidence/grade5/timeseries.csv @@ -0,0 +1,144 @@ +phase,second,successful,failed +saturation-c1,0,252,0 +saturation-c1,1,266,0 +saturation-c1,2,257,0 +saturation-c1,3,262,0 +saturation-c1,4,253,0 +saturation-c1,5,249,0 +saturation-c1,6,228,0 +saturation-c1,7,240,0 +saturation-c1,8,219,0 +saturation-c1,9,244,0 +saturation-c1,10,2,0 +saturation-c2,0,444,0 +saturation-c2,1,377,0 +saturation-c2,2,352,0 +saturation-c2,3,297,0 +saturation-c2,4,383,0 +saturation-c2,5,324,0 +saturation-c2,6,275,0 +saturation-c2,7,323,0 +saturation-c2,8,333,0 +saturation-c2,9,381,0 +saturation-c2,10,6,0 +saturation-c4,0,526,0 +saturation-c4,1,453,0 +saturation-c4,2,425,0 +saturation-c4,3,478,0 +saturation-c4,4,510,0 +saturation-c4,5,468,0 +saturation-c4,6,458,0 +saturation-c4,7,428,0 +saturation-c4,8,490,0 +saturation-c4,9,474,0 +saturation-c4,10,11,0 +saturation-c8,0,532,0 +saturation-c8,1,476,0 +saturation-c8,2,530,0 +saturation-c8,3,529,0 +saturation-c8,4,490,0 +saturation-c8,5,527,0 +saturation-c8,6,445,0 +saturation-c8,7,524,0 +saturation-c8,8,507,0 +saturation-c8,9,519,0 +saturation-c8,10,16,0 +saturation-c16,0,349,0 +saturation-c16,1,592,0 +saturation-c16,2,541,0 +saturation-c16,3,491,0 +saturation-c16,4,545,0 +saturation-c16,5,420,0 +saturation-c16,6,499,0 +saturation-c16,7,529,0 +saturation-c16,8,548,0 +saturation-c16,9,601,0 +saturation-c16,10,28,0 +saturation-c32,0,661,0 +saturation-c32,1,604,0 +saturation-c32,2,610,0 +saturation-c32,3,589,0 +saturation-c32,4,571,0 +saturation-c32,5,627,0 +saturation-c32,6,587,0 +saturation-c32,7,619,0 +saturation-c32,8,629,0 +saturation-c32,9,548,0 +saturation-c32,10,51,0 +fixed-r1,0,511,0 +fixed-r1,1,580,0 +fixed-r1,2,659,0 +fixed-r1,3,687,0 +fixed-r1,4,588,0 +fixed-r1,5,675,0 +fixed-r1,6,647,0 +fixed-r1,7,701,0 +fixed-r1,8,690,0 +fixed-r1,9,521,0 +fixed-r1,10,32,0 +fixed-r2,0,452,0 +fixed-r2,1,545,0 +fixed-r2,2,604,0 +fixed-r2,3,652,0 +fixed-r2,4,521,0 +fixed-r2,5,627,0 +fixed-r2,6,571,0 +fixed-r2,7,611,0 +fixed-r2,8,573,0 +fixed-r2,9,586,0 +fixed-r2,10,32,0 +fixed-r4,0,133,0 +fixed-r4,1,276,0 +fixed-r4,2,461,0 +fixed-r4,3,406,0 +fixed-r4,4,503,0 +fixed-r4,5,487,0 +fixed-r4,6,377,0 +fixed-r4,7,427,0 +fixed-r4,8,551,0 +fixed-r4,9,547,0 +fixed-r4,10,50,0 +growth-r1,0,527,0 +growth-r1,1,596,0 +growth-r1,2,604,0 +growth-r1,3,616,0 +growth-r1,4,516,0 +growth-r1,5,508,0 +growth-r1,6,606,0 +growth-r1,7,620,0 +growth-r1,8,628,0 +growth-r1,9,591,0 +growth-r1,10,32,0 +growth-r2,0,461,0 +growth-r2,1,662,0 +growth-r2,2,670,0 +growth-r2,3,655,0 +growth-r2,4,640,0 +growth-r2,5,495,0 +growth-r2,6,635,0 +growth-r2,7,626,0 +growth-r2,8,598,0 +growth-r2,9,609,0 +growth-r2,10,32,0 +growth-r4,0,106,0 +growth-r4,1,514,0 +growth-r4,2,587,0 +growth-r4,3,506,0 +growth-r4,4,423,0 +growth-r4,5,409,0 +growth-r4,6,459,0 +growth-r4,7,462,0 +growth-r4,8,618,0 +growth-r4,9,549,0 +growth-r4,10,55,0 +improved-db-cpu-r4,0,492,0 +improved-db-cpu-r4,1,583,0 +improved-db-cpu-r4,2,629,0 +improved-db-cpu-r4,3,681,0 +improved-db-cpu-r4,4,667,0 +improved-db-cpu-r4,5,632,0 +improved-db-cpu-r4,6,585,0 +improved-db-cpu-r4,7,677,0 +improved-db-cpu-r4,8,602,0 +improved-db-cpu-r4,9,629,0 +improved-db-cpu-r4,10,52,0 diff --git a/evidence/lab3-summary.md b/evidence/lab3-summary.md index ac792635..8501342e 100644 --- a/evidence/lab3-summary.md +++ b/evidence/lab3-summary.md @@ -1,6 +1,6 @@ # Lab 3 verification summary -Дата проверки: 2026-10-20. +Дата проверки: 2026-10-23. Файл содержит обезличенную сводку воспроизводимых проверок. Bearer tokens, пароли, ключи идемпотентности, UUID пользователей/чатов/сообщений и содержимое @@ -12,7 +12,8 @@ - Docker Engine 29.7.2; - Docker Compose 5.5.0; - две Compose-реплики сервиса `api`; -- единственный опубликованный endpoint: Nginx на `127.0.0.1:8080`; +- единственный опубликованный endpoint: HAProxy на `127.0.0.1:8080`; +- Nginx доступен только внутри Compose network и отдаёт статический React; - PostgreSQL и Redis доступны только внутри Compose network. ## Статические проверки @@ -42,6 +43,16 @@ shell syntax for hooks PASS - преподавательский `python3 tests/smoke.py` завершился строкой `PASS: lab3 public API scenario`. +## Readiness-aware routing + +`make test-readiness-routing` добавил в network namespace одной API-реплики +правило, запрещающее TCP-доступ к PostgreSQL, не останавливая сам процесс. Его +`/health/ready` вернул 503; HAProxy исключил backend за 2,861 с. После этого +40 запросов `/health/live` получили 200, а 20 корректных регистраций — 201; +не готовая реплика не получила ни одного запроса. После удаления fault-правила +она вернулась в ротацию. Машиночитаемый результат сохранён в +`evidence/readiness-routing.json`. + ## Failover API Команда: @@ -54,30 +65,40 @@ make test-failover ACTION_SCRIPT=./scripts/stop-one.sh ```text run 1: - stopped container: api-2 + observed instances before action: 2 + successful writes / failures: 141 / 0 + first recovery: 2.668 seconds +run 2, after restoring the desired two replicas: observed instances before action: 2 successful writes / failures: 142 / 0 - first recovery: 0.963 seconds -run 2, after restoring api-2: - stopped container: api-1 - observed instances before action: 2 - successful writes / failures: 143 / 0 - first recovery: 0.750 seconds + first recovery: 2.653 seconds both results: PASS ``` -После первого запуска реплика была восстановлена; один health-запрос сдвинул -очередь LB, чтобы следующий synthetic message выбрал второй target. Hook -сопоставил `TARGET_INSTANCE_ID` с hostname контейнера и в каждом запуске -остановил только выбранную API-реплику. После тестов желаемое число -реплик было восстановлено командой `docker compose up -d --wait api`. +После каждого запуска желаемое число реплик было восстановлено командой +`docker compose up -d --wait api`. Hook сопоставляет `TARGET_INSTANCE_ID` с +hostname контейнера и останавливает только выбранную API-реплику. Отдельный +`make grade4-experiment` целенаправленно остановил обе исходные реплики по +очереди: 7853/0 и 7970/0 успешных/ошибочных запросов, recovery 2,893/2,938 с. ## Persistence и идемпотентность - `make test-persistence ACTION_SCRIPT=./scripts/restart.sh`: PASS; после - рестарта PostgreSQL, Redis, всех API-реплик и Nginx прежние bearer session и + рестарта PostgreSQL, Redis, всех API-реплик, frontend и HAProxy прежние bearer session и созданные данные сохранились; - 32 конкурентных POST сообщения с одинаковыми semantic payload и - `Idempotency-Key` через Nginx вернули один Message ID; + `Idempotency-Key` через HAProxy вернули один Message ID; - прямой запрос PostgreSQL подтвердил ровно одну строку сообщения; - повтор с тем же scope/key и другим payload возвращает HTTP 409. + +## Негативные и browser-security проверки + +- сырой `Authorization: Bearer` с UTF-8 `é` вернул 401, Bearer challenge и + стандартный `Error`; процесс не завершился с panic/500; +- тело JSON из одного байта `FF` вернуло 400 `invalid_request`, а + `error.request_id` совпал с `X-Request-Id`; +- login установил host-only `HttpOnly; SameSite=Strict` cookie; +- cookie без Bearer на контрактном `/api/v1/users/me` дала 401, через + `/web-api/v1/users/me` — 200; +- logout через `/web-api` вернул 204 и инвалидировал серверную сессию; +- статический frontend вернул 200 и `Content-Security-Policy`. diff --git a/evidence/readiness-routing.json b/evidence/readiness-routing.json new file mode 100644 index 00000000..abb6ab7f --- /dev/null +++ b/evidence/readiness-routing.json @@ -0,0 +1,11 @@ +{ + "target_instance": "bd53aadefab0", + "detection_seconds": 2.861, + "health_requests": { + "200": 40 + }, + "registration_requests": { + "201": 20 + }, + "recovered": true +} diff --git a/infrastructure/haproxy/Dockerfile b/infrastructure/haproxy/Dockerfile new file mode 100644 index 00000000..a2befcec --- /dev/null +++ b/infrastructure/haproxy/Dockerfile @@ -0,0 +1,7 @@ +FROM haproxy:3.2-alpine@sha256:dfba86f031bd4c9039f10f8cc5de75991d1b6766d5460f0335b980408503baad + +COPY infrastructure/haproxy/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg + +USER haproxy + +EXPOSE 8080 diff --git a/infrastructure/haproxy/haproxy.cfg b/infrastructure/haproxy/haproxy.cfg new file mode 100644 index 00000000..3a4f34ab --- /dev/null +++ b/infrastructure/haproxy/haproxy.cfg @@ -0,0 +1,66 @@ +global + log stdout format raw local0 + maxconn 2048 + +defaults + log global + mode http + option httplog + timeout connect 1s + timeout client 30s + timeout server 30s + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold other 1s + hold refused 1s + hold nx 1s + hold timeout 1s + hold obsolete 1s + +frontend public + bind :8080 + + acl api_request path_beg /api/ /web-api/ /health/ + acl web_api path_beg /web-api/ + acl has_session req.cook(mtusi_session) -m found + acl message_create method POST + acl message_path path_reg ^/(api|web-api)/v1/chats/[^/]+/messages$ + + http-request del-header Authorization if web_api + http-request set-header Authorization "Bearer %[req.cook(mtusi_session)]" if web_api has_session + http-request del-header Cookie if api_request + http-request replace-path ^/web-api/(.*)$ /api/\1 if web_api + + use_backend message_backend if message_create message_path + use_backend api_backend if api_request + default_backend frontend_backend + +backend api_backend + balance leastconn + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none + server-template api 8 api:8080 + +backend message_backend + balance leastconn + option httpchk + option redispatch + option http-buffer-request + retries 3 + retry-on conn-failure empty-response response-timeout 502 503 504 + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api + http-check expect status 200 + default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none + server-template api 8 api:8080 + +backend frontend_backend + option httpchk GET / + http-check expect status 200 + server frontend frontend:8080 check inter 2s fall 2 rise 2 resolvers docker init-addr libc,none diff --git a/infrastructure/nginx/nginx.conf b/infrastructure/nginx/nginx.conf index 9132e623..ec07590b 100644 --- a/infrastructure/nginx/nginx.conf +++ b/infrastructure/nginx/nginx.conf @@ -1,24 +1,12 @@ worker_processes auto; events { - worker_connections 1024; + worker_connections 512; } http { include /etc/nginx/mime.types; default_type application/octet-stream; - resolver 127.0.0.11 valid=1s ipv6=off; - - map $cookie_mtusi_session $web_authorization { - default "Bearer $cookie_mtusi_session"; - "" ""; - } - - upstream api_backend { - zone api_backend 64k; - least_conn; - server api:8080 resolve max_fails=2 fail_timeout=2s; - } server { listen 8080; @@ -31,90 +19,6 @@ http { add_header X-Frame-Options "DENY" always; add_header Content-Security-Policy "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always; - # Message creation carries an Idempotency-Key. It is therefore safe to - # retry a POST on another replica when the selected backend is draining. - location ~ ^/api/v1/chats/[^/]+/messages$ { - proxy_pass http://api_backend; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Connection ""; - proxy_set_header Cookie ""; - proxy_connect_timeout 1s; - proxy_send_timeout 20s; - proxy_read_timeout 20s; - proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; - proxy_next_upstream_tries 3; - } - - location ~ ^/web-api/v1/chats/[^/]+/messages$ { - rewrite ^/web-api/(.*)$ /api/$1 break; - proxy_pass http://api_backend; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Connection ""; - proxy_set_header Authorization $web_authorization; - proxy_set_header Cookie ""; - proxy_connect_timeout 1s; - proxy_send_timeout 20s; - proxy_read_timeout 20s; - proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; - proxy_next_upstream_tries 3; - } - - location /api/ { - proxy_pass http://api_backend; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Connection ""; - proxy_set_header Cookie ""; - proxy_connect_timeout 1s; - proxy_send_timeout 20s; - proxy_read_timeout 20s; - proxy_next_upstream error timeout http_502 http_503 http_504; - proxy_next_upstream_tries 2; - } - - location /web-api/ { - rewrite ^/web-api/(.*)$ /api/$1 break; - proxy_pass http://api_backend; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Connection ""; - proxy_set_header Authorization $web_authorization; - proxy_set_header Cookie ""; - proxy_connect_timeout 1s; - proxy_send_timeout 20s; - proxy_read_timeout 20s; - proxy_next_upstream error timeout http_502 http_503 http_504; - proxy_next_upstream_tries 2; - } - - location /health/ { - proxy_pass http://api_backend; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header Connection ""; - proxy_connect_timeout 1s; - proxy_read_timeout 5s; - proxy_next_upstream error timeout http_502 http_503 http_504; - proxy_next_upstream_tries 2; - } - location /assets/ { try_files $uri =404; expires 1y; diff --git a/scripts/grade4_experiment.py b/scripts/grade4_experiment.py index 4d938472..c22eb68b 100755 --- a/scripts/grade4_experiment.py +++ b/scripts/grade4_experiment.py @@ -77,7 +77,7 @@ def memory_mib(raw): def service(name): - for candidate in ("load-balancer", "postgres", "redis", "api"): + for candidate in ("load-balancer", "frontend", "postgres", "redis", "api"): if f"-{candidate}-" in name: return candidate return "other" @@ -357,7 +357,7 @@ def main(): "action_after_seconds": args.action_after_seconds, "post_action_seconds": args.post_action_seconds, "warmup_requests": 12, - "workload": "closed-loop POST messages via Nginx; no automatic retries", + "workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries", }, "phases": phases, "idempotency": idem, @@ -371,7 +371,7 @@ def main(): "single Docker host", "closed-loop coordinated omission", "successful-response latency percentiles only", - "PostgreSQL and Nginx remain single points of failure", + "PostgreSQL and HAProxy remain single points of failure", ], } destination = ROOT / args.output diff --git a/scripts/grade5_experiment.py b/scripts/grade5_experiment.py new file mode 100755 index 00000000..925d8b69 --- /dev/null +++ b/scripts/grade5_experiment.py @@ -0,0 +1,577 @@ +#!/usr/bin/env python3 +"""Measure the price of horizontal API scaling for lab 3 grade 5.""" + +import argparse +from collections import Counter, defaultdict +import datetime as dt +import json +import math +import os +from pathlib import Path +import platform +import subprocess +import sys +import threading +import time + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def command(args, env=None, timeout=240, check=True): + result = subprocess.run( + [str(value) for value in args], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + if check and result.returncode: + detail = (result.stderr or result.stdout).strip().splitlines() + suffix = f": {detail[-1]}" if detail else "" + raise RuntimeError( + f"{' '.join(map(str, args[:4]))} exited {result.returncode}{suffix}" + ) + return result + + +def compose(env_file, *args, env=None, timeout=240, check=True): + return command( + ["docker", "compose", "--env-file", env_file, *args], + env=env, + timeout=timeout, + check=check, + ) + + +def inspect(container_id): + raw = command( + ["docker", "inspect", "--format", "{{json .}}", container_id] + ).stdout + return json.loads(raw) + + +def service_ids(env_file, service=None): + args = ["ps", "-q"] + if service: + args.append(service) + return [line for line in compose(env_file, *args).stdout.splitlines() if line] + + +def wait_healthy(env_file, service, expected=1, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + healthy = [] + for container_id in service_ids(env_file, service): + state = inspect(container_id)["State"] + if state.get("Health", {}).get("Status") == "healthy": + healthy.append(container_id) + if len(healthy) == expected: + return healthy + time.sleep(0.5) + raise RuntimeError(f"{service}: expected {expected} healthy containers") + + +def percentile(values, ratio): + if not values: + return None + ordered = sorted(values) + return round(ordered[max(0, math.ceil(ratio * len(ordered)) - 1)], 3) + + +def memory_mib(raw): + value = raw.split("/", 1)[0].strip() + units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3} + for suffix in ("GiB", "MiB", "KiB", "B"): + if value.endswith(suffix): + return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3) + return 0.0 + + +def profile_environment(profile): + env = dict(os.environ) + env.update({ + "API_REPLICAS": str(profile["replicas"]), + "API_CPU_LIMIT": f"{profile['api_cpu']:.2f}", + "API_MEMORY_LIMIT": profile["api_memory"], + "DB_MAX_CONNS": str(profile["db_max_conns"]), + "DB_MIN_CONNS": str(profile["db_min_conns"]), + "POSTGRES_CPU_LIMIT": f"{profile['postgres_cpu']:.2f}", + "POSTGRES_MEMORY_LIMIT": profile["postgres_memory"], + }) + return env + + +def configure_profile(env_file, profile): + env = profile_environment(profile) + compose( + env_file, + "up", "-d", "--no-deps", "postgres", + env=env, + timeout=180, + ) + wait_healthy(env_file, "postgres") + compose( + env_file, + "up", "-d", "--no-deps", + "--scale", f"api={profile['replicas']}", + "--force-recreate", "api", + env=env, + timeout=240, + ) + api = wait_healthy(env_file, "api", profile["replicas"]) + wait_healthy(env_file, "load-balancer") + time.sleep(1.5) + + expected_cpu = int(profile["api_cpu"] * 1_000_000_000) + expected_memory = parse_memory_limit(profile["api_memory"]) + applied = [] + for container_id in api: + data = inspect(container_id) + actual_env = dict(item.split("=", 1) for item in data["Config"]["Env"] if "=" in item) + host = data["HostConfig"] + if host.get("NanoCpus") != expected_cpu or host.get("Memory") != expected_memory: + raise RuntimeError("API container limits do not match the selected profile") + if actual_env.get("DB_MAX_CONNS") != str(profile["db_max_conns"]): + raise RuntimeError("API DB pool size does not match the selected profile") + applied.append({ + "instance": data["Config"]["Hostname"], + "cpu_limit": host.get("NanoCpus", 0) / 1_000_000_000, + "memory_limit_mib": round(host.get("Memory", 0) / 1024**2, 1), + "db_max_conns": int(actual_env["DB_MAX_CONNS"]), + }) + + postgres = inspect(service_ids(env_file, "postgres")[0]) + expected_pg_cpu = int(profile["postgres_cpu"] * 1_000_000_000) + if postgres["HostConfig"].get("NanoCpus") != expected_pg_cpu: + raise RuntimeError("PostgreSQL CPU limit does not match the selected profile") + return { + "api": sorted(applied, key=lambda item: item["instance"]), + "postgres_cpu_limit": postgres["HostConfig"].get("NanoCpus", 0) / 1_000_000_000, + "postgres_memory_limit_mib": round(postgres["HostConfig"].get("Memory", 0) / 1024**2, 1), + } + + +def parse_memory_limit(value): + suffix = value[-1].upper() + multiplier = {"K": 1024, "M": 1024**2, "G": 1024**3}[suffix] + return int(value[:-1]) * multiplier + + +def resource_targets(env_file): + targets = {} + for container_id in service_ids(env_file): + data = inspect(container_id) + service = data["Config"]["Labels"].get("com.docker.compose.service", "other") + if service in {"api", "postgres", "redis", "load-balancer", "frontend"}: + targets[container_id] = { + "service": service, + "name": data["Name"].lstrip("/"), + } + return targets + + +def stats_snapshot(targets, elapsed): + if not targets: + return {"t": round(elapsed, 3), "services": {}} + result = command( + ["docker", "stats", "--no-stream", "--format", "{{json .}}", *targets], + timeout=15, + check=False, + ) + grouped = defaultdict(lambda: {"cpu_percent": 0.0, "memory_mib": 0.0, "containers": 0}) + for line in result.stdout.splitlines(): + try: + row = json.loads(line) + container_id = row.get("ID", "") + match = next((key for key in targets if key.startswith(container_id)), None) + if not match: + continue + values = grouped[targets[match]["service"]] + values["cpu_percent"] += float(row.get("CPUPerc", "0%").rstrip("%")) + values["memory_mib"] += memory_mib(row.get("MemUsage", "0B / 0B")) + values["containers"] += 1 + except (ValueError, json.JSONDecodeError): + continue + return { + "t": round(elapsed, 3), + "services": { + name: { + "cpu_percent": round(values["cpu_percent"], 3), + "memory_mib": round(values["memory_mib"], 3), + "containers": values["containers"], + } + for name, values in sorted(grouped.items()) + }, + } + + +def summarize_resources(snapshots): + grouped = defaultdict(lambda: {"cpu": [], "memory": [], "containers": []}) + for snapshot in snapshots: + for service, values in snapshot["services"].items(): + grouped[service]["cpu"].append(values["cpu_percent"]) + grouped[service]["memory"].append(values["memory_mib"]) + grouped[service]["containers"].append(values["containers"]) + return { + service: { + "cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3), + "cpu_percent_max": round(max(values["cpu"]), 3), + "memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3), + "memory_mib_max": round(max(values["memory"]), 3), + "containers": max(values["containers"]), + } + for service, values in sorted(grouped.items()) + if values["cpu"] + } + + +def run_phase(name, kind, actor, chat, base_url, duration, concurrency, env_file, profile): + records = [] + snapshots = [] + lock = threading.Lock() + stop = threading.Event() + barrier = threading.Barrier(concurrency + 1) + targets = resource_targets(env_file) + started = time.monotonic() + + def worker(worker_id): + client = Client(base_url, timeout=5) + index = 0 + barrier.wait() + while not stop.is_set(): + request_started = time.monotonic() + try: + response = client.message( + actor, + chat, + f"grade5-{name}-{worker_id}-{index}", + ) + row = { + "t": round(time.monotonic() - started, 3), + "ok": True, + "latency_ms": round((time.monotonic() - request_started) * 1000, 3), + "instance": response["headers"].get("X-Instance-Id", ""), + } + except Failure as exc: + row = { + "t": round(time.monotonic() - started, 3), + "ok": False, + "latency_ms": round((time.monotonic() - request_started) * 1000, 3), + "error": str(exc), + } + with lock: + records.append(row) + index += 1 + + def sampler(): + while not stop.is_set(): + snapshots.append(stats_snapshot(targets, time.monotonic() - started)) + stop.wait(1) + + workers = [ + threading.Thread(target=worker, args=(index,), daemon=True) + for index in range(concurrency) + ] + for thread in workers: + thread.start() + stats_thread = threading.Thread(target=sampler, daemon=True) + stats_thread.start() + started = time.monotonic() + barrier.wait() + time.sleep(duration) + stop.set() + for thread in workers: + thread.join(10) + stats_thread.join(15) + + elapsed = time.monotonic() - started + successes = [row for row in records if row["ok"]] + failures = [row for row in records if not row["ok"]] + latencies = [row["latency_ms"] for row in successes] + buckets = defaultdict(lambda: {"successful": 0, "failed": 0}) + for row in records: + key = str(max(0, int(row["t"]))) + buckets[key]["successful" if row["ok"] else "failed"] += 1 + total = len(records) + return { + "name": name, + "kind": kind, + "profile": profile["name"], + "replicas": profile["replicas"], + "concurrency": concurrency, + "elapsed_seconds": round(elapsed, 3), + "successful_requests": len(successes), + "failed_requests": len(failures), + "error_ratio": round(len(failures) / total, 6) if total else None, + "errors": dict(sorted(Counter(row["error"] for row in failures).items())), + "successful_rps": round(len(successes) / elapsed, 3), + "successful_latency_ms": { + "p50": percentile(latencies, 0.50), + "p95": percentile(latencies, 0.95), + "p99": percentile(latencies, 0.99), + }, + "instances": sorted({row["instance"] for row in successes if row.get("instance")}), + "per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))), + "resource_samples": snapshots, + "resources": summarize_resources(snapshots), + } + + +def dataset(env_file): + sql = ( + "select count(*)::text || '|' || pg_database_size(current_database())::text " + "from messages" + ) + result = compose( + env_file, + "exec", "-T", "postgres", "sh", "-c", + 'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"', + "sh", sql, + ) + count, size = result.stdout.strip().split("|", 1) + return {"messages": int(count), "database_size_bytes": int(size)} + + +def host_environment(env_file): + mem_total_kib = 0 + with open("/proc/meminfo", encoding="utf-8") as source: + for line in source: + if line.startswith("MemTotal:"): + mem_total_kib = int(line.split()[1]) + break + images = compose(env_file, "config", "--images").stdout.splitlines() + return { + "utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(), + "git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(), + "working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()), + "logical_cpu_count": os.cpu_count(), + "host_memory_mib": round(mem_total_kib / 1024, 1), + "kernel": platform.release(), + "python": platform.python_version(), + "docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(), + "docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(), + "images": images, + } + + +def analyze(phases): + sweep = sorted( + (phase for phase in phases if phase["kind"] == "saturation"), + key=lambda phase: phase["concurrency"], + ) + saturation = None + previous = None + for phase in sweep: + if previous: + gain = (phase["successful_rps"] - previous["successful_rps"]) / max(previous["successful_rps"], 0.001) + latency_growth = phase["successful_latency_ms"]["p95"] / max(previous["successful_latency_ms"]["p95"], 0.001) + if saturation is None and gain < 0.10 and latency_growth >= 1.50: + saturation = { + "concurrency": phase["concurrency"], + "previous_concurrency": previous["concurrency"], + "rps_gain_percent": round(gain * 100, 2), + "p95_growth_ratio": round(latency_growth, 3), + } + previous = phase + + baseline = next(phase for phase in phases if phase["name"] == "growth-r4") + improved = next(phase for phase in phases if phase["name"] == "improved-db-cpu-r4") + rps_change = (improved["successful_rps"] - baseline["successful_rps"]) / max(baseline["successful_rps"], 0.001) * 100 + p95_change = (improved["successful_latency_ms"]["p95"] - baseline["successful_latency_ms"]["p95"]) / max(baseline["successful_latency_ms"]["p95"], 0.001) * 100 + confirmed = ( + rps_change >= 5 + and improved["error_ratio"] <= baseline["error_ratio"] + 0.001 + ) or p95_change <= -10 + + resources = baseline["resources"] + pg = resources.get("postgres", {}) + lb = resources.get("load-balancer", {}) + bottleneck = "undetermined" + if pg.get("cpu_percent_avg", 0) >= 70 and pg.get("cpu_percent_max", 0) >= 95: + bottleneck = "postgres_cpu" + elif lb.get("cpu_percent_avg", 0) >= 20 and lb.get("cpu_percent_max", 0) >= 24: + bottleneck = "load_balancer_cpu" + improved_resources = improved["resources"] + improved_lb = improved_resources.get("load-balancer", {}) + remaining_bottleneck = "undetermined" + if improved_lb.get("cpu_percent_avg", 0) >= 20 and improved_lb.get("cpu_percent_max", 0) >= 24: + remaining_bottleneck = "load_balancer_cpu" + return { + "saturation_point": saturation, + "observed_bottleneck": bottleneck, + "baseline_resource_snapshot": resources, + "improved_resource_snapshot": improved_resources, + "remaining_bottleneck_after_improvement": remaining_bottleneck, + "improvement": { + "change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged", + "baseline_phase": baseline["name"], + "improved_phase": improved["name"], + "rps_change_percent": round(rps_change, 2), + "p95_change_percent": round(p95_change, 2), + "confirmed": confirmed, + }, + } + + +def profiles(): + base = {"postgres_cpu": 1.0, "postgres_memory": "512M"} + return { + "fixed-r1": {**base, "name": "fixed-r1", "replicas": 1, "api_cpu": 1.0, "api_memory": "512M", "db_max_conns": 16, "db_min_conns": 2}, + "fixed-r2": {**base, "name": "fixed-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1}, + "fixed-r4": {**base, "name": "fixed-r4", "replicas": 4, "api_cpu": 0.25, "api_memory": "128M", "db_max_conns": 4, "db_min_conns": 1}, + "growth-r1": {**base, "name": "growth-r1", "replicas": 1, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1}, + "growth-r2": {**base, "name": "growth-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1}, + "growth-r4": {**base, "name": "growth-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1}, + "improved-db-cpu-r4": {**base, "name": "improved-db-cpu-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1, "postgres_cpu": 2.0}, + } + + +def run_warmup(actor, chat, args, profile): + phase = run_phase( + f"warmup-{profile['name']}", + "warmup", + actor, + chat, + args.base_url, + args.warmup_seconds, + min(4, args.comparison_concurrency), + args.env_file, + profile, + ) + if phase["failed_requests"]: + raise RuntimeError(f"warmup failed for {profile['name']}") + return phase["successful_requests"] + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--env-file", default=".env") + parser.add_argument("--duration", type=float, default=10) + parser.add_argument("--warmup-seconds", type=float, default=2) + parser.add_argument("--comparison-concurrency", type=int, default=32) + parser.add_argument("--sweep", default="1,2,4,8,16,32") + parser.add_argument("--output", default="evidence/grade5/results.json") + args = parser.parse_args() + sweep = [int(value) for value in args.sweep.split(",")] + if args.duration <= 0 or args.warmup_seconds <= 0: + raise SystemExit("durations must be positive") + if not sweep or min(sweep) < 1 or max(sweep + [args.comparison_concurrency]) > 64: + raise SystemExit("concurrency must be 1..64") + + all_profiles = profiles() + restore = { + "name": "restore-default", + "replicas": 2, + "api_cpu": 0.5, + "api_memory": "256M", + "db_max_conns": 10, + "db_min_conns": 2, + "postgres_cpu": 1.0, + "postgres_memory": "512M", + } + phases = [] + applied = {} + warmup_requests = 0 + restoration = {"attempted": False, "succeeded": False} + environment = host_environment(args.env_file) + start_dataset = dataset(args.env_file) + try: + client = Client(args.base_url, timeout=8) + alice, _, _, chat = client.bootstrap() + + saturation_profile = all_profiles["growth-r2"] + applied[saturation_profile["name"]] = configure_profile(args.env_file, saturation_profile) + warmup_requests += run_warmup(alice, chat, args, saturation_profile) + for concurrency in sweep: + print(f"saturation: concurrency={concurrency}", flush=True) + phases.append(run_phase( + f"saturation-c{concurrency}", "saturation", alice, chat, + args.base_url, args.duration, concurrency, args.env_file, + saturation_profile, + )) + + for profile_name in ( + "fixed-r1", "fixed-r2", "fixed-r4", + "growth-r1", "growth-r2", "growth-r4", + "improved-db-cpu-r4", + ): + profile = all_profiles[profile_name] + print(f"profile: {profile_name}", flush=True) + applied[profile_name] = configure_profile(args.env_file, profile) + warmup_requests += run_warmup(alice, chat, args, profile) + kind = "improvement" if profile_name.startswith("improved") else profile_name.split("-", 1)[0] + phases.append(run_phase( + profile_name, kind, alice, chat, args.base_url, + args.duration, args.comparison_concurrency, args.env_file, + profile, + )) + end_dataset = dataset(args.env_file) + finally: + restoration["attempted"] = True + try: + restoration["applied"] = configure_profile(args.env_file, restore) + restoration["succeeded"] = True + except Exception as exc: + restoration["error"] = f"{type(exc).__name__}: {exc}" + + if not restoration["succeeded"]: + raise RuntimeError("experiment completed but default Compose profile was not restored") + + report = { + "schema_version": 1, + "environment": environment, + "hypothesis": ( + "API replicas stop improving throughput when shared PostgreSQL CPU saturates; " + "adding PostgreSQL CPU should improve the unchanged four-replica profile." + ), + "methodology": { + "workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries", + "duration_seconds_per_phase": args.duration, + "warmup_seconds_per_profile": args.warmup_seconds, + "comparison_concurrency": args.comparison_concurrency, + "saturation_concurrency": sweep, + "fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections", + "growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica", + "improvement": "only PostgreSQL CPU changes from 1.00 to 2.00", + "saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step", + "cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer", + }, + "profiles": all_profiles, + "applied_limits": applied, + "phases": phases, + "analysis": analyze(phases), + "dataset": { + "before": start_dataset, + "after": end_dataset, + "warmup_successful_requests": warmup_requests, + }, + "restoration": restoration, + "limitations": [ + "single Docker host: replicas do not prove host-level availability", + "closed-loop workload has coordinated omission", + "successful-response latency percentiles are reported separately from failures", + "one measured run per profile; host scheduling noise is possible", + ], + } + destination = ROOT / args.output + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + try: + displayed_output = destination.relative_to(ROOT) + except ValueError: + displayed_output = destination + print(json.dumps({ + "output": str(displayed_output), + "analysis": report["analysis"], + "restoration": restoration["succeeded"], + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_grade4.py b/scripts/render_grade4.py index 9c62886d..e510ffdf 100755 --- a/scripts/render_grade4.py +++ b/scripts/render_grade4.py @@ -148,7 +148,7 @@ def summary(report, output): "## Method", "", f"- concurrency: {method['concurrency']};", f"- baseline duration: {method['baseline_seconds']} s;", - "- load: closed-loop POST messages through Nginx;", + "- load: closed-loop POST messages through the public HAProxy gateway;", "- failover: both original API containers stopped and restored in turn;", "- rolling update: one surge replica, old replicas replaced one by one;", "- percentiles cover successful responses; failures are separate.", "", @@ -178,7 +178,7 @@ def summary(report, output): "![Resources](resources.svg)", "", "## Scope", "", "Single-host process failure and rolling replacement only. PostgreSQL " - "and Nginx remain single points of failure.", "", + "and HAProxy remain single points of failure.", "", ] output.write_text("\n".join(lines)) diff --git a/scripts/render_grade5.py b/scripts/render_grade5.py new file mode 100755 index 00000000..60014c30 --- /dev/null +++ b/scripts/render_grade5.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Render grade-5 scaling evidence as SVG, CSV and Markdown.""" + +import argparse +import csv +import html +import json +from pathlib import Path + + +STYLE = """ +text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; } +.title { font-size: 26px; font-weight: 650; } +.subtitle { font-size: 14px; fill: #aeb2b8; } +.label { font-size: 14px; fill: #c8cbd0; } +.small { font-size: 12px; fill: #a2a5aa; } +.grid { stroke: #34373d; stroke-width: 1; } +.axis { stroke: #747981; stroke-width: 1.5; } +.line { fill: none; stroke: #f4f5f6; stroke-width: 3; } +.line2 { fill: none; stroke: #92979f; stroke-width: 3; } +.point { fill: #f4f5f6; } +.point2 { fill: #92979f; } +.bar { fill: #d7d9dc; } +.bar2 { fill: #777c84; } +.bar3 { fill: #4c5057; } +.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; } +""" + + +def svg(title, subtitle, body, width=1120, height=620): + return f""" + + +{html.escape(title)} +{html.escape(subtitle)} +{body} + +""" + + +def grid(maximum, left, top, width, height, suffix=""): + maximum = maximum or 1 + parts = [] + for step in range(6): + value = maximum * step / 5 + y = top + height - height * step / 5 + parts.append(f'') + parts.append(f'{value:.0f}{suffix}') + return parts + + +def phase_map(report): + return {phase["name"]: phase for phase in report["phases"]} + + +def saturation_graph(report, output): + phases = sorted( + (phase for phase in report["phases"] if phase["kind"] == "saturation"), + key=lambda phase: phase["concurrency"], + ) + left, top, width, height = 90, 110, 940, 390 + max_rps = max(phase["successful_rps"] for phase in phases) * 1.1 + max_p95 = max(phase["successful_latency_ms"]["p95"] for phase in phases) * 1.1 + parts = grid(max_rps, left, top, width, height) + points_rps, points_p95 = [], [] + for index, phase in enumerate(phases): + x = left + width * index / max(1, len(phases) - 1) + y_rps = top + height - height * phase["successful_rps"] / max_rps + y_p95 = top + height - height * phase["successful_latency_ms"]["p95"] / max_p95 + points_rps.append(f"{x:.1f},{y_rps:.1f}") + points_p95.append(f"{x:.1f},{y_p95:.1f}") + parts.append(f'') + parts.append(f'') + parts.append(f'{phase["concurrency"]}') + parts.append(f'{phase["successful_rps"]:.0f}') + parts.append(f'') + parts.append(f'') + parts.append(f'Concurrency') + parts.append(f'white: RPS · gray: p95 normalized') + parts.append(f'Successful requests per second') + output.write_text(svg( + "Saturation curve", + "Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU", + "\n".join(parts), + ), encoding="utf-8") + + +def comparison_graph(report, kind, output): + phases = sorted( + (phase for phase in report["phases"] if phase["kind"] == kind), + key=lambda phase: phase["replicas"], + ) + left, top, width, height = 90, 110, 940, 370 + max_rps = max(phase["successful_rps"] for phase in phases) * 1.15 + parts = grid(max_rps, left, top, width, height) + slot = width / len(phases) + for index, phase in enumerate(phases): + bar_width = min(150, slot * 0.52) + x = left + index * slot + (slot - bar_width) / 2 + bar_height = height * phase["successful_rps"] / max_rps + y = top + height - bar_height + parts.append(f'') + parts.append(f'{phase["successful_rps"]:.1f} RPS') + parts.append(f'{phase["replicas"]} replica(s)') + parts.append(f'p95 {phase["successful_latency_ms"]["p95"]:.1f} ms') + parts.append(f'errors {phase["error_ratio"]*100:.3f}%') + if kind == "fixed": + title = "Fixed total API budget" + subtitle = "1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas" + else: + title = "Growing API budget" + subtitle = "Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU" + output.write_text(svg(title, subtitle, "\n".join(parts)), encoding="utf-8") + + +def bottleneck_graph(report, output): + phases = sorted( + (phase for phase in report["phases"] if phase["kind"] == "growth"), + key=lambda phase: phase["replicas"], + ) + services = ("api", "postgres", "load-balancer", "frontend", "redis") + left, top, width, height = 90, 110, 940, 370 + maximum = max( + phase["resources"].get(service, {}).get("cpu_percent_avg", 0) + for phase in phases for service in services + ) * 1.15 or 1 + parts = grid(maximum, left, top, width, height) + slot = width / len(phases) + colors = ("bar", "bar2", "bar3", "bar3", "bar3") + for phase_index, phase in enumerate(phases): + group_left = left + phase_index * slot + bar_width = min(48, slot / 6) + for service_index, service in enumerate(services): + value = phase["resources"].get(service, {}).get("cpu_percent_avg", 0) + x = group_left + slot * 0.12 + service_index * (bar_width + 9) + bar_height = height * value / maximum + y = top + height - bar_height + parts.append(f'') + parts.append(f'{value:.0f}') + parts.append(f'{service[:3]}') + parts.append(f'{phase["replicas"]} replica(s)') + parts.append(f'Average aggregate CPU, %') + output.write_text(svg( + "Where scaling stops", + "Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis)", + "\n".join(parts), + ), encoding="utf-8") + + +def improvement_graph(report, output): + phases = phase_map(report) + baseline = phases["growth-r4"] + improved = phases["improved-db-cpu-r4"] + pair = [baseline, improved] + left, top, panel_width, height = 90, 120, 390, 340 + parts = [] + panels = [ + ("Successful RPS", "successful_rps", left), + ("p95 latency, ms", "p95", 630), + ] + for title, metric, panel_left in panels: + values = [ + phase["successful_rps"] if metric == "successful_rps" else phase["successful_latency_ms"]["p95"] + for phase in pair + ] + maximum = max(values) * 1.2 or 1 + parts.append(f'{title}') + for index, (phase, value) in enumerate(zip(pair, values)): + bar_width = 120 + x = panel_left + 35 + index * 185 + bar_height = height * value / maximum + y = top + height - bar_height + parts.append(f'') + parts.append(f'{value:.1f}') + parts.append(f'PG {1 if index == 0 else 2} CPU') + analysis = report["analysis"]["improvement"] + parts.append(f'RPS change: {analysis["rps_change_percent"]:+.2f}% · p95 change: {analysis["p95_change_percent"]:+.2f}% · confirmed: {str(analysis["confirmed"]).lower()}') + output.write_text(svg( + "Confirmed improvement", + "Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2", + "\n".join(parts), + ), encoding="utf-8") + + +def write_csv(report, destination): + with (destination / "measurements.csv").open("w", newline="", encoding="utf-8") as stream: + fields = [ + "phase", "kind", "replicas", "concurrency", "elapsed_seconds", + "successful_requests", "failed_requests", "error_ratio", "successful_rps", + "p50_ms", "p95_ms", "p99_ms", "api_cpu_avg", "postgres_cpu_avg", + "load_balancer_cpu_avg", "frontend_cpu_avg", "redis_cpu_avg", + ] + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + for phase in report["phases"]: + writer.writerow({ + "phase": phase["name"], + "kind": phase["kind"], + "replicas": phase["replicas"], + "concurrency": phase["concurrency"], + "elapsed_seconds": phase["elapsed_seconds"], + "successful_requests": phase["successful_requests"], + "failed_requests": phase["failed_requests"], + "error_ratio": phase["error_ratio"], + "successful_rps": phase["successful_rps"], + "p50_ms": phase["successful_latency_ms"]["p50"], + "p95_ms": phase["successful_latency_ms"]["p95"], + "p99_ms": phase["successful_latency_ms"]["p99"], + "api_cpu_avg": phase["resources"].get("api", {}).get("cpu_percent_avg"), + "postgres_cpu_avg": phase["resources"].get("postgres", {}).get("cpu_percent_avg"), + "load_balancer_cpu_avg": phase["resources"].get("load-balancer", {}).get("cpu_percent_avg"), + "frontend_cpu_avg": phase["resources"].get("frontend", {}).get("cpu_percent_avg"), + "redis_cpu_avg": phase["resources"].get("redis", {}).get("cpu_percent_avg"), + }) + + with (destination / "timeseries.csv").open("w", newline="", encoding="utf-8") as stream: + fields = ["phase", "second", "successful", "failed"] + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + for phase in report["phases"]: + for second, counts in phase["per_second"].items(): + writer.writerow({"phase": phase["name"], "second": second, **counts}) + + with (destination / "resources.csv").open("w", newline="", encoding="utf-8") as stream: + fields = ["phase", "t", "service", "containers", "cpu_percent", "memory_mib"] + writer = csv.DictWriter(stream, fieldnames=fields) + writer.writeheader() + for phase in report["phases"]: + for snapshot in phase["resource_samples"]: + for service, values in snapshot["services"].items(): + writer.writerow({ + "phase": phase["name"], "t": snapshot["t"], "service": service, + **values, + }) + + +def markdown_summary(report, output): + method = report["methodology"] + analysis = report["analysis"] + improvement = analysis["improvement"] + lines = [ + "# Lab 3 grade-5 scaling experiment", "", + f"Generated: {report['environment']['utc_started_at']}.", "", + "## Hypothesis", "", report["hypothesis"], "", + "## Reproduction", "", + f"- workload: {method['workload']};", + f"- phase duration: {method['duration_seconds_per_phase']} s;", + f"- comparison concurrency: {method['comparison_concurrency']};", + f"- saturation sweep: {method['saturation_concurrency']};", + f"- fixed budget: {method['fixed_total_budget']};", + f"- growth series: {method['growth_budget']};", + f"- saturation rule: {method['saturation_rule']};", + f"- host: {report['environment']['logical_cpu_count']} logical CPU, {report['environment']['host_memory_mib']} MiB RAM;", + f"- Docker {report['environment']['docker_engine']}, Compose {report['environment']['docker_compose']};", + "- command: `make grade5-experiment`; warmup runs before every changed profile.", "", + "## Measurements", "", + "| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + for phase in report["phases"]: + latency = phase["successful_latency_ms"] + resources = phase["resources"] + lines.append( + f"| {phase['name']} | {phase['replicas']} | {phase['concurrency']} | " + f"{phase['successful_rps']} | {phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | " + f"{latency['p50']} | {latency['p95']} | {latency['p99']} | " + f"{resources.get('postgres', {}).get('cpu_percent_avg', 'n/a')} | " + f"{resources.get('api', {}).get('cpu_percent_avg', 'n/a')} |" + ) + saturation = analysis["saturation_point"] + lines += ["", "## Conclusion", ""] + if saturation: + lines.append( + f"The first saturation signal appears at concurrency {saturation['concurrency']}: " + f"RPS changes by {saturation['rps_gain_percent']:+.2f}% while p95 changes by " + f"{saturation['p95_growth_ratio']:.2f}x versus concurrency {saturation['previous_concurrency']}." + ) + else: + lines.append("The configured sweep did not locate a saturation point by the declared rule.") + lines += [ + "", + f"Observed bottleneck classification: `{analysis['observed_bottleneck']}`.", "", + f"The PostgreSQL CPU change produced {improvement['rps_change_percent']:+.2f}% RPS and " + f"{improvement['p95_change_percent']:+.2f}% p95; confirmed improvement: " + f"`{str(improvement['confirmed']).lower()}`.", "", + f"After the improvement the remaining bottleneck classification is " + f"`{analysis['remaining_bottleneck_after_improvement']}`.", "", + "## Graphs", "", + "![Saturation](01-saturation.svg)", "", + "![Fixed budget](02-fixed-total.svg)", "", + "![Growing budget](03-resource-growth.svg)", "", + "![Bottleneck](04-bottleneck.svg)", "", + "![Improvement](05-improvement.svg)", "", + "## Raw data and limits", "", + "Raw aggregates are in `measurements.csv`; one-second request buckets are in " + "`timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and " + "applied limits are in `results.json`.", "", + ] + lines.extend(f"- {item}" for item in report["limitations"]) + lines.append("") + output.write_text("\n".join(lines), encoding="utf-8") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("results", nargs="?", default="evidence/grade5/results.json") + args = parser.parse_args() + source = Path(args.results) + report = json.loads(source.read_text(encoding="utf-8")) + destination = source.parent + saturation_graph(report, destination / "01-saturation.svg") + comparison_graph(report, "fixed", destination / "02-fixed-total.svg") + comparison_graph(report, "growth", destination / "03-resource-growth.svg") + bottleneck_graph(report, destination / "04-bottleneck.svg") + improvement_graph(report, destination / "05-improvement.svg") + write_csv(report, destination) + markdown_summary(report, destination / "README.md") + print(f"rendered grade-5 evidence in {destination}") + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh index 80ea344d..fd04a052 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -19,6 +19,10 @@ compose up -d --no-deps api compose restart api compose up -d --wait --no-deps api -# Публичная точка входа перезапускается последней, когда API уже готовы. +# Статический frontend возвращается до публичной точки входа. +compose restart frontend +compose up -d --wait --no-deps frontend + +# Публичная точка входа перезапускается последней, когда API и frontend готовы. compose restart load-balancer compose up -d --wait --no-deps load-balancer diff --git a/scripts/test_readiness_routing.py b/scripts/test_readiness_routing.py new file mode 100755 index 00000000..257b3246 --- /dev/null +++ b/scripts/test_readiness_routing.py @@ -0,0 +1,198 @@ +#!/usr/bin/env python3 +"""Prove that the gateway excludes an API replica which loses readiness.""" + +import argparse +from collections import Counter +import http.client +import json +import os +from pathlib import Path +import subprocess +import time +import uuid + + +ROOT = Path(__file__).resolve().parents[1] +FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f" + + +def command(args, timeout=180, check=True): + result = subprocess.run( + [str(value) for value in args], + cwd=ROOT, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + if check and result.returncode: + detail = (result.stderr or result.stdout).strip().splitlines() + suffix = f": {detail[-1]}" if detail else "" + raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}") + return result + + +def compose(env_file, *args): + return command(["docker", "compose", "--env-file", env_file, *args]) + + +def inspect(container_id): + return json.loads(command([ + "docker", "inspect", "--format", "{{json .}}", container_id, + ]).stdout) + + +def call(host, port, method, path, body=None, headers=None, timeout=5): + connection = http.client.HTTPConnection(host, port, timeout=timeout) + connection.request(method, path, body=body, headers=headers or {}) + response = connection.getresponse() + payload = response.read() + result = { + "status": response.status, + "instance": response.getheader("X-Instance-Id", ""), + "body": payload, + } + connection.close() + return result + + +def observe_instances(base_host, base_port, requests=16): + instances = Counter() + statuses = Counter() + for _ in range(requests): + result = call(base_host, base_port, "GET", "/health/live") + statuses[result["status"]] += 1 + instances[result["instance"]] += 1 + return statuses, instances + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--env-file", default=".env") + parser.add_argument("--host", default="localhost") + parser.add_argument("--port", type=int, default=8080) + parser.add_argument("--output", default="") + args = parser.parse_args() + + api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line] + if len(api_ids) != 2: + raise RuntimeError("readiness routing test requires exactly two API replicas") + target_id = api_ids[0] + target = inspect(target_id) + target_instance = target["Config"]["Hostname"] + target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"] + + postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip() + postgres = inspect(postgres_id) + postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"] + + before_statuses, before_instances = observe_instances(args.host, args.port, 40) + if before_statuses != {200: 40} or len(before_instances) != 2: + raise RuntimeError("both API replicas must serve traffic before fault injection") + + helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}" + rule = [ + "iptables", "-D", "OUTPUT", "-d", postgres_ip, + "-p", "tcp", "--dport", "5432", "-j", "REJECT", + "--reject-with", "tcp-reset", + ] + detection_started = time.monotonic() + try: + script = ( + "apk add --no-cache iptables >/dev/null && " + f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 " + "-j REJECT --reject-with tcp-reset && " + "touch /tmp/fault-ready && sleep 180" + ) + command([ + "docker", "run", "-d", "--name", helper, + "--network", f"container:{target_id}", + "--cap-add", "NET_ADMIN", FAULT_IMAGE, + "sh", "-c", script, + ]) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + ready = command( + ["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"], + check=False, + ) + if ready.returncode == 0: + break + time.sleep(0.2) + else: + raise RuntimeError("fault helper did not become ready") + + direct = call(target_ip, 8080, "GET", "/health/ready") + if direct["status"] != 503: + raise RuntimeError(f"target readiness is {direct['status']}, want 503") + + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + statuses, instances = observe_instances(args.host, args.port, 16) + if statuses == {200: 16} and target_instance not in instances: + break + time.sleep(0.25) + else: + raise RuntimeError("gateway did not exclude the unready API within 10 seconds") + detection_seconds = time.monotonic() - detection_started + + health_statuses, health_instances = observe_instances(args.host, args.port, 40) + registration_statuses = Counter() + registration_instances = Counter() + for _ in range(20): + body = json.dumps({ + "username": "readiness_" + uuid.uuid4().hex[:16], + "display_name": "Readiness audit", + "password": "T3st-" + uuid.uuid4().hex, + }).encode() + response = call( + args.host, + args.port, + "POST", + "/api/v1/users", + body=body, + headers={"Content-Type": "application/json"}, + timeout=8, + ) + registration_statuses[response["status"]] += 1 + registration_instances[response["instance"]] += 1 + + if health_statuses != {200: 40} or target_instance in health_instances: + raise RuntimeError("unready API received liveness traffic through gateway") + if registration_statuses != {201: 20} or target_instance in registration_instances: + raise RuntimeError("unready API received registration traffic through gateway") + finally: + command(["docker", "exec", helper, *rule], check=False) + command(["docker", "rm", "-f", helper], check=False) + + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + direct = call(target_ip, 8080, "GET", "/health/ready") + if direct["status"] == 200: + _, instances = observe_instances(args.host, args.port, 24) + if target_instance in instances: + break + time.sleep(0.25) + else: + raise RuntimeError("recovered API did not return to gateway rotation") + + result = { + "target_instance": target_instance, + "detection_seconds": round(detection_seconds, 3), + "health_requests": dict(health_statuses), + "registration_requests": dict(registration_statuses), + "recovered": True, + } + rendered = json.dumps(result, ensure_ascii=False, indent=2) + if args.output: + output = Path(args.output) + if not output.is_absolute(): + output = ROOT / output + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(rendered + "\n", encoding="utf-8") + print(rendered) + print("PASS: readiness-aware gateway excluded and restored one API replica") + + +if __name__ == "__main__": + main() diff --git a/student_tests/auth_test.go b/student_tests/auth_test.go index 6b5e6e31..08e43ff6 100644 --- a/student_tests/auth_test.go +++ b/student_tests/auth_test.go @@ -209,6 +209,11 @@ func TestSessionAuthenticationLifecycle(t *testing.T) { authorization: "Basic " + base64.StdEncoding.EncodeToString([]byte(alice.Username+":"+testPassword)), wantStatus: http.StatusUnauthorized, }, + { + name: "non-ASCII Bearer is rejected without panic", + authorization: "Bearer é", + wantStatus: http.StatusUnauthorized, + }, { name: "Bearer identity wins over X-User-Id", authorization: "Bearer " + session.Token,