на 5
@@ -9,6 +9,8 @@ APP_ENV=lab
|
||||
|
||||
POSTGRES_USER=messenger
|
||||
POSTGRES_DB=messenger
|
||||
POSTGRES_CPU_LIMIT=1.00
|
||||
POSTGRES_MEMORY_LIMIT=512M
|
||||
|
||||
SESSION_TTL_SECONDS=3600
|
||||
SESSION_STORE=postgres
|
||||
|
||||
@@ -7,6 +7,8 @@ BASE_URL ?= http://localhost:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
GRADE4_OUTPUT ?= evidence/grade4/results.json
|
||||
GRADE5_OUTPUT ?= evidence/grade5/results.json
|
||||
READINESS_OUTPUT ?= evidence/readiness-routing.json
|
||||
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
|
||||
@@ -27,8 +29,10 @@ COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
|
||||
load \
|
||||
test-persistence \
|
||||
test-failover \
|
||||
test-readiness-routing \
|
||||
rolling-update \
|
||||
grade4-experiment \
|
||||
grade5-experiment \
|
||||
compose-config \
|
||||
compose-build \
|
||||
compose-up \
|
||||
@@ -51,8 +55,10 @@ help:
|
||||
@echo "load: run synthetic writes against the LB entrypoint"
|
||||
@echo "test-persistence: run persistence test with ACTION_SCRIPT"
|
||||
@echo "test-failover: run replica failover test with ACTION_SCRIPT"
|
||||
@echo "test-readiness-routing: isolate PostgreSQL from one API and verify active LB health checks"
|
||||
@echo "rolling-update: replace API replicas one by one with surge capacity"
|
||||
@echo "grade4-experiment: run failovers, idempotency and rolling update with evidence"
|
||||
@echo "grade5-experiment: measure fixed-budget scaling, saturation and one improvement"
|
||||
@echo "compose-config: validate resolved Compose configuration"
|
||||
@echo "compose-build: build application images"
|
||||
@echo "compose-up: start the stack"
|
||||
@@ -114,6 +120,9 @@ test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
test-readiness-routing:
|
||||
$(PYTHON) scripts/test_readiness_routing.py --env-file "$(ENV_FILE)" --output "$(READINESS_OUTPUT)"
|
||||
|
||||
rolling-update:
|
||||
ENV_FILE="$(ENV_FILE)" ./scripts/rolling-update.sh
|
||||
|
||||
@@ -121,6 +130,10 @@ grade4-experiment:
|
||||
$(PYTHON) scripts/grade4_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE4_OUTPUT)"
|
||||
$(PYTHON) scripts/render_grade4.py "$(GRADE4_OUTPUT)"
|
||||
|
||||
grade5-experiment:
|
||||
$(PYTHON) scripts/grade5_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE5_OUTPUT)"
|
||||
$(PYTHON) scripts/render_grade5.py "$(GRADE5_OUTPUT)"
|
||||
|
||||
compose-config:
|
||||
@test -f "$(ENV_FILE)" || \
|
||||
(echo "$(ENV_FILE) is missing; run: make init-env"; exit 2)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: Чурилин, БВТ2301
|
||||
- Текущий завершённый уровень: оценка «4»
|
||||
- Итоговая цель: оценка «5»; исследовательский трек выполняется следующим этапом
|
||||
- Текущий завершённый уровень: оценка «5», трек 1 «Цена масштабирования»
|
||||
- Итоговая цель: выполнена; уровень «4» и законченный исследовательский трек
|
||||
- Commit/PR: заполняется автором после локального коммита и публикации
|
||||
- Использованные источники и библиотеки: преподавательские README/OpenAPI/tests,
|
||||
документация Go, pgx, PostgreSQL, Docker Compose, Nginx, React, TypeScript,
|
||||
документация Go, pgx, PostgreSQL, Docker Compose, HAProxy, Nginx, React, TypeScript,
|
||||
Vite, TanStack Query, React Router, Lucide и Vitest. Для разбора архитектуры,
|
||||
рефакторинга, реализации и проверок использовался OpenAI Codex; решения
|
||||
проверены тестами и описанными ниже воспроизводимыми сценариями.
|
||||
@@ -35,13 +35,16 @@ cp .env.example .env
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
make test
|
||||
make test-readiness-routing
|
||||
make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
|
||||
docker compose --env-file .env up -d --wait api
|
||||
make test-persistence ACTION_SCRIPT=./scripts/restart.sh
|
||||
make grade4-experiment
|
||||
make grade5-experiment
|
||||
```
|
||||
|
||||
По этому же адресу Nginx отдаёт production-сборку React. Переход на
|
||||
По этому же адресу HAProxy направляет обычные browser-запросы во внутренний
|
||||
Nginx, который отдаёт production-сборку React. Переход на
|
||||
`http://localhost:8080` открывает Messenger, а `/api/v1/*` и health endpoints
|
||||
остаются доступны автоматическим проверкам. Отдельный Node.js на машине для
|
||||
обычного запуска не нужен: multi-stage image сам выполняет `npm ci`, Vitest,
|
||||
@@ -84,15 +87,22 @@ PostgreSQL остаётся источником истины для польз
|
||||
PostgreSQL.
|
||||
|
||||
Compose запускает один сервис `api` с `API_REPLICAS=2`. Реплики не имеют host
|
||||
ports и разделяют PostgreSQL, включая таблицу сессий. Nginx — единственная
|
||||
публичная точка входа; он использует `least_conn` и Docker DNS resolver с
|
||||
повторным разрешением имени `api` раз в секунду. Sticky sessions не применяются.
|
||||
ports и разделяют PostgreSQL, включая таблицу сессий. HAProxy — единственная
|
||||
публичная точка входа; он использует `leastconn`, Docker DNS resolver и активные
|
||||
HTTP-проверки `/health/ready` раз в секунду. Реплика исключается после двух
|
||||
неуспешных readiness-проверок и возвращается после двух успешных. Поэтому живой,
|
||||
но потерявший PostgreSQL и ставший неготовым процесс перестаёт получать трафик;
|
||||
Docker healthcheck не является единственным механизмом обнаружения. Sticky
|
||||
sessions не применяются. Nginx остаётся отдельным внутренним контейнером и
|
||||
обслуживает только статические файлы React.
|
||||
|
||||
Произвольные POST балансировщик автоматически не повторяет. Исключение сделано
|
||||
только для маршрута отправки сообщения: обязательный `Idempotency-Key` позволяет
|
||||
Nginx безопасно повторить запрос на другой реплике при 502/503/504 во время
|
||||
drain. Клиент также сохраняет ключ для повтора после неопределённого результата.
|
||||
Создание пользователя и чата контрактом не идемпотентно, поэтому автоматический
|
||||
retry для этих маршрутов запрещён.
|
||||
HAProxy безопасно повторить запрос на другой реплике при connection failure,
|
||||
empty response, timeout или 502/503/504 во время обнаружения отказа и drain.
|
||||
Для этого маршрута тело запроса буферизуется отдельным backend; регистрации
|
||||
пользователя и создание чата контрактом не идемпотентны, поэтому retry для них
|
||||
запрещён. Клиент также сохраняет ключ для повтора после неопределённого результата.
|
||||
|
||||
Миграции вынесены в одноразовый сервис `migrate`, завершение которого является
|
||||
предусловием запуска API. Поэтому две реплики не конкурируют за изменение схемы.
|
||||
@@ -154,7 +164,7 @@ React Router — за маршруты login/register, выбранного ча
|
||||
`SameSite=Strict`. Флаг `Secure` управляется `SESSION_COOKIE_SECURE`: `false`
|
||||
нужен только локальному HTTP, на HTTPS должен быть `true`.
|
||||
|
||||
Контрактные `/api/v1/*` сознательно остались Bearer-only. Nginx предоставляет
|
||||
Контрактные `/api/v1/*` сознательно остались Bearer-only. HAProxy предоставляет
|
||||
отдельный браузерный адаптер `/web-api/v1/*`: берёт token из HttpOnly-cookie,
|
||||
формирует `Authorization: Bearer ...`, удаляет исходный `Cookie` и проксирует на
|
||||
обычный `/api/v1/*`. Поэтому преподавательские тесты и внешние API-клиенты не
|
||||
@@ -173,18 +183,23 @@ cookie.
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Воспроизводимая конфигурация и качество Go | `make check && make go-check && make validate PYTHON=../mtusi2026lab2/.venv/bin/python` | форматирование, tests, race detector, vet, Compose config и OpenAPI validation завершились без ошибок | `evidence/lab3-summary.md` |
|
||||
| Две API-реплики за одним LB | `docker compose --env-file .env up -d --build --wait`; 20 health-запросов через `:8080` | наружу опубликован только Nginx; два разных `X-Instance-Id`, по 10 ответов от каждого | `compose.yaml`, `infrastructure/nginx/nginx.conf`, `evidence/lab3-summary.md` |
|
||||
| Две API-реплики за одним LB | `docker compose --env-file .env up -d --build --wait`; 20 health-запросов через `:8080` | наружу опубликован только HAProxy; два разных `X-Instance-Id`, по 10 ответов от каждого | `compose.yaml`, `infrastructure/haproxy/haproxy.cfg`, `evidence/lab3-summary.md` |
|
||||
| Публичный smoke lab3 | `make test` | `tests/smoke.py` завершился `PASS: lab3 public API scenario` | `evidence/lab3-summary.md` |
|
||||
| Общая сессия без sticky routing | smoke, затем failover и persistence через тот же bearer token | обе реплики обслуживали запросы; сессия работает после остановки API и рестартов | `evidence/lab3-summary.md` |
|
||||
| Идемпотентность сообщения | `go test ./student_tests -run Idempotency`; 32 конкурентных POST с одним ключом через LB | один Message и ровно одна строка в PostgreSQL; другой payload даёт 409 | `student_tests/idempotency_test.go`, `evidence/lab3-summary.md` |
|
||||
| Failover каждой API-реплики | два последовательных запуска `make test-failover ACTION_SCRIPT=./scripts/stop-one.sh` с восстановлением между ними | реально остановлены `api-2`, затем `api-1`; получено 142/143 успешных записи, 0 ошибок, восстановление 0.963/0.750 с | `scripts/stop-one.sh`, `evidence/lab3-summary.md` |
|
||||
| Persistence после рестартов | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS; прежняя сессия и данные доступны после рестарта PostgreSQL, Redis, API и LB | `scripts/restart.sh`, `evidence/lab3-summary.md` |
|
||||
| Миграции перед API | `docker compose --env-file .env ps -a` и логи `migrate` | миграции 1 и 2 применены; one-shot контейнер завершился с кодом 0; API healthy | `migrations/000002_message_idempotency.up.sql`, `evidence/lab3-summary.md` |
|
||||
| React frontend и HttpOnly BFF | `docker compose build load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/Dockerfile`, `infrastructure/nginx/nginx.conf` |
|
||||
| React frontend и HttpOnly BFF | `docker compose build frontend load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/`, `infrastructure/haproxy/` |
|
||||
| Согласованные health и shutdown | SIGTERM каждой реплики под нагрузкой | API сразу закрывает availability gate, 2 с дренирует запросы и укладывается в 10 с shutdown; `/health/live` остаётся доступен | `internal/transport/http/middleware/availability.go`, `evidence/grade4/` |
|
||||
| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только Nginx `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
|
||||
| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 3,464/3,476 с, одна краткая ошибка из 4614 запросов в первом прогоне и 0 из 5815 во втором | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
|
||||
| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 20 350 успешных запросов, 1 краткий 502 (0,005%), все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
|
||||
| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только HAProxy `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
|
||||
| Readiness-aware routing | `make test-readiness-routing` | одной API-реплике заблокирован PostgreSQL: HAProxy исключил её за 2,861 с; 40 health-запросов получили 200, 20 регистраций — 201; после снятия fault реплика вернулась | `scripts/test_readiness_routing.py`, `evidence/readiness-routing.json` |
|
||||
| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 2,893/2,938 с; во всех фазах 0 ошибочных запросов | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
|
||||
| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 22 212 успешных запросов, 0 ошибок; все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
|
||||
| Фиксированный бюджет 1/2/4 API | `make grade5-experiment`; 32 closed-loop клиента | при суммарных 1 CPU, 512 MiB и 16 соединениях получено 578,242 / 529,499 / 353,955 RPS; четыре процесса при том же бюджете дали на 38,79% меньше одной реплики | `evidence/grade5/02-fixed-total.svg`, `measurements.csv` |
|
||||
| Рост ресурсов 1/2/4 API | тот же workload; по 0,5 CPU, 256 MiB и 8 соединений на реплику | 533,656 / 563,460 / 438,868 RPS; нелинейная серия показывает отсутствие пропорционального выигрыша при общей БД в 1 CPU | `evidence/grade5/03-resource-growth.svg`, `results.json` |
|
||||
| Saturation point и bottleneck | concurrency 1/2/4/8/16/32, по 10 с после warm-up | первый сигнал насыщения — 8: относительно 4 клиентов RPS вырос только на 8,03%, p95 — в 1,92 раза; PostgreSQL в профиле четырёх реплик достиг 101,91% одного CPU | `evidence/grade5/01-saturation.svg`, `04-bottleneck.svg` |
|
||||
| Подтверждённое улучшение | четыре API без изменений; PostgreSQL 1 → 2 CPU | RPS 438,868 → 587,574 (+33,88%), ошибок 0, p95 снизился на 14,12%; следующим ограничением стал HAProxy с лимитом 0,25 CPU | `evidence/grade5/05-improvement.svg`, `resources.csv` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
@@ -192,8 +207,8 @@ cookie.
|
||||
Docker Engine 29.7.2, Compose 5.5.0, 8 логических CPU и 3813,6 MiB RAM,
|
||||
commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиченными
|
||||
изменениями текущего уровня. В Compose каждой API-реплике выделено 0,5 CPU и
|
||||
256 MiB; Nginx — 0,25 CPU/128 MiB; PostgreSQL — 1 CPU/512 MiB; Redis —
|
||||
0,25 CPU/128 MiB.
|
||||
256 MiB; HAProxy — 0,25 CPU/128 MiB; frontend Nginx — 0,10 CPU/64 MiB;
|
||||
PostgreSQL — 1 CPU/512 MiB; Redis — 0,25 CPU/128 MiB.
|
||||
|
||||
Команда `make grade4-experiment` создаёт синтетических пользователей и чат,
|
||||
делает 12 warm-up записей, затем запускает 8 closed-loop клиентов без повторов
|
||||
@@ -202,44 +217,144 @@ commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиче
|
||||
следующей, после действия нагрузка продолжается ещё 8 с. Rolling update
|
||||
временно увеличивает сервис до трёх реплик, ждёт healthcheck и два периода
|
||||
Docker DNS, заменяет старые контейнеры по одному и возвращает масштаб к двум.
|
||||
Минимальный подтверждённый размер набора после прогона — 34 808 сообщений.
|
||||
Минимальный подтверждённый размер набора после прогона — 43 810 сообщений.
|
||||
|
||||
| Фаза | Успешно / ошибок | RPS | Доля ошибок | p50 / p95 / p99, мс | Восстановление |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| baseline | 4017 / 0 | 319,463 | 0% | 16,799 / 55,515 / 93,803 | — |
|
||||
| остановка реплики 1 | 4613 / 1 | 301,040 | 0,0217% | 17,145 / 69,558 / 89,752 | 3,464 с |
|
||||
| остановка реплики 2 | 5815 / 0 | 374,449 | 0% | 14,211 / 49,542 / 81,040 | 3,476 с |
|
||||
| rolling update | 20 350 / 1 | 376,959 | 0,0049% | 15,725 / 48,724 / 79,337 | непрерывная нагрузка |
|
||||
| baseline | 5762 / 0 | 479,581 | 0% | 9,178 / 56,402 / 79,978 | — |
|
||||
| остановка реплики 1 | 7853 / 0 | 537,548 | 0% | 8,812 / 50,050 / 66,373 | 2,893 с |
|
||||
| остановка реплики 2 | 7970 / 0 | 533,154 | 0% | 8,726 / 48,789 / 64,333 | 2,938 с |
|
||||
| rolling update | 22 212 / 0 | 428,860 | 0% | 10,560 / 58,713 / 83,982 | непрерывная нагрузка |
|
||||
|
||||
На baseline средняя выборка одного API-контейнера составила 28,453% CPU и
|
||||
42,413 MiB RAM, максимум — 36,1% и 66,07 MiB. PostgreSQL оказался самым
|
||||
нагруженным компонентом: в rolling-фазе среднее 98,172% CPU при лимите одного
|
||||
ядра, максимум RAM 145,2 MiB. Это не capacity-тест и не доказательство
|
||||
На baseline средняя выборка одного API-контейнера составила 27,144% CPU и
|
||||
31,008 MiB RAM, максимум — 30,93% и 66,17 MiB. PostgreSQL оказался самым
|
||||
нагруженным компонентом: среднее 89,392% CPU при лимите одного ядра, максимум
|
||||
110,59%; HAProxy использовал в среднем 26,262% CPU. Это не capacity-тест и не доказательство
|
||||
saturation point, но важный ориентир для исследования уровня 5.
|
||||
|
||||
Nginx обновляет Docker DNS раз в 1 с, исключает peer после двух неудач за 2 с,
|
||||
а connect timeout равен 1 с. При постоянной нагрузке ожидаемое максимальное
|
||||
обнаружение — около 4 с; измеренные 3,464 и 3,476 с укладываются в требование
|
||||
10 с. Одна 504 и одна 502 сохранены в отчёте честно: README прямо допускает
|
||||
краткие ошибки обнаружения. Подтверждённых потерь и дублей нет. Параллельные
|
||||
32 POST с одним ключом прошли через обе реплики и вернули один Message ID.
|
||||
HAProxy обновляет Docker DNS раз в 1 с и исключает peer после двух неуспешных
|
||||
активных `/health/ready`-проверок. Измеренные 2,893 и 2,938 с укладываются в
|
||||
требование 10 с. Отдельный тест потери доступа одной реплики к PostgreSQL
|
||||
подтвердил исключение неготового backend за 2,861 с без 5xx: 40 health-запросов
|
||||
получили 200, а 20 регистраций — 201. Во всех четырёх фазах grade-4 эксперимента
|
||||
ошибок нет. Параллельные 32 POST с одним ключом прошли через обе реплики и
|
||||
вернули один Message ID.
|
||||
|
||||
Исходные агрегаты, environment/limits, посекундные таблицы и четыре SVG-графика
|
||||
лежат в `evidence/grade4/`. Percentile рассчитаны только по успешным ответам;
|
||||
профиль closed-loop имеет coordinated omission. CPU для API агрегируется как
|
||||
средняя/максимальная выборка отдельного контейнера, а не сумма двух реплик.
|
||||
|
||||
## Исследование уровня 5: цена масштабирования
|
||||
|
||||
Трек 1 выполнен командой `make grade5-experiment` 23.10.2026 на том же
|
||||
однохостовом стенде Windows + WSL2: Docker Engine 29.7.2, Compose 5.5.0,
|
||||
8 логических CPU и 3813,6 MiB RAM. Базовым commit до изменений уровня 5 был
|
||||
`5300a1a`; измерения ожидаемо зафиксировали dirty working tree, поскольку сам
|
||||
скрипт и параметризация лимитов ещё не были закоммичены. Все запросы проходили
|
||||
через единственную публичную точку HAProxy, создавали сообщения с уникальными
|
||||
`Idempotency-Key` и не повторялись нагрузочным клиентом автоматически.
|
||||
|
||||
### Методика и воспроизводимость
|
||||
|
||||
Каждая измеряемая фаза длилась 10 секунд. После каждой смены профиля выполнялся
|
||||
двухсекундный warm-up, затем контейнеры и реально применённые `NanoCpus`, RAM и
|
||||
`DB_MAX_CONNS` проверялись через `docker inspect`. Скрипт сохраняет посекундные
|
||||
request buckets и каждую выборку `docker stats`; CPU/RAM API сначала суммируются
|
||||
по всем репликам в одной временной точке и только затем усредняются. Это устраняет
|
||||
ошибку, при которой средняя загрузка одного из четырёх контейнеров выдавалась бы
|
||||
за стоимость всего API-сервиса.
|
||||
|
||||
В серии с фиксированным бюджетом сохранялись суммарные 1 CPU, 512 MiB RAM и
|
||||
16 соединений пула API:
|
||||
|
||||
| Реплики | CPU на реплику | RAM на реплику | `DB_MAX_CONNS` на реплику | Суммарный пул |
|
||||
| ---: | ---: | ---: | ---: | ---: |
|
||||
| 1 | 1,00 | 512 MiB | 16 | 16 |
|
||||
| 2 | 0,50 | 256 MiB | 8 | 16 |
|
||||
| 4 | 0,25 | 128 MiB | 4 | 16 |
|
||||
|
||||
В серии роста каждая реплика получала 0,5 CPU, 256 MiB и 8 соединений, поэтому
|
||||
суммарный API-бюджет менялся от 0,5 CPU/256 MiB/8 соединений до
|
||||
2 CPU/1024 MiB/32 соединений. Во всех основных профилях PostgreSQL оставался на
|
||||
1 CPU/512 MiB, HAProxy — 0,25 CPU/128 MiB, frontend Nginx — 0,10 CPU/64 MiB,
|
||||
Redis — 0,25 CPU/128 MiB. Сравнение
|
||||
1/2/4 выполнялось при одинаковом concurrency 32. До и после эксперимента
|
||||
зафиксированы размер набора и базы: 225 626 → 299 524 сообщений и
|
||||
175 406 103 → 228 793 367 байт. После любого исхода `finally` возвращает стенд к
|
||||
двум API по 0,5 CPU/256 MiB, пулу 10 на реплику и PostgreSQL с 1 CPU; успешное
|
||||
восстановление также записано в `results.json`.
|
||||
|
||||
### Насыщение и цена реплик
|
||||
|
||||
Для поиска saturation point две API-реплики с суммарным 1 CPU и пулом 16
|
||||
нагружались последовательностью concurrency 1/2/4/8/16/32:
|
||||
|
||||
| Concurrency | RPS | p95, мс | PostgreSQL CPU avg/max | Ошибки |
|
||||
| ---: | ---: | ---: | ---: | ---: |
|
||||
| 1 | 230,369 | 5,065 | 23,900% / 29,41% | 0 |
|
||||
| 2 | 320,190 | 7,196 | 54,987% / 62,60% | 0 |
|
||||
| 4 | 435,822 | 29,304 | 63,093% / 81,80% | 0 |
|
||||
| 8 | 470,828 | 56,230 | 65,945% / 92,98% | 0 |
|
||||
| 16 | 472,892 | 73,628 | 65,080% / 88,93% | 0 |
|
||||
| 32 | 564,012 | 94,785 | 83,825% / 107,62% | 0 |
|
||||
|
||||
Заранее зафиксированное правило считает первым saturation signal шаг, где прирост
|
||||
RPS меньше 10%, а p95 одновременно вырастает не менее чем в 1,5 раза. Это
|
||||
произошло при переходе 4 → 8: throughput вырос только на 8,03%, а p95 — в
|
||||
1,92 раза. При 16 клиентах прирост относительно 8 почти отсутствует (+0,44%),
|
||||
а p95 продолжает расти; при 32 клиентах RPS увеличивается ценой ещё большей
|
||||
очереди. Поэтому concurrency 8 — первый воспроизводимый saturation signal по
|
||||
заранее заданному правилу, а не утверждение об абсолютном максимуме системы.
|
||||
|
||||
При фиксированном суммарном бюджете 1/2/4 реплики дали соответственно
|
||||
578,242 / 529,499 / 353,955 RPS. Четыре процесса при том же общем бюджете дали
|
||||
на 38,79% меньше одной реплики: дробление ресурсов добавило scheduling и pool
|
||||
overhead, но не увеличило общую ёмкость. В серии роста ресурсов получены
|
||||
533,656 / 563,460 / 438,868 RPS. Разброс и нестрогая монотонность честно
|
||||
сохранены: это один прогон на одном Docker-host, где контейнеры конкурируют за
|
||||
общий scheduler и одну БД. Главный результат здесь не выбор «лучшего» количества
|
||||
реплик, а отсутствие пропорционального роста: четырёхкратный API-бюджет дал на
|
||||
17,76% меньше throughput одной реплики в этом конкретном прогоне.
|
||||
|
||||
Доминирующее ограничение локализовано в PostgreSQL. В базовом профиле четырёх
|
||||
реплик API суммарно использовал в среднем 35,690% CPU из доступных 200%, HAProxy —
|
||||
15,595% из 25%, тогда как PostgreSQL использовал 84,847% в среднем и достигал
|
||||
101,91% лимита одного ядра. Само по себе это корреляция, поэтому выполнено
|
||||
контрольное изменение только одного параметра.
|
||||
|
||||
### Подтверждённое улучшение
|
||||
|
||||
Для тех же четырёх API по 0,5 CPU/256 MiB, того же пула и concurrency 32 лимит
|
||||
PostgreSQL увеличен с 1 до 2 CPU. RPS вырос с 438,868 до 587,574 — на 33,88%;
|
||||
ошибок в обеих фазах не было, p95 снизился с 109,446 до 93,991 мс (-14,12%).
|
||||
Таким образом, снятие именно ограничения CPU БД существенно увеличило throughput
|
||||
и улучшило tail latency, что подтверждает исходную гипотезу.
|
||||
|
||||
После улучшения PostgreSQL использовал в среднем 95,573% из доступных 200%, а
|
||||
HAProxy — 22,350% при лимите 0,25 CPU и достигал 26,73%. То есть следующее
|
||||
ограничение ожидаемо сместилось к публичному
|
||||
балансировщику. Увеличивать API дальше без пересмотра лимита или топологии LB уже
|
||||
нерационально — это и есть практическая «цена масштабирования».
|
||||
|
||||
Полный JSON с окружением, профилями, фактически применёнными лимитами и сырыми
|
||||
выборками находится в `evidence/grade5/results.json`; агрегаты — в
|
||||
`measurements.csv`, посекундные запросы — в `timeseries.csv`, ресурсы — в
|
||||
`resources.csv`. Пять SVG показывают saturation curve, обе серии 1/2/4,
|
||||
локализацию bottleneck и контрольное улучшение.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
- Топология однохостовая: Nginx и PostgreSQL остаются single points of failure.
|
||||
- Топология однохостовая: HAProxy и PostgreSQL остаются single points of failure.
|
||||
Репликация БД и несколько балансировщиков в требования уровня «4» не входят.
|
||||
- Redis содержит только кэш, поэтому его потеря допустима. Отдельная
|
||||
персистентность Redis для корректности API не требуется.
|
||||
- Истёкшие записи идемпотентности удаляются лениво при повторном обращении к той
|
||||
же области ключа. Индекс по `expires_at` оставлен для будущей периодической
|
||||
очистки без изменения схемы.
|
||||
- Уровень «5» ещё не реализован: сравнение 1/2/4 реплик при фиксированных
|
||||
ресурсах и поиск saturation point выполняются следующим отдельным этапом.
|
||||
- Исследование уровня «5» выполняет по одному измеряемому прогону на профиль.
|
||||
Для production capacity planning нужны несколько повторов, доверительные
|
||||
интервалы и отдельный генератор нагрузки вне измеряемого Docker-host.
|
||||
- Rolling-сценарий заменяет бинарник без изменения схемы. Миграции выполняются
|
||||
однократно до API; для будущей несовместимой схемы потребуется expand/contract,
|
||||
поэтому одновременная работа разных schema-version сейчас не заявляется.
|
||||
|
||||
@@ -2,13 +2,15 @@ services:
|
||||
load-balancer:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infrastructure/nginx/Dockerfile
|
||||
image: mtusi2026lab3-web
|
||||
dockerfile: infrastructure/haproxy/Dockerfile
|
||||
image: mtusi2026lab3-load-balancer
|
||||
ports:
|
||||
- "127.0.0.1:${APP_PORT:-8080}:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
frontend:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
@@ -27,6 +29,27 @@ services:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
frontend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: infrastructure/nginx/Dockerfile
|
||||
image: mtusi2026lab3-web
|
||||
expose:
|
||||
- "8080"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/ >/dev/null || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 3s
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.10"
|
||||
memory: 64M
|
||||
|
||||
postgres:
|
||||
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
|
||||
environment:
|
||||
@@ -51,8 +74,8 @@ services:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
cpus: "${POSTGRES_CPU_LIMIT:-1.00}"
|
||||
memory: ${POSTGRES_MEMORY_LIMIT:-512M}
|
||||
|
||||
redis:
|
||||
image: redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0
|
||||
|
||||
@@ -3,3 +3,12 @@
|
||||
Храните небольшие обезличенные результаты: команды и stdout проверок, JSON/CSV агрегатов нагрузки, EXPLAIN, dashboard JSON, текст разбора отказа. Укажите дату, commit, аппаратные лимиты и профиль нагрузки.
|
||||
|
||||
Не добавляйте .env, session tokens, cookies, приватные ключи, credentials, реальные сообщения, raw dumps и signed URLs. Для скриншотов скрывайте секреты. Большие логи/бинарные traces сохраняйте отдельно по правилам преподавателя; одного скриншота без методики недостаточно.
|
||||
|
||||
## Сохранённые эксперименты
|
||||
|
||||
- `grade4/` — failover обеих API-реплик и rolling update под нагрузкой;
|
||||
- `grade5/` — трек «Цена масштабирования»: saturation sweep, сравнение 1/2/4
|
||||
реплик с фиксированным и растущим бюджетом, сырые JSON/CSV, пять SVG-графиков
|
||||
и подтверждённое улучшение PostgreSQL CPU;
|
||||
- `readiness-routing.json` — результат изоляции PostgreSQL от одной живой
|
||||
API-реплики: время исключения backend, статусы запросов и восстановление.
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# Lab 3 grade-4 experiment
|
||||
|
||||
Generated: 2026-10-23T12:31:12.749687+00:00.
|
||||
Generated: 2026-10-23T15:23:05.031807+00:00.
|
||||
|
||||
## Method
|
||||
|
||||
- concurrency: 8;
|
||||
- baseline duration: 12 s;
|
||||
- load: closed-loop POST messages through Nginx;
|
||||
- load: closed-loop POST messages through the public HAProxy gateway;
|
||||
- failover: both original API containers stopped and restored in turn;
|
||||
- rolling update: one surge replica, old replicas replaced one by one;
|
||||
- percentiles cover successful responses; failures are separate.
|
||||
@@ -15,10 +15,10 @@ Generated: 2026-10-23T12:31:12.749687+00:00.
|
||||
|
||||
| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| baseline | 319.463 | 0 (0.000%) | 16.799 | 55.515 | 93.803 | n/a |
|
||||
| failover-1 | 301.04 | 1 (0.022%) | 17.145 | 69.558 | 89.752 | 3.464 |
|
||||
| failover-2 | 374.449 | 0 (0.000%) | 14.211 | 49.542 | 81.04 | 3.476 |
|
||||
| rolling-update | 376.959 | 1 (0.005%) | 15.725 | 48.724 | 79.337 | n/a |
|
||||
| baseline | 479.581 | 0 (0.000%) | 9.178 | 56.402 | 79.978 | n/a |
|
||||
| failover-1 | 537.548 | 0 (0.000%) | 8.812 | 50.05 | 66.373 | 2.893 |
|
||||
| failover-2 | 533.154 | 0 (0.000%) | 8.726 | 48.789 | 64.333 | 2.938 |
|
||||
| rolling-update | 428.86 | 0 (0.000%) | 10.56 | 58.713 | 83.982 | n/a |
|
||||
|
||||
## Parallel idempotency
|
||||
|
||||
@@ -36,4 +36,4 @@ Generated: 2026-10-23T12:31:12.749687+00:00.
|
||||
|
||||
## Scope
|
||||
|
||||
Single-host process failure and rolling replacement only. PostgreSQL and Nginx remain single points of failure.
|
||||
Single-host process failure and rolling replacement only. PostgreSQL and HAProxy remain single points of failure.
|
||||
|
||||
@@ -16,53 +16,53 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
<line x1="80" y1="450.0" x2="1040" y2="450.0" class="grid"/>
|
||||
<text x="68" y="454.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="80" y1="378.0" x2="1040" y2="378.0" class="grid"/>
|
||||
<text x="68" y="382.0" text-anchor="end" class="small">19</text>
|
||||
<text x="68" y="382.0" text-anchor="end" class="small">17</text>
|
||||
<line x1="80" y1="306.0" x2="1040" y2="306.0" class="grid"/>
|
||||
<text x="68" y="310.0" text-anchor="end" class="small">38</text>
|
||||
<text x="68" y="310.0" text-anchor="end" class="small">34</text>
|
||||
<line x1="80" y1="234.0" x2="1040" y2="234.0" class="grid"/>
|
||||
<text x="68" y="238.0" text-anchor="end" class="small">56</text>
|
||||
<text x="68" y="238.0" text-anchor="end" class="small">50</text>
|
||||
<line x1="80" y1="162.0" x2="1040" y2="162.0" class="grid"/>
|
||||
<text x="68" y="166.0" text-anchor="end" class="small">75</text>
|
||||
<text x="68" y="166.0" text-anchor="end" class="small">67</text>
|
||||
<line x1="80" y1="90.0" x2="1040" y2="90.0" class="grid"/>
|
||||
<text x="68" y="94.0" text-anchor="end" class="small">94</text>
|
||||
<rect x="128.0" y="385.5" width="42.0" height="64.5" rx="7" class="bar"/>
|
||||
<text x="149.0" y="378.5" text-anchor="middle" class="small">17</text>
|
||||
<text x="68" y="94.0" text-anchor="end" class="small">84</text>
|
||||
<rect x="128.0" y="410.7" width="42.0" height="39.3" rx="7" class="bar"/>
|
||||
<text x="149.0" y="403.7" text-anchor="middle" class="small">9</text>
|
||||
<text x="149.0" y="470" text-anchor="middle" class="small">p50</text>
|
||||
<rect x="178.0" y="236.9" width="42.0" height="213.1" rx="7" class="bar2"/>
|
||||
<text x="199.0" y="229.9" text-anchor="middle" class="small">56</text>
|
||||
<rect x="178.0" y="208.2" width="42.0" height="241.8" rx="7" class="bar2"/>
|
||||
<text x="199.0" y="201.2" text-anchor="middle" class="small">56</text>
|
||||
<text x="199.0" y="470" text-anchor="middle" class="small">p95</text>
|
||||
<rect x="228.0" y="90.0" width="42.0" height="360.0" rx="7" class="bar2"/>
|
||||
<text x="249.0" y="83.0" text-anchor="middle" class="small">94</text>
|
||||
<rect x="228.0" y="107.2" width="42.0" height="342.8" rx="7" class="bar2"/>
|
||||
<text x="249.0" y="100.2" text-anchor="middle" class="small">80</text>
|
||||
<text x="249.0" y="470" text-anchor="middle" class="small">p99</text>
|
||||
<text x="200.0" y="498" text-anchor="middle" class="label">baseline</text>
|
||||
<rect x="368.0" y="384.2" width="42.0" height="65.8" rx="7" class="bar"/>
|
||||
<text x="389.0" y="377.2" text-anchor="middle" class="small">17</text>
|
||||
<rect x="368.0" y="412.2" width="42.0" height="37.8" rx="7" class="bar"/>
|
||||
<text x="389.0" y="405.2" text-anchor="middle" class="small">9</text>
|
||||
<text x="389.0" y="470" text-anchor="middle" class="small">p50</text>
|
||||
<rect x="418.0" y="183.0" width="42.0" height="267.0" rx="7" class="bar2"/>
|
||||
<text x="439.0" y="176.0" text-anchor="middle" class="small">70</text>
|
||||
<rect x="418.0" y="235.5" width="42.0" height="214.5" rx="7" class="bar2"/>
|
||||
<text x="439.0" y="228.5" text-anchor="middle" class="small">50</text>
|
||||
<text x="439.0" y="470" text-anchor="middle" class="small">p95</text>
|
||||
<rect x="468.0" y="105.5" width="42.0" height="344.5" rx="7" class="bar2"/>
|
||||
<text x="489.0" y="98.5" text-anchor="middle" class="small">90</text>
|
||||
<rect x="468.0" y="165.5" width="42.0" height="284.5" rx="7" class="bar2"/>
|
||||
<text x="489.0" y="158.5" text-anchor="middle" class="small">66</text>
|
||||
<text x="489.0" y="470" text-anchor="middle" class="small">p99</text>
|
||||
<text x="440.0" y="498" text-anchor="middle" class="label">failover-1</text>
|
||||
<rect x="608.0" y="395.5" width="42.0" height="54.5" rx="7" class="bar"/>
|
||||
<text x="629.0" y="388.5" text-anchor="middle" class="small">14</text>
|
||||
<rect x="608.0" y="412.6" width="42.0" height="37.4" rx="7" class="bar"/>
|
||||
<text x="629.0" y="405.6" text-anchor="middle" class="small">9</text>
|
||||
<text x="629.0" y="470" text-anchor="middle" class="small">p50</text>
|
||||
<rect x="658.0" y="259.9" width="42.0" height="190.1" rx="7" class="bar2"/>
|
||||
<text x="679.0" y="252.9" text-anchor="middle" class="small">50</text>
|
||||
<rect x="658.0" y="240.9" width="42.0" height="209.1" rx="7" class="bar2"/>
|
||||
<text x="679.0" y="233.9" text-anchor="middle" class="small">49</text>
|
||||
<text x="679.0" y="470" text-anchor="middle" class="small">p95</text>
|
||||
<rect x="708.0" y="139.0" width="42.0" height="311.0" rx="7" class="bar2"/>
|
||||
<text x="729.0" y="132.0" text-anchor="middle" class="small">81</text>
|
||||
<rect x="708.0" y="174.2" width="42.0" height="275.8" rx="7" class="bar2"/>
|
||||
<text x="729.0" y="167.2" text-anchor="middle" class="small">64</text>
|
||||
<text x="729.0" y="470" text-anchor="middle" class="small">p99</text>
|
||||
<text x="680.0" y="498" text-anchor="middle" class="label">failover-2</text>
|
||||
<rect x="848.0" y="389.7" width="42.0" height="60.3" rx="7" class="bar"/>
|
||||
<text x="869.0" y="382.7" text-anchor="middle" class="small">16</text>
|
||||
<rect x="848.0" y="404.7" width="42.0" height="45.3" rx="7" class="bar"/>
|
||||
<text x="869.0" y="397.7" text-anchor="middle" class="small">11</text>
|
||||
<text x="869.0" y="470" text-anchor="middle" class="small">p50</text>
|
||||
<rect x="898.0" y="263.0" width="42.0" height="187.0" rx="7" class="bar2"/>
|
||||
<text x="919.0" y="256.0" text-anchor="middle" class="small">49</text>
|
||||
<rect x="898.0" y="198.3" width="42.0" height="251.7" rx="7" class="bar2"/>
|
||||
<text x="919.0" y="191.3" text-anchor="middle" class="small">59</text>
|
||||
<text x="919.0" y="470" text-anchor="middle" class="small">p95</text>
|
||||
<rect x="948.0" y="145.5" width="42.0" height="304.5" rx="7" class="bar2"/>
|
||||
<text x="969.0" y="138.5" text-anchor="middle" class="small">79</text>
|
||||
<rect x="948.0" y="90.0" width="42.0" height="360.0" rx="7" class="bar2"/>
|
||||
<text x="969.0" y="83.0" text-anchor="middle" class="small">84</text>
|
||||
<text x="969.0" y="470" text-anchor="middle" class="small">p99</text>
|
||||
<text x="920.0" y="498" text-anchor="middle" class="label">rolling-update</text>
|
||||
<text x="24" y="270.0" transform="rotate(-90 24 270.0)" class="label">Successful latency, ms</text>
|
||||
|
||||
|
Before Width: | Height: | Size: 4.4 KiB After Width: | Height: | Size: 4.4 KiB |
@@ -14,29 +14,35 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
</style>
|
||||
<text x="48" y="42" class="title">Grade 4: peak container resources</text>
|
||||
<text x="80" y="72" class="label">Peak CPU, %</text>
|
||||
<rect x="103.6" y="270.1" width="57.8" height="179.9" rx="9" class="bar"/>
|
||||
<text x="132.5" y="262.1" text-anchor="middle" class="small">52.6</text>
|
||||
<text x="132.5" y="475" text-anchor="middle" class="small">api</text>
|
||||
<rect x="208.6" y="370.1" width="57.8" height="79.9" rx="9" class="bar"/>
|
||||
<text x="237.5" y="362.1" text-anchor="middle" class="small">23.4</text>
|
||||
<text x="237.5" y="475" text-anchor="middle" class="small">load-balancer</text>
|
||||
<rect x="313.6" y="90.0" width="57.8" height="360.0" rx="9" class="bar"/>
|
||||
<text x="342.5" y="82.0" text-anchor="middle" class="small">105.2</text>
|
||||
<text x="342.5" y="475" text-anchor="middle" class="small">postgres</text>
|
||||
<rect x="418.6" y="429.9" width="57.8" height="20.1" rx="9" class="bar"/>
|
||||
<text x="447.5" y="421.9" text-anchor="middle" class="small">5.9</text>
|
||||
<text x="447.5" y="475" text-anchor="middle" class="small">redis</text>
|
||||
<rect x="98.9" y="288.9" width="46.2" height="161.1" rx="9" class="bar"/>
|
||||
<text x="122.0" y="280.9" text-anchor="middle" class="small">49.5</text>
|
||||
<text x="122.0" y="475" text-anchor="middle" class="small">api</text>
|
||||
<rect x="182.9" y="423.8" width="46.2" height="26.2" rx="9" class="bar"/>
|
||||
<text x="206.0" y="415.8" text-anchor="middle" class="small">8.1</text>
|
||||
<text x="206.0" y="475" text-anchor="middle" class="small">frontend</text>
|
||||
<rect x="266.9" y="354.2" width="46.2" height="95.8" rx="9" class="bar"/>
|
||||
<text x="290.0" y="346.2" text-anchor="middle" class="small">29.4</text>
|
||||
<text x="290.0" y="475" text-anchor="middle" class="small">load-balancer</text>
|
||||
<rect x="350.9" y="90.0" width="46.2" height="360.0" rx="9" class="bar"/>
|
||||
<text x="374.0" y="82.0" text-anchor="middle" class="small">110.6</text>
|
||||
<text x="374.0" y="475" text-anchor="middle" class="small">postgres</text>
|
||||
<rect x="434.9" y="428.6" width="46.2" height="21.4" rx="9" class="bar"/>
|
||||
<text x="458.0" y="420.6" text-anchor="middle" class="small">6.6</text>
|
||||
<text x="458.0" y="475" text-anchor="middle" class="small">redis</text>
|
||||
<text x="600" y="72" class="label">Peak memory, MiB</text>
|
||||
<rect x="623.6" y="286.2" width="57.8" height="163.8" rx="9" class="bar"/>
|
||||
<text x="652.5" y="278.2" text-anchor="middle" class="small">66.1</text>
|
||||
<text x="652.5" y="475" text-anchor="middle" class="small">api</text>
|
||||
<rect x="728.6" y="416.5" width="57.8" height="33.5" rx="9" class="bar"/>
|
||||
<text x="757.5" y="408.5" text-anchor="middle" class="small">13.5</text>
|
||||
<text x="757.5" y="475" text-anchor="middle" class="small">load-balancer</text>
|
||||
<rect x="833.6" y="90.0" width="57.8" height="360.0" rx="9" class="bar"/>
|
||||
<text x="862.5" y="82.0" text-anchor="middle" class="small">145.2</text>
|
||||
<text x="862.5" y="475" text-anchor="middle" class="small">postgres</text>
|
||||
<rect x="938.6" y="419.6" width="57.8" height="30.4" rx="9" class="bar"/>
|
||||
<text x="967.5" y="411.6" text-anchor="middle" class="small">12.3</text>
|
||||
<text x="967.5" y="475" text-anchor="middle" class="small">redis</text>
|
||||
<rect x="618.9" y="321.9" width="46.2" height="128.1" rx="9" class="bar"/>
|
||||
<text x="642.0" y="313.9" text-anchor="middle" class="small">66.2</text>
|
||||
<text x="642.0" y="475" text-anchor="middle" class="small">api</text>
|
||||
<rect x="702.9" y="426.4" width="46.2" height="23.6" rx="9" class="bar"/>
|
||||
<text x="726.0" y="418.4" text-anchor="middle" class="small">12.2</text>
|
||||
<text x="726.0" y="475" text-anchor="middle" class="small">frontend</text>
|
||||
<rect x="786.9" y="412.0" width="46.2" height="38.0" rx="9" class="bar"/>
|
||||
<text x="810.0" y="404.0" text-anchor="middle" class="small">19.6</text>
|
||||
<text x="810.0" y="475" text-anchor="middle" class="small">load-balancer</text>
|
||||
<rect x="870.9" y="90.0" width="46.2" height="360.0" rx="9" class="bar"/>
|
||||
<text x="894.0" y="82.0" text-anchor="middle" class="small">185.9</text>
|
||||
<text x="894.0" y="475" text-anchor="middle" class="small">postgres</text>
|
||||
<rect x="954.9" y="417.8" width="46.2" height="32.2" rx="9" class="bar"/>
|
||||
<text x="978.0" y="409.8" text-anchor="middle" class="small">16.6</text>
|
||||
<text x="978.0" y="475" text-anchor="middle" class="small">redis</text>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 2.5 KiB After Width: | Height: | Size: 3.0 KiB |
@@ -1,8 +1,8 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"environment": {
|
||||
"utc_started_at": "2026-10-23T12:31:12.749687+00:00",
|
||||
"git_commit": "34618bd4237bbb8bf6a3a303056fc95d799577f9",
|
||||
"utc_started_at": "2026-10-23T15:23:05.031807+00:00",
|
||||
"git_commit": "5300a1a1e0e5497f2d30097bb1542b2743a8aad5",
|
||||
"working_tree_dirty": true,
|
||||
"logical_cpu_count": 8,
|
||||
"host_memory_mib": 3813.6,
|
||||
@@ -19,6 +19,11 @@
|
||||
"nano_cpus": 500000000,
|
||||
"memory_limit_bytes": 268435456
|
||||
},
|
||||
{
|
||||
"service": "frontend",
|
||||
"nano_cpus": 100000000,
|
||||
"memory_limit_bytes": 67108864
|
||||
},
|
||||
{
|
||||
"service": "load-balancer",
|
||||
"nano_cpus": 250000000,
|
||||
@@ -43,611 +48,611 @@
|
||||
"action_after_seconds": 3,
|
||||
"post_action_seconds": 8,
|
||||
"warmup_requests": 12,
|
||||
"workload": "closed-loop POST messages via Nginx; no automatic retries"
|
||||
"workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries"
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"name": "baseline",
|
||||
"elapsed_seconds": 12.574,
|
||||
"successful_requests": 4017,
|
||||
"elapsed_seconds": 12.015,
|
||||
"successful_requests": 5762,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"errors": {},
|
||||
"successful_rps": 319.463,
|
||||
"successful_rps": 479.581,
|
||||
"successful_latency_ms": {
|
||||
"p50": 16.799,
|
||||
"p95": 55.515,
|
||||
"p99": 93.803
|
||||
"p50": 9.178,
|
||||
"p95": 56.402,
|
||||
"p99": 79.978
|
||||
},
|
||||
"instances": [
|
||||
"0b933a46fccb",
|
||||
"55aaffcf508a"
|
||||
"a85567c48d67",
|
||||
"df8a69558af0"
|
||||
],
|
||||
"recovery_seconds": null,
|
||||
"action": null,
|
||||
"per_second": {
|
||||
"0": {
|
||||
"successful": 118,
|
||||
"successful": 434,
|
||||
"failed": 0
|
||||
},
|
||||
"1": {
|
||||
"successful": 308,
|
||||
"successful": 280,
|
||||
"failed": 0
|
||||
},
|
||||
"2": {
|
||||
"successful": 386,
|
||||
"successful": 372,
|
||||
"failed": 0
|
||||
},
|
||||
"3": {
|
||||
"successful": 378,
|
||||
"successful": 436,
|
||||
"failed": 0
|
||||
},
|
||||
"4": {
|
||||
"successful": 344,
|
||||
"successful": 541,
|
||||
"failed": 0
|
||||
},
|
||||
"5": {
|
||||
"successful": 322,
|
||||
"successful": 474,
|
||||
"failed": 0
|
||||
},
|
||||
"6": {
|
||||
"successful": 316,
|
||||
"successful": 526,
|
||||
"failed": 0
|
||||
},
|
||||
"7": {
|
||||
"successful": 384,
|
||||
"successful": 539,
|
||||
"failed": 0
|
||||
},
|
||||
"8": {
|
||||
"successful": 380,
|
||||
"successful": 554,
|
||||
"failed": 0
|
||||
},
|
||||
"9": {
|
||||
"successful": 402,
|
||||
"successful": 559,
|
||||
"failed": 0
|
||||
},
|
||||
"10": {
|
||||
"successful": 377,
|
||||
"successful": 448,
|
||||
"failed": 0
|
||||
},
|
||||
"11": {
|
||||
"successful": 288,
|
||||
"successful": 584,
|
||||
"failed": 0
|
||||
},
|
||||
"12": {
|
||||
"successful": 14,
|
||||
"successful": 15,
|
||||
"failed": 0
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"cpu_percent_avg": 28.453,
|
||||
"cpu_percent_max": 36.1,
|
||||
"memory_mib_avg": 42.413,
|
||||
"memory_mib_max": 66.07
|
||||
"cpu_percent_avg": 27.144,
|
||||
"cpu_percent_max": 30.93,
|
||||
"memory_mib_avg": 31.008,
|
||||
"memory_mib_max": 66.17
|
||||
},
|
||||
"frontend": {
|
||||
"cpu_percent_avg": 1.812,
|
||||
"cpu_percent_max": 3.76,
|
||||
"memory_mib_avg": 7.212,
|
||||
"memory_mib_max": 10.15
|
||||
},
|
||||
"load-balancer": {
|
||||
"cpu_percent_avg": 17.448,
|
||||
"cpu_percent_max": 20.8,
|
||||
"memory_mib_avg": 9.214,
|
||||
"memory_mib_max": 9.559
|
||||
"cpu_percent_avg": 26.262,
|
||||
"cpu_percent_max": 27.37,
|
||||
"memory_mib_avg": 15.645,
|
||||
"memory_mib_max": 15.97
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_percent_avg": 85.888,
|
||||
"cpu_percent_max": 105.25,
|
||||
"memory_mib_avg": 97.975,
|
||||
"memory_mib_max": 100.8
|
||||
"cpu_percent_avg": 89.392,
|
||||
"cpu_percent_max": 110.59,
|
||||
"memory_mib_avg": 130.125,
|
||||
"memory_mib_max": 132.2
|
||||
},
|
||||
"redis": {
|
||||
"cpu_percent_avg": 2.552,
|
||||
"cpu_percent_max": 5.79,
|
||||
"memory_mib_avg": 8.796,
|
||||
"memory_mib_max": 11.74
|
||||
"cpu_percent_avg": 2.488,
|
||||
"cpu_percent_max": 6.58,
|
||||
"memory_mib_avg": 15.072,
|
||||
"memory_mib_max": 15.25
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "failover-1",
|
||||
"elapsed_seconds": 15.324,
|
||||
"successful_requests": 4613,
|
||||
"failed_requests": 1,
|
||||
"error_ratio": 0.000217,
|
||||
"errors": {
|
||||
"POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 504": 1
|
||||
},
|
||||
"successful_rps": 301.04,
|
||||
"elapsed_seconds": 14.609,
|
||||
"successful_requests": 7853,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"errors": {},
|
||||
"successful_rps": 537.548,
|
||||
"successful_latency_ms": {
|
||||
"p50": 17.145,
|
||||
"p95": 69.558,
|
||||
"p99": 89.752
|
||||
"p50": 8.812,
|
||||
"p95": 50.05,
|
||||
"p99": 66.373
|
||||
},
|
||||
"instances": [
|
||||
"0b933a46fccb",
|
||||
"55aaffcf508a"
|
||||
"a85567c48d67",
|
||||
"df8a69558af0"
|
||||
],
|
||||
"recovery_seconds": 3.464,
|
||||
"recovery_seconds": 2.893,
|
||||
"action": {
|
||||
"started_at": 3.006,
|
||||
"ended_at": 6.463,
|
||||
"duration_seconds": 3.457,
|
||||
"started_at": 3.003,
|
||||
"ended_at": 5.888,
|
||||
"duration_seconds": 2.885,
|
||||
"exit_code": 0,
|
||||
"summary": "stopped API instance 55aaffcf508a"
|
||||
"summary": "stopped API instance df8a69558af0"
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"successful": 404,
|
||||
"successful": 551,
|
||||
"failed": 0
|
||||
},
|
||||
"1": {
|
||||
"successful": 426,
|
||||
"successful": 605,
|
||||
"failed": 0
|
||||
},
|
||||
"2": {
|
||||
"successful": 358,
|
||||
"successful": 558,
|
||||
"failed": 0
|
||||
},
|
||||
"3": {
|
||||
"successful": 399,
|
||||
"successful": 443,
|
||||
"failed": 0
|
||||
},
|
||||
"4": {
|
||||
"successful": 392,
|
||||
"successful": 575,
|
||||
"failed": 0
|
||||
},
|
||||
"5": {
|
||||
"successful": 322,
|
||||
"successful": 538,
|
||||
"failed": 0
|
||||
},
|
||||
"6": {
|
||||
"successful": 299,
|
||||
"successful": 593,
|
||||
"failed": 0
|
||||
},
|
||||
"7": {
|
||||
"successful": 329,
|
||||
"failed": 1
|
||||
"successful": 623,
|
||||
"failed": 0
|
||||
},
|
||||
"8": {
|
||||
"successful": 313,
|
||||
"successful": 507,
|
||||
"failed": 0
|
||||
},
|
||||
"9": {
|
||||
"successful": 229,
|
||||
"successful": 626,
|
||||
"failed": 0
|
||||
},
|
||||
"10": {
|
||||
"successful": 155,
|
||||
"successful": 608,
|
||||
"failed": 0
|
||||
},
|
||||
"11": {
|
||||
"successful": 187,
|
||||
"successful": 601,
|
||||
"failed": 0
|
||||
},
|
||||
"12": {
|
||||
"successful": 299,
|
||||
"successful": 556,
|
||||
"failed": 0
|
||||
},
|
||||
"13": {
|
||||
"successful": 317,
|
||||
"failed": 0
|
||||
},
|
||||
"14": {
|
||||
"successful": 184,
|
||||
"successful": 469,
|
||||
"failed": 0
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"cpu_percent_avg": 27.62,
|
||||
"cpu_percent_max": 52.59,
|
||||
"memory_mib_avg": 11.58,
|
||||
"memory_mib_max": 12.66
|
||||
"cpu_percent_avg": 35.916,
|
||||
"cpu_percent_max": 49.48,
|
||||
"memory_mib_avg": 11.892,
|
||||
"memory_mib_max": 13.27
|
||||
},
|
||||
"frontend": {
|
||||
"cpu_percent_avg": 0.762,
|
||||
"cpu_percent_max": 3.75,
|
||||
"memory_mib_avg": 6.256,
|
||||
"memory_mib_max": 6.434
|
||||
},
|
||||
"load-balancer": {
|
||||
"cpu_percent_avg": 18.536,
|
||||
"cpu_percent_max": 21.25,
|
||||
"memory_mib_avg": 9.351,
|
||||
"memory_mib_max": 9.523
|
||||
"cpu_percent_avg": 22.21,
|
||||
"cpu_percent_max": 26.89,
|
||||
"memory_mib_avg": 16.312,
|
||||
"memory_mib_max": 16.93
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_percent_avg": 81.822,
|
||||
"cpu_percent_max": 104.67,
|
||||
"memory_mib_avg": 100.512,
|
||||
"memory_mib_max": 109.1
|
||||
"cpu_percent_avg": 73.128,
|
||||
"cpu_percent_max": 89.46,
|
||||
"memory_mib_avg": 135.64,
|
||||
"memory_mib_max": 141.9
|
||||
},
|
||||
"redis": {
|
||||
"cpu_percent_avg": 1.486,
|
||||
"cpu_percent_max": 5.88,
|
||||
"memory_mib_avg": 8.04,
|
||||
"memory_mib_max": 8.25
|
||||
"cpu_percent_avg": 0.782,
|
||||
"cpu_percent_max": 3.27,
|
||||
"memory_mib_avg": 15.258,
|
||||
"memory_mib_max": 15.73
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "failover-2",
|
||||
"elapsed_seconds": 15.529,
|
||||
"successful_requests": 5815,
|
||||
"elapsed_seconds": 14.949,
|
||||
"successful_requests": 7970,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"errors": {},
|
||||
"successful_rps": 374.449,
|
||||
"successful_rps": 533.154,
|
||||
"successful_latency_ms": {
|
||||
"p50": 14.211,
|
||||
"p95": 49.542,
|
||||
"p99": 81.04
|
||||
"p50": 8.726,
|
||||
"p95": 48.789,
|
||||
"p99": 64.333
|
||||
},
|
||||
"instances": [
|
||||
"0b933a46fccb",
|
||||
"55aaffcf508a"
|
||||
"a85567c48d67",
|
||||
"df8a69558af0"
|
||||
],
|
||||
"recovery_seconds": 3.476,
|
||||
"recovery_seconds": 2.938,
|
||||
"action": {
|
||||
"started_at": 3.013,
|
||||
"ended_at": 6.485,
|
||||
"duration_seconds": 3.472,
|
||||
"started_at": 3.006,
|
||||
"ended_at": 5.933,
|
||||
"duration_seconds": 2.927,
|
||||
"exit_code": 0,
|
||||
"summary": "stopped API instance 0b933a46fccb"
|
||||
"summary": "stopped API instance a85567c48d67"
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"successful": 396,
|
||||
"successful": 584,
|
||||
"failed": 0
|
||||
},
|
||||
"1": {
|
||||
"successful": 439,
|
||||
"successful": 613,
|
||||
"failed": 0
|
||||
},
|
||||
"2": {
|
||||
"successful": 412,
|
||||
"successful": 570,
|
||||
"failed": 0
|
||||
},
|
||||
"3": {
|
||||
"successful": 355,
|
||||
"successful": 465,
|
||||
"failed": 0
|
||||
},
|
||||
"4": {
|
||||
"successful": 378,
|
||||
"successful": 564,
|
||||
"failed": 0
|
||||
},
|
||||
"5": {
|
||||
"successful": 440,
|
||||
"successful": 605,
|
||||
"failed": 0
|
||||
},
|
||||
"6": {
|
||||
"successful": 362,
|
||||
"successful": 610,
|
||||
"failed": 0
|
||||
},
|
||||
"7": {
|
||||
"successful": 419,
|
||||
"successful": 579,
|
||||
"failed": 0
|
||||
},
|
||||
"8": {
|
||||
"successful": 385,
|
||||
"successful": 497,
|
||||
"failed": 0
|
||||
},
|
||||
"9": {
|
||||
"successful": 372,
|
||||
"successful": 575,
|
||||
"failed": 0
|
||||
},
|
||||
"10": {
|
||||
"successful": 420,
|
||||
"successful": 569,
|
||||
"failed": 0
|
||||
},
|
||||
"11": {
|
||||
"successful": 423,
|
||||
"successful": 600,
|
||||
"failed": 0
|
||||
},
|
||||
"12": {
|
||||
"successful": 432,
|
||||
"successful": 586,
|
||||
"failed": 0
|
||||
},
|
||||
"13": {
|
||||
"successful": 375,
|
||||
"failed": 0
|
||||
},
|
||||
"14": {
|
||||
"successful": 207,
|
||||
"successful": 553,
|
||||
"failed": 0
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"cpu_percent_avg": 31.411,
|
||||
"cpu_percent_max": 51.63,
|
||||
"memory_mib_avg": 10.373,
|
||||
"memory_mib_max": 11.6
|
||||
"cpu_percent_avg": 28.073,
|
||||
"cpu_percent_max": 48.59,
|
||||
"memory_mib_avg": 10.391,
|
||||
"memory_mib_max": 13.47
|
||||
},
|
||||
"frontend": {
|
||||
"cpu_percent_avg": 0.696,
|
||||
"cpu_percent_max": 3.44,
|
||||
"memory_mib_avg": 6.25,
|
||||
"memory_mib_max": 6.398
|
||||
},
|
||||
"load-balancer": {
|
||||
"cpu_percent_avg": 15.778,
|
||||
"cpu_percent_max": 21.79,
|
||||
"memory_mib_avg": 9.62,
|
||||
"memory_mib_max": 9.781
|
||||
"cpu_percent_avg": 25.966,
|
||||
"cpu_percent_max": 26.65,
|
||||
"memory_mib_avg": 16.918,
|
||||
"memory_mib_max": 17.25
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_percent_avg": 76.87,
|
||||
"cpu_percent_max": 104.51,
|
||||
"memory_mib_avg": 106.016,
|
||||
"memory_mib_max": 120.0
|
||||
"cpu_percent_avg": 72.558,
|
||||
"cpu_percent_max": 93.6,
|
||||
"memory_mib_avg": 146.22,
|
||||
"memory_mib_max": 160.2
|
||||
},
|
||||
"redis": {
|
||||
"cpu_percent_avg": 0.786,
|
||||
"cpu_percent_max": 2.83,
|
||||
"memory_mib_avg": 8.844,
|
||||
"memory_mib_max": 12.28
|
||||
"cpu_percent_avg": 0.168,
|
||||
"cpu_percent_max": 0.2,
|
||||
"memory_mib_avg": 15.646,
|
||||
"memory_mib_max": 16.64
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "rolling-update",
|
||||
"elapsed_seconds": 53.985,
|
||||
"successful_requests": 20350,
|
||||
"failed_requests": 1,
|
||||
"error_ratio": 4.9e-05,
|
||||
"errors": {
|
||||
"POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 502": 1
|
||||
},
|
||||
"successful_rps": 376.959,
|
||||
"elapsed_seconds": 51.793,
|
||||
"successful_requests": 22212,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"errors": {},
|
||||
"successful_rps": 428.86,
|
||||
"successful_latency_ms": {
|
||||
"p50": 15.725,
|
||||
"p95": 48.724,
|
||||
"p99": 79.337
|
||||
"p50": 10.56,
|
||||
"p95": 58.713,
|
||||
"p99": 83.982
|
||||
},
|
||||
"instances": [
|
||||
"0b933a46fccb",
|
||||
"3adbcad9e34b",
|
||||
"55aaffcf508a",
|
||||
"ceca3f6f1faf",
|
||||
"f0a3bedda85a"
|
||||
"872dbac02cd3",
|
||||
"a85567c48d67",
|
||||
"dac03e904506",
|
||||
"df00f21e6f41",
|
||||
"df8a69558af0"
|
||||
],
|
||||
"recovery_seconds": null,
|
||||
"action": {
|
||||
"started_at": 3.016,
|
||||
"ended_at": 45.961,
|
||||
"duration_seconds": 42.945,
|
||||
"started_at": 3.005,
|
||||
"ended_at": 42.955,
|
||||
"duration_seconds": 39.95,
|
||||
"exit_code": 0,
|
||||
"summary": "rolling update completed with 2 healthy API replicas"
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"successful": 340,
|
||||
"successful": 509,
|
||||
"failed": 0
|
||||
},
|
||||
"1": {
|
||||
"successful": 369,
|
||||
"successful": 577,
|
||||
"failed": 0
|
||||
},
|
||||
"2": {
|
||||
"successful": 426,
|
||||
"successful": 660,
|
||||
"failed": 0
|
||||
},
|
||||
"3": {
|
||||
"successful": 382,
|
||||
"successful": 438,
|
||||
"failed": 0
|
||||
},
|
||||
"4": {
|
||||
"successful": 371,
|
||||
"successful": 577,
|
||||
"failed": 0
|
||||
},
|
||||
"5": {
|
||||
"successful": 376,
|
||||
"successful": 392,
|
||||
"failed": 0
|
||||
},
|
||||
"6": {
|
||||
"successful": 328,
|
||||
"successful": 326,
|
||||
"failed": 0
|
||||
},
|
||||
"7": {
|
||||
"successful": 350,
|
||||
"successful": 232,
|
||||
"failed": 0
|
||||
},
|
||||
"8": {
|
||||
"successful": 434,
|
||||
"successful": 199,
|
||||
"failed": 0
|
||||
},
|
||||
"9": {
|
||||
"successful": 411,
|
||||
"successful": 170,
|
||||
"failed": 0
|
||||
},
|
||||
"10": {
|
||||
"successful": 399,
|
||||
"successful": 263,
|
||||
"failed": 0
|
||||
},
|
||||
"11": {
|
||||
"successful": 417,
|
||||
"successful": 323,
|
||||
"failed": 0
|
||||
},
|
||||
"12": {
|
||||
"successful": 374,
|
||||
"successful": 308,
|
||||
"failed": 0
|
||||
},
|
||||
"13": {
|
||||
"successful": 427,
|
||||
"failed": 0
|
||||
},
|
||||
"14": {
|
||||
"successful": 414,
|
||||
"failed": 0
|
||||
},
|
||||
"15": {
|
||||
"successful": 418,
|
||||
"failed": 0
|
||||
},
|
||||
"16": {
|
||||
"successful": 408,
|
||||
"failed": 0
|
||||
},
|
||||
"17": {
|
||||
"successful": 352,
|
||||
"failed": 0
|
||||
},
|
||||
"18": {
|
||||
"successful": 391,
|
||||
"failed": 0
|
||||
},
|
||||
"19": {
|
||||
"successful": 388,
|
||||
"failed": 0
|
||||
},
|
||||
"20": {
|
||||
"successful": 413,
|
||||
"failed": 0
|
||||
},
|
||||
"21": {
|
||||
"successful": 417,
|
||||
"failed": 0
|
||||
},
|
||||
"22": {
|
||||
"successful": 388,
|
||||
"failed": 0
|
||||
},
|
||||
"23": {
|
||||
"successful": 396,
|
||||
"failed": 0
|
||||
},
|
||||
"24": {
|
||||
"successful": 400,
|
||||
"failed": 0
|
||||
},
|
||||
"25": {
|
||||
"successful": 339,
|
||||
"failed": 0
|
||||
},
|
||||
"26": {
|
||||
"successful": 225,
|
||||
"failed": 0
|
||||
},
|
||||
"27": {
|
||||
"successful": 376,
|
||||
"failed": 0
|
||||
},
|
||||
"28": {
|
||||
"successful": 385,
|
||||
"failed": 0
|
||||
},
|
||||
"29": {
|
||||
"successful": 396,
|
||||
"failed": 0
|
||||
},
|
||||
"30": {
|
||||
"successful": 403,
|
||||
"failed": 0
|
||||
},
|
||||
"31": {
|
||||
"successful": 262,
|
||||
"failed": 0
|
||||
},
|
||||
"32": {
|
||||
"successful": 428,
|
||||
"failed": 0
|
||||
},
|
||||
"33": {
|
||||
"successful": 362,
|
||||
"failed": 0
|
||||
},
|
||||
"34": {
|
||||
"successful": 286,
|
||||
"14": {
|
||||
"successful": 443,
|
||||
"failed": 0
|
||||
},
|
||||
"35": {
|
||||
"successful": 337,
|
||||
"15": {
|
||||
"successful": 495,
|
||||
"failed": 0
|
||||
},
|
||||
"36": {
|
||||
"successful": 328,
|
||||
"16": {
|
||||
"successful": 429,
|
||||
"failed": 0
|
||||
},
|
||||
"37": {
|
||||
"successful": 424,
|
||||
"17": {
|
||||
"successful": 370,
|
||||
"failed": 0
|
||||
},
|
||||
"38": {
|
||||
"successful": 356,
|
||||
"18": {
|
||||
"successful": 451,
|
||||
"failed": 0
|
||||
},
|
||||
"39": {
|
||||
"successful": 386,
|
||||
"19": {
|
||||
"successful": 377,
|
||||
"failed": 0
|
||||
},
|
||||
"40": {
|
||||
"20": {
|
||||
"successful": 422,
|
||||
"failed": 0
|
||||
},
|
||||
"21": {
|
||||
"successful": 485,
|
||||
"failed": 0
|
||||
},
|
||||
"22": {
|
||||
"successful": 473,
|
||||
"failed": 0
|
||||
},
|
||||
"23": {
|
||||
"successful": 441,
|
||||
"failed": 0
|
||||
},
|
||||
"24": {
|
||||
"successful": 470,
|
||||
"failed": 0
|
||||
},
|
||||
"25": {
|
||||
"successful": 487,
|
||||
"failed": 0
|
||||
},
|
||||
"26": {
|
||||
"successful": 504,
|
||||
"failed": 0
|
||||
},
|
||||
"27": {
|
||||
"successful": 516,
|
||||
"failed": 0
|
||||
},
|
||||
"28": {
|
||||
"successful": 441,
|
||||
"failed": 0
|
||||
},
|
||||
"29": {
|
||||
"successful": 390,
|
||||
"failed": 0
|
||||
},
|
||||
"30": {
|
||||
"successful": 502,
|
||||
"failed": 0
|
||||
},
|
||||
"31": {
|
||||
"successful": 450,
|
||||
"failed": 0
|
||||
},
|
||||
"32": {
|
||||
"successful": 500,
|
||||
"failed": 0
|
||||
},
|
||||
"33": {
|
||||
"successful": 363,
|
||||
"failed": 0
|
||||
},
|
||||
"34": {
|
||||
"successful": 470,
|
||||
"failed": 0
|
||||
},
|
||||
"35": {
|
||||
"successful": 434,
|
||||
"failed": 0
|
||||
},
|
||||
"36": {
|
||||
"successful": 424,
|
||||
"failed": 0
|
||||
},
|
||||
"37": {
|
||||
"successful": 373,
|
||||
"failed": 0
|
||||
},
|
||||
"38": {
|
||||
"successful": 460,
|
||||
"failed": 0
|
||||
},
|
||||
"39": {
|
||||
"successful": 516,
|
||||
"failed": 0
|
||||
},
|
||||
"40": {
|
||||
"successful": 375,
|
||||
"failed": 0
|
||||
},
|
||||
"41": {
|
||||
"successful": 285,
|
||||
"successful": 492,
|
||||
"failed": 0
|
||||
},
|
||||
"42": {
|
||||
"successful": 279,
|
||||
"successful": 477,
|
||||
"failed": 0
|
||||
},
|
||||
"43": {
|
||||
"successful": 222,
|
||||
"failed": 1
|
||||
"successful": 503,
|
||||
"failed": 0
|
||||
},
|
||||
"44": {
|
||||
"successful": 316,
|
||||
"successful": 506,
|
||||
"failed": 0
|
||||
},
|
||||
"45": {
|
||||
"successful": 359,
|
||||
"successful": 465,
|
||||
"failed": 0
|
||||
},
|
||||
"46": {
|
||||
"successful": 405,
|
||||
"successful": 501,
|
||||
"failed": 0
|
||||
},
|
||||
"47": {
|
||||
"successful": 398,
|
||||
"successful": 502,
|
||||
"failed": 0
|
||||
},
|
||||
"48": {
|
||||
"successful": 395,
|
||||
"successful": 446,
|
||||
"failed": 0
|
||||
},
|
||||
"49": {
|
||||
"successful": 430,
|
||||
"successful": 446,
|
||||
"failed": 0
|
||||
},
|
||||
"50": {
|
||||
"successful": 453,
|
||||
"failed": 0
|
||||
},
|
||||
"51": {
|
||||
"successful": 409,
|
||||
"failed": 0
|
||||
},
|
||||
"52": {
|
||||
"successful": 445,
|
||||
"failed": 0
|
||||
},
|
||||
"53": {
|
||||
"successful": 450,
|
||||
"successful": 477,
|
||||
"failed": 0
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"cpu_percent_avg": 25.598,
|
||||
"cpu_percent_max": 37.65,
|
||||
"memory_mib_avg": 8.956,
|
||||
"memory_mib_max": 10.97
|
||||
"cpu_percent_avg": 20.954,
|
||||
"cpu_percent_max": 31.69,
|
||||
"memory_mib_avg": 8.855,
|
||||
"memory_mib_max": 12.74
|
||||
},
|
||||
"frontend": {
|
||||
"cpu_percent_avg": 1.326,
|
||||
"cpu_percent_max": 8.06,
|
||||
"memory_mib_avg": 7.43,
|
||||
"memory_mib_max": 12.17
|
||||
},
|
||||
"load-balancer": {
|
||||
"cpu_percent_avg": 19.935,
|
||||
"cpu_percent_max": 23.36,
|
||||
"memory_mib_avg": 9.836,
|
||||
"memory_mib_max": 13.52
|
||||
"cpu_percent_avg": 24.581,
|
||||
"cpu_percent_max": 29.42,
|
||||
"memory_mib_avg": 17.398,
|
||||
"memory_mib_max": 19.6
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_percent_avg": 98.172,
|
||||
"cpu_percent_max": 105.11,
|
||||
"memory_mib_avg": 127.975,
|
||||
"memory_mib_max": 145.2
|
||||
"cpu_percent_avg": 77.881,
|
||||
"cpu_percent_max": 102.6,
|
||||
"memory_mib_avg": 170.081,
|
||||
"memory_mib_max": 185.9
|
||||
},
|
||||
"redis": {
|
||||
"cpu_percent_avg": 0.722,
|
||||
"cpu_percent_max": 3.94,
|
||||
"memory_mib_avg": 7.661,
|
||||
"memory_mib_max": 11.84
|
||||
"cpu_percent_avg": 0.63,
|
||||
"cpu_percent_max": 3.84,
|
||||
"memory_mib_avg": 15.178,
|
||||
"memory_mib_max": 15.91
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -658,18 +663,18 @@
|
||||
"failed_responses": 0,
|
||||
"unique_message_ids": 1,
|
||||
"observed_instances": [
|
||||
"0b933a46fccb",
|
||||
"55aaffcf508a"
|
||||
"a85567c48d67",
|
||||
"df8a69558af0"
|
||||
]
|
||||
},
|
||||
"dataset": {
|
||||
"warmup_requests": 12,
|
||||
"minimum_confirmed_messages": 34808
|
||||
"minimum_confirmed_messages": 43810
|
||||
},
|
||||
"limitations": [
|
||||
"single Docker host",
|
||||
"closed-loop coordinated omission",
|
||||
"successful-response latency percentiles only",
|
||||
"PostgreSQL and Nginx remain single points of failure"
|
||||
"PostgreSQL and HAProxy remain single points of failure"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,29 +16,29 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
<line x1="80" y1="450.0" x2="1040" y2="450.0" class="grid"/>
|
||||
<text x="68" y="454.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="80" y1="378.0" x2="1040" y2="378.0" class="grid"/>
|
||||
<text x="68" y="382.0" text-anchor="end" class="small">75</text>
|
||||
<text x="68" y="382.0" text-anchor="end" class="small">108</text>
|
||||
<line x1="80" y1="306.0" x2="1040" y2="306.0" class="grid"/>
|
||||
<text x="68" y="310.0" text-anchor="end" class="small">151</text>
|
||||
<text x="68" y="310.0" text-anchor="end" class="small">215</text>
|
||||
<line x1="80" y1="234.0" x2="1040" y2="234.0" class="grid"/>
|
||||
<text x="68" y="238.0" text-anchor="end" class="small">226</text>
|
||||
<text x="68" y="238.0" text-anchor="end" class="small">323</text>
|
||||
<line x1="80" y1="162.0" x2="1040" y2="162.0" class="grid"/>
|
||||
<text x="68" y="166.0" text-anchor="end" class="small">302</text>
|
||||
<text x="68" y="166.0" text-anchor="end" class="small">430</text>
|
||||
<line x1="80" y1="90.0" x2="1040" y2="90.0" class="grid"/>
|
||||
<text x="68" y="94.0" text-anchor="end" class="small">377</text>
|
||||
<rect x="140.0" y="144.9" width="120.0" height="305.1" rx="10" class="bar"/>
|
||||
<text x="200.0" y="134.9" text-anchor="middle" class="label">319.5 RPS</text>
|
||||
<text x="68" y="94.0" text-anchor="end" class="small">538</text>
|
||||
<rect x="140.0" y="128.8" width="120.0" height="321.2" rx="10" class="bar"/>
|
||||
<text x="200.0" y="118.8" text-anchor="middle" class="label">479.6 RPS</text>
|
||||
<text x="200.0" y="478" text-anchor="middle" class="label">baseline</text>
|
||||
<text x="200.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
|
||||
<rect x="380.0" y="162.5" width="120.0" height="287.5" rx="10" class="bar"/>
|
||||
<text x="440.0" y="152.5" text-anchor="middle" class="label">301.0 RPS</text>
|
||||
<rect x="380.0" y="90.0" width="120.0" height="360.0" rx="10" class="bar"/>
|
||||
<text x="440.0" y="80.0" text-anchor="middle" class="label">537.5 RPS</text>
|
||||
<text x="440.0" y="478" text-anchor="middle" class="label">failover-1</text>
|
||||
<text x="440.0" y="498" text-anchor="middle" class="small">errors 0.02%</text>
|
||||
<rect x="620.0" y="92.4" width="120.0" height="357.6" rx="10" class="bar"/>
|
||||
<text x="680.0" y="82.4" text-anchor="middle" class="label">374.4 RPS</text>
|
||||
<text x="440.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
|
||||
<rect x="620.0" y="92.9" width="120.0" height="357.1" rx="10" class="bar"/>
|
||||
<text x="680.0" y="82.9" text-anchor="middle" class="label">533.2 RPS</text>
|
||||
<text x="680.0" y="478" text-anchor="middle" class="label">failover-2</text>
|
||||
<text x="680.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
|
||||
<rect x="860.0" y="90.0" width="120.0" height="360.0" rx="10" class="bar"/>
|
||||
<text x="920.0" y="80.0" text-anchor="middle" class="label">377.0 RPS</text>
|
||||
<rect x="860.0" y="162.8" width="120.0" height="287.2" rx="10" class="bar"/>
|
||||
<text x="920.0" y="152.8" text-anchor="middle" class="label">428.9 RPS</text>
|
||||
<text x="920.0" y="478" text-anchor="middle" class="label">rolling-update</text>
|
||||
<text x="920.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
|
||||
<text x="24" y="270.0" transform="rotate(-90 24 270.0)" class="label">Successful requests per second</text>
|
||||
|
||||
|
Before Width: | Height: | Size: 2.8 KiB After Width: | Height: | Size: 2.8 KiB |
@@ -16,23 +16,21 @@ text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
<line x1="70" y1="480.0" x2="1050" y2="480.0" class="grid"/>
|
||||
<text x="58" y="484.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="70" y1="402.0" x2="1050" y2="402.0" class="grid"/>
|
||||
<text x="58" y="406.0" text-anchor="end" class="small">91</text>
|
||||
<text x="58" y="406.0" text-anchor="end" class="small">132</text>
|
||||
<line x1="70" y1="324.0" x2="1050" y2="324.0" class="grid"/>
|
||||
<text x="58" y="328.0" text-anchor="end" class="small">181</text>
|
||||
<text x="58" y="328.0" text-anchor="end" class="small">264</text>
|
||||
<line x1="70" y1="246.0" x2="1050" y2="246.0" class="grid"/>
|
||||
<text x="58" y="250.0" text-anchor="end" class="small">272</text>
|
||||
<text x="58" y="250.0" text-anchor="end" class="small">396</text>
|
||||
<line x1="70" y1="168.0" x2="1050" y2="168.0" class="grid"/>
|
||||
<text x="58" y="172.0" text-anchor="end" class="small">362</text>
|
||||
<text x="58" y="172.0" text-anchor="end" class="small">528</text>
|
||||
<line x1="70" y1="90.0" x2="1050" y2="90.0" class="grid"/>
|
||||
<text x="58" y="94.0" text-anchor="end" class="small">453</text>
|
||||
<circle cx="277.9" cy="472" r="4" class="error"/>
|
||||
<circle cx="951.0" cy="472" r="4" class="error"/>
|
||||
<polyline points="70.0,378.4 79.9,214.8 89.8,147.7 99.7,154.6 109.6,183.8 119.5,202.8 129.4,207.9 139.3,149.4 149.2,152.8 159.1,133.9 169.0,155.4 178.9,232.1 188.8,467.9 208.6,132.2 218.5,113.2 228.4,171.8 238.3,136.5 248.2,142.5 258.1,202.8 268.0,222.6 277.9,196.8 287.8,210.5 297.7,282.8 307.6,346.6 317.5,319.0 327.4,222.6 337.3,207.1 347.2,321.6 367.0,139.1 376.9,102.1 386.8,125.3 396.7,174.4 406.6,154.6 416.5,101.2 426.4,168.3 436.3,119.3 446.2,148.5 456.1,159.7 466.0,118.4 475.9,115.8 485.8,108.1 495.7,157.2 505.6,301.8 525.4,187.3 535.3,162.3 545.2,113.2 555.1,151.1 564.9,160.6 574.8,156.3 584.7,197.6 594.6,178.7 604.5,106.4 614.4,126.2 624.3,136.5 634.2,121.0 644.1,158.0 654.0,112.4 663.9,123.6 673.8,120.1 683.7,128.7 693.6,177.0 703.5,143.4 713.4,146.0 723.3,124.4 733.2,121.0 743.1,146.0 753.0,139.1 762.9,135.6 772.8,188.1 782.7,286.3 792.6,156.3 802.5,148.5 812.4,139.1 822.3,133.0 832.2,254.4 842.1,111.5 852.0,168.3 861.9,233.8 871.8,189.9 881.7,197.6 891.6,115.0 901.5,173.5 911.4,147.7 921.3,116.7 931.2,234.6 941.1,239.8 951.0,288.9 960.9,207.9 970.8,170.9 980.7,131.3 990.6,137.4 1000.5,139.9 1010.4,109.8 1020.3,90.0 1030.2,127.9 1040.1,96.9 1050.0,92.6" class="line"/>
|
||||
<line x1="238.3" y1="90" x2="238.3" y2="480" class="event"/>
|
||||
<text x="244.3" y="108" class="small">failover-1 action</text>
|
||||
<line x1="396.8" y1="90" x2="396.8" y2="480" class="event"/>
|
||||
<text x="402.8" y="108" class="small">failover-2 action</text>
|
||||
<line x1="555.2" y1="90" x2="555.2" y2="480" class="event"/>
|
||||
<text x="561.2" y="108" class="small">rolling-update action</text>
|
||||
<text x="58" y="94.0" text-anchor="end" class="small">660</text>
|
||||
<polyline points="70.0,223.5 80.4,314.5 90.9,260.2 101.3,222.4 111.7,160.3 122.1,199.9 132.6,169.2 143.0,161.5 153.4,152.6 163.8,149.7 174.3,215.3 184.7,134.9 195.1,471.1 216.0,154.4 226.4,122.5 236.8,150.3 247.2,218.2 257.7,140.2 268.1,162.1 278.5,129.6 288.9,111.9 299.4,180.4 309.8,110.1 320.2,120.7 330.6,124.9 341.1,151.5 351.5,202.9 372.3,134.9 382.8,117.8 393.2,143.2 403.6,205.2 414.0,146.7 424.5,122.5 434.9,119.5 445.3,137.9 455.7,186.3 466.2,140.2 476.6,143.8 487.0,125.5 497.4,133.7 507.9,153.2 528.7,179.2 539.1,139.0 549.6,90.0 560.0,221.2 570.4,139.0 580.9,248.4 591.3,287.4 601.7,342.9 612.1,362.4 622.6,379.5 633.0,324.6 643.4,289.1 653.8,298.0 664.3,266.1 674.7,218.2 685.1,187.5 695.5,226.5 706.0,261.4 716.4,213.5 726.8,257.2 737.2,230.6 747.7,193.4 758.1,200.5 768.5,219.4 778.9,202.3 789.4,192.2 799.8,182.2 810.2,175.1 820.6,219.4 831.1,249.5 841.5,183.4 851.9,214.1 862.3,184.5 872.8,265.5 883.2,202.3 893.6,223.5 904.0,229.5 914.5,259.6 924.9,208.2 935.3,175.1 945.7,258.4 956.2,189.3 966.6,198.1 977.0,182.8 987.4,181.0 997.9,205.2 1008.3,184.0 1018.7,183.4 1029.1,216.5 1039.6,216.5 1050.0,198.1" class="line"/>
|
||||
<line x1="247.3" y1="90" x2="247.3" y2="480" class="event"/>
|
||||
<text x="253.3" y="108" class="small">failover-1 action</text>
|
||||
<line x1="403.7" y1="90" x2="403.7" y2="480" class="event"/>
|
||||
<text x="409.7" y="108" class="small">failover-2 action</text>
|
||||
<line x1="560.1" y1="90" x2="560.1" y2="480" class="event"/>
|
||||
<text x="566.1" y="108" class="small">rolling-update action</text>
|
||||
<text x="70" y="512" class="label">One-second buckets; circles mark failed requests</text>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 3.2 KiB After Width: | Height: | Size: 3.0 KiB |
@@ -0,0 +1,63 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.subtitle { font-size: 14px; fill: #aeb2b8; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.axis { stroke: #747981; stroke-width: 1.5; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
|
||||
.point { fill: #f4f5f6; }
|
||||
.point2 { fill: #92979f; }
|
||||
.bar { fill: #d7d9dc; }
|
||||
.bar2 { fill: #777c84; }
|
||||
.bar3 { fill: #4c5057; }
|
||||
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
|
||||
</style>
|
||||
<text x="50" y="44" class="title">Saturation curve</text>
|
||||
<text x="50" y="70" class="subtitle">Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU</text>
|
||||
<line x1="90" y1="500.0" x2="1030" y2="500.0" class="grid"/>
|
||||
<text x="78" y="504.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="90" y1="422.0" x2="1030" y2="422.0" class="grid"/>
|
||||
<text x="78" y="426.0" text-anchor="end" class="small">124</text>
|
||||
<line x1="90" y1="344.0" x2="1030" y2="344.0" class="grid"/>
|
||||
<text x="78" y="348.0" text-anchor="end" class="small">248</text>
|
||||
<line x1="90" y1="266.0" x2="1030" y2="266.0" class="grid"/>
|
||||
<text x="78" y="270.0" text-anchor="end" class="small">372</text>
|
||||
<line x1="90" y1="188.0" x2="1030" y2="188.0" class="grid"/>
|
||||
<text x="78" y="192.0" text-anchor="end" class="small">496</text>
|
||||
<line x1="90" y1="110.0" x2="1030" y2="110.0" class="grid"/>
|
||||
<text x="78" y="114.0" text-anchor="end" class="small">620</text>
|
||||
<circle cx="90.0" cy="355.2" r="6" class="point"/>
|
||||
<circle cx="90.0" cy="481.1" r="6" class="point2"/>
|
||||
<text x="90.0" y="528" text-anchor="middle" class="label">1</text>
|
||||
<text x="90.0" y="343.2" text-anchor="middle" class="small">230</text>
|
||||
<circle cx="278.0" cy="298.7" r="6" class="point"/>
|
||||
<circle cx="278.0" cy="473.1" r="6" class="point2"/>
|
||||
<text x="278.0" y="528" text-anchor="middle" class="label">2</text>
|
||||
<text x="278.0" y="286.7" text-anchor="middle" class="small">320</text>
|
||||
<circle cx="466.0" cy="226.0" r="6" class="point"/>
|
||||
<circle cx="466.0" cy="390.4" r="6" class="point2"/>
|
||||
<text x="466.0" y="528" text-anchor="middle" class="label">4</text>
|
||||
<text x="466.0" y="214.0" text-anchor="middle" class="small">436</text>
|
||||
<circle cx="654.0" cy="204.0" r="6" class="point"/>
|
||||
<circle cx="654.0" cy="289.7" r="6" class="point2"/>
|
||||
<text x="654.0" y="528" text-anchor="middle" class="label">8</text>
|
||||
<text x="654.0" y="192.0" text-anchor="middle" class="small">471</text>
|
||||
<circle cx="842.0" cy="202.7" r="6" class="point"/>
|
||||
<circle cx="842.0" cy="224.6" r="6" class="point2"/>
|
||||
<text x="842.0" y="528" text-anchor="middle" class="label">16</text>
|
||||
<text x="842.0" y="190.7" text-anchor="middle" class="small">473</text>
|
||||
<circle cx="1030.0" cy="145.5" r="6" class="point"/>
|
||||
<circle cx="1030.0" cy="145.5" r="6" class="point2"/>
|
||||
<text x="1030.0" y="528" text-anchor="middle" class="label">32</text>
|
||||
<text x="1030.0" y="133.5" text-anchor="middle" class="small">564</text>
|
||||
<polyline points="90.0,355.2 278.0,298.7 466.0,226.0 654.0,204.0 842.0,202.7 1030.0,145.5" class="line"/>
|
||||
<polyline points="90.0,481.1 278.0,473.1 466.0,390.4 654.0,289.7 842.0,224.6 1030.0,145.5" class="line2"/>
|
||||
<text x="90" y="562" class="label">Concurrency</text>
|
||||
<text x="810" y="562" class="label">white: RPS · gray: p95 normalized</text>
|
||||
<text x="30" y="305.0" transform="rotate(-90 30 305.0)" class="label">Successful requests per second</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.6 KiB |
@@ -0,0 +1,49 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.subtitle { font-size: 14px; fill: #aeb2b8; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.axis { stroke: #747981; stroke-width: 1.5; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
|
||||
.point { fill: #f4f5f6; }
|
||||
.point2 { fill: #92979f; }
|
||||
.bar { fill: #d7d9dc; }
|
||||
.bar2 { fill: #777c84; }
|
||||
.bar3 { fill: #4c5057; }
|
||||
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
|
||||
</style>
|
||||
<text x="50" y="44" class="title">Fixed total API budget</text>
|
||||
<text x="50" y="70" class="subtitle">1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas</text>
|
||||
<line x1="90" y1="480.0" x2="1030" y2="480.0" class="grid"/>
|
||||
<text x="78" y="484.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="90" y1="406.0" x2="1030" y2="406.0" class="grid"/>
|
||||
<text x="78" y="410.0" text-anchor="end" class="small">133</text>
|
||||
<line x1="90" y1="332.0" x2="1030" y2="332.0" class="grid"/>
|
||||
<text x="78" y="336.0" text-anchor="end" class="small">266</text>
|
||||
<line x1="90" y1="258.0" x2="1030" y2="258.0" class="grid"/>
|
||||
<text x="78" y="262.0" text-anchor="end" class="small">399</text>
|
||||
<line x1="90" y1="184.0" x2="1030" y2="184.0" class="grid"/>
|
||||
<text x="78" y="188.0" text-anchor="end" class="small">532</text>
|
||||
<line x1="90" y1="110.0" x2="1030" y2="110.0" class="grid"/>
|
||||
<text x="78" y="114.0" text-anchor="end" class="small">665</text>
|
||||
<rect x="171.7" y="158.3" width="150.0" height="321.7" rx="14" class="bar"/>
|
||||
<text x="246.7" y="146.3" text-anchor="middle" class="label">578.2 RPS</text>
|
||||
<text x="246.7" y="510" text-anchor="middle" class="label">1 replica(s)</text>
|
||||
<text x="246.7" y="532" text-anchor="middle" class="small">p95 95.8 ms</text>
|
||||
<text x="246.7" y="550" text-anchor="middle" class="small">errors 0.000%</text>
|
||||
<rect x="485.0" y="185.4" width="150.0" height="294.6" rx="14" class="bar"/>
|
||||
<text x="560.0" y="173.4" text-anchor="middle" class="label">529.5 RPS</text>
|
||||
<text x="560.0" y="510" text-anchor="middle" class="label">2 replica(s)</text>
|
||||
<text x="560.0" y="532" text-anchor="middle" class="small">p95 100.1 ms</text>
|
||||
<text x="560.0" y="550" text-anchor="middle" class="small">errors 0.000%</text>
|
||||
<rect x="798.3" y="283.1" width="150.0" height="196.9" rx="14" class="bar"/>
|
||||
<text x="873.3" y="271.1" text-anchor="middle" class="label">354.0 RPS</text>
|
||||
<text x="873.3" y="510" text-anchor="middle" class="label">4 replica(s)</text>
|
||||
<text x="873.3" y="532" text-anchor="middle" class="small">p95 139.4 ms</text>
|
||||
<text x="873.3" y="550" text-anchor="middle" class="small">errors 0.000%</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.9 KiB |
@@ -0,0 +1,49 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.subtitle { font-size: 14px; fill: #aeb2b8; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.axis { stroke: #747981; stroke-width: 1.5; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
|
||||
.point { fill: #f4f5f6; }
|
||||
.point2 { fill: #92979f; }
|
||||
.bar { fill: #d7d9dc; }
|
||||
.bar2 { fill: #777c84; }
|
||||
.bar3 { fill: #4c5057; }
|
||||
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
|
||||
</style>
|
||||
<text x="50" y="44" class="title">Growing API budget</text>
|
||||
<text x="50" y="70" class="subtitle">Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU</text>
|
||||
<line x1="90" y1="480.0" x2="1030" y2="480.0" class="grid"/>
|
||||
<text x="78" y="484.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="90" y1="406.0" x2="1030" y2="406.0" class="grid"/>
|
||||
<text x="78" y="410.0" text-anchor="end" class="small">130</text>
|
||||
<line x1="90" y1="332.0" x2="1030" y2="332.0" class="grid"/>
|
||||
<text x="78" y="336.0" text-anchor="end" class="small">259</text>
|
||||
<line x1="90" y1="258.0" x2="1030" y2="258.0" class="grid"/>
|
||||
<text x="78" y="262.0" text-anchor="end" class="small">389</text>
|
||||
<line x1="90" y1="184.0" x2="1030" y2="184.0" class="grid"/>
|
||||
<text x="78" y="188.0" text-anchor="end" class="small">518</text>
|
||||
<line x1="90" y1="110.0" x2="1030" y2="110.0" class="grid"/>
|
||||
<text x="78" y="114.0" text-anchor="end" class="small">648</text>
|
||||
<rect x="171.7" y="175.3" width="150.0" height="304.7" rx="14" class="bar"/>
|
||||
<text x="246.7" y="163.3" text-anchor="middle" class="label">533.7 RPS</text>
|
||||
<text x="246.7" y="510" text-anchor="middle" class="label">1 replica(s)</text>
|
||||
<text x="246.7" y="532" text-anchor="middle" class="small">p95 93.5 ms</text>
|
||||
<text x="246.7" y="550" text-anchor="middle" class="small">errors 0.000%</text>
|
||||
<rect x="485.0" y="158.3" width="150.0" height="321.7" rx="14" class="bar"/>
|
||||
<text x="560.0" y="146.3" text-anchor="middle" class="label">563.5 RPS</text>
|
||||
<text x="560.0" y="510" text-anchor="middle" class="label">2 replica(s)</text>
|
||||
<text x="560.0" y="532" text-anchor="middle" class="small">p95 96.4 ms</text>
|
||||
<text x="560.0" y="550" text-anchor="middle" class="small">errors 0.000%</text>
|
||||
<rect x="798.3" y="229.4" width="150.0" height="250.6" rx="14" class="bar"/>
|
||||
<text x="873.3" y="217.4" text-anchor="middle" class="label">438.9 RPS</text>
|
||||
<text x="873.3" y="510" text-anchor="middle" class="label">4 replica(s)</text>
|
||||
<text x="873.3" y="532" text-anchor="middle" class="small">p95 109.4 ms</text>
|
||||
<text x="873.3" y="550" text-anchor="middle" class="small">errors 0.000%</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.9 KiB |
@@ -0,0 +1,83 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.subtitle { font-size: 14px; fill: #aeb2b8; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.axis { stroke: #747981; stroke-width: 1.5; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
|
||||
.point { fill: #f4f5f6; }
|
||||
.point2 { fill: #92979f; }
|
||||
.bar { fill: #d7d9dc; }
|
||||
.bar2 { fill: #777c84; }
|
||||
.bar3 { fill: #4c5057; }
|
||||
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
|
||||
</style>
|
||||
<text x="50" y="44" class="title">Where scaling stops</text>
|
||||
<text x="50" y="70" class="subtitle">Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis)</text>
|
||||
<line x1="90" y1="480.0" x2="1030" y2="480.0" class="grid"/>
|
||||
<text x="78" y="484.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="90" y1="406.0" x2="1030" y2="406.0" class="grid"/>
|
||||
<text x="78" y="410.0" text-anchor="end" class="small">20</text>
|
||||
<line x1="90" y1="332.0" x2="1030" y2="332.0" class="grid"/>
|
||||
<text x="78" y="336.0" text-anchor="end" class="small">39</text>
|
||||
<line x1="90" y1="258.0" x2="1030" y2="258.0" class="grid"/>
|
||||
<text x="78" y="262.0" text-anchor="end" class="small">59</text>
|
||||
<line x1="90" y1="184.0" x2="1030" y2="184.0" class="grid"/>
|
||||
<text x="78" y="188.0" text-anchor="end" class="small">79</text>
|
||||
<line x1="90" y1="110.0" x2="1030" y2="110.0" class="grid"/>
|
||||
<text x="78" y="114.0" text-anchor="end" class="small">98</text>
|
||||
<rect x="127.6" y="336.1" width="48.0" height="143.9" rx="7" class="bar"/>
|
||||
<text x="151.6" y="329.1" text-anchor="middle" class="small">38</text>
|
||||
<text x="151.6" y="502" text-anchor="middle" class="small">api</text>
|
||||
<rect x="184.6" y="189.6" width="48.0" height="290.4" rx="7" class="bar2"/>
|
||||
<text x="208.6" y="182.6" text-anchor="middle" class="small">77</text>
|
||||
<text x="208.6" y="502" text-anchor="middle" class="small">pos</text>
|
||||
<rect x="241.6" y="404.6" width="48.0" height="75.4" rx="7" class="bar3"/>
|
||||
<text x="265.6" y="397.6" text-anchor="middle" class="small">20</text>
|
||||
<text x="265.6" y="502" text-anchor="middle" class="small">loa</text>
|
||||
<rect x="298.6" y="477.4" width="48.0" height="2.6" rx="7" class="bar3"/>
|
||||
<text x="322.6" y="470.4" text-anchor="middle" class="small">1</text>
|
||||
<text x="322.6" y="502" text-anchor="middle" class="small">fro</text>
|
||||
<rect x="355.6" y="476.4" width="48.0" height="3.6" rx="7" class="bar3"/>
|
||||
<text x="379.6" y="469.4" text-anchor="middle" class="small">1</text>
|
||||
<text x="379.6" y="502" text-anchor="middle" class="small">red</text>
|
||||
<text x="246.7" y="532" text-anchor="middle" class="label">1 replica(s)</text>
|
||||
<rect x="440.9" y="304.6" width="48.0" height="175.4" rx="7" class="bar"/>
|
||||
<text x="464.9" y="297.6" text-anchor="middle" class="small">47</text>
|
||||
<text x="464.9" y="502" text-anchor="middle" class="small">api</text>
|
||||
<rect x="497.9" y="158.3" width="48.0" height="321.7" rx="7" class="bar2"/>
|
||||
<text x="521.9" y="151.3" text-anchor="middle" class="small">86</text>
|
||||
<text x="521.9" y="502" text-anchor="middle" class="small">pos</text>
|
||||
<rect x="554.9" y="400.3" width="48.0" height="79.7" rx="7" class="bar3"/>
|
||||
<text x="578.9" y="393.3" text-anchor="middle" class="small">21</text>
|
||||
<text x="578.9" y="502" text-anchor="middle" class="small">loa</text>
|
||||
<rect x="611.9" y="477.1" width="48.0" height="2.9" rx="7" class="bar3"/>
|
||||
<text x="635.9" y="470.1" text-anchor="middle" class="small">1</text>
|
||||
<text x="635.9" y="502" text-anchor="middle" class="small">fro</text>
|
||||
<rect x="668.9" y="476.7" width="48.0" height="3.3" rx="7" class="bar3"/>
|
||||
<text x="692.9" y="469.7" text-anchor="middle" class="small">1</text>
|
||||
<text x="692.9" y="502" text-anchor="middle" class="small">red</text>
|
||||
<text x="560.0" y="532" text-anchor="middle" class="label">2 replica(s)</text>
|
||||
<rect x="754.3" y="345.8" width="48.0" height="134.2" rx="7" class="bar"/>
|
||||
<text x="778.3" y="338.8" text-anchor="middle" class="small">36</text>
|
||||
<text x="778.3" y="502" text-anchor="middle" class="small">api</text>
|
||||
<rect x="811.3" y="161.1" width="48.0" height="318.9" rx="7" class="bar2"/>
|
||||
<text x="835.3" y="154.1" text-anchor="middle" class="small">85</text>
|
||||
<text x="835.3" y="502" text-anchor="middle" class="small">pos</text>
|
||||
<rect x="868.3" y="421.4" width="48.0" height="58.6" rx="7" class="bar3"/>
|
||||
<text x="892.3" y="414.4" text-anchor="middle" class="small">16</text>
|
||||
<text x="892.3" y="502" text-anchor="middle" class="small">loa</text>
|
||||
<rect x="925.3" y="477.5" width="48.0" height="2.5" rx="7" class="bar3"/>
|
||||
<text x="949.3" y="470.5" text-anchor="middle" class="small">1</text>
|
||||
<text x="949.3" y="502" text-anchor="middle" class="small">fro</text>
|
||||
<rect x="982.3" y="476.7" width="48.0" height="3.3" rx="7" class="bar3"/>
|
||||
<text x="1006.3" y="469.7" text-anchor="middle" class="small">1</text>
|
||||
<text x="1006.3" y="502" text-anchor="middle" class="small">red</text>
|
||||
<text x="873.3" y="532" text-anchor="middle" class="label">4 replica(s)</text>
|
||||
<text x="30" y="295.0" transform="rotate(-90 30 295.0)" class="label">Average aggregate CPU, %</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 5.2 KiB |
@@ -0,0 +1,37 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1120" height="620" viewBox="0 0 1120 620">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.subtitle { font-size: 14px; fill: #aeb2b8; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.axis { stroke: #747981; stroke-width: 1.5; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
|
||||
.point { fill: #f4f5f6; }
|
||||
.point2 { fill: #92979f; }
|
||||
.bar { fill: #d7d9dc; }
|
||||
.bar2 { fill: #777c84; }
|
||||
.bar3 { fill: #4c5057; }
|
||||
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
|
||||
</style>
|
||||
<text x="50" y="44" class="title">Confirmed improvement</text>
|
||||
<text x="50" y="70" class="subtitle">Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2</text>
|
||||
<text x="90" y="98" class="label">Successful RPS</text>
|
||||
<rect x="125" y="248.4" width="120" height="211.6" rx="14" class="bar2"/>
|
||||
<text x="185" y="238.4" text-anchor="middle" class="label">438.9</text>
|
||||
<text x="185" y="488" text-anchor="middle" class="small">PG 1 CPU</text>
|
||||
<rect x="310" y="176.7" width="120" height="283.3" rx="14" class="bar"/>
|
||||
<text x="370" y="166.7" text-anchor="middle" class="label">587.6</text>
|
||||
<text x="370" y="488" text-anchor="middle" class="small">PG 2 CPU</text>
|
||||
<text x="630" y="98" class="label">p95 latency, ms</text>
|
||||
<rect x="665" y="176.7" width="120" height="283.3" rx="14" class="bar2"/>
|
||||
<text x="725" y="166.7" text-anchor="middle" class="label">109.4</text>
|
||||
<text x="725" y="488" text-anchor="middle" class="small">PG 1 CPU</text>
|
||||
<rect x="850" y="216.7" width="120" height="243.3" rx="14" class="bar"/>
|
||||
<text x="910" y="206.7" text-anchor="middle" class="label">94.0</text>
|
||||
<text x="910" y="488" text-anchor="middle" class="small">PG 2 CPU</text>
|
||||
<text x="90" y="545" class="label">RPS change: +33.88% · p95 change: -14.12% · confirmed: true</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.0 KiB |
@@ -0,0 +1,69 @@
|
||||
# Lab 3 grade-5 scaling experiment
|
||||
|
||||
Generated: 2026-10-23T15:23:12.734339+00:00.
|
||||
|
||||
## Hypothesis
|
||||
|
||||
API replicas stop improving throughput when shared PostgreSQL CPU saturates; adding PostgreSQL CPU should improve the unchanged four-replica profile.
|
||||
|
||||
## Reproduction
|
||||
|
||||
- workload: closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries;
|
||||
- phase duration: 10 s;
|
||||
- comparison concurrency: 32;
|
||||
- saturation sweep: [1, 2, 4, 8, 16, 32];
|
||||
- fixed budget: 1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections;
|
||||
- growth series: 0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica;
|
||||
- saturation rule: RPS gain below 10% and p95 growth at least 1.5x versus the previous step;
|
||||
- host: 8 logical CPU, 3813.6 MiB RAM;
|
||||
- Docker 29.7.2, Compose 5.5.0;
|
||||
- command: `make grade5-experiment`; warmup runs before every changed profile.
|
||||
|
||||
## Measurements
|
||||
|
||||
| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| saturation-c1 | 2 | 1 | 230.369 | 0 (0.000%) | 3.839 | 5.065 | 6.282 | 23.9 | 26.85 |
|
||||
| saturation-c2 | 2 | 2 | 320.19 | 0 (0.000%) | 5.192 | 7.196 | 15.11 | 54.987 | 48.3 |
|
||||
| saturation-c4 | 2 | 4 | 435.822 | 0 (0.000%) | 6.171 | 29.304 | 47.665 | 63.093 | 54.453 |
|
||||
| saturation-c8 | 2 | 8 | 470.828 | 0 (0.000%) | 9.159 | 56.23 | 70.71 | 65.945 | 48.505 |
|
||||
| saturation-c16 | 2 | 16 | 472.892 | 0 (0.000%) | 19.862 | 73.628 | 98.773 | 65.08 | 38.028 |
|
||||
| saturation-c32 | 2 | 32 | 564.012 | 0 (0.000%) | 51.089 | 94.785 | 109.138 | 83.825 | 51.965 |
|
||||
| fixed-r1 | 1 | 32 | 578.242 | 0 (0.000%) | 48.579 | 95.757 | 115.549 | 79.058 | 36.575 |
|
||||
| fixed-r2 | 2 | 32 | 529.499 | 0 (0.000%) | 53.316 | 100.145 | 129.953 | 81.065 | 44.7 |
|
||||
| fixed-r4 | 4 | 32 | 353.955 | 0 (0.000%) | 69.354 | 139.408 | 291.312 | 74.677 | 46.25 |
|
||||
| growth-r1 | 1 | 32 | 533.656 | 0 (0.000%) | 53.378 | 93.541 | 115.58 | 77.252 | 38.273 |
|
||||
| growth-r2 | 2 | 32 | 563.46 | 0 (0.000%) | 50.516 | 96.428 | 129.46 | 85.597 | 46.653 |
|
||||
| growth-r4 | 4 | 32 | 438.868 | 0 (0.000%) | 68.331 | 109.446 | 199.616 | 84.847 | 35.69 |
|
||||
| improved-db-cpu-r4 | 4 | 32 | 587.574 | 0 (0.000%) | 49.466 | 93.991 | 112.002 | 95.573 | 55.2 |
|
||||
|
||||
## Conclusion
|
||||
|
||||
The first saturation signal appears at concurrency 8: RPS changes by +8.03% while p95 changes by 1.92x versus concurrency 4.
|
||||
|
||||
Observed bottleneck classification: `postgres_cpu`.
|
||||
|
||||
The PostgreSQL CPU change produced +33.88% RPS and -14.12% p95; confirmed improvement: `true`.
|
||||
|
||||
After the improvement the remaining bottleneck classification is `load_balancer_cpu`.
|
||||
|
||||
## Graphs
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||

|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
## Raw data and limits
|
||||
|
||||
Raw aggregates are in `measurements.csv`; one-second request buckets are in `timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and applied limits are in `results.json`.
|
||||
|
||||
- single Docker host: replicas do not prove host-level availability
|
||||
- closed-loop workload has coordinated omission
|
||||
- successful-response latency percentiles are reported separately from failures
|
||||
- one measured run per profile; host scheduling noise is possible
|
||||
@@ -0,0 +1,14 @@
|
||||
phase,kind,replicas,concurrency,elapsed_seconds,successful_requests,failed_requests,error_ratio,successful_rps,p50_ms,p95_ms,p99_ms,api_cpu_avg,postgres_cpu_avg,load_balancer_cpu_avg,frontend_cpu_avg,redis_cpu_avg
|
||||
saturation-c1,saturation,2,1,10.731,2472,0,0.0,230.369,3.839,5.065,6.282,26.85,23.9,11.662,0.688,0.8
|
||||
saturation-c2,saturation,2,2,10.915,3495,0,0.0,320.19,5.192,7.196,15.11,48.3,54.987,19.13,0.325,0.193
|
||||
saturation-c4,saturation,2,4,10.832,4721,0,0.0,435.822,6.171,29.304,47.665,54.453,63.093,20.843,0.95,0.81
|
||||
saturation-c8,saturation,2,8,10.821,5095,0,0.0,470.828,9.159,56.23,70.71,48.505,65.945,20.867,0.86,0.953
|
||||
saturation-c16,saturation,2,16,10.876,5143,0,0.0,472.892,19.862,73.628,98.773,38.028,65.08,20.98,1.113,0.172
|
||||
saturation-c32,saturation,2,32,10.808,6096,0,0.0,564.012,51.089,94.785,109.138,51.965,83.825,21.163,1.837,2.475
|
||||
fixed-r1,fixed,1,32,10.88,6291,0,0.0,578.242,48.579,95.757,115.549,36.575,79.058,20.732,0.005,1.232
|
||||
fixed-r2,fixed,2,32,10.905,5774,0,0.0,529.499,53.316,100.145,129.953,44.7,81.065,21.15,0.823,1.093
|
||||
fixed-r4,fixed,4,32,11.917,4218,0,0.0,353.955,69.354,139.408,291.312,46.25,74.677,21.18,0.637,2.715
|
||||
growth-r1,growth,1,32,10.951,5844,0,0.0,533.656,53.378,93.541,115.58,38.273,77.252,20.062,0.693,0.958
|
||||
growth-r2,growth,2,32,10.796,6083,0,0.0,563.46,50.516,96.428,129.46,46.653,85.597,21.192,0.765,0.865
|
||||
growth-r4,growth,4,32,10.682,4688,0,0.0,438.868,68.331,109.446,199.616,35.69,84.847,15.595,0.667,0.873
|
||||
improved-db-cpu-r4,improvement,4,32,10.601,6229,0,0.0,587.574,49.466,93.991,112.002,55.2,95.573,22.35,0.83,0.458
|
||||
|
@@ -0,0 +1,261 @@
|
||||
phase,t,service,containers,cpu_percent,memory_mib
|
||||
saturation-c1,0.0,api,2,26.09,17.976
|
||||
saturation-c1,0.0,frontend,1,1.61,7.578
|
||||
saturation-c1,0.0,load-balancer,1,13.0,17.18
|
||||
saturation-c1,0.0,postgres,1,28.57,169.0
|
||||
saturation-c1,0.0,redis,1,0.15,15.28
|
||||
saturation-c1,2.356,api,2,32.65,16.886
|
||||
saturation-c1,2.356,frontend,1,0.02,7.617
|
||||
saturation-c1,2.356,load-balancer,1,13.32,17.23
|
||||
saturation-c1,2.356,postgres,1,28.96,169.5
|
||||
saturation-c1,2.356,redis,1,2.75,15.4
|
||||
saturation-c1,5.485,api,2,27.72,17.586
|
||||
saturation-c1,5.485,frontend,1,0.0,7.383
|
||||
saturation-c1,5.485,load-balancer,1,13.69,17.41
|
||||
saturation-c1,5.485,postgres,1,29.41,170.1
|
||||
saturation-c1,5.485,redis,1,0.15,15.28
|
||||
saturation-c1,8.605,api,2,20.94,17.41
|
||||
saturation-c1,8.605,frontend,1,1.12,12.06
|
||||
saturation-c1,8.605,load-balancer,1,6.64,17.24
|
||||
saturation-c1,8.605,postgres,1,8.66,170.5
|
||||
saturation-c1,8.605,redis,1,0.15,15.53
|
||||
saturation-c2,0.001,api,2,51.0,17.761
|
||||
saturation-c2,0.001,frontend,1,0.03,7.617
|
||||
saturation-c2,0.001,load-balancer,1,24.78,17.71
|
||||
saturation-c2,0.001,postgres,1,57.44,171.2
|
||||
saturation-c2,0.001,redis,1,0.17,16.03
|
||||
saturation-c2,2.361,api,2,45.61,17.516
|
||||
saturation-c2,2.361,frontend,1,0.03,7.367
|
||||
saturation-c2,2.361,load-balancer,1,24.6,17.41
|
||||
saturation-c2,2.361,postgres,1,49.85,172.6
|
||||
saturation-c2,2.361,redis,1,0.18,15.28
|
||||
saturation-c2,5.543,api,2,51.19,16.258
|
||||
saturation-c2,5.543,frontend,1,0.02,7.367
|
||||
saturation-c2,5.543,load-balancer,1,24.27,17.91
|
||||
saturation-c2,5.543,postgres,1,62.6,173.0
|
||||
saturation-c2,5.543,redis,1,0.21,15.28
|
||||
saturation-c2,8.736,api,2,45.4,17.082
|
||||
saturation-c2,8.736,frontend,1,1.22,8.121
|
||||
saturation-c2,8.736,load-balancer,1,2.87,17.2
|
||||
saturation-c2,8.736,postgres,1,50.06,173.8
|
||||
saturation-c2,8.736,redis,1,0.21,15.78
|
||||
saturation-c4,0.001,api,2,65.41,17.297
|
||||
saturation-c4,0.001,frontend,1,0.03,7.367
|
||||
saturation-c4,0.001,load-balancer,1,25.95,17.71
|
||||
saturation-c4,0.001,postgres,1,81.8,174.4
|
||||
saturation-c4,0.001,redis,1,0.19,15.28
|
||||
saturation-c4,2.372,api,2,60.01,18.188
|
||||
saturation-c4,2.372,frontend,1,3.75,7.965
|
||||
saturation-c4,2.372,load-balancer,1,26.04,17.46
|
||||
saturation-c4,2.372,postgres,1,79.42,175.4
|
||||
saturation-c4,2.372,redis,1,0.16,15.53
|
||||
saturation-c4,5.501,api,2,56.53,17.433
|
||||
saturation-c4,5.501,frontend,1,0.02,7.367
|
||||
saturation-c4,5.501,load-balancer,1,25.9,21.22
|
||||
saturation-c4,5.501,postgres,1,73.71,175.7
|
||||
saturation-c4,5.501,redis,1,0.18,15.28
|
||||
saturation-c4,8.662,api,2,35.86,16.699
|
||||
saturation-c4,8.662,frontend,1,0.0,7.367
|
||||
saturation-c4,8.662,load-balancer,1,5.48,17.53
|
||||
saturation-c4,8.662,postgres,1,17.44,177.2
|
||||
saturation-c4,8.662,redis,1,2.71,16.1
|
||||
saturation-c8,0.006,api,2,48.34,17.403
|
||||
saturation-c8,0.006,frontend,1,0.0,7.367
|
||||
saturation-c8,0.006,load-balancer,1,25.42,20.12
|
||||
saturation-c8,0.006,postgres,1,77.51,185.1
|
||||
saturation-c8,0.006,redis,1,0.18,15.28
|
||||
saturation-c8,2.372,api,2,57.14,18.602
|
||||
saturation-c8,2.372,frontend,1,3.44,7.734
|
||||
saturation-c8,2.372,load-balancer,1,26.83,17.14
|
||||
saturation-c8,2.372,postgres,1,92.98,188.5
|
||||
saturation-c8,2.372,redis,1,3.34,15.28
|
||||
saturation-c8,5.527,api,2,48.26,18.192
|
||||
saturation-c8,5.527,frontend,1,0.0,7.363
|
||||
saturation-c8,5.527,load-balancer,1,25.47,17.28
|
||||
saturation-c8,5.527,postgres,1,76.05,189.4
|
||||
saturation-c8,5.527,redis,1,0.15,15.28
|
||||
saturation-c8,8.685,api,2,40.28,21.813
|
||||
saturation-c8,8.685,frontend,1,0.0,7.363
|
||||
saturation-c8,8.685,load-balancer,1,5.75,17.64
|
||||
saturation-c8,8.685,postgres,1,17.24,190.4
|
||||
saturation-c8,8.685,redis,1,0.14,15.28
|
||||
saturation-c16,0.015,api,2,44.39,18.796
|
||||
saturation-c16,0.015,frontend,1,0.0,7.363
|
||||
saturation-c16,0.015,load-balancer,1,26.92,17.44
|
||||
saturation-c16,0.015,postgres,1,82.81,205.9
|
||||
saturation-c16,0.015,redis,1,0.17,15.28
|
||||
saturation-c16,2.354,api,2,56.54,17.856
|
||||
saturation-c16,2.354,frontend,1,0.03,7.375
|
||||
saturation-c16,2.354,load-balancer,1,26.66,17.69
|
||||
saturation-c16,2.354,postgres,1,88.93,207.7
|
||||
saturation-c16,2.354,redis,1,0.19,15.75
|
||||
saturation-c16,5.525,api,2,41.39,17.73
|
||||
saturation-c16,5.525,frontend,1,4.4,7.379
|
||||
saturation-c16,5.525,load-balancer,1,25.76,17.7
|
||||
saturation-c16,5.525,postgres,1,72.49,208.6
|
||||
saturation-c16,5.525,redis,1,0.18,15.25
|
||||
saturation-c16,8.731,api,2,9.79,18.257
|
||||
saturation-c16,8.731,frontend,1,0.02,7.375
|
||||
saturation-c16,8.731,load-balancer,1,4.58,17.55
|
||||
saturation-c16,8.731,postgres,1,16.09,209.7
|
||||
saturation-c16,8.731,redis,1,0.15,15.33
|
||||
saturation-c32,0.039,api,2,55.84,18.75
|
||||
saturation-c32,0.039,frontend,1,6.63,8.125
|
||||
saturation-c32,0.039,load-balancer,1,25.97,18.0
|
||||
saturation-c32,0.039,postgres,1,104.2,210.6
|
||||
saturation-c32,0.039,redis,1,5.85,15.05
|
||||
saturation-c32,2.29,api,2,56.87,18.933
|
||||
saturation-c32,2.29,frontend,1,0.0,7.625
|
||||
saturation-c32,2.29,load-balancer,1,27.03,19.23
|
||||
saturation-c32,2.29,postgres,1,105.41,212.1
|
||||
saturation-c32,2.29,redis,1,0.19,15.55
|
||||
saturation-c32,5.446,api,2,64.13,19.003
|
||||
saturation-c32,5.446,frontend,1,0.7,7.375
|
||||
saturation-c32,5.446,load-balancer,1,26.33,18.27
|
||||
saturation-c32,5.446,postgres,1,107.62,213.1
|
||||
saturation-c32,5.446,redis,1,3.7,15.27
|
||||
saturation-c32,8.653,api,2,31.02,19.379
|
||||
saturation-c32,8.653,frontend,1,0.02,7.625
|
||||
saturation-c32,8.653,load-balancer,1,5.32,18.36
|
||||
saturation-c32,8.653,postgres,1,18.07,213.5
|
||||
saturation-c32,8.653,redis,1,0.16,15.26
|
||||
fixed-r1,0.034,api,1,45.68,10.61
|
||||
fixed-r1,0.034,frontend,1,0.02,7.359
|
||||
fixed-r1,0.034,load-balancer,1,25.97,18.08
|
||||
fixed-r1,0.034,postgres,1,106.86,214.7
|
||||
fixed-r1,0.034,redis,1,4.41,15.3
|
||||
fixed-r1,2.424,api,1,48.23,11.36
|
||||
fixed-r1,2.424,frontend,1,0.0,7.699
|
||||
fixed-r1,2.424,load-balancer,1,27.02,22.21
|
||||
fixed-r1,2.424,postgres,1,96.33,215.7
|
||||
fixed-r1,2.424,redis,1,0.14,15.44
|
||||
fixed-r1,5.588,api,1,48.84,12.07
|
||||
fixed-r1,5.588,frontend,1,0.0,7.359
|
||||
fixed-r1,5.588,load-balancer,1,26.38,18.43
|
||||
fixed-r1,5.588,postgres,1,104.71,217.4
|
||||
fixed-r1,5.588,redis,1,0.19,15.92
|
||||
fixed-r1,8.743,api,1,3.55,11.93
|
||||
fixed-r1,8.743,frontend,1,0.0,7.508
|
||||
fixed-r1,8.743,load-balancer,1,3.56,17.96
|
||||
fixed-r1,8.743,postgres,1,8.33,218.1
|
||||
fixed-r1,8.743,redis,1,0.19,16.07
|
||||
fixed-r2,0.041,api,2,52.24,18.086
|
||||
fixed-r2,0.041,frontend,1,0.0,7.359
|
||||
fixed-r2,0.041,load-balancer,1,26.74,18.2
|
||||
fixed-r2,0.041,postgres,1,100.69,219.4
|
||||
fixed-r2,0.041,redis,1,3.81,15.18
|
||||
fixed-r2,2.379,api,2,65.39,18.801
|
||||
fixed-r2,2.379,frontend,1,1.92,7.859
|
||||
fixed-r2,2.379,load-balancer,1,26.23,18.61
|
||||
fixed-r2,2.379,postgres,1,106.33,221.3
|
||||
fixed-r2,2.379,redis,1,0.16,15.16
|
||||
fixed-r2,5.564,api,2,54.12,18.379
|
||||
fixed-r2,5.564,frontend,1,1.37,10.54
|
||||
fixed-r2,5.564,load-balancer,1,25.33,18.35
|
||||
fixed-r2,5.564,postgres,1,103.06,222.7
|
||||
fixed-r2,5.564,redis,1,0.2,14.93
|
||||
fixed-r2,8.728,api,2,7.05,18.086
|
||||
fixed-r2,8.728,frontend,1,0.0,7.363
|
||||
fixed-r2,8.728,load-balancer,1,6.3,17.82
|
||||
fixed-r2,8.728,postgres,1,14.18,223.3
|
||||
fixed-r2,8.728,redis,1,0.2,15.43
|
||||
fixed-r4,0.095,api,4,51.98,27.492
|
||||
fixed-r4,0.095,frontend,1,0.0,7.363
|
||||
fixed-r4,0.095,load-balancer,1,27.15,19.06
|
||||
fixed-r4,0.095,postgres,1,90.1,223.1
|
||||
fixed-r4,0.095,redis,1,5.94,16.87
|
||||
fixed-r4,3.244,api,4,64.6,33.144
|
||||
fixed-r4,3.244,frontend,1,0.02,7.359
|
||||
fixed-r4,3.244,load-balancer,1,26.34,18.58
|
||||
fixed-r4,3.244,postgres,1,101.51,225.1
|
||||
fixed-r4,3.244,redis,1,0.2,16.96
|
||||
fixed-r4,6.45,api,4,65.35,34.281
|
||||
fixed-r4,6.45,frontend,1,0.02,7.359
|
||||
fixed-r4,6.45,load-balancer,1,26.27,18.56
|
||||
fixed-r4,6.45,postgres,1,106.68,225.7
|
||||
fixed-r4,6.45,redis,1,4.55,16.7
|
||||
fixed-r4,9.725,api,4,3.07,32.867
|
||||
fixed-r4,9.725,frontend,1,2.51,7.762
|
||||
fixed-r4,9.725,load-balancer,1,4.96,17.89
|
||||
fixed-r4,9.725,postgres,1,0.42,226.7
|
||||
fixed-r4,9.725,redis,1,0.17,16.95
|
||||
growth-r1,0.045,api,1,44.44,9.355
|
||||
growth-r1,0.045,frontend,1,0.0,7.359
|
||||
growth-r1,0.045,load-balancer,1,25.9,18.3
|
||||
growth-r1,0.045,postgres,1,93.36,207.6
|
||||
growth-r1,0.045,redis,1,3.22,18.55
|
||||
growth-r1,2.404,api,1,50.32,9.379
|
||||
growth-r1,2.404,frontend,1,0.0,7.363
|
||||
growth-r1,2.404,load-balancer,1,25.42,18.36
|
||||
growth-r1,2.404,postgres,1,96.7,209.4
|
||||
growth-r1,2.404,redis,1,0.22,18.11
|
||||
growth-r1,5.579,api,1,50.68,9.66
|
||||
growth-r1,5.579,frontend,1,0.03,7.613
|
||||
growth-r1,5.579,load-balancer,1,25.44,18.67
|
||||
growth-r1,5.579,postgres,1,105.06,210.9
|
||||
growth-r1,5.579,redis,1,0.22,17.84
|
||||
growth-r1,8.767,api,1,7.65,9.262
|
||||
growth-r1,8.767,frontend,1,2.74,8.02
|
||||
growth-r1,8.767,load-balancer,1,3.49,17.86
|
||||
growth-r1,8.767,postgres,1,13.89,211.8
|
||||
growth-r1,8.767,redis,1,0.17,17.93
|
||||
growth-r2,0.047,api,2,52.25,17.629
|
||||
growth-r2,0.047,frontend,1,0.02,7.363
|
||||
growth-r2,0.047,load-balancer,1,24.23,18.59
|
||||
growth-r2,0.047,postgres,1,100.9,233.3
|
||||
growth-r2,0.047,redis,1,0.14,17.16
|
||||
growth-r2,2.292,api,2,62.86,17.606
|
||||
growth-r2,2.292,frontend,1,0.0,7.355
|
||||
growth-r2,2.292,load-balancer,1,25.57,18.19
|
||||
growth-r2,2.292,postgres,1,105.07,235.3
|
||||
growth-r2,2.292,redis,1,0.16,17.16
|
||||
growth-r2,5.459,api,2,57.29,18.789
|
||||
growth-r2,5.459,frontend,1,0.04,7.363
|
||||
growth-r2,5.459,load-balancer,1,25.89,18.43
|
||||
growth-r2,5.459,postgres,1,104.58,235.1
|
||||
growth-r2,5.459,redis,1,0.17,17.73
|
||||
growth-r2,8.628,api,2,14.21,17.832
|
||||
growth-r2,8.628,frontend,1,3.0,8.062
|
||||
growth-r2,8.628,load-balancer,1,9.08,18.0
|
||||
growth-r2,8.628,postgres,1,31.84,234.3
|
||||
growth-r2,8.628,redis,1,2.99,22.08
|
||||
growth-r4,0.043,api,4,25.03,29.672
|
||||
growth-r4,0.043,frontend,1,0.02,7.316
|
||||
growth-r4,0.043,load-balancer,1,9.16,18.25
|
||||
growth-r4,0.043,postgres,1,99.03,269.2
|
||||
growth-r4,0.043,redis,1,0.15,16.5
|
||||
growth-r4,2.197,api,4,48.89,31.184
|
||||
growth-r4,2.197,frontend,1,2.63,7.305
|
||||
growth-r4,2.197,load-balancer,1,23.31,18.25
|
||||
growth-r4,2.197,postgres,1,100.3,284.5
|
||||
growth-r4,2.197,redis,1,2.97,20.3
|
||||
growth-r4,5.406,api,4,43.96,32.164
|
||||
growth-r4,5.406,frontend,1,0.0,7.305
|
||||
growth-r4,5.406,load-balancer,1,19.48,17.79
|
||||
growth-r4,5.406,postgres,1,101.91,290.1
|
||||
growth-r4,5.406,redis,1,0.19,16.68
|
||||
growth-r4,8.538,api,4,24.88,32.855
|
||||
growth-r4,8.538,frontend,1,0.02,7.551
|
||||
growth-r4,8.538,load-balancer,1,10.43,17.5
|
||||
growth-r4,8.538,postgres,1,38.15,279.9
|
||||
growth-r4,8.538,redis,1,0.18,16.91
|
||||
improved-db-cpu-r4,0.054,api,4,52.63,30.188
|
||||
improved-db-cpu-r4,0.054,frontend,1,0.03,7.301
|
||||
improved-db-cpu-r4,0.054,load-balancer,1,24.59,18.03
|
||||
improved-db-cpu-r4,0.054,postgres,1,121.18,130.1
|
||||
improved-db-cpu-r4,0.054,redis,1,1.35,16.63
|
||||
improved-db-cpu-r4,2.129,api,4,70.26,36.812
|
||||
improved-db-cpu-r4,2.129,frontend,1,0.0,7.301
|
||||
improved-db-cpu-r4,2.129,load-balancer,1,25.64,18.99
|
||||
improved-db-cpu-r4,2.129,postgres,1,106.99,141.4
|
||||
improved-db-cpu-r4,2.129,redis,1,0.16,16.88
|
||||
improved-db-cpu-r4,5.271,api,4,65.05,33.652
|
||||
improved-db-cpu-r4,5.271,frontend,1,3.29,7.43
|
||||
improved-db-cpu-r4,5.271,load-balancer,1,26.73,18.41
|
||||
improved-db-cpu-r4,5.271,postgres,1,102.2,145.9
|
||||
improved-db-cpu-r4,5.271,redis,1,0.14,16.63
|
||||
improved-db-cpu-r4,8.446,api,4,32.86,34.023
|
||||
improved-db-cpu-r4,8.446,frontend,1,0.0,7.301
|
||||
improved-db-cpu-r4,8.446,load-balancer,1,12.44,18.0
|
||||
improved-db-cpu-r4,8.446,postgres,1,51.92,147.6
|
||||
improved-db-cpu-r4,8.446,redis,1,0.18,16.88
|
||||
|
@@ -0,0 +1,144 @@
|
||||
phase,second,successful,failed
|
||||
saturation-c1,0,252,0
|
||||
saturation-c1,1,266,0
|
||||
saturation-c1,2,257,0
|
||||
saturation-c1,3,262,0
|
||||
saturation-c1,4,253,0
|
||||
saturation-c1,5,249,0
|
||||
saturation-c1,6,228,0
|
||||
saturation-c1,7,240,0
|
||||
saturation-c1,8,219,0
|
||||
saturation-c1,9,244,0
|
||||
saturation-c1,10,2,0
|
||||
saturation-c2,0,444,0
|
||||
saturation-c2,1,377,0
|
||||
saturation-c2,2,352,0
|
||||
saturation-c2,3,297,0
|
||||
saturation-c2,4,383,0
|
||||
saturation-c2,5,324,0
|
||||
saturation-c2,6,275,0
|
||||
saturation-c2,7,323,0
|
||||
saturation-c2,8,333,0
|
||||
saturation-c2,9,381,0
|
||||
saturation-c2,10,6,0
|
||||
saturation-c4,0,526,0
|
||||
saturation-c4,1,453,0
|
||||
saturation-c4,2,425,0
|
||||
saturation-c4,3,478,0
|
||||
saturation-c4,4,510,0
|
||||
saturation-c4,5,468,0
|
||||
saturation-c4,6,458,0
|
||||
saturation-c4,7,428,0
|
||||
saturation-c4,8,490,0
|
||||
saturation-c4,9,474,0
|
||||
saturation-c4,10,11,0
|
||||
saturation-c8,0,532,0
|
||||
saturation-c8,1,476,0
|
||||
saturation-c8,2,530,0
|
||||
saturation-c8,3,529,0
|
||||
saturation-c8,4,490,0
|
||||
saturation-c8,5,527,0
|
||||
saturation-c8,6,445,0
|
||||
saturation-c8,7,524,0
|
||||
saturation-c8,8,507,0
|
||||
saturation-c8,9,519,0
|
||||
saturation-c8,10,16,0
|
||||
saturation-c16,0,349,0
|
||||
saturation-c16,1,592,0
|
||||
saturation-c16,2,541,0
|
||||
saturation-c16,3,491,0
|
||||
saturation-c16,4,545,0
|
||||
saturation-c16,5,420,0
|
||||
saturation-c16,6,499,0
|
||||
saturation-c16,7,529,0
|
||||
saturation-c16,8,548,0
|
||||
saturation-c16,9,601,0
|
||||
saturation-c16,10,28,0
|
||||
saturation-c32,0,661,0
|
||||
saturation-c32,1,604,0
|
||||
saturation-c32,2,610,0
|
||||
saturation-c32,3,589,0
|
||||
saturation-c32,4,571,0
|
||||
saturation-c32,5,627,0
|
||||
saturation-c32,6,587,0
|
||||
saturation-c32,7,619,0
|
||||
saturation-c32,8,629,0
|
||||
saturation-c32,9,548,0
|
||||
saturation-c32,10,51,0
|
||||
fixed-r1,0,511,0
|
||||
fixed-r1,1,580,0
|
||||
fixed-r1,2,659,0
|
||||
fixed-r1,3,687,0
|
||||
fixed-r1,4,588,0
|
||||
fixed-r1,5,675,0
|
||||
fixed-r1,6,647,0
|
||||
fixed-r1,7,701,0
|
||||
fixed-r1,8,690,0
|
||||
fixed-r1,9,521,0
|
||||
fixed-r1,10,32,0
|
||||
fixed-r2,0,452,0
|
||||
fixed-r2,1,545,0
|
||||
fixed-r2,2,604,0
|
||||
fixed-r2,3,652,0
|
||||
fixed-r2,4,521,0
|
||||
fixed-r2,5,627,0
|
||||
fixed-r2,6,571,0
|
||||
fixed-r2,7,611,0
|
||||
fixed-r2,8,573,0
|
||||
fixed-r2,9,586,0
|
||||
fixed-r2,10,32,0
|
||||
fixed-r4,0,133,0
|
||||
fixed-r4,1,276,0
|
||||
fixed-r4,2,461,0
|
||||
fixed-r4,3,406,0
|
||||
fixed-r4,4,503,0
|
||||
fixed-r4,5,487,0
|
||||
fixed-r4,6,377,0
|
||||
fixed-r4,7,427,0
|
||||
fixed-r4,8,551,0
|
||||
fixed-r4,9,547,0
|
||||
fixed-r4,10,50,0
|
||||
growth-r1,0,527,0
|
||||
growth-r1,1,596,0
|
||||
growth-r1,2,604,0
|
||||
growth-r1,3,616,0
|
||||
growth-r1,4,516,0
|
||||
growth-r1,5,508,0
|
||||
growth-r1,6,606,0
|
||||
growth-r1,7,620,0
|
||||
growth-r1,8,628,0
|
||||
growth-r1,9,591,0
|
||||
growth-r1,10,32,0
|
||||
growth-r2,0,461,0
|
||||
growth-r2,1,662,0
|
||||
growth-r2,2,670,0
|
||||
growth-r2,3,655,0
|
||||
growth-r2,4,640,0
|
||||
growth-r2,5,495,0
|
||||
growth-r2,6,635,0
|
||||
growth-r2,7,626,0
|
||||
growth-r2,8,598,0
|
||||
growth-r2,9,609,0
|
||||
growth-r2,10,32,0
|
||||
growth-r4,0,106,0
|
||||
growth-r4,1,514,0
|
||||
growth-r4,2,587,0
|
||||
growth-r4,3,506,0
|
||||
growth-r4,4,423,0
|
||||
growth-r4,5,409,0
|
||||
growth-r4,6,459,0
|
||||
growth-r4,7,462,0
|
||||
growth-r4,8,618,0
|
||||
growth-r4,9,549,0
|
||||
growth-r4,10,55,0
|
||||
improved-db-cpu-r4,0,492,0
|
||||
improved-db-cpu-r4,1,583,0
|
||||
improved-db-cpu-r4,2,629,0
|
||||
improved-db-cpu-r4,3,681,0
|
||||
improved-db-cpu-r4,4,667,0
|
||||
improved-db-cpu-r4,5,632,0
|
||||
improved-db-cpu-r4,6,585,0
|
||||
improved-db-cpu-r4,7,677,0
|
||||
improved-db-cpu-r4,8,602,0
|
||||
improved-db-cpu-r4,9,629,0
|
||||
improved-db-cpu-r4,10,52,0
|
||||
|
@@ -1,6 +1,6 @@
|
||||
# Lab 3 verification summary
|
||||
|
||||
Дата проверки: 2026-10-20.
|
||||
Дата проверки: 2026-10-23.
|
||||
|
||||
Файл содержит обезличенную сводку воспроизводимых проверок. Bearer tokens,
|
||||
пароли, ключи идемпотентности, UUID пользователей/чатов/сообщений и содержимое
|
||||
@@ -12,7 +12,8 @@
|
||||
- Docker Engine 29.7.2;
|
||||
- Docker Compose 5.5.0;
|
||||
- две Compose-реплики сервиса `api`;
|
||||
- единственный опубликованный endpoint: Nginx на `127.0.0.1:8080`;
|
||||
- единственный опубликованный endpoint: HAProxy на `127.0.0.1:8080`;
|
||||
- Nginx доступен только внутри Compose network и отдаёт статический React;
|
||||
- PostgreSQL и Redis доступны только внутри Compose network.
|
||||
|
||||
## Статические проверки
|
||||
@@ -42,6 +43,16 @@ shell syntax for hooks PASS
|
||||
- преподавательский `python3 tests/smoke.py` завершился строкой
|
||||
`PASS: lab3 public API scenario`.
|
||||
|
||||
## Readiness-aware routing
|
||||
|
||||
`make test-readiness-routing` добавил в network namespace одной API-реплики
|
||||
правило, запрещающее TCP-доступ к PostgreSQL, не останавливая сам процесс. Его
|
||||
`/health/ready` вернул 503; HAProxy исключил backend за 2,861 с. После этого
|
||||
40 запросов `/health/live` получили 200, а 20 корректных регистраций — 201;
|
||||
не готовая реплика не получила ни одного запроса. После удаления fault-правила
|
||||
она вернулась в ротацию. Машиночитаемый результат сохранён в
|
||||
`evidence/readiness-routing.json`.
|
||||
|
||||
## Failover API
|
||||
|
||||
Команда:
|
||||
@@ -54,30 +65,40 @@ make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
|
||||
|
||||
```text
|
||||
run 1:
|
||||
stopped container: api-2
|
||||
observed instances before action: 2
|
||||
successful writes / failures: 141 / 0
|
||||
first recovery: 2.668 seconds
|
||||
run 2, after restoring the desired two replicas:
|
||||
observed instances before action: 2
|
||||
successful writes / failures: 142 / 0
|
||||
first recovery: 0.963 seconds
|
||||
run 2, after restoring api-2:
|
||||
stopped container: api-1
|
||||
observed instances before action: 2
|
||||
successful writes / failures: 143 / 0
|
||||
first recovery: 0.750 seconds
|
||||
first recovery: 2.653 seconds
|
||||
both results: PASS
|
||||
```
|
||||
|
||||
После первого запуска реплика была восстановлена; один health-запрос сдвинул
|
||||
очередь LB, чтобы следующий synthetic message выбрал второй target. Hook
|
||||
сопоставил `TARGET_INSTANCE_ID` с hostname контейнера и в каждом запуске
|
||||
остановил только выбранную API-реплику. После тестов желаемое число
|
||||
реплик было восстановлено командой `docker compose up -d --wait api`.
|
||||
После каждого запуска желаемое число реплик было восстановлено командой
|
||||
`docker compose up -d --wait api`. Hook сопоставляет `TARGET_INSTANCE_ID` с
|
||||
hostname контейнера и останавливает только выбранную API-реплику. Отдельный
|
||||
`make grade4-experiment` целенаправленно остановил обе исходные реплики по
|
||||
очереди: 7853/0 и 7970/0 успешных/ошибочных запросов, recovery 2,893/2,938 с.
|
||||
|
||||
## Persistence и идемпотентность
|
||||
|
||||
- `make test-persistence ACTION_SCRIPT=./scripts/restart.sh`: PASS; после
|
||||
рестарта PostgreSQL, Redis, всех API-реплик и Nginx прежние bearer session и
|
||||
рестарта PostgreSQL, Redis, всех API-реплик, frontend и HAProxy прежние bearer session и
|
||||
созданные данные сохранились;
|
||||
- 32 конкурентных POST сообщения с одинаковыми semantic payload и
|
||||
`Idempotency-Key` через Nginx вернули один Message ID;
|
||||
`Idempotency-Key` через HAProxy вернули один Message ID;
|
||||
- прямой запрос PostgreSQL подтвердил ровно одну строку сообщения;
|
||||
- повтор с тем же scope/key и другим payload возвращает HTTP 409.
|
||||
|
||||
## Негативные и browser-security проверки
|
||||
|
||||
- сырой `Authorization: Bearer` с UTF-8 `é` вернул 401, Bearer challenge и
|
||||
стандартный `Error`; процесс не завершился с panic/500;
|
||||
- тело JSON из одного байта `FF` вернуло 400 `invalid_request`, а
|
||||
`error.request_id` совпал с `X-Request-Id`;
|
||||
- login установил host-only `HttpOnly; SameSite=Strict` cookie;
|
||||
- cookie без Bearer на контрактном `/api/v1/users/me` дала 401, через
|
||||
`/web-api/v1/users/me` — 200;
|
||||
- logout через `/web-api` вернул 204 и инвалидировал серверную сессию;
|
||||
- статический frontend вернул 200 и `Content-Security-Policy`.
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"target_instance": "bd53aadefab0",
|
||||
"detection_seconds": 2.861,
|
||||
"health_requests": {
|
||||
"200": 40
|
||||
},
|
||||
"registration_requests": {
|
||||
"201": 20
|
||||
},
|
||||
"recovered": true
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
FROM haproxy:3.2-alpine@sha256:dfba86f031bd4c9039f10f8cc5de75991d1b6766d5460f0335b980408503baad
|
||||
|
||||
COPY infrastructure/haproxy/haproxy.cfg /usr/local/etc/haproxy/haproxy.cfg
|
||||
|
||||
USER haproxy
|
||||
|
||||
EXPOSE 8080
|
||||
@@ -0,0 +1,66 @@
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 2048
|
||||
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
option httplog
|
||||
timeout connect 1s
|
||||
timeout client 30s
|
||||
timeout server 30s
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 1s
|
||||
hold other 1s
|
||||
hold refused 1s
|
||||
hold nx 1s
|
||||
hold timeout 1s
|
||||
hold obsolete 1s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
|
||||
acl api_request path_beg /api/ /web-api/ /health/
|
||||
acl web_api path_beg /web-api/
|
||||
acl has_session req.cook(mtusi_session) -m found
|
||||
acl message_create method POST
|
||||
acl message_path path_reg ^/(api|web-api)/v1/chats/[^/]+/messages$
|
||||
|
||||
http-request del-header Authorization if web_api
|
||||
http-request set-header Authorization "Bearer %[req.cook(mtusi_session)]" if web_api has_session
|
||||
http-request del-header Cookie if api_request
|
||||
http-request replace-path ^/web-api/(.*)$ /api/\1 if web_api
|
||||
|
||||
use_backend message_backend if message_create message_path
|
||||
use_backend api_backend if api_request
|
||||
default_backend frontend_backend
|
||||
|
||||
backend api_backend
|
||||
balance leastconn
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
server-template api 8 api:8080
|
||||
|
||||
backend message_backend
|
||||
balance leastconn
|
||||
option httpchk
|
||||
option redispatch
|
||||
option http-buffer-request
|
||||
retries 3
|
||||
retry-on conn-failure empty-response response-timeout 502 503 504
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
server-template api 8 api:8080
|
||||
|
||||
backend frontend_backend
|
||||
option httpchk GET /
|
||||
http-check expect status 200
|
||||
server frontend frontend:8080 check inter 2s fall 2 rise 2 resolvers docker init-addr libc,none
|
||||
@@ -1,24 +1,12 @@
|
||||
worker_processes auto;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
worker_connections 512;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
|
||||
map $cookie_mtusi_session $web_authorization {
|
||||
default "Bearer $cookie_mtusi_session";
|
||||
"" "";
|
||||
}
|
||||
|
||||
upstream api_backend {
|
||||
zone api_backend 64k;
|
||||
least_conn;
|
||||
server api:8080 resolve max_fails=2 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
@@ -31,90 +19,6 @@ http {
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header Content-Security-Policy "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always;
|
||||
|
||||
# Message creation carries an Idempotency-Key. It is therefore safe to
|
||||
# retry a POST on another replica when the selected backend is draining.
|
||||
location ~ ^/api/v1/chats/[^/]+/messages$ {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Cookie "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
|
||||
proxy_next_upstream_tries 3;
|
||||
}
|
||||
|
||||
location ~ ^/web-api/v1/chats/[^/]+/messages$ {
|
||||
rewrite ^/web-api/(.*)$ /api/$1 break;
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Authorization $web_authorization;
|
||||
proxy_set_header Cookie "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
|
||||
proxy_next_upstream_tries 3;
|
||||
}
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Cookie "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
location /web-api/ {
|
||||
rewrite ^/web-api/(.*)$ /api/$1 break;
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Authorization $web_authorization;
|
||||
proxy_set_header Cookie "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
location /health/ {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 5s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504;
|
||||
proxy_next_upstream_tries 2;
|
||||
}
|
||||
|
||||
location /assets/ {
|
||||
try_files $uri =404;
|
||||
expires 1y;
|
||||
|
||||
@@ -77,7 +77,7 @@ def memory_mib(raw):
|
||||
|
||||
|
||||
def service(name):
|
||||
for candidate in ("load-balancer", "postgres", "redis", "api"):
|
||||
for candidate in ("load-balancer", "frontend", "postgres", "redis", "api"):
|
||||
if f"-{candidate}-" in name:
|
||||
return candidate
|
||||
return "other"
|
||||
@@ -357,7 +357,7 @@ def main():
|
||||
"action_after_seconds": args.action_after_seconds,
|
||||
"post_action_seconds": args.post_action_seconds,
|
||||
"warmup_requests": 12,
|
||||
"workload": "closed-loop POST messages via Nginx; no automatic retries",
|
||||
"workload": "closed-loop POST messages via the public HAProxy gateway; no automatic retries",
|
||||
},
|
||||
"phases": phases,
|
||||
"idempotency": idem,
|
||||
@@ -371,7 +371,7 @@ def main():
|
||||
"single Docker host",
|
||||
"closed-loop coordinated omission",
|
||||
"successful-response latency percentiles only",
|
||||
"PostgreSQL and Nginx remain single points of failure",
|
||||
"PostgreSQL and HAProxy remain single points of failure",
|
||||
],
|
||||
}
|
||||
destination = ROOT / args.output
|
||||
|
||||
@@ -0,0 +1,577 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Measure the price of horizontal API scaling for lab 3 grade 5."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
import datetime as dt
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def command(args, env=None, timeout=240, check=True):
|
||||
result = subprocess.run(
|
||||
[str(value) for value in args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
suffix = f": {detail[-1]}" if detail else ""
|
||||
raise RuntimeError(
|
||||
f"{' '.join(map(str, args[:4]))} exited {result.returncode}{suffix}"
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def compose(env_file, *args, env=None, timeout=240, check=True):
|
||||
return command(
|
||||
["docker", "compose", "--env-file", env_file, *args],
|
||||
env=env,
|
||||
timeout=timeout,
|
||||
check=check,
|
||||
)
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
raw = command(
|
||||
["docker", "inspect", "--format", "{{json .}}", container_id]
|
||||
).stdout
|
||||
return json.loads(raw)
|
||||
|
||||
|
||||
def service_ids(env_file, service=None):
|
||||
args = ["ps", "-q"]
|
||||
if service:
|
||||
args.append(service)
|
||||
return [line for line in compose(env_file, *args).stdout.splitlines() if line]
|
||||
|
||||
|
||||
def wait_healthy(env_file, service, expected=1, timeout=120):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
healthy = []
|
||||
for container_id in service_ids(env_file, service):
|
||||
state = inspect(container_id)["State"]
|
||||
if state.get("Health", {}).get("Status") == "healthy":
|
||||
healthy.append(container_id)
|
||||
if len(healthy) == expected:
|
||||
return healthy
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"{service}: expected {expected} healthy containers")
|
||||
|
||||
|
||||
def percentile(values, ratio):
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
return round(ordered[max(0, math.ceil(ratio * len(ordered)) - 1)], 3)
|
||||
|
||||
|
||||
def memory_mib(raw):
|
||||
value = raw.split("/", 1)[0].strip()
|
||||
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
||||
for suffix in ("GiB", "MiB", "KiB", "B"):
|
||||
if value.endswith(suffix):
|
||||
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
|
||||
return 0.0
|
||||
|
||||
|
||||
def profile_environment(profile):
|
||||
env = dict(os.environ)
|
||||
env.update({
|
||||
"API_REPLICAS": str(profile["replicas"]),
|
||||
"API_CPU_LIMIT": f"{profile['api_cpu']:.2f}",
|
||||
"API_MEMORY_LIMIT": profile["api_memory"],
|
||||
"DB_MAX_CONNS": str(profile["db_max_conns"]),
|
||||
"DB_MIN_CONNS": str(profile["db_min_conns"]),
|
||||
"POSTGRES_CPU_LIMIT": f"{profile['postgres_cpu']:.2f}",
|
||||
"POSTGRES_MEMORY_LIMIT": profile["postgres_memory"],
|
||||
})
|
||||
return env
|
||||
|
||||
|
||||
def configure_profile(env_file, profile):
|
||||
env = profile_environment(profile)
|
||||
compose(
|
||||
env_file,
|
||||
"up", "-d", "--no-deps", "postgres",
|
||||
env=env,
|
||||
timeout=180,
|
||||
)
|
||||
wait_healthy(env_file, "postgres")
|
||||
compose(
|
||||
env_file,
|
||||
"up", "-d", "--no-deps",
|
||||
"--scale", f"api={profile['replicas']}",
|
||||
"--force-recreate", "api",
|
||||
env=env,
|
||||
timeout=240,
|
||||
)
|
||||
api = wait_healthy(env_file, "api", profile["replicas"])
|
||||
wait_healthy(env_file, "load-balancer")
|
||||
time.sleep(1.5)
|
||||
|
||||
expected_cpu = int(profile["api_cpu"] * 1_000_000_000)
|
||||
expected_memory = parse_memory_limit(profile["api_memory"])
|
||||
applied = []
|
||||
for container_id in api:
|
||||
data = inspect(container_id)
|
||||
actual_env = dict(item.split("=", 1) for item in data["Config"]["Env"] if "=" in item)
|
||||
host = data["HostConfig"]
|
||||
if host.get("NanoCpus") != expected_cpu or host.get("Memory") != expected_memory:
|
||||
raise RuntimeError("API container limits do not match the selected profile")
|
||||
if actual_env.get("DB_MAX_CONNS") != str(profile["db_max_conns"]):
|
||||
raise RuntimeError("API DB pool size does not match the selected profile")
|
||||
applied.append({
|
||||
"instance": data["Config"]["Hostname"],
|
||||
"cpu_limit": host.get("NanoCpus", 0) / 1_000_000_000,
|
||||
"memory_limit_mib": round(host.get("Memory", 0) / 1024**2, 1),
|
||||
"db_max_conns": int(actual_env["DB_MAX_CONNS"]),
|
||||
})
|
||||
|
||||
postgres = inspect(service_ids(env_file, "postgres")[0])
|
||||
expected_pg_cpu = int(profile["postgres_cpu"] * 1_000_000_000)
|
||||
if postgres["HostConfig"].get("NanoCpus") != expected_pg_cpu:
|
||||
raise RuntimeError("PostgreSQL CPU limit does not match the selected profile")
|
||||
return {
|
||||
"api": sorted(applied, key=lambda item: item["instance"]),
|
||||
"postgres_cpu_limit": postgres["HostConfig"].get("NanoCpus", 0) / 1_000_000_000,
|
||||
"postgres_memory_limit_mib": round(postgres["HostConfig"].get("Memory", 0) / 1024**2, 1),
|
||||
}
|
||||
|
||||
|
||||
def parse_memory_limit(value):
|
||||
suffix = value[-1].upper()
|
||||
multiplier = {"K": 1024, "M": 1024**2, "G": 1024**3}[suffix]
|
||||
return int(value[:-1]) * multiplier
|
||||
|
||||
|
||||
def resource_targets(env_file):
|
||||
targets = {}
|
||||
for container_id in service_ids(env_file):
|
||||
data = inspect(container_id)
|
||||
service = data["Config"]["Labels"].get("com.docker.compose.service", "other")
|
||||
if service in {"api", "postgres", "redis", "load-balancer", "frontend"}:
|
||||
targets[container_id] = {
|
||||
"service": service,
|
||||
"name": data["Name"].lstrip("/"),
|
||||
}
|
||||
return targets
|
||||
|
||||
|
||||
def stats_snapshot(targets, elapsed):
|
||||
if not targets:
|
||||
return {"t": round(elapsed, 3), "services": {}}
|
||||
result = command(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}", *targets],
|
||||
timeout=15,
|
||||
check=False,
|
||||
)
|
||||
grouped = defaultdict(lambda: {"cpu_percent": 0.0, "memory_mib": 0.0, "containers": 0})
|
||||
for line in result.stdout.splitlines():
|
||||
try:
|
||||
row = json.loads(line)
|
||||
container_id = row.get("ID", "")
|
||||
match = next((key for key in targets if key.startswith(container_id)), None)
|
||||
if not match:
|
||||
continue
|
||||
values = grouped[targets[match]["service"]]
|
||||
values["cpu_percent"] += float(row.get("CPUPerc", "0%").rstrip("%"))
|
||||
values["memory_mib"] += memory_mib(row.get("MemUsage", "0B / 0B"))
|
||||
values["containers"] += 1
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
return {
|
||||
"t": round(elapsed, 3),
|
||||
"services": {
|
||||
name: {
|
||||
"cpu_percent": round(values["cpu_percent"], 3),
|
||||
"memory_mib": round(values["memory_mib"], 3),
|
||||
"containers": values["containers"],
|
||||
}
|
||||
for name, values in sorted(grouped.items())
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def summarize_resources(snapshots):
|
||||
grouped = defaultdict(lambda: {"cpu": [], "memory": [], "containers": []})
|
||||
for snapshot in snapshots:
|
||||
for service, values in snapshot["services"].items():
|
||||
grouped[service]["cpu"].append(values["cpu_percent"])
|
||||
grouped[service]["memory"].append(values["memory_mib"])
|
||||
grouped[service]["containers"].append(values["containers"])
|
||||
return {
|
||||
service: {
|
||||
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
|
||||
"cpu_percent_max": round(max(values["cpu"]), 3),
|
||||
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
|
||||
"memory_mib_max": round(max(values["memory"]), 3),
|
||||
"containers": max(values["containers"]),
|
||||
}
|
||||
for service, values in sorted(grouped.items())
|
||||
if values["cpu"]
|
||||
}
|
||||
|
||||
|
||||
def run_phase(name, kind, actor, chat, base_url, duration, concurrency, env_file, profile):
|
||||
records = []
|
||||
snapshots = []
|
||||
lock = threading.Lock()
|
||||
stop = threading.Event()
|
||||
barrier = threading.Barrier(concurrency + 1)
|
||||
targets = resource_targets(env_file)
|
||||
started = time.monotonic()
|
||||
|
||||
def worker(worker_id):
|
||||
client = Client(base_url, timeout=5)
|
||||
index = 0
|
||||
barrier.wait()
|
||||
while not stop.is_set():
|
||||
request_started = time.monotonic()
|
||||
try:
|
||||
response = client.message(
|
||||
actor,
|
||||
chat,
|
||||
f"grade5-{name}-{worker_id}-{index}",
|
||||
)
|
||||
row = {
|
||||
"t": round(time.monotonic() - started, 3),
|
||||
"ok": True,
|
||||
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
||||
"instance": response["headers"].get("X-Instance-Id", ""),
|
||||
}
|
||||
except Failure as exc:
|
||||
row = {
|
||||
"t": round(time.monotonic() - started, 3),
|
||||
"ok": False,
|
||||
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
||||
"error": str(exc),
|
||||
}
|
||||
with lock:
|
||||
records.append(row)
|
||||
index += 1
|
||||
|
||||
def sampler():
|
||||
while not stop.is_set():
|
||||
snapshots.append(stats_snapshot(targets, time.monotonic() - started))
|
||||
stop.wait(1)
|
||||
|
||||
workers = [
|
||||
threading.Thread(target=worker, args=(index,), daemon=True)
|
||||
for index in range(concurrency)
|
||||
]
|
||||
for thread in workers:
|
||||
thread.start()
|
||||
stats_thread = threading.Thread(target=sampler, daemon=True)
|
||||
stats_thread.start()
|
||||
started = time.monotonic()
|
||||
barrier.wait()
|
||||
time.sleep(duration)
|
||||
stop.set()
|
||||
for thread in workers:
|
||||
thread.join(10)
|
||||
stats_thread.join(15)
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
successes = [row for row in records if row["ok"]]
|
||||
failures = [row for row in records if not row["ok"]]
|
||||
latencies = [row["latency_ms"] for row in successes]
|
||||
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
|
||||
for row in records:
|
||||
key = str(max(0, int(row["t"])))
|
||||
buckets[key]["successful" if row["ok"] else "failed"] += 1
|
||||
total = len(records)
|
||||
return {
|
||||
"name": name,
|
||||
"kind": kind,
|
||||
"profile": profile["name"],
|
||||
"replicas": profile["replicas"],
|
||||
"concurrency": concurrency,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"successful_requests": len(successes),
|
||||
"failed_requests": len(failures),
|
||||
"error_ratio": round(len(failures) / total, 6) if total else None,
|
||||
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
|
||||
"successful_rps": round(len(successes) / elapsed, 3),
|
||||
"successful_latency_ms": {
|
||||
"p50": percentile(latencies, 0.50),
|
||||
"p95": percentile(latencies, 0.95),
|
||||
"p99": percentile(latencies, 0.99),
|
||||
},
|
||||
"instances": sorted({row["instance"] for row in successes if row.get("instance")}),
|
||||
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
|
||||
"resource_samples": snapshots,
|
||||
"resources": summarize_resources(snapshots),
|
||||
}
|
||||
|
||||
|
||||
def dataset(env_file):
|
||||
sql = (
|
||||
"select count(*)::text || '|' || pg_database_size(current_database())::text "
|
||||
"from messages"
|
||||
)
|
||||
result = compose(
|
||||
env_file,
|
||||
"exec", "-T", "postgres", "sh", "-c",
|
||||
'psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -Atc "$1"',
|
||||
"sh", sql,
|
||||
)
|
||||
count, size = result.stdout.strip().split("|", 1)
|
||||
return {"messages": int(count), "database_size_bytes": int(size)}
|
||||
|
||||
|
||||
def host_environment(env_file):
|
||||
mem_total_kib = 0
|
||||
with open("/proc/meminfo", encoding="utf-8") as source:
|
||||
for line in source:
|
||||
if line.startswith("MemTotal:"):
|
||||
mem_total_kib = int(line.split()[1])
|
||||
break
|
||||
images = compose(env_file, "config", "--images").stdout.splitlines()
|
||||
return {
|
||||
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
|
||||
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
|
||||
"logical_cpu_count": os.cpu_count(),
|
||||
"host_memory_mib": round(mem_total_kib / 1024, 1),
|
||||
"kernel": platform.release(),
|
||||
"python": platform.python_version(),
|
||||
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
|
||||
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
|
||||
"images": images,
|
||||
}
|
||||
|
||||
|
||||
def analyze(phases):
|
||||
sweep = sorted(
|
||||
(phase for phase in phases if phase["kind"] == "saturation"),
|
||||
key=lambda phase: phase["concurrency"],
|
||||
)
|
||||
saturation = None
|
||||
previous = None
|
||||
for phase in sweep:
|
||||
if previous:
|
||||
gain = (phase["successful_rps"] - previous["successful_rps"]) / max(previous["successful_rps"], 0.001)
|
||||
latency_growth = phase["successful_latency_ms"]["p95"] / max(previous["successful_latency_ms"]["p95"], 0.001)
|
||||
if saturation is None and gain < 0.10 and latency_growth >= 1.50:
|
||||
saturation = {
|
||||
"concurrency": phase["concurrency"],
|
||||
"previous_concurrency": previous["concurrency"],
|
||||
"rps_gain_percent": round(gain * 100, 2),
|
||||
"p95_growth_ratio": round(latency_growth, 3),
|
||||
}
|
||||
previous = phase
|
||||
|
||||
baseline = next(phase for phase in phases if phase["name"] == "growth-r4")
|
||||
improved = next(phase for phase in phases if phase["name"] == "improved-db-cpu-r4")
|
||||
rps_change = (improved["successful_rps"] - baseline["successful_rps"]) / max(baseline["successful_rps"], 0.001) * 100
|
||||
p95_change = (improved["successful_latency_ms"]["p95"] - baseline["successful_latency_ms"]["p95"]) / max(baseline["successful_latency_ms"]["p95"], 0.001) * 100
|
||||
confirmed = (
|
||||
rps_change >= 5
|
||||
and improved["error_ratio"] <= baseline["error_ratio"] + 0.001
|
||||
) or p95_change <= -10
|
||||
|
||||
resources = baseline["resources"]
|
||||
pg = resources.get("postgres", {})
|
||||
lb = resources.get("load-balancer", {})
|
||||
bottleneck = "undetermined"
|
||||
if pg.get("cpu_percent_avg", 0) >= 70 and pg.get("cpu_percent_max", 0) >= 95:
|
||||
bottleneck = "postgres_cpu"
|
||||
elif lb.get("cpu_percent_avg", 0) >= 20 and lb.get("cpu_percent_max", 0) >= 24:
|
||||
bottleneck = "load_balancer_cpu"
|
||||
improved_resources = improved["resources"]
|
||||
improved_lb = improved_resources.get("load-balancer", {})
|
||||
remaining_bottleneck = "undetermined"
|
||||
if improved_lb.get("cpu_percent_avg", 0) >= 20 and improved_lb.get("cpu_percent_max", 0) >= 24:
|
||||
remaining_bottleneck = "load_balancer_cpu"
|
||||
return {
|
||||
"saturation_point": saturation,
|
||||
"observed_bottleneck": bottleneck,
|
||||
"baseline_resource_snapshot": resources,
|
||||
"improved_resource_snapshot": improved_resources,
|
||||
"remaining_bottleneck_after_improvement": remaining_bottleneck,
|
||||
"improvement": {
|
||||
"change": "PostgreSQL CPU limit: 1.00 -> 2.00; API profile unchanged",
|
||||
"baseline_phase": baseline["name"],
|
||||
"improved_phase": improved["name"],
|
||||
"rps_change_percent": round(rps_change, 2),
|
||||
"p95_change_percent": round(p95_change, 2),
|
||||
"confirmed": confirmed,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def profiles():
|
||||
base = {"postgres_cpu": 1.0, "postgres_memory": "512M"}
|
||||
return {
|
||||
"fixed-r1": {**base, "name": "fixed-r1", "replicas": 1, "api_cpu": 1.0, "api_memory": "512M", "db_max_conns": 16, "db_min_conns": 2},
|
||||
"fixed-r2": {**base, "name": "fixed-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
|
||||
"fixed-r4": {**base, "name": "fixed-r4", "replicas": 4, "api_cpu": 0.25, "api_memory": "128M", "db_max_conns": 4, "db_min_conns": 1},
|
||||
"growth-r1": {**base, "name": "growth-r1", "replicas": 1, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
|
||||
"growth-r2": {**base, "name": "growth-r2", "replicas": 2, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
|
||||
"growth-r4": {**base, "name": "growth-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1},
|
||||
"improved-db-cpu-r4": {**base, "name": "improved-db-cpu-r4", "replicas": 4, "api_cpu": 0.5, "api_memory": "256M", "db_max_conns": 8, "db_min_conns": 1, "postgres_cpu": 2.0},
|
||||
}
|
||||
|
||||
|
||||
def run_warmup(actor, chat, args, profile):
|
||||
phase = run_phase(
|
||||
f"warmup-{profile['name']}",
|
||||
"warmup",
|
||||
actor,
|
||||
chat,
|
||||
args.base_url,
|
||||
args.warmup_seconds,
|
||||
min(4, args.comparison_concurrency),
|
||||
args.env_file,
|
||||
profile,
|
||||
)
|
||||
if phase["failed_requests"]:
|
||||
raise RuntimeError(f"warmup failed for {profile['name']}")
|
||||
return phase["successful_requests"]
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--env-file", default=".env")
|
||||
parser.add_argument("--duration", type=float, default=10)
|
||||
parser.add_argument("--warmup-seconds", type=float, default=2)
|
||||
parser.add_argument("--comparison-concurrency", type=int, default=32)
|
||||
parser.add_argument("--sweep", default="1,2,4,8,16,32")
|
||||
parser.add_argument("--output", default="evidence/grade5/results.json")
|
||||
args = parser.parse_args()
|
||||
sweep = [int(value) for value in args.sweep.split(",")]
|
||||
if args.duration <= 0 or args.warmup_seconds <= 0:
|
||||
raise SystemExit("durations must be positive")
|
||||
if not sweep or min(sweep) < 1 or max(sweep + [args.comparison_concurrency]) > 64:
|
||||
raise SystemExit("concurrency must be 1..64")
|
||||
|
||||
all_profiles = profiles()
|
||||
restore = {
|
||||
"name": "restore-default",
|
||||
"replicas": 2,
|
||||
"api_cpu": 0.5,
|
||||
"api_memory": "256M",
|
||||
"db_max_conns": 10,
|
||||
"db_min_conns": 2,
|
||||
"postgres_cpu": 1.0,
|
||||
"postgres_memory": "512M",
|
||||
}
|
||||
phases = []
|
||||
applied = {}
|
||||
warmup_requests = 0
|
||||
restoration = {"attempted": False, "succeeded": False}
|
||||
environment = host_environment(args.env_file)
|
||||
start_dataset = dataset(args.env_file)
|
||||
try:
|
||||
client = Client(args.base_url, timeout=8)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
|
||||
saturation_profile = all_profiles["growth-r2"]
|
||||
applied[saturation_profile["name"]] = configure_profile(args.env_file, saturation_profile)
|
||||
warmup_requests += run_warmup(alice, chat, args, saturation_profile)
|
||||
for concurrency in sweep:
|
||||
print(f"saturation: concurrency={concurrency}", flush=True)
|
||||
phases.append(run_phase(
|
||||
f"saturation-c{concurrency}", "saturation", alice, chat,
|
||||
args.base_url, args.duration, concurrency, args.env_file,
|
||||
saturation_profile,
|
||||
))
|
||||
|
||||
for profile_name in (
|
||||
"fixed-r1", "fixed-r2", "fixed-r4",
|
||||
"growth-r1", "growth-r2", "growth-r4",
|
||||
"improved-db-cpu-r4",
|
||||
):
|
||||
profile = all_profiles[profile_name]
|
||||
print(f"profile: {profile_name}", flush=True)
|
||||
applied[profile_name] = configure_profile(args.env_file, profile)
|
||||
warmup_requests += run_warmup(alice, chat, args, profile)
|
||||
kind = "improvement" if profile_name.startswith("improved") else profile_name.split("-", 1)[0]
|
||||
phases.append(run_phase(
|
||||
profile_name, kind, alice, chat, args.base_url,
|
||||
args.duration, args.comparison_concurrency, args.env_file,
|
||||
profile,
|
||||
))
|
||||
end_dataset = dataset(args.env_file)
|
||||
finally:
|
||||
restoration["attempted"] = True
|
||||
try:
|
||||
restoration["applied"] = configure_profile(args.env_file, restore)
|
||||
restoration["succeeded"] = True
|
||||
except Exception as exc:
|
||||
restoration["error"] = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
if not restoration["succeeded"]:
|
||||
raise RuntimeError("experiment completed but default Compose profile was not restored")
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"environment": environment,
|
||||
"hypothesis": (
|
||||
"API replicas stop improving throughput when shared PostgreSQL CPU saturates; "
|
||||
"adding PostgreSQL CPU should improve the unchanged four-replica profile."
|
||||
),
|
||||
"methodology": {
|
||||
"workload": "closed-loop POST messages through the public HAProxy gateway; unique idempotency keys; no client retries",
|
||||
"duration_seconds_per_phase": args.duration,
|
||||
"warmup_seconds_per_profile": args.warmup_seconds,
|
||||
"comparison_concurrency": args.comparison_concurrency,
|
||||
"saturation_concurrency": sweep,
|
||||
"fixed_total_budget": "1.00 API CPU, 512 MiB API RAM, 16 total DB pool connections",
|
||||
"growth_budget": "0.50 API CPU, 256 MiB RAM and 8 DB pool connections per replica",
|
||||
"improvement": "only PostgreSQL CPU changes from 1.00 to 2.00",
|
||||
"saturation_rule": "RPS gain below 10% and p95 growth at least 1.5x versus the previous step",
|
||||
"cpu_bottleneck_rule": "average at least 70% and peak at least 95% of PostgreSQL CPU limit; service-specific threshold for the 0.25 CPU load balancer",
|
||||
},
|
||||
"profiles": all_profiles,
|
||||
"applied_limits": applied,
|
||||
"phases": phases,
|
||||
"analysis": analyze(phases),
|
||||
"dataset": {
|
||||
"before": start_dataset,
|
||||
"after": end_dataset,
|
||||
"warmup_successful_requests": warmup_requests,
|
||||
},
|
||||
"restoration": restoration,
|
||||
"limitations": [
|
||||
"single Docker host: replicas do not prove host-level availability",
|
||||
"closed-loop workload has coordinated omission",
|
||||
"successful-response latency percentiles are reported separately from failures",
|
||||
"one measured run per profile; host scheduling noise is possible",
|
||||
],
|
||||
}
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
try:
|
||||
displayed_output = destination.relative_to(ROOT)
|
||||
except ValueError:
|
||||
displayed_output = destination
|
||||
print(json.dumps({
|
||||
"output": str(displayed_output),
|
||||
"analysis": report["analysis"],
|
||||
"restoration": restoration["succeeded"],
|
||||
}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -148,7 +148,7 @@ def summary(report, output):
|
||||
"## Method", "",
|
||||
f"- concurrency: {method['concurrency']};",
|
||||
f"- baseline duration: {method['baseline_seconds']} s;",
|
||||
"- load: closed-loop POST messages through Nginx;",
|
||||
"- load: closed-loop POST messages through the public HAProxy gateway;",
|
||||
"- failover: both original API containers stopped and restored in turn;",
|
||||
"- rolling update: one surge replica, old replicas replaced one by one;",
|
||||
"- percentiles cover successful responses; failures are separate.", "",
|
||||
@@ -178,7 +178,7 @@ def summary(report, output):
|
||||
"", "",
|
||||
"## Scope", "",
|
||||
"Single-host process failure and rolling replacement only. PostgreSQL "
|
||||
"and Nginx remain single points of failure.", "",
|
||||
"and HAProxy remain single points of failure.", "",
|
||||
]
|
||||
output.write_text("\n".join(lines))
|
||||
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render grade-5 scaling evidence as SVG, CSV and Markdown."""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
STYLE = """
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.subtitle { font-size: 14px; fill: #aeb2b8; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.axis { stroke: #747981; stroke-width: 1.5; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.line2 { fill: none; stroke: #92979f; stroke-width: 3; }
|
||||
.point { fill: #f4f5f6; }
|
||||
.point2 { fill: #92979f; }
|
||||
.bar { fill: #d7d9dc; }
|
||||
.bar2 { fill: #777c84; }
|
||||
.bar3 { fill: #4c5057; }
|
||||
.threshold { stroke: #8f949c; stroke-width: 2; stroke-dasharray: 7 5; }
|
||||
"""
|
||||
|
||||
|
||||
def svg(title, subtitle, body, width=1120, height=620):
|
||||
return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>{STYLE}</style>
|
||||
<text x="50" y="44" class="title">{html.escape(title)}</text>
|
||||
<text x="50" y="70" class="subtitle">{html.escape(subtitle)}</text>
|
||||
{body}
|
||||
</svg>
|
||||
"""
|
||||
|
||||
|
||||
def grid(maximum, left, top, width, height, suffix=""):
|
||||
maximum = maximum or 1
|
||||
parts = []
|
||||
for step in range(6):
|
||||
value = maximum * step / 5
|
||||
y = top + height - height * step / 5
|
||||
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{left+width}" y2="{y:.1f}" class="grid"/>')
|
||||
parts.append(f'<text x="{left-12}" y="{y+4:.1f}" text-anchor="end" class="small">{value:.0f}{suffix}</text>')
|
||||
return parts
|
||||
|
||||
|
||||
def phase_map(report):
|
||||
return {phase["name"]: phase for phase in report["phases"]}
|
||||
|
||||
|
||||
def saturation_graph(report, output):
|
||||
phases = sorted(
|
||||
(phase for phase in report["phases"] if phase["kind"] == "saturation"),
|
||||
key=lambda phase: phase["concurrency"],
|
||||
)
|
||||
left, top, width, height = 90, 110, 940, 390
|
||||
max_rps = max(phase["successful_rps"] for phase in phases) * 1.1
|
||||
max_p95 = max(phase["successful_latency_ms"]["p95"] for phase in phases) * 1.1
|
||||
parts = grid(max_rps, left, top, width, height)
|
||||
points_rps, points_p95 = [], []
|
||||
for index, phase in enumerate(phases):
|
||||
x = left + width * index / max(1, len(phases) - 1)
|
||||
y_rps = top + height - height * phase["successful_rps"] / max_rps
|
||||
y_p95 = top + height - height * phase["successful_latency_ms"]["p95"] / max_p95
|
||||
points_rps.append(f"{x:.1f},{y_rps:.1f}")
|
||||
points_p95.append(f"{x:.1f},{y_p95:.1f}")
|
||||
parts.append(f'<circle cx="{x:.1f}" cy="{y_rps:.1f}" r="6" class="point"/>')
|
||||
parts.append(f'<circle cx="{x:.1f}" cy="{y_p95:.1f}" r="6" class="point2"/>')
|
||||
parts.append(f'<text x="{x:.1f}" y="{top+height+28}" text-anchor="middle" class="label">{phase["concurrency"]}</text>')
|
||||
parts.append(f'<text x="{x:.1f}" y="{y_rps-12:.1f}" text-anchor="middle" class="small">{phase["successful_rps"]:.0f}</text>')
|
||||
parts.append(f'<polyline points="{" ".join(points_rps)}" class="line"/>')
|
||||
parts.append(f'<polyline points="{" ".join(points_p95)}" class="line2"/>')
|
||||
parts.append(f'<text x="{left}" y="{top+height+62}" class="label">Concurrency</text>')
|
||||
parts.append(f'<text x="{left+720}" y="{top+height+62}" class="label">white: RPS · gray: p95 normalized</text>')
|
||||
parts.append(f'<text x="30" y="{top+height/2}" transform="rotate(-90 30 {top+height/2})" class="label">Successful requests per second</text>')
|
||||
output.write_text(svg(
|
||||
"Saturation curve",
|
||||
"Two API replicas; 1 CPU total; 16 total DB connections; PostgreSQL limited to 1 CPU",
|
||||
"\n".join(parts),
|
||||
), encoding="utf-8")
|
||||
|
||||
|
||||
def comparison_graph(report, kind, output):
|
||||
phases = sorted(
|
||||
(phase for phase in report["phases"] if phase["kind"] == kind),
|
||||
key=lambda phase: phase["replicas"],
|
||||
)
|
||||
left, top, width, height = 90, 110, 940, 370
|
||||
max_rps = max(phase["successful_rps"] for phase in phases) * 1.15
|
||||
parts = grid(max_rps, left, top, width, height)
|
||||
slot = width / len(phases)
|
||||
for index, phase in enumerate(phases):
|
||||
bar_width = min(150, slot * 0.52)
|
||||
x = left + index * slot + (slot - bar_width) / 2
|
||||
bar_height = height * phase["successful_rps"] / max_rps
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="14" class="bar"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-12:.1f}" text-anchor="middle" class="label">{phase["successful_rps"]:.1f} RPS</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+30}" text-anchor="middle" class="label">{phase["replicas"]} replica(s)</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+52}" text-anchor="middle" class="small">p95 {phase["successful_latency_ms"]["p95"]:.1f} ms</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+70}" text-anchor="middle" class="small">errors {phase["error_ratio"]*100:.3f}%</text>')
|
||||
if kind == "fixed":
|
||||
title = "Fixed total API budget"
|
||||
subtitle = "1 CPU · 512 MiB · 16 DB connections split across 1, 2 and 4 replicas"
|
||||
else:
|
||||
title = "Growing API budget"
|
||||
subtitle = "Each replica adds 0.5 CPU · 256 MiB · 8 DB connections; PostgreSQL remains at 1 CPU"
|
||||
output.write_text(svg(title, subtitle, "\n".join(parts)), encoding="utf-8")
|
||||
|
||||
|
||||
def bottleneck_graph(report, output):
|
||||
phases = sorted(
|
||||
(phase for phase in report["phases"] if phase["kind"] == "growth"),
|
||||
key=lambda phase: phase["replicas"],
|
||||
)
|
||||
services = ("api", "postgres", "load-balancer", "frontend", "redis")
|
||||
left, top, width, height = 90, 110, 940, 370
|
||||
maximum = max(
|
||||
phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
|
||||
for phase in phases for service in services
|
||||
) * 1.15 or 1
|
||||
parts = grid(maximum, left, top, width, height)
|
||||
slot = width / len(phases)
|
||||
colors = ("bar", "bar2", "bar3", "bar3", "bar3")
|
||||
for phase_index, phase in enumerate(phases):
|
||||
group_left = left + phase_index * slot
|
||||
bar_width = min(48, slot / 6)
|
||||
for service_index, service in enumerate(services):
|
||||
value = phase["resources"].get(service, {}).get("cpu_percent_avg", 0)
|
||||
x = group_left + slot * 0.12 + service_index * (bar_width + 9)
|
||||
bar_height = height * value / maximum
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="7" class="{colors[service_index]}"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-7:.1f}" text-anchor="middle" class="small">{value:.0f}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+22}" text-anchor="middle" class="small">{service[:3]}</text>')
|
||||
parts.append(f'<text x="{group_left+slot/2:.1f}" y="{top+height+52}" text-anchor="middle" class="label">{phase["replicas"]} replica(s)</text>')
|
||||
parts.append(f'<text x="30" y="{top+height/2}" transform="rotate(-90 30 {top+height/2})" class="label">Average aggregate CPU, %</text>')
|
||||
output.write_text(svg(
|
||||
"Where scaling stops",
|
||||
"Aggregate service CPU during the proportional-growth series (api · postgres · HAProxy · frontend · redis)",
|
||||
"\n".join(parts),
|
||||
), encoding="utf-8")
|
||||
|
||||
|
||||
def improvement_graph(report, output):
|
||||
phases = phase_map(report)
|
||||
baseline = phases["growth-r4"]
|
||||
improved = phases["improved-db-cpu-r4"]
|
||||
pair = [baseline, improved]
|
||||
left, top, panel_width, height = 90, 120, 390, 340
|
||||
parts = []
|
||||
panels = [
|
||||
("Successful RPS", "successful_rps", left),
|
||||
("p95 latency, ms", "p95", 630),
|
||||
]
|
||||
for title, metric, panel_left in panels:
|
||||
values = [
|
||||
phase["successful_rps"] if metric == "successful_rps" else phase["successful_latency_ms"]["p95"]
|
||||
for phase in pair
|
||||
]
|
||||
maximum = max(values) * 1.2 or 1
|
||||
parts.append(f'<text x="{panel_left}" y="{top-22}" class="label">{title}</text>')
|
||||
for index, (phase, value) in enumerate(zip(pair, values)):
|
||||
bar_width = 120
|
||||
x = panel_left + 35 + index * 185
|
||||
bar_height = height * value / maximum
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x}" y="{y:.1f}" width="{bar_width}" height="{bar_height:.1f}" rx="14" class="{"bar" if index else "bar2"}"/>')
|
||||
parts.append(f'<text x="{x+60}" y="{y-10:.1f}" text-anchor="middle" class="label">{value:.1f}</text>')
|
||||
parts.append(f'<text x="{x+60}" y="{top+height+28}" text-anchor="middle" class="small">PG {1 if index == 0 else 2} CPU</text>')
|
||||
analysis = report["analysis"]["improvement"]
|
||||
parts.append(f'<text x="90" y="545" class="label">RPS change: {analysis["rps_change_percent"]:+.2f}% · p95 change: {analysis["p95_change_percent"]:+.2f}% · confirmed: {str(analysis["confirmed"]).lower()}</text>')
|
||||
output.write_text(svg(
|
||||
"Confirmed improvement",
|
||||
"Four API replicas unchanged; only the PostgreSQL CPU limit changes from 1 to 2",
|
||||
"\n".join(parts),
|
||||
), encoding="utf-8")
|
||||
|
||||
|
||||
def write_csv(report, destination):
|
||||
with (destination / "measurements.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
fields = [
|
||||
"phase", "kind", "replicas", "concurrency", "elapsed_seconds",
|
||||
"successful_requests", "failed_requests", "error_ratio", "successful_rps",
|
||||
"p50_ms", "p95_ms", "p99_ms", "api_cpu_avg", "postgres_cpu_avg",
|
||||
"load_balancer_cpu_avg", "frontend_cpu_avg", "redis_cpu_avg",
|
||||
]
|
||||
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for phase in report["phases"]:
|
||||
writer.writerow({
|
||||
"phase": phase["name"],
|
||||
"kind": phase["kind"],
|
||||
"replicas": phase["replicas"],
|
||||
"concurrency": phase["concurrency"],
|
||||
"elapsed_seconds": phase["elapsed_seconds"],
|
||||
"successful_requests": phase["successful_requests"],
|
||||
"failed_requests": phase["failed_requests"],
|
||||
"error_ratio": phase["error_ratio"],
|
||||
"successful_rps": phase["successful_rps"],
|
||||
"p50_ms": phase["successful_latency_ms"]["p50"],
|
||||
"p95_ms": phase["successful_latency_ms"]["p95"],
|
||||
"p99_ms": phase["successful_latency_ms"]["p99"],
|
||||
"api_cpu_avg": phase["resources"].get("api", {}).get("cpu_percent_avg"),
|
||||
"postgres_cpu_avg": phase["resources"].get("postgres", {}).get("cpu_percent_avg"),
|
||||
"load_balancer_cpu_avg": phase["resources"].get("load-balancer", {}).get("cpu_percent_avg"),
|
||||
"frontend_cpu_avg": phase["resources"].get("frontend", {}).get("cpu_percent_avg"),
|
||||
"redis_cpu_avg": phase["resources"].get("redis", {}).get("cpu_percent_avg"),
|
||||
})
|
||||
|
||||
with (destination / "timeseries.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
fields = ["phase", "second", "successful", "failed"]
|
||||
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for phase in report["phases"]:
|
||||
for second, counts in phase["per_second"].items():
|
||||
writer.writerow({"phase": phase["name"], "second": second, **counts})
|
||||
|
||||
with (destination / "resources.csv").open("w", newline="", encoding="utf-8") as stream:
|
||||
fields = ["phase", "t", "service", "containers", "cpu_percent", "memory_mib"]
|
||||
writer = csv.DictWriter(stream, fieldnames=fields)
|
||||
writer.writeheader()
|
||||
for phase in report["phases"]:
|
||||
for snapshot in phase["resource_samples"]:
|
||||
for service, values in snapshot["services"].items():
|
||||
writer.writerow({
|
||||
"phase": phase["name"], "t": snapshot["t"], "service": service,
|
||||
**values,
|
||||
})
|
||||
|
||||
|
||||
def markdown_summary(report, output):
|
||||
method = report["methodology"]
|
||||
analysis = report["analysis"]
|
||||
improvement = analysis["improvement"]
|
||||
lines = [
|
||||
"# Lab 3 grade-5 scaling experiment", "",
|
||||
f"Generated: {report['environment']['utc_started_at']}.", "",
|
||||
"## Hypothesis", "", report["hypothesis"], "",
|
||||
"## Reproduction", "",
|
||||
f"- workload: {method['workload']};",
|
||||
f"- phase duration: {method['duration_seconds_per_phase']} s;",
|
||||
f"- comparison concurrency: {method['comparison_concurrency']};",
|
||||
f"- saturation sweep: {method['saturation_concurrency']};",
|
||||
f"- fixed budget: {method['fixed_total_budget']};",
|
||||
f"- growth series: {method['growth_budget']};",
|
||||
f"- saturation rule: {method['saturation_rule']};",
|
||||
f"- host: {report['environment']['logical_cpu_count']} logical CPU, {report['environment']['host_memory_mib']} MiB RAM;",
|
||||
f"- Docker {report['environment']['docker_engine']}, Compose {report['environment']['docker_compose']};",
|
||||
"- command: `make grade5-experiment`; warmup runs before every changed profile.", "",
|
||||
"## Measurements", "",
|
||||
"| Phase | Replicas | Concurrency | RPS | Errors | p50 ms | p95 ms | p99 ms | PG CPU avg | API CPU avg |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
for phase in report["phases"]:
|
||||
latency = phase["successful_latency_ms"]
|
||||
resources = phase["resources"]
|
||||
lines.append(
|
||||
f"| {phase['name']} | {phase['replicas']} | {phase['concurrency']} | "
|
||||
f"{phase['successful_rps']} | {phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
|
||||
f"{latency['p50']} | {latency['p95']} | {latency['p99']} | "
|
||||
f"{resources.get('postgres', {}).get('cpu_percent_avg', 'n/a')} | "
|
||||
f"{resources.get('api', {}).get('cpu_percent_avg', 'n/a')} |"
|
||||
)
|
||||
saturation = analysis["saturation_point"]
|
||||
lines += ["", "## Conclusion", ""]
|
||||
if saturation:
|
||||
lines.append(
|
||||
f"The first saturation signal appears at concurrency {saturation['concurrency']}: "
|
||||
f"RPS changes by {saturation['rps_gain_percent']:+.2f}% while p95 changes by "
|
||||
f"{saturation['p95_growth_ratio']:.2f}x versus concurrency {saturation['previous_concurrency']}."
|
||||
)
|
||||
else:
|
||||
lines.append("The configured sweep did not locate a saturation point by the declared rule.")
|
||||
lines += [
|
||||
"",
|
||||
f"Observed bottleneck classification: `{analysis['observed_bottleneck']}`.", "",
|
||||
f"The PostgreSQL CPU change produced {improvement['rps_change_percent']:+.2f}% RPS and "
|
||||
f"{improvement['p95_change_percent']:+.2f}% p95; confirmed improvement: "
|
||||
f"`{str(improvement['confirmed']).lower()}`.", "",
|
||||
f"After the improvement the remaining bottleneck classification is "
|
||||
f"`{analysis['remaining_bottleneck_after_improvement']}`.", "",
|
||||
"## Graphs", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"## Raw data and limits", "",
|
||||
"Raw aggregates are in `measurements.csv`; one-second request buckets are in "
|
||||
"`timeseries.csv`; every resource sample is in `resources.csv`; complete metadata and "
|
||||
"applied limits are in `results.json`.", "",
|
||||
]
|
||||
lines.extend(f"- {item}" for item in report["limitations"])
|
||||
lines.append("")
|
||||
output.write_text("\n".join(lines), encoding="utf-8")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("results", nargs="?", default="evidence/grade5/results.json")
|
||||
args = parser.parse_args()
|
||||
source = Path(args.results)
|
||||
report = json.loads(source.read_text(encoding="utf-8"))
|
||||
destination = source.parent
|
||||
saturation_graph(report, destination / "01-saturation.svg")
|
||||
comparison_graph(report, "fixed", destination / "02-fixed-total.svg")
|
||||
comparison_graph(report, "growth", destination / "03-resource-growth.svg")
|
||||
bottleneck_graph(report, destination / "04-bottleneck.svg")
|
||||
improvement_graph(report, destination / "05-improvement.svg")
|
||||
write_csv(report, destination)
|
||||
markdown_summary(report, destination / "README.md")
|
||||
print(f"rendered grade-5 evidence in {destination}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -19,6 +19,10 @@ compose up -d --no-deps api
|
||||
compose restart api
|
||||
compose up -d --wait --no-deps api
|
||||
|
||||
# Публичная точка входа перезапускается последней, когда API уже готовы.
|
||||
# Статический frontend возвращается до публичной точки входа.
|
||||
compose restart frontend
|
||||
compose up -d --wait --no-deps frontend
|
||||
|
||||
# Публичная точка входа перезапускается последней, когда API и frontend готовы.
|
||||
compose restart load-balancer
|
||||
compose up -d --wait --no-deps load-balancer
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prove that the gateway excludes an API replica which loses readiness."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
FAULT_IMAGE = "node:24-alpine@sha256:44fce696f38ba8c987c4db89954051000bbd05ffd8d8659ba15ed4a8602e937f"
|
||||
|
||||
|
||||
def command(args, timeout=180, check=True):
|
||||
result = subprocess.run(
|
||||
[str(value) for value in args],
|
||||
cwd=ROOT,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode:
|
||||
detail = (result.stderr or result.stdout).strip().splitlines()
|
||||
suffix = f": {detail[-1]}" if detail else ""
|
||||
raise RuntimeError(f"{' '.join(map(str, args[:4]))} failed{suffix}")
|
||||
return result
|
||||
|
||||
|
||||
def compose(env_file, *args):
|
||||
return command(["docker", "compose", "--env-file", env_file, *args])
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
return json.loads(command([
|
||||
"docker", "inspect", "--format", "{{json .}}", container_id,
|
||||
]).stdout)
|
||||
|
||||
|
||||
def call(host, port, method, path, body=None, headers=None, timeout=5):
|
||||
connection = http.client.HTTPConnection(host, port, timeout=timeout)
|
||||
connection.request(method, path, body=body, headers=headers or {})
|
||||
response = connection.getresponse()
|
||||
payload = response.read()
|
||||
result = {
|
||||
"status": response.status,
|
||||
"instance": response.getheader("X-Instance-Id", ""),
|
||||
"body": payload,
|
||||
}
|
||||
connection.close()
|
||||
return result
|
||||
|
||||
|
||||
def observe_instances(base_host, base_port, requests=16):
|
||||
instances = Counter()
|
||||
statuses = Counter()
|
||||
for _ in range(requests):
|
||||
result = call(base_host, base_port, "GET", "/health/live")
|
||||
statuses[result["status"]] += 1
|
||||
instances[result["instance"]] += 1
|
||||
return statuses, instances
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--env-file", default=".env")
|
||||
parser.add_argument("--host", default="localhost")
|
||||
parser.add_argument("--port", type=int, default=8080)
|
||||
parser.add_argument("--output", default="")
|
||||
args = parser.parse_args()
|
||||
|
||||
api_ids = [line for line in compose(args.env_file, "ps", "-q", "api").stdout.splitlines() if line]
|
||||
if len(api_ids) != 2:
|
||||
raise RuntimeError("readiness routing test requires exactly two API replicas")
|
||||
target_id = api_ids[0]
|
||||
target = inspect(target_id)
|
||||
target_instance = target["Config"]["Hostname"]
|
||||
target_ip = next(iter(target["NetworkSettings"]["Networks"].values()))["IPAddress"]
|
||||
|
||||
postgres_id = compose(args.env_file, "ps", "-q", "postgres").stdout.strip()
|
||||
postgres = inspect(postgres_id)
|
||||
postgres_ip = next(iter(postgres["NetworkSettings"]["Networks"].values()))["IPAddress"]
|
||||
|
||||
before_statuses, before_instances = observe_instances(args.host, args.port, 40)
|
||||
if before_statuses != {200: 40} or len(before_instances) != 2:
|
||||
raise RuntimeError("both API replicas must serve traffic before fault injection")
|
||||
|
||||
helper = f"mtusi2026lab3-readiness-fault-{os.getpid()}"
|
||||
rule = [
|
||||
"iptables", "-D", "OUTPUT", "-d", postgres_ip,
|
||||
"-p", "tcp", "--dport", "5432", "-j", "REJECT",
|
||||
"--reject-with", "tcp-reset",
|
||||
]
|
||||
detection_started = time.monotonic()
|
||||
try:
|
||||
script = (
|
||||
"apk add --no-cache iptables >/dev/null && "
|
||||
f"iptables -I OUTPUT -d {postgres_ip} -p tcp --dport 5432 "
|
||||
"-j REJECT --reject-with tcp-reset && "
|
||||
"touch /tmp/fault-ready && sleep 180"
|
||||
)
|
||||
command([
|
||||
"docker", "run", "-d", "--name", helper,
|
||||
"--network", f"container:{target_id}",
|
||||
"--cap-add", "NET_ADMIN", FAULT_IMAGE,
|
||||
"sh", "-c", script,
|
||||
])
|
||||
deadline = time.monotonic() + 30
|
||||
while time.monotonic() < deadline:
|
||||
ready = command(
|
||||
["docker", "exec", helper, "test", "-f", "/tmp/fault-ready"],
|
||||
check=False,
|
||||
)
|
||||
if ready.returncode == 0:
|
||||
break
|
||||
time.sleep(0.2)
|
||||
else:
|
||||
raise RuntimeError("fault helper did not become ready")
|
||||
|
||||
direct = call(target_ip, 8080, "GET", "/health/ready")
|
||||
if direct["status"] != 503:
|
||||
raise RuntimeError(f"target readiness is {direct['status']}, want 503")
|
||||
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
statuses, instances = observe_instances(args.host, args.port, 16)
|
||||
if statuses == {200: 16} and target_instance not in instances:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
else:
|
||||
raise RuntimeError("gateway did not exclude the unready API within 10 seconds")
|
||||
detection_seconds = time.monotonic() - detection_started
|
||||
|
||||
health_statuses, health_instances = observe_instances(args.host, args.port, 40)
|
||||
registration_statuses = Counter()
|
||||
registration_instances = Counter()
|
||||
for _ in range(20):
|
||||
body = json.dumps({
|
||||
"username": "readiness_" + uuid.uuid4().hex[:16],
|
||||
"display_name": "Readiness audit",
|
||||
"password": "T3st-" + uuid.uuid4().hex,
|
||||
}).encode()
|
||||
response = call(
|
||||
args.host,
|
||||
args.port,
|
||||
"POST",
|
||||
"/api/v1/users",
|
||||
body=body,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=8,
|
||||
)
|
||||
registration_statuses[response["status"]] += 1
|
||||
registration_instances[response["instance"]] += 1
|
||||
|
||||
if health_statuses != {200: 40} or target_instance in health_instances:
|
||||
raise RuntimeError("unready API received liveness traffic through gateway")
|
||||
if registration_statuses != {201: 20} or target_instance in registration_instances:
|
||||
raise RuntimeError("unready API received registration traffic through gateway")
|
||||
finally:
|
||||
command(["docker", "exec", helper, *rule], check=False)
|
||||
command(["docker", "rm", "-f", helper], check=False)
|
||||
|
||||
deadline = time.monotonic() + 15
|
||||
while time.monotonic() < deadline:
|
||||
direct = call(target_ip, 8080, "GET", "/health/ready")
|
||||
if direct["status"] == 200:
|
||||
_, instances = observe_instances(args.host, args.port, 24)
|
||||
if target_instance in instances:
|
||||
break
|
||||
time.sleep(0.25)
|
||||
else:
|
||||
raise RuntimeError("recovered API did not return to gateway rotation")
|
||||
|
||||
result = {
|
||||
"target_instance": target_instance,
|
||||
"detection_seconds": round(detection_seconds, 3),
|
||||
"health_requests": dict(health_statuses),
|
||||
"registration_requests": dict(registration_statuses),
|
||||
"recovered": True,
|
||||
}
|
||||
rendered = json.dumps(result, ensure_ascii=False, indent=2)
|
||||
if args.output:
|
||||
output = Path(args.output)
|
||||
if not output.is_absolute():
|
||||
output = ROOT / output
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(rendered + "\n", encoding="utf-8")
|
||||
print(rendered)
|
||||
print("PASS: readiness-aware gateway excluded and restored one API replica")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -209,6 +209,11 @@ func TestSessionAuthenticationLifecycle(t *testing.T) {
|
||||
authorization: "Basic " + base64.StdEncoding.EncodeToString([]byte(alice.Username+":"+testPassword)),
|
||||
wantStatus: http.StatusUnauthorized,
|
||||
},
|
||||
{
|
||||
name: "non-ASCII Bearer is rejected without panic",
|
||||
authorization: "Bearer é",
|
||||
wantStatus: http.StatusUnauthorized,
|
||||
},
|
||||
{
|
||||
name: "Bearer identity wins over X-User-Id",
|
||||
authorization: "Bearer " + session.Token,
|
||||
|
||||