оценка 4

This commit is contained in:
2026-10-23 16:20:43 +03:00
parent 34618bd423
commit 5300a1a1e0
18 changed files with 1861 additions and 29 deletions
+3
View File
@@ -17,6 +17,8 @@ SESSION_COOKIE_SECURE=false
CURSOR_SIGNING_KEY_FILE=/run/secrets/cursor_signing_key
API_REPLICAS=2
API_CPU_LIMIT=0.50
API_MEMORY_LIMIT=256M
IDEMPOTENCY_TTL_SECONDS=86400
# File paths contain no secret values. Create the referenced local secret files
@@ -32,6 +34,7 @@ DB_CONNECT_TIMEOUT=5s
DB_QUERY_TIMEOUT=3s
READINESS_TIMEOUT=2s
SHUTDOWN_DRAIN_DELAY=2s
SHUTDOWN_TIMEOUT=10s
# Redis stores a disposable shared chat cache; PostgreSQL remains authoritative.
+12
View File
@@ -6,6 +6,7 @@ ENV_FILE ?= .env
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
GRADE4_OUTPUT ?= evidence/grade4/results.json
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
@@ -26,6 +27,8 @@ COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
load \
test-persistence \
test-failover \
rolling-update \
grade4-experiment \
compose-config \
compose-build \
compose-up \
@@ -48,6 +51,8 @@ help:
@echo "load: run synthetic writes against the LB entrypoint"
@echo "test-persistence: run persistence test with ACTION_SCRIPT"
@echo "test-failover: run replica failover test with ACTION_SCRIPT"
@echo "rolling-update: replace API replicas one by one with surge capacity"
@echo "grade4-experiment: run failovers, idempotency and rolling update with evidence"
@echo "compose-config: validate resolved Compose configuration"
@echo "compose-build: build application images"
@echo "compose-up: start the stack"
@@ -109,6 +114,13 @@ test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
rolling-update:
ENV_FILE="$(ENV_FILE)" ./scripts/rolling-update.sh
grade4-experiment:
$(PYTHON) scripts/grade4_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE4_OUTPUT)"
$(PYTHON) scripts/render_grade4.py "$(GRADE4_OUTPUT)"
compose-config:
@test -f "$(ENV_FILE)" || \
(echo "$(ENV_FILE) is missing; run: make init-env"; exit 2)
+62 -24
View File
@@ -1,8 +1,8 @@
# Отчёт — лабораторная 3
- Автор, группа: Чурилин, БВТ2301
- Текущий завершённый уровень: оценка «3»
- Итоговая цель: оценка «5»; трек будет выбран после реализации уровня «4»
- Текущий завершённый уровень: оценка «4»
- Итоговая цель: оценка «5»; исследовательский трек выполняется следующим этапом
- Commit/PR: заполняется автором после локального коммита и публикации
- Использованные источники и библиотеки: преподавательские README/OpenAPI/tests,
документация Go, pgx, PostgreSQL, Docker Compose, Nginx, React, TypeScript,
@@ -38,6 +38,7 @@ make test
make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
docker compose --env-file .env up -d --wait api
make test-persistence ACTION_SCRIPT=./scripts/restart.sh
make grade4-experiment
```
По этому же адресу Nginx отдаёт production-сборку React. Переход на
@@ -86,10 +87,12 @@ Compose запускает один сервис `api` с `API_REPLICAS=2`. Ре
ports и разделяют PostgreSQL, включая таблицу сессий. Nginx — единственная
публичная точка входа; он использует `least_conn` и Docker DNS resolver с
повторным разрешением имени `api` раз в секунду. Sticky sessions не применяются.
Безопасные методы могут быть перенаправлены на другую реплику при сетевой ошибке.
Произвольные POST балансировщик автоматически не повторяет: после
неопределённого результата повтор сообщения выполняет клиент с тем же
`Idempotency-Key`, а создание пользователя/чата контрактом не идемпотентно.
Произвольные POST балансировщик автоматически не повторяет. Исключение сделано
только для маршрута отправки сообщения: обязательный `Idempotency-Key` позволяет
Nginx безопасно повторить запрос на другой реплике при 502/503/504 во время
drain. Клиент также сохраняет ключ для повтора после неопределённого результата.
Создание пользователя и чата контрактом не идемпотентно, поэтому автоматический
retry для этих маршрутов запрещён.
Миграции вынесены в одноразовый сервис `migrate`, завершение которого является
предусловием запуска API. Поэтому две реплики не конкурируют за изменение схемы.
@@ -104,7 +107,8 @@ secret `cursor_signing_key`. Репозитории получают тольк
`.env.example` перечисляет реально читаемые приложением параметры PostgreSQL,
пула, readiness/shutdown, Redis-кэша, числа реплик, TTL идемпотентности и ключа
курсора. `API_REPLICAS` используется Compose, а `IDEMPOTENCY_TTL_SECONDS` читает
курсора. Там же задаются CPU/RAM API и задержка drain перед shutdown.
`API_REPLICAS` используется Compose, а `IDEMPOTENCY_TTL_SECONDS` читает
Go-приложение и не допускает значение меньше 86400 секунд. Секретных значений в
шаблоне нет: там указаны только пути внутри контейнера.
@@ -177,37 +181,71 @@ cookie.
| Persistence после рестартов | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS; прежняя сессия и данные доступны после рестарта PostgreSQL, Redis, API и LB | `scripts/restart.sh`, `evidence/lab3-summary.md` |
| Миграции перед API | `docker compose --env-file .env ps -a` и логи `migrate` | миграции 1 и 2 применены; one-shot контейнер завершился с кодом 0; API healthy | `migrations/000002_message_idempotency.up.sql`, `evidence/lab3-summary.md` |
| React frontend и HttpOnly BFF | `docker compose build load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/Dockerfile`, `infrastructure/nginx/nginx.conf` |
| Согласованные health и shutdown | SIGTERM каждой реплики под нагрузкой | API сразу закрывает availability gate, 2 с дренирует запросы и укладывается в 10 с shutdown; `/health/live` остаётся доступен | `internal/transport/http/middleware/availability.go`, `evidence/grade4/` |
| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только Nginx `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 3,464/3,476 с, одна краткая ошибка из 4614 запросов в первом прогоне и 0 из 5815 во втором | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 20 350 успешных запросов, 1 краткий 502 (0,005%), все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
## Эксперименты и отказы
Проведены функциональные эксперименты устойчивости, а не capacity benchmark.
Окружение: Windows + WSL2, Docker Engine 29.7.2, Docker Compose 5.5.0, две
API-реплики; лимиты CPU/RAM пока не задавались, максимальный пул БД —
10 соединений на реплику. Использовались синтетические данные; отдельного прогрева
и серии повторов не было. Поэтому результаты подтверждают корректность failover,
идемпотентности и сохранности данных, но не предельный RPS или p50/p95/p99.
Нагрузочный профиль с лимитами и повторами относится к следующему уровню.
Итоговый эксперимент уровня 4 выполнен 23.10.2026 на Windows + WSL2:
Docker Engine 29.7.2, Compose 5.5.0, 8 логических CPU и 3813,6 MiB RAM,
commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиченными
изменениями текущего уровня. В Compose каждой API-реплике выделено 0,5 CPU и
256 MiB; Nginx — 0,25 CPU/128 MiB; PostgreSQL — 1 CPU/512 MiB; Redis —
0,25 CPU/128 MiB.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| Последовательная остановка `api-2` и `api-1` с восстановлением между тестами | LB обслуживает через оставшуюся реплику не позднее 10 с | 142 и 143 успешные записи; в обоих запусках 0 ошибок | первое восстановление 0.963 и 0.750 с; подтверждённой потери или дублей нет |
| Рестарт PostgreSQL, Redis, API и LB без удаления volumes | прежний token и данные сохраняются | `test-persistence` завершился PASS | уложилось в тайм-аут 90 с; потери данных нет |
| 32 одновременных POST с одним `Idempotency-Key` | один бизнес-эффект | 32 ответа 201 с одним Message ID; в БД одно сообщение | дубли отсутствуют |
Команда `make grade4-experiment` создаёт синтетических пользователей и чат,
делает 12 warm-up записей, затем запускает 8 closed-loop клиентов без повторов
на стороне клиента. Baseline длится 12 с. В каждом failover воздействие
начинается через 3 с, предыдущая реплика восстанавливается перед остановкой
следующей, после действия нагрузка продолжается ещё 8 с. Rolling update
временно увеличивает сервис до трёх реплик, ждёт healthcheck и два периода
Docker DNS, заменяет старые контейнеры по одному и возвращает масштаб к двум.
Минимальный подтверждённый размер набора после прогона — 34 808 сообщений.
| Фаза | Успешно / ошибок | RPS | Доля ошибок | p50 / p95 / p99, мс | Восстановление |
| --- | ---: | ---: | ---: | ---: | ---: |
| baseline | 4017 / 0 | 319,463 | 0% | 16,799 / 55,515 / 93,803 | — |
| остановка реплики 1 | 4613 / 1 | 301,040 | 0,0217% | 17,145 / 69,558 / 89,752 | 3,464 с |
| остановка реплики 2 | 5815 / 0 | 374,449 | 0% | 14,211 / 49,542 / 81,040 | 3,476 с |
| rolling update | 20 350 / 1 | 376,959 | 0,0049% | 15,725 / 48,724 / 79,337 | непрерывная нагрузка |
На baseline средняя выборка одного API-контейнера составила 28,453% CPU и
42,413 MiB RAM, максимум — 36,1% и 66,07 MiB. PostgreSQL оказался самым
нагруженным компонентом: в rolling-фазе среднее 98,172% CPU при лимите одного
ядра, максимум RAM 145,2 MiB. Это не capacity-тест и не доказательство
saturation point, но важный ориентир для исследования уровня 5.
Nginx обновляет Docker DNS раз в 1 с, исключает peer после двух неудач за 2 с,
а connect timeout равен 1 с. При постоянной нагрузке ожидаемое максимальное
обнаружение — около 4 с; измеренные 3,464 и 3,476 с укладываются в требование
10 с. Одна 504 и одна 502 сохранены в отчёте честно: README прямо допускает
краткие ошибки обнаружения. Подтверждённых потерь и дублей нет. Параллельные
32 POST с одним ключом прошли через обе реплики и вернули один Message ID.
Исходные агрегаты, environment/limits, посекундные таблицы и четыре SVG-графика
лежат в `evidence/grade4/`. Percentile рассчитаны только по успешным ответам;
профиль closed-loop имеет coordinated omission. CPU для API агрегируется как
средняя/максимальная выборка отдельного контейнера, а не сумма двух реплик.
## Ограничения и следующие шаги
- Топология однохостовая: Nginx и PostgreSQL остаются single points of failure.
Репликация БД и несколько балансировщиков в требования уровня «3» не входят.
Репликация БД и несколько балансировщиков в требования уровня «4» не входят.
- Redis содержит только кэш, поэтому его потеря допустима. Отдельная
персистентность Redis для корректности API не требуется.
- Истёкшие записи идемпотентности удаляются лениво при повторном обращении к той
же области ключа. Индекс по `expires_at` оставлен для будущей периодической
очистки без изменения схемы.
- Resource limits, количественный нагрузочный эксперимент и сценарий rolling
update ещё не реализованы; это следующий этап уровня «4».
- Уровень «5» ещё не реализован: сравнение 1/2/4 реплик при фиксированных
ресурсах и поиск saturation point выполняются следующим отдельным этапом.
- Rolling-сценарий заменяет бинарник без изменения схемы. Миграции выполняются
однократно до API; для будущей несовместимой схемы потребуется expand/contract,
поэтому одновременная работа разных schema-version сейчас не заявляется.
- Текущий API не содержит endpoint поиска пользователей, аватаров, изменения
имени/пароля и агрегатов last message/unread. Поэтому UI принимает UUID
участников при создании чата, а профиль честно показывает только доступные
поля. Эти функции требуют сначала расширить и зафиксировать API-контракт.
- Трек уровня «5» пока не заявлен: сначала необходимо завершить и измерить
обязательный уровень «4».
- Однохостовая среда WSL2 не доказывает устойчивость к потере Docker-host;
измерен только отказ и замена процесса API.
+9
View File
@@ -154,6 +154,8 @@ func main() {
var rootHandler http.Handler = mux
rootHandler = middleware.Recovery(rootHandler)
rootHandler = middleware.Logger(rootHandler)
availabilityGate := middleware.NewAvailabilityGate()
rootHandler = availabilityGate.Middleware(rootHandler)
rootHandler = middleware.RequestID(rootHandler)
rootHandler = middleware.InstanceID(cfg.InstanceID, rootHandler)
@@ -196,6 +198,13 @@ func main() {
signalValue,
)
availabilityGate.BeginDrain()
log.Printf(
"[INFO] Readiness disabled; draining new traffic for %s",
cfg.ShutdownDrainDelay,
)
time.Sleep(cfg.ShutdownDrainDelay)
shutdownContext, shutdownCancel := context.WithTimeout(
context.Background(),
cfg.ShutdownTimeout,
+35
View File
@@ -21,6 +21,11 @@ services:
start_period: 3s
stop_grace_period: 10s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
postgres:
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
@@ -43,6 +48,11 @@ services:
retries: 10
start_period: 5s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
redis:
image: redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0
@@ -66,6 +76,11 @@ services:
start_period: 3s
stop_grace_period: 10s
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.25"
memory: 128M
migrate:
image: migrate/migrate:v4.20.1@sha256:76cc2074cb6642631f34a898ced71e6aeaa6b1a4d78c4daa743275a22e0c5be7
@@ -84,6 +99,11 @@ services:
postgres:
condition: service_healthy
restart: "no"
deploy:
resources:
limits:
cpus: "0.25"
memory: 64M
api:
build:
@@ -116,6 +136,7 @@ services:
DB_QUERY_TIMEOUT: ${DB_QUERY_TIMEOUT:-3s}
READINESS_TIMEOUT: ${READINESS_TIMEOUT:-2s}
SHUTDOWN_DRAIN_DELAY: ${SHUTDOWN_DRAIN_DELAY:-2s}
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10s}
secrets:
- database_url
@@ -131,6 +152,10 @@ services:
condition: service_completed_successfully
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "${API_CPU_LIMIT:-0.50}"
memory: ${API_MEMORY_LIMIT:-256M}
healthcheck:
test:
[
@@ -155,6 +180,11 @@ services:
redis:
condition: service_healthy
restart: unless-stopped
deploy:
resources:
limits:
cpus: "0.50"
memory: 256M
postgres-restore:
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
@@ -178,6 +208,11 @@ services:
retries: 10
start_period: 5s
restart: "no"
deploy:
resources:
limits:
cpus: "1.00"
memory: 512M
volumes:
postgres_data:
+39
View File
@@ -0,0 +1,39 @@
# Lab 3 grade-4 experiment
Generated: 2026-10-23T12:31:12.749687+00:00.
## Method
- concurrency: 8;
- baseline duration: 12 s;
- load: closed-loop POST messages through Nginx;
- failover: both original API containers stopped and restored in turn;
- rolling update: one surge replica, old replicas replaced one by one;
- percentiles cover successful responses; failures are separate.
## Results
| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| baseline | 319.463 | 0 (0.000%) | 16.799 | 55.515 | 93.803 | n/a |
| failover-1 | 301.04 | 1 (0.022%) | 17.145 | 69.558 | 89.752 | 3.464 |
| failover-2 | 374.449 | 0 (0.000%) | 14.211 | 49.542 | 81.04 | 3.476 |
| rolling-update | 376.959 | 1 (0.005%) | 15.725 | 48.724 | 79.337 | n/a |
## Parallel idempotency
32 requests produced 1 unique message ID across 2 API instances.
## Graphs
![Throughput and errors](throughput-errors.svg)
![Latency](latency.svg)
![Timeline](timeline.svg)
![Resources](resources.svg)
## Scope
Single-host process failure and rolling replacement only. PostgreSQL and Nginx remain single points of failure.
+69
View File
@@ -0,0 +1,69 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="560" viewBox="0 0 1100 560">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.bar { fill: #cfd2d6; }
.bar2 { fill: #73777f; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
</style>
<text x="48" y="42" class="title">Grade 4: successful latency percentiles</text>
<line x1="80" y1="450.0" x2="1040" y2="450.0" class="grid"/>
<text x="68" y="454.0" text-anchor="end" class="small">0</text>
<line x1="80" y1="378.0" x2="1040" y2="378.0" class="grid"/>
<text x="68" y="382.0" text-anchor="end" class="small">19</text>
<line x1="80" y1="306.0" x2="1040" y2="306.0" class="grid"/>
<text x="68" y="310.0" text-anchor="end" class="small">38</text>
<line x1="80" y1="234.0" x2="1040" y2="234.0" class="grid"/>
<text x="68" y="238.0" text-anchor="end" class="small">56</text>
<line x1="80" y1="162.0" x2="1040" y2="162.0" class="grid"/>
<text x="68" y="166.0" text-anchor="end" class="small">75</text>
<line x1="80" y1="90.0" x2="1040" y2="90.0" class="grid"/>
<text x="68" y="94.0" text-anchor="end" class="small">94</text>
<rect x="128.0" y="385.5" width="42.0" height="64.5" rx="7" class="bar"/>
<text x="149.0" y="378.5" text-anchor="middle" class="small">17</text>
<text x="149.0" y="470" text-anchor="middle" class="small">p50</text>
<rect x="178.0" y="236.9" width="42.0" height="213.1" rx="7" class="bar2"/>
<text x="199.0" y="229.9" text-anchor="middle" class="small">56</text>
<text x="199.0" y="470" text-anchor="middle" class="small">p95</text>
<rect x="228.0" y="90.0" width="42.0" height="360.0" rx="7" class="bar2"/>
<text x="249.0" y="83.0" text-anchor="middle" class="small">94</text>
<text x="249.0" y="470" text-anchor="middle" class="small">p99</text>
<text x="200.0" y="498" text-anchor="middle" class="label">baseline</text>
<rect x="368.0" y="384.2" width="42.0" height="65.8" rx="7" class="bar"/>
<text x="389.0" y="377.2" text-anchor="middle" class="small">17</text>
<text x="389.0" y="470" text-anchor="middle" class="small">p50</text>
<rect x="418.0" y="183.0" width="42.0" height="267.0" rx="7" class="bar2"/>
<text x="439.0" y="176.0" text-anchor="middle" class="small">70</text>
<text x="439.0" y="470" text-anchor="middle" class="small">p95</text>
<rect x="468.0" y="105.5" width="42.0" height="344.5" rx="7" class="bar2"/>
<text x="489.0" y="98.5" text-anchor="middle" class="small">90</text>
<text x="489.0" y="470" text-anchor="middle" class="small">p99</text>
<text x="440.0" y="498" text-anchor="middle" class="label">failover-1</text>
<rect x="608.0" y="395.5" width="42.0" height="54.5" rx="7" class="bar"/>
<text x="629.0" y="388.5" text-anchor="middle" class="small">14</text>
<text x="629.0" y="470" text-anchor="middle" class="small">p50</text>
<rect x="658.0" y="259.9" width="42.0" height="190.1" rx="7" class="bar2"/>
<text x="679.0" y="252.9" text-anchor="middle" class="small">50</text>
<text x="679.0" y="470" text-anchor="middle" class="small">p95</text>
<rect x="708.0" y="139.0" width="42.0" height="311.0" rx="7" class="bar2"/>
<text x="729.0" y="132.0" text-anchor="middle" class="small">81</text>
<text x="729.0" y="470" text-anchor="middle" class="small">p99</text>
<text x="680.0" y="498" text-anchor="middle" class="label">failover-2</text>
<rect x="848.0" y="389.7" width="42.0" height="60.3" rx="7" class="bar"/>
<text x="869.0" y="382.7" text-anchor="middle" class="small">16</text>
<text x="869.0" y="470" text-anchor="middle" class="small">p50</text>
<rect x="898.0" y="263.0" width="42.0" height="187.0" rx="7" class="bar2"/>
<text x="919.0" y="256.0" text-anchor="middle" class="small">49</text>
<text x="919.0" y="470" text-anchor="middle" class="small">p95</text>
<rect x="948.0" y="145.5" width="42.0" height="304.5" rx="7" class="bar2"/>
<text x="969.0" y="138.5" text-anchor="middle" class="small">79</text>
<text x="969.0" y="470" text-anchor="middle" class="small">p99</text>
<text x="920.0" y="498" text-anchor="middle" class="label">rolling-update</text>
<text x="24" y="270.0" transform="rotate(-90 24 270.0)" class="label">Successful latency, ms</text>
</svg>

After

Width:  |  Height:  |  Size: 4.4 KiB

+42
View File
@@ -0,0 +1,42 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="600" viewBox="0 0 1100 600">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.bar { fill: #cfd2d6; }
.bar2 { fill: #73777f; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
</style>
<text x="48" y="42" class="title">Grade 4: peak container resources</text>
<text x="80" y="72" class="label">Peak CPU, %</text>
<rect x="103.6" y="270.1" width="57.8" height="179.9" rx="9" class="bar"/>
<text x="132.5" y="262.1" text-anchor="middle" class="small">52.6</text>
<text x="132.5" y="475" text-anchor="middle" class="small">api</text>
<rect x="208.6" y="370.1" width="57.8" height="79.9" rx="9" class="bar"/>
<text x="237.5" y="362.1" text-anchor="middle" class="small">23.4</text>
<text x="237.5" y="475" text-anchor="middle" class="small">load-balancer</text>
<rect x="313.6" y="90.0" width="57.8" height="360.0" rx="9" class="bar"/>
<text x="342.5" y="82.0" text-anchor="middle" class="small">105.2</text>
<text x="342.5" y="475" text-anchor="middle" class="small">postgres</text>
<rect x="418.6" y="429.9" width="57.8" height="20.1" rx="9" class="bar"/>
<text x="447.5" y="421.9" text-anchor="middle" class="small">5.9</text>
<text x="447.5" y="475" text-anchor="middle" class="small">redis</text>
<text x="600" y="72" class="label">Peak memory, MiB</text>
<rect x="623.6" y="286.2" width="57.8" height="163.8" rx="9" class="bar"/>
<text x="652.5" y="278.2" text-anchor="middle" class="small">66.1</text>
<text x="652.5" y="475" text-anchor="middle" class="small">api</text>
<rect x="728.6" y="416.5" width="57.8" height="33.5" rx="9" class="bar"/>
<text x="757.5" y="408.5" text-anchor="middle" class="small">13.5</text>
<text x="757.5" y="475" text-anchor="middle" class="small">load-balancer</text>
<rect x="833.6" y="90.0" width="57.8" height="360.0" rx="9" class="bar"/>
<text x="862.5" y="82.0" text-anchor="middle" class="small">145.2</text>
<text x="862.5" y="475" text-anchor="middle" class="small">postgres</text>
<rect x="938.6" y="419.6" width="57.8" height="30.4" rx="9" class="bar"/>
<text x="967.5" y="411.6" text-anchor="middle" class="small">12.3</text>
<text x="967.5" y="475" text-anchor="middle" class="small">redis</text>
</svg>

After

Width:  |  Height:  |  Size: 2.5 KiB

+675
View File
@@ -0,0 +1,675 @@
{
"schema_version": 1,
"environment": {
"utc_started_at": "2026-10-23T12:31:12.749687+00:00",
"git_commit": "34618bd4237bbb8bf6a3a303056fc95d799577f9",
"working_tree_dirty": true,
"logical_cpu_count": 8,
"host_memory_mib": 3813.6,
"docker_engine": "29.7.2",
"docker_compose": "5.5.0",
"container_limits": [
{
"service": "api",
"nano_cpus": 500000000,
"memory_limit_bytes": 268435456
},
{
"service": "api",
"nano_cpus": 500000000,
"memory_limit_bytes": 268435456
},
{
"service": "load-balancer",
"nano_cpus": 250000000,
"memory_limit_bytes": 134217728
},
{
"service": "postgres",
"nano_cpus": 1000000000,
"memory_limit_bytes": 536870912
},
{
"service": "redis",
"nano_cpus": 250000000,
"memory_limit_bytes": 134217728
}
]
},
"methodology": {
"base_url": "http://localhost:8080",
"concurrency": 8,
"baseline_seconds": 12,
"action_after_seconds": 3,
"post_action_seconds": 8,
"warmup_requests": 12,
"workload": "closed-loop POST messages via Nginx; no automatic retries"
},
"phases": [
{
"name": "baseline",
"elapsed_seconds": 12.574,
"successful_requests": 4017,
"failed_requests": 0,
"error_ratio": 0.0,
"errors": {},
"successful_rps": 319.463,
"successful_latency_ms": {
"p50": 16.799,
"p95": 55.515,
"p99": 93.803
},
"instances": [
"0b933a46fccb",
"55aaffcf508a"
],
"recovery_seconds": null,
"action": null,
"per_second": {
"0": {
"successful": 118,
"failed": 0
},
"1": {
"successful": 308,
"failed": 0
},
"2": {
"successful": 386,
"failed": 0
},
"3": {
"successful": 378,
"failed": 0
},
"4": {
"successful": 344,
"failed": 0
},
"5": {
"successful": 322,
"failed": 0
},
"6": {
"successful": 316,
"failed": 0
},
"7": {
"successful": 384,
"failed": 0
},
"8": {
"successful": 380,
"failed": 0
},
"9": {
"successful": 402,
"failed": 0
},
"10": {
"successful": 377,
"failed": 0
},
"11": {
"successful": 288,
"failed": 0
},
"12": {
"successful": 14,
"failed": 0
}
},
"resources": {
"api": {
"cpu_percent_avg": 28.453,
"cpu_percent_max": 36.1,
"memory_mib_avg": 42.413,
"memory_mib_max": 66.07
},
"load-balancer": {
"cpu_percent_avg": 17.448,
"cpu_percent_max": 20.8,
"memory_mib_avg": 9.214,
"memory_mib_max": 9.559
},
"postgres": {
"cpu_percent_avg": 85.888,
"cpu_percent_max": 105.25,
"memory_mib_avg": 97.975,
"memory_mib_max": 100.8
},
"redis": {
"cpu_percent_avg": 2.552,
"cpu_percent_max": 5.79,
"memory_mib_avg": 8.796,
"memory_mib_max": 11.74
}
}
},
{
"name": "failover-1",
"elapsed_seconds": 15.324,
"successful_requests": 4613,
"failed_requests": 1,
"error_ratio": 0.000217,
"errors": {
"POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 504": 1
},
"successful_rps": 301.04,
"successful_latency_ms": {
"p50": 17.145,
"p95": 69.558,
"p99": 89.752
},
"instances": [
"0b933a46fccb",
"55aaffcf508a"
],
"recovery_seconds": 3.464,
"action": {
"started_at": 3.006,
"ended_at": 6.463,
"duration_seconds": 3.457,
"exit_code": 0,
"summary": "stopped API instance 55aaffcf508a"
},
"per_second": {
"0": {
"successful": 404,
"failed": 0
},
"1": {
"successful": 426,
"failed": 0
},
"2": {
"successful": 358,
"failed": 0
},
"3": {
"successful": 399,
"failed": 0
},
"4": {
"successful": 392,
"failed": 0
},
"5": {
"successful": 322,
"failed": 0
},
"6": {
"successful": 299,
"failed": 0
},
"7": {
"successful": 329,
"failed": 1
},
"8": {
"successful": 313,
"failed": 0
},
"9": {
"successful": 229,
"failed": 0
},
"10": {
"successful": 155,
"failed": 0
},
"11": {
"successful": 187,
"failed": 0
},
"12": {
"successful": 299,
"failed": 0
},
"13": {
"successful": 317,
"failed": 0
},
"14": {
"successful": 184,
"failed": 0
}
},
"resources": {
"api": {
"cpu_percent_avg": 27.62,
"cpu_percent_max": 52.59,
"memory_mib_avg": 11.58,
"memory_mib_max": 12.66
},
"load-balancer": {
"cpu_percent_avg": 18.536,
"cpu_percent_max": 21.25,
"memory_mib_avg": 9.351,
"memory_mib_max": 9.523
},
"postgres": {
"cpu_percent_avg": 81.822,
"cpu_percent_max": 104.67,
"memory_mib_avg": 100.512,
"memory_mib_max": 109.1
},
"redis": {
"cpu_percent_avg": 1.486,
"cpu_percent_max": 5.88,
"memory_mib_avg": 8.04,
"memory_mib_max": 8.25
}
}
},
{
"name": "failover-2",
"elapsed_seconds": 15.529,
"successful_requests": 5815,
"failed_requests": 0,
"error_ratio": 0.0,
"errors": {},
"successful_rps": 374.449,
"successful_latency_ms": {
"p50": 14.211,
"p95": 49.542,
"p99": 81.04
},
"instances": [
"0b933a46fccb",
"55aaffcf508a"
],
"recovery_seconds": 3.476,
"action": {
"started_at": 3.013,
"ended_at": 6.485,
"duration_seconds": 3.472,
"exit_code": 0,
"summary": "stopped API instance 0b933a46fccb"
},
"per_second": {
"0": {
"successful": 396,
"failed": 0
},
"1": {
"successful": 439,
"failed": 0
},
"2": {
"successful": 412,
"failed": 0
},
"3": {
"successful": 355,
"failed": 0
},
"4": {
"successful": 378,
"failed": 0
},
"5": {
"successful": 440,
"failed": 0
},
"6": {
"successful": 362,
"failed": 0
},
"7": {
"successful": 419,
"failed": 0
},
"8": {
"successful": 385,
"failed": 0
},
"9": {
"successful": 372,
"failed": 0
},
"10": {
"successful": 420,
"failed": 0
},
"11": {
"successful": 423,
"failed": 0
},
"12": {
"successful": 432,
"failed": 0
},
"13": {
"successful": 375,
"failed": 0
},
"14": {
"successful": 207,
"failed": 0
}
},
"resources": {
"api": {
"cpu_percent_avg": 31.411,
"cpu_percent_max": 51.63,
"memory_mib_avg": 10.373,
"memory_mib_max": 11.6
},
"load-balancer": {
"cpu_percent_avg": 15.778,
"cpu_percent_max": 21.79,
"memory_mib_avg": 9.62,
"memory_mib_max": 9.781
},
"postgres": {
"cpu_percent_avg": 76.87,
"cpu_percent_max": 104.51,
"memory_mib_avg": 106.016,
"memory_mib_max": 120.0
},
"redis": {
"cpu_percent_avg": 0.786,
"cpu_percent_max": 2.83,
"memory_mib_avg": 8.844,
"memory_mib_max": 12.28
}
}
},
{
"name": "rolling-update",
"elapsed_seconds": 53.985,
"successful_requests": 20350,
"failed_requests": 1,
"error_ratio": 4.9e-05,
"errors": {
"POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 502": 1
},
"successful_rps": 376.959,
"successful_latency_ms": {
"p50": 15.725,
"p95": 48.724,
"p99": 79.337
},
"instances": [
"0b933a46fccb",
"3adbcad9e34b",
"55aaffcf508a",
"ceca3f6f1faf",
"f0a3bedda85a"
],
"recovery_seconds": null,
"action": {
"started_at": 3.016,
"ended_at": 45.961,
"duration_seconds": 42.945,
"exit_code": 0,
"summary": "rolling update completed with 2 healthy API replicas"
},
"per_second": {
"0": {
"successful": 340,
"failed": 0
},
"1": {
"successful": 369,
"failed": 0
},
"2": {
"successful": 426,
"failed": 0
},
"3": {
"successful": 382,
"failed": 0
},
"4": {
"successful": 371,
"failed": 0
},
"5": {
"successful": 376,
"failed": 0
},
"6": {
"successful": 328,
"failed": 0
},
"7": {
"successful": 350,
"failed": 0
},
"8": {
"successful": 434,
"failed": 0
},
"9": {
"successful": 411,
"failed": 0
},
"10": {
"successful": 399,
"failed": 0
},
"11": {
"successful": 417,
"failed": 0
},
"12": {
"successful": 374,
"failed": 0
},
"13": {
"successful": 427,
"failed": 0
},
"14": {
"successful": 414,
"failed": 0
},
"15": {
"successful": 418,
"failed": 0
},
"16": {
"successful": 408,
"failed": 0
},
"17": {
"successful": 352,
"failed": 0
},
"18": {
"successful": 391,
"failed": 0
},
"19": {
"successful": 388,
"failed": 0
},
"20": {
"successful": 413,
"failed": 0
},
"21": {
"successful": 417,
"failed": 0
},
"22": {
"successful": 388,
"failed": 0
},
"23": {
"successful": 396,
"failed": 0
},
"24": {
"successful": 400,
"failed": 0
},
"25": {
"successful": 339,
"failed": 0
},
"26": {
"successful": 225,
"failed": 0
},
"27": {
"successful": 376,
"failed": 0
},
"28": {
"successful": 385,
"failed": 0
},
"29": {
"successful": 396,
"failed": 0
},
"30": {
"successful": 403,
"failed": 0
},
"31": {
"successful": 262,
"failed": 0
},
"32": {
"successful": 428,
"failed": 0
},
"33": {
"successful": 362,
"failed": 0
},
"34": {
"successful": 286,
"failed": 0
},
"35": {
"successful": 337,
"failed": 0
},
"36": {
"successful": 328,
"failed": 0
},
"37": {
"successful": 424,
"failed": 0
},
"38": {
"successful": 356,
"failed": 0
},
"39": {
"successful": 386,
"failed": 0
},
"40": {
"successful": 422,
"failed": 0
},
"41": {
"successful": 285,
"failed": 0
},
"42": {
"successful": 279,
"failed": 0
},
"43": {
"successful": 222,
"failed": 1
},
"44": {
"successful": 316,
"failed": 0
},
"45": {
"successful": 359,
"failed": 0
},
"46": {
"successful": 405,
"failed": 0
},
"47": {
"successful": 398,
"failed": 0
},
"48": {
"successful": 395,
"failed": 0
},
"49": {
"successful": 430,
"failed": 0
},
"50": {
"successful": 453,
"failed": 0
},
"51": {
"successful": 409,
"failed": 0
},
"52": {
"successful": 445,
"failed": 0
},
"53": {
"successful": 450,
"failed": 0
}
},
"resources": {
"api": {
"cpu_percent_avg": 25.598,
"cpu_percent_max": 37.65,
"memory_mib_avg": 8.956,
"memory_mib_max": 10.97
},
"load-balancer": {
"cpu_percent_avg": 19.935,
"cpu_percent_max": 23.36,
"memory_mib_avg": 9.836,
"memory_mib_max": 13.52
},
"postgres": {
"cpu_percent_avg": 98.172,
"cpu_percent_max": 105.11,
"memory_mib_avg": 127.975,
"memory_mib_max": 145.2
},
"redis": {
"cpu_percent_avg": 0.722,
"cpu_percent_max": 3.94,
"memory_mib_avg": 7.661,
"memory_mib_max": 11.84
}
}
}
],
"idempotency": {
"parallel_requests": 32,
"successful_responses": 32,
"failed_responses": 0,
"unique_message_ids": 1,
"observed_instances": [
"0b933a46fccb",
"55aaffcf508a"
]
},
"dataset": {
"warmup_requests": 12,
"minimum_confirmed_messages": 34808
},
"limitations": [
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
"PostgreSQL and Nginx remain single points of failure"
]
}
+45
View File
@@ -0,0 +1,45 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="560" viewBox="0 0 1100 560">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.bar { fill: #cfd2d6; }
.bar2 { fill: #73777f; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
</style>
<text x="48" y="42" class="title">Grade 4: throughput and errors</text>
<line x1="80" y1="450.0" x2="1040" y2="450.0" class="grid"/>
<text x="68" y="454.0" text-anchor="end" class="small">0</text>
<line x1="80" y1="378.0" x2="1040" y2="378.0" class="grid"/>
<text x="68" y="382.0" text-anchor="end" class="small">75</text>
<line x1="80" y1="306.0" x2="1040" y2="306.0" class="grid"/>
<text x="68" y="310.0" text-anchor="end" class="small">151</text>
<line x1="80" y1="234.0" x2="1040" y2="234.0" class="grid"/>
<text x="68" y="238.0" text-anchor="end" class="small">226</text>
<line x1="80" y1="162.0" x2="1040" y2="162.0" class="grid"/>
<text x="68" y="166.0" text-anchor="end" class="small">302</text>
<line x1="80" y1="90.0" x2="1040" y2="90.0" class="grid"/>
<text x="68" y="94.0" text-anchor="end" class="small">377</text>
<rect x="140.0" y="144.9" width="120.0" height="305.1" rx="10" class="bar"/>
<text x="200.0" y="134.9" text-anchor="middle" class="label">319.5 RPS</text>
<text x="200.0" y="478" text-anchor="middle" class="label">baseline</text>
<text x="200.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
<rect x="380.0" y="162.5" width="120.0" height="287.5" rx="10" class="bar"/>
<text x="440.0" y="152.5" text-anchor="middle" class="label">301.0 RPS</text>
<text x="440.0" y="478" text-anchor="middle" class="label">failover-1</text>
<text x="440.0" y="498" text-anchor="middle" class="small">errors 0.02%</text>
<rect x="620.0" y="92.4" width="120.0" height="357.6" rx="10" class="bar"/>
<text x="680.0" y="82.4" text-anchor="middle" class="label">374.4 RPS</text>
<text x="680.0" y="478" text-anchor="middle" class="label">failover-2</text>
<text x="680.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
<rect x="860.0" y="90.0" width="120.0" height="360.0" rx="10" class="bar"/>
<text x="920.0" y="80.0" text-anchor="middle" class="label">377.0 RPS</text>
<text x="920.0" y="478" text-anchor="middle" class="label">rolling-update</text>
<text x="920.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
<text x="24" y="270.0" transform="rotate(-90 24 270.0)" class="label">Successful requests per second</text>
</svg>

After

Width:  |  Height:  |  Size: 2.8 KiB

+38
View File
@@ -0,0 +1,38 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="600" viewBox="0 0 1100 600">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.bar { fill: #cfd2d6; }
.bar2 { fill: #73777f; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
</style>
<text x="48" y="42" class="title">Grade 4: request timeline</text>
<line x1="70" y1="480.0" x2="1050" y2="480.0" class="grid"/>
<text x="58" y="484.0" text-anchor="end" class="small">0</text>
<line x1="70" y1="402.0" x2="1050" y2="402.0" class="grid"/>
<text x="58" y="406.0" text-anchor="end" class="small">91</text>
<line x1="70" y1="324.0" x2="1050" y2="324.0" class="grid"/>
<text x="58" y="328.0" text-anchor="end" class="small">181</text>
<line x1="70" y1="246.0" x2="1050" y2="246.0" class="grid"/>
<text x="58" y="250.0" text-anchor="end" class="small">272</text>
<line x1="70" y1="168.0" x2="1050" y2="168.0" class="grid"/>
<text x="58" y="172.0" text-anchor="end" class="small">362</text>
<line x1="70" y1="90.0" x2="1050" y2="90.0" class="grid"/>
<text x="58" y="94.0" text-anchor="end" class="small">453</text>
<circle cx="277.9" cy="472" r="4" class="error"/>
<circle cx="951.0" cy="472" r="4" class="error"/>
<polyline points="70.0,378.4 79.9,214.8 89.8,147.7 99.7,154.6 109.6,183.8 119.5,202.8 129.4,207.9 139.3,149.4 149.2,152.8 159.1,133.9 169.0,155.4 178.9,232.1 188.8,467.9 208.6,132.2 218.5,113.2 228.4,171.8 238.3,136.5 248.2,142.5 258.1,202.8 268.0,222.6 277.9,196.8 287.8,210.5 297.7,282.8 307.6,346.6 317.5,319.0 327.4,222.6 337.3,207.1 347.2,321.6 367.0,139.1 376.9,102.1 386.8,125.3 396.7,174.4 406.6,154.6 416.5,101.2 426.4,168.3 436.3,119.3 446.2,148.5 456.1,159.7 466.0,118.4 475.9,115.8 485.8,108.1 495.7,157.2 505.6,301.8 525.4,187.3 535.3,162.3 545.2,113.2 555.1,151.1 564.9,160.6 574.8,156.3 584.7,197.6 594.6,178.7 604.5,106.4 614.4,126.2 624.3,136.5 634.2,121.0 644.1,158.0 654.0,112.4 663.9,123.6 673.8,120.1 683.7,128.7 693.6,177.0 703.5,143.4 713.4,146.0 723.3,124.4 733.2,121.0 743.1,146.0 753.0,139.1 762.9,135.6 772.8,188.1 782.7,286.3 792.6,156.3 802.5,148.5 812.4,139.1 822.3,133.0 832.2,254.4 842.1,111.5 852.0,168.3 861.9,233.8 871.8,189.9 881.7,197.6 891.6,115.0 901.5,173.5 911.4,147.7 921.3,116.7 931.2,234.6 941.1,239.8 951.0,288.9 960.9,207.9 970.8,170.9 980.7,131.3 990.6,137.4 1000.5,139.9 1010.4,109.8 1020.3,90.0 1030.2,127.9 1040.1,96.9 1050.0,92.6" class="line"/>
<line x1="238.3" y1="90" x2="238.3" y2="480" class="event"/>
<text x="244.3" y="108" class="small">failover-1 action</text>
<line x1="396.8" y1="90" x2="396.8" y2="480" class="event"/>
<text x="402.8" y="108" class="small">failover-2 action</text>
<line x1="555.2" y1="90" x2="555.2" y2="480" class="event"/>
<text x="561.2" y="108" class="small">rolling-update action</text>
<text x="70" y="512" class="label">One-second buckets; circles mark failed requests</text>
</svg>

After

Width:  |  Height:  |  Size: 3.2 KiB

+37 -1
View File
@@ -17,7 +17,7 @@ http {
upstream api_backend {
zone api_backend 64k;
least_conn;
server api:8080 resolve max_fails=1 fail_timeout=1s;
server api:8080 resolve max_fails=2 fail_timeout=2s;
}
server {
@@ -31,6 +31,42 @@ http {
add_header X-Frame-Options "DENY" always;
add_header Content-Security-Policy "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always;
# Message creation carries an Idempotency-Key. It is therefore safe to
# retry a POST on another replica when the selected backend is draining.
location ~ ^/api/v1/chats/[^/]+/messages$ {
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";
proxy_set_header Cookie "";
proxy_connect_timeout 1s;
proxy_send_timeout 20s;
proxy_read_timeout 20s;
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
proxy_next_upstream_tries 3;
}
location ~ ^/web-api/v1/chats/[^/]+/messages$ {
rewrite ^/web-api/(.*)$ /api/$1 break;
proxy_pass http://api_backend;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header Connection "";
proxy_set_header Authorization $web_authorization;
proxy_set_header Cookie "";
proxy_connect_timeout 1s;
proxy_send_timeout 20s;
proxy_read_timeout 20s;
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
proxy_next_upstream_tries 3;
}
location /api/ {
proxy_pass http://api_backend;
proxy_http_version 1.1;
+20 -4
View File
@@ -35,8 +35,9 @@ type Config struct {
RedisOperationTimeout time.Duration
RedisPoolSize int
ReadinessTimeout time.Duration
ShutdownTimeout time.Duration
ReadinessTimeout time.Duration
ShutdownDrainDelay time.Duration
ShutdownTimeout time.Duration
}
func Load() (Config, error) {
@@ -165,6 +166,14 @@ func Load() (Config, error) {
return Config{}, err
}
shutdownDrainDelay, err := durationFromEnv(
"SHUTDOWN_DRAIN_DELAY",
2*time.Second,
)
if err != nil {
return Config{}, err
}
shutdownTimeout, err := durationFromEnv(
"SHUTDOWN_TIMEOUT",
10*time.Second,
@@ -185,6 +194,12 @@ func Load() (Config, error) {
)
}
if shutdownDrainDelay >= shutdownTimeout {
return Config{}, errors.New(
"SHUTDOWN_DRAIN_DELAY must be less than SHUTDOWN_TIMEOUT",
)
}
return Config{
HTTPHost: envOrDefault("APP_HOST", "0.0.0.0"),
HTTPPort: envOrDefault("APP_PORT", "8080"),
@@ -208,8 +223,9 @@ func Load() (Config, error) {
RedisOperationTimeout: redisOperationTimeout,
RedisPoolSize: int(redisPoolSize),
ReadinessTimeout: readinessTimeout,
ShutdownTimeout: shutdownTimeout,
ReadinessTimeout: readinessTimeout,
ShutdownDrainDelay: shutdownDrainDelay,
ShutdownTimeout: shutdownTimeout,
}, nil
}
@@ -0,0 +1,48 @@
package middleware
import (
"net/http"
"sync/atomic"
"mtusi2026lab3/internal/transport/http/response"
)
// AvailabilityGate lets a process reject new work before its HTTP listener is
// closed. Existing requests are still drained by http.Server.Shutdown.
type AvailabilityGate struct {
accepting atomic.Bool
}
func NewAvailabilityGate() *AvailabilityGate {
gate := &AvailabilityGate{}
gate.accepting.Store(true)
return gate
}
func (g *AvailabilityGate) BeginDrain() {
g.accepting.Store(false)
}
func (g *AvailabilityGate) IsAccepting() bool {
return g.accepting.Load()
}
func (g *AvailabilityGate) Middleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Liveness deliberately stays available while the process drains. The
// readiness endpoint and all business traffic receive 503 immediately.
if g.IsAccepting() || r.URL.Path == "/health/live" {
next.ServeHTTP(w, r)
return
}
response.WriteError(
w,
http.StatusServiceUnavailable,
response.CodeUnavailable,
"service is draining",
GetRequestID(r.Context()),
)
})
}
@@ -0,0 +1,38 @@
package middleware
import (
"net/http"
"net/http/httptest"
"testing"
)
func TestAvailabilityGateRejectsNewWorkWhileKeepingLiveness(t *testing.T) {
gate := NewAvailabilityGate()
next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusNoContent)
})
handler := RequestID(gate.Middleware(next))
before := httptest.NewRecorder()
handler.ServeHTTP(before, httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil))
if before.Code != http.StatusNoContent {
t.Fatalf("before drain status = %d, want %d", before.Code, http.StatusNoContent)
}
gate.BeginDrain()
draining := httptest.NewRecorder()
handler.ServeHTTP(draining, httptest.NewRequest(http.MethodGet, "/health/ready", nil))
if draining.Code != http.StatusServiceUnavailable {
t.Fatalf("ready while draining status = %d, want %d", draining.Code, http.StatusServiceUnavailable)
}
if draining.Header().Get("X-Request-Id") == "" {
t.Fatal("draining response is missing X-Request-Id")
}
live := httptest.NewRecorder()
handler.ServeHTTP(live, httptest.NewRequest(http.MethodGet, "/health/live", nil))
if live.Code != http.StatusNoContent {
t.Fatalf("live while draining status = %d, want %d", live.Code, http.StatusNoContent)
}
}
+397
View File
@@ -0,0 +1,397 @@
#!/usr/bin/env python3
"""Run the repeatable, secret-free resilience experiment required for grade 4."""
import argparse
from collections import Counter, defaultdict
import datetime as dt
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
import uuid
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
from client import Client, Failure # noqa: E402
def command(args, env=None, timeout=240, check=True):
result = subprocess.run(
[str(value) for value in args],
cwd=ROOT,
env=env,
text=True,
capture_output=True,
timeout=timeout,
check=False,
)
if check and result.returncode:
raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}")
return result
def compose(env_file, *args):
return command(["docker", "compose", "--env-file", env_file, *args])
def api_ids(env_file):
return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line]
def inspect(container_id):
data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout
return json.loads(data)
def wait_api(env_file, expected, timeout=90):
deadline = time.monotonic() + timeout
while time.monotonic() < deadline:
healthy = []
for container_id in api_ids(env_file):
state = inspect(container_id)["State"]
if state.get("Health", {}).get("Status") == "healthy":
healthy.append(container_id)
if len(healthy) >= expected:
return healthy
time.sleep(0.5)
raise RuntimeError(f"fewer than {expected} API replicas became healthy")
def percentile(values, ratio):
if not values:
return None
values = sorted(values)
return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3)
def memory_mib(raw):
value = raw.split("/", 1)[0].strip()
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
for suffix in ("GiB", "MiB", "KiB", "B"):
if value.endswith(suffix):
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
return 0.0
def service(name):
for candidate in ("load-balancer", "postgres", "redis", "api"):
if f"-{candidate}-" in name:
return candidate
return "other"
def sample_stats():
result = command(
["docker", "stats", "--no-stream", "--format", "{{json .}}"],
timeout=10,
check=False,
)
samples = []
if result.returncode:
return samples
for line in result.stdout.splitlines():
try:
item = json.loads(line)
samples.append({
"service": service(item.get("Name", "")),
"cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")),
"memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")),
})
except (ValueError, json.JSONDecodeError):
continue
return samples
def aggregate_resources(samples):
grouped = defaultdict(lambda: {"cpu": [], "memory": []})
for item in samples:
if item["service"] != "other":
grouped[item["service"]]["cpu"].append(item["cpu_percent"])
grouped[item["service"]]["memory"].append(item["memory_mib"])
return {
name: {
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
"cpu_percent_max": round(max(values["cpu"]), 3),
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
"memory_mib_max": round(max(values["memory"]), 3),
}
for name, values in sorted(grouped.items())
}
def load_phase(name, actor, chat, args, action=None, target=None):
records, resources = [], []
lock = threading.Lock()
stop = threading.Event()
barrier = threading.Barrier(args.concurrency)
started = time.monotonic()
def worker(worker_id):
client = Client(args.base_url, timeout=4)
index = 0
barrier.wait()
while not stop.is_set():
request_started = time.monotonic()
try:
response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}")
row = {
"t": round(time.monotonic() - started, 3),
"ok": True,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"instance": response["headers"].get("X-Instance-Id", ""),
}
except Failure as exc:
row = {
"t": round(time.monotonic() - started, 3),
"ok": False,
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
"error": str(exc),
}
with lock:
records.append(row)
index += 1
def sampler():
while not stop.is_set():
resources.extend(sample_stats())
stop.wait(1)
workers = [threading.Thread(target=worker, args=(index,), daemon=True)
for index in range(args.concurrency)]
stats_thread = threading.Thread(target=sampler, daemon=True)
stats_thread.start()
for thread in workers:
thread.start()
action_data = None
action_error = None
try:
if action is None:
time.sleep(args.baseline_seconds)
else:
time.sleep(args.action_after_seconds)
action_started = time.monotonic()
try:
summary = action()
exit_code = 0
except Exception as exc:
summary, exit_code, action_error = type(exc).__name__, 1, exc
action_ended = time.monotonic()
action_data = {
"started_at": round(action_started - started, 3),
"ended_at": round(action_ended - started, 3),
"duration_seconds": round(action_ended - action_started, 3),
"exit_code": exit_code,
"summary": summary,
}
time.sleep(args.post_action_seconds)
finally:
stop.set()
for thread in workers:
thread.join(10)
stats_thread.join(5)
if action_error:
raise action_error
elapsed = time.monotonic() - started
successes = [row for row in records if row["ok"]]
failures = [row for row in records if not row["ok"]]
latencies = [row["latency_ms"] for row in successes]
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
for row in records:
buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1
recovery = None
if action_data and target:
recovery_row = next((
row for row in successes
if row["t"] >= action_data["ended_at"] and row.get("instance") != target
), None)
if recovery_row:
recovery = round(recovery_row["t"] - action_data["started_at"], 3)
total = len(records)
return {
"name": name,
"elapsed_seconds": round(elapsed, 3),
"successful_requests": len(successes),
"failed_requests": len(failures),
"error_ratio": round(len(failures) / total, 6) if total else None,
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
"successful_rps": round(len(successes) / elapsed, 3),
"successful_latency_ms": {
"p50": percentile(latencies, 0.50),
"p95": percentile(latencies, 0.95),
"p99": percentile(latencies, 0.99),
},
"instances": sorted({row.get("instance") for row in successes if row.get("instance")}),
"recovery_seconds": recovery,
"action": action_data,
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
"resources": aggregate_resources(resources),
}
def idempotency(actor, chat, base_url, count=32):
key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex
barrier, lock = threading.Barrier(count), threading.Lock()
rows, errors = [], []
def request():
client = Client(base_url, timeout=8)
barrier.wait()
try:
response = client.message(actor, chat, body, key=key)
row = (response["json"]["id"], response["headers"].get("X-Instance-Id", ""))
with lock:
rows.append(row)
except Failure as exc:
with lock:
errors.append(str(exc))
threads = [threading.Thread(target=request) for _ in range(count)]
for thread in threads:
thread.start()
for thread in threads:
thread.join(30)
message_ids = {row[0] for row in rows}
if errors or len(rows) != count or len(message_ids) != 1:
raise RuntimeError("parallel idempotency check failed")
return {
"parallel_requests": count,
"successful_responses": len(rows),
"failed_responses": len(errors),
"unique_message_ids": len(message_ids),
"observed_instances": sorted({row[1] for row in rows}),
}
def environment(env_file):
mem_total_kib = 0
with open("/proc/meminfo", encoding="utf-8") as meminfo:
for line in meminfo:
if line.startswith("MemTotal:"):
mem_total_kib = int(line.split()[1])
break
limits = []
for container_id in compose(env_file, "ps", "-q").stdout.splitlines():
data = inspect(container_id)
limits.append({
"service": service(data["Name"].lstrip("/")),
"nano_cpus": data["HostConfig"].get("NanoCpus", 0),
"memory_limit_bytes": data["HostConfig"].get("Memory", 0),
})
return {
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
"logical_cpu_count": os.cpu_count(),
"host_memory_mib": round(mem_total_kib / 1024, 1),
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
"container_limits": limits,
}
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base-url", default="http://localhost:8080")
parser.add_argument("--env-file", default=".env")
parser.add_argument("--concurrency", type=int, default=8)
parser.add_argument("--baseline-seconds", type=float, default=12)
parser.add_argument("--action-after-seconds", type=float, default=3)
parser.add_argument("--post-action-seconds", type=float, default=8)
parser.add_argument("--output", default="evidence/grade4/results.json")
args = parser.parse_args()
if not 1 <= args.concurrency <= 64:
raise SystemExit("concurrency must be 1..64")
original = wait_api(args.env_file, 2)
if len(original) != 2:
raise RuntimeError("experiment requires exactly two initial API replicas")
client = Client(args.base_url, timeout=5)
alice, _, _, chat = client.bootstrap()
for index in range(12):
client.message(alice, chat, f"grade4-warmup-{index}")
phases = [load_phase("baseline", alice, chat, args)]
targets = [(container_id, inspect(container_id)["Config"]["Hostname"])
for container_id in original]
for ordinal, (container_id, target) in enumerate(targets, start=1):
def stop_target(instance=target):
hook_env = dict(os.environ)
hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance})
result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30)
return result.stdout.strip().splitlines()[-1]
phases.append(load_phase(
f"failover-{ordinal}", alice, chat, args,
action=stop_target, target=target,
))
command(["docker", "start", container_id])
wait_api(args.env_file, 2)
idem = idempotency(alice, chat, args.base_url)
def rolling():
hook_env = dict(os.environ)
hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"})
result = command(
[ROOT / "scripts" / "rolling-update.sh"],
env=hook_env,
timeout=240,
)
return result.stdout.strip().splitlines()[-1]
phases.append(load_phase("rolling-update", alice, chat, args, action=rolling))
wait_api(args.env_file, 2)
report = {
"schema_version": 1,
"environment": environment(args.env_file),
"methodology": {
"base_url": args.base_url,
"concurrency": args.concurrency,
"baseline_seconds": args.baseline_seconds,
"action_after_seconds": args.action_after_seconds,
"post_action_seconds": args.post_action_seconds,
"warmup_requests": 12,
"workload": "closed-loop POST messages via Nginx; no automatic retries",
},
"phases": phases,
"idempotency": idem,
"dataset": {
"warmup_requests": 12,
"minimum_confirmed_messages": 13 + sum(
phase["successful_requests"] for phase in phases
),
},
"limitations": [
"single Docker host",
"closed-loop coordinated omission",
"successful-response latency percentiles only",
"PostgreSQL and Nginx remain single points of failure",
],
}
destination = ROOT / args.output
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
try:
displayed_output = destination.relative_to(ROOT)
except ValueError:
displayed_output = destination
print(json.dumps({
"output": str(displayed_output),
"phases": [{
"name": phase["name"],
"successful_rps": phase["successful_rps"],
"failed_requests": phase["failed_requests"],
"recovery_seconds": phase["recovery_seconds"],
} for phase in phases],
"idempotency": idem,
}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env python3
"""Render portable SVG graphs and a Markdown summary from grade-4 results."""
import argparse
from collections import defaultdict
import html
import json
from pathlib import Path
STYLE = """
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
.title { font-size: 26px; font-weight: 650; }
.label { font-size: 14px; fill: #c8cbd0; }
.small { font-size: 12px; fill: #a2a5aa; }
.grid { stroke: #34373d; stroke-width: 1; }
.bar { fill: #cfd2d6; }
.bar2 { fill: #73777f; }
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
"""
def svg(title, body, width=1100, height=560):
return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">
<rect width="100%" height="100%" fill="#0e1013"/>
<style>{STYLE}</style>
<text x="48" y="42" class="title">{html.escape(title)}</text>
{body}
</svg>
"""
def grid(maximum, left, top, width, height):
parts = []
for step in range(6):
value = maximum * step / 5
y = top + height - height * step / 5
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{left+width}" y2="{y:.1f}" class="grid"/>')
parts.append(f'<text x="{left-12}" y="{y+4:.1f}" text-anchor="end" class="small">{value:.0f}</text>')
return parts
def throughput(phases, output):
left, top, width, height = 80, 90, 960, 360
maximum = max((phase["successful_rps"] for phase in phases), default=1) or 1
parts = grid(maximum, left, top, width, height)
slot = width / max(1, len(phases))
for index, phase in enumerate(phases):
bar_width = min(120, slot * 0.5)
x = left + slot * index + (slot - bar_width) / 2
bar_height = height * phase["successful_rps"] / maximum
y = top + height - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="10" class="bar"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-10:.1f}" text-anchor="middle" class="label">{phase["successful_rps"]:.1f} RPS</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+28}" text-anchor="middle" class="label">{html.escape(phase["name"])}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+48}" text-anchor="middle" class="small">errors {phase["error_ratio"]*100:.2f}%</text>')
parts.append(f'<text x="24" y="{top+height/2}" transform="rotate(-90 24 {top+height/2})" class="label">Successful requests per second</text>')
output.write_text(svg("Grade 4: throughput and errors", "\n".join(parts)))
def latency(phases, output):
left, top, width, height = 80, 90, 960, 360
values = [value for phase in phases
for value in phase["successful_latency_ms"].values()
if value is not None]
maximum = max(values, default=1) or 1
parts = grid(maximum, left, top, width, height)
slot = width / max(1, len(phases))
for phase_index, phase in enumerate(phases):
group_x = left + slot * phase_index
bar_width = min(42, slot / 5)
for metric_index, metric in enumerate(("p50", "p95", "p99")):
value = phase["successful_latency_ms"][metric] or 0
bar_height = height * value / maximum
x = group_x + slot * 0.2 + metric_index * (bar_width + 8)
y = top + height - bar_height
css = "bar" if metric_index == 0 else "bar2"
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="7" class="{css}"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-7:.1f}" text-anchor="middle" class="small">{value:.0f}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+20}" text-anchor="middle" class="small">{metric}</text>')
parts.append(f'<text x="{group_x+slot/2:.1f}" y="{top+height+48}" text-anchor="middle" class="label">{html.escape(phase["name"])}</text>')
parts.append(f'<text x="24" y="{top+height/2}" transform="rotate(-90 24 {top+height/2})" class="label">Successful latency, ms</text>')
output.write_text(svg("Grade 4: successful latency percentiles", "\n".join(parts)))
def timeline(phases, output):
left, top, width, height = 70, 90, 980, 390
rows, events, offset = [], [], 0
for phase in phases:
buckets = phase["per_second"]
for second, counts in sorted(buckets.items(), key=lambda item: int(item[0])):
rows.append((offset + int(second), counts))
if phase.get("action"):
events.append((offset + phase["action"]["started_at"], phase["name"]))
offset += max([int(key) for key in buckets] or [0]) + 2
maximum = max((counts["successful"] for _, counts in rows), default=1) or 1
max_x = max((second for second, _ in rows), default=1) or 1
parts, points = grid(maximum, left, top, width, height), []
for second, counts in rows:
x = left + width * second / max_x
y = top + height - height * counts["successful"] / maximum
points.append(f"{x:.1f},{y:.1f}")
if counts["failed"]:
radius = min(12, 3 + counts["failed"])
parts.append(f'<circle cx="{x:.1f}" cy="{top+height-8}" r="{radius}" class="error"/>')
parts.append(f'<polyline points="{" ".join(points)}" class="line"/>')
for second, label in events:
x = left + width * second / max_x
parts.append(f'<line x1="{x:.1f}" y1="{top}" x2="{x:.1f}" y2="{top+height}" class="event"/>')
parts.append(f'<text x="{x+6:.1f}" y="{top+18}" class="small">{html.escape(label)} action</text>')
parts.append(f'<text x="{left}" y="{top+height+32}" class="label">One-second buckets; circles mark failed requests</text>')
output.write_text(svg("Grade 4: request timeline", "\n".join(parts), height=600))
def resources(phases, output):
combined = defaultdict(lambda: {"cpu": [], "memory": []})
for phase in phases:
for name, values in phase["resources"].items():
combined[name]["cpu"].append(values["cpu_percent_max"])
combined[name]["memory"].append(values["memory_mib_max"])
names, parts = sorted(combined), []
panels = [
("Peak CPU, %", "cpu", 80, 90, 420, 360),
("Peak memory, MiB", "memory", 600, 90, 420, 360),
]
for title, metric, left, top, width, height in panels:
maximum = max((max(combined[name][metric]) for name in names), default=1) or 1
parts.append(f'<text x="{left}" y="{top-18}" class="label">{title}</text>')
slot = width / max(1, len(names))
for index, name in enumerate(names):
value = max(combined[name][metric])
bar_width = min(70, slot * 0.55)
x = left + slot * index + (slot - bar_width) / 2
bar_height = height * value / maximum
y = top + height - bar_height
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="9" class="bar"/>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-8:.1f}" text-anchor="middle" class="small">{value:.1f}</text>')
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+25}" text-anchor="middle" class="small">{html.escape(name)}</text>')
output.write_text(svg("Grade 4: peak container resources", "\n".join(parts), height=600))
def summary(report, output):
method = report["methodology"]
lines = [
"# Lab 3 grade-4 experiment", "",
f"Generated: {report['environment']['utc_started_at']}.", "",
"## Method", "",
f"- concurrency: {method['concurrency']};",
f"- baseline duration: {method['baseline_seconds']} s;",
"- load: closed-loop POST messages through Nginx;",
"- failover: both original API containers stopped and restored in turn;",
"- rolling update: one surge replica, old replicas replaced one by one;",
"- percentiles cover successful responses; failures are separate.", "",
"## Results", "",
"| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for phase in report["phases"]:
lat = phase["successful_latency_ms"]
recovery = phase["recovery_seconds"]
lines.append(
f"| {phase['name']} | {phase['successful_rps']} | "
f"{phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
f"{lat['p50']} | {lat['p95']} | {lat['p99']} | "
f"{recovery if recovery is not None else 'n/a'} |"
)
idem = report["idempotency"]
lines += [
"", "## Parallel idempotency", "",
f"{idem['parallel_requests']} requests produced "
f"{idem['unique_message_ids']} unique message ID across "
f"{len(idem['observed_instances'])} API instances.", "",
"## Graphs", "",
"![Throughput and errors](throughput-errors.svg)", "",
"![Latency](latency.svg)", "",
"![Timeline](timeline.svg)", "",
"![Resources](resources.svg)", "",
"## Scope", "",
"Single-host process failure and rolling replacement only. PostgreSQL "
"and Nginx remain single points of failure.", "",
]
output.write_text("\n".join(lines))
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("results", nargs="?", default="evidence/grade4/results.json")
args = parser.parse_args()
source = Path(args.results)
report = json.loads(source.read_text())
destination = source.parent
phases = report["phases"]
throughput(phases, destination / "throughput-errors.svg")
latency(phases, destination / "latency.svg")
timeline(phases, destination / "timeline.svg")
resources(phases, destination / "resources.svg")
summary(report, destination / "README.md")
print(f"rendered grade-4 evidence in {destination}")
if __name__ == "__main__":
main()
+89
View File
@@ -0,0 +1,89 @@
#!/bin/sh
set -eu
project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
cd "$project_root"
env_file=${ENV_FILE:-.env}
desired_replicas=${DESIRED_REPLICAS:-2}
wait_timeout=${ROLLING_WAIT_TIMEOUT:-90}
stop_timeout=${ROLLING_STOP_TIMEOUT:-15}
dns_settle_delay=${ROLLING_DNS_SETTLE_DELAY:-2}
case "$desired_replicas" in
''|*[!0-9]*)
printf '%s\n' "DESIRED_REPLICAS must be an integer" >&2
exit 2
;;
esac
case "$dns_settle_delay" in
''|*[!0-9]*)
printf '%s\n' "ROLLING_DNS_SETTLE_DELAY must be a non-negative integer" >&2
exit 2
;;
esac
if [ "$desired_replicas" -lt 2 ]; then
printf '%s\n' "rolling update requires at least two desired replicas" >&2
exit 2
fi
compose() {
docker compose --env-file "$env_file" "$@"
}
healthy_api_count() {
healthy=0
for container_id in $(compose ps -q api); do
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")
if [ "$status" = "healthy" ] || [ "$status" = "running" ]; then
healthy=$((healthy + 1))
fi
done
printf '%s\n' "$healthy"
}
wait_for_healthy() {
expected=$1
deadline=$(( $(date +%s) + wait_timeout ))
while [ "$(healthy_api_count)" -lt "$expected" ]; do
if [ "$(date +%s)" -ge "$deadline" ]; then
printf '%s\n' "timed out waiting for $expected healthy API replicas" >&2
return 1
fi
sleep 1
done
}
old_containers=$(compose ps -q api)
old_count=$(printf '%s\n' "$old_containers" | awk 'NF { count++ } END { print count+0 }')
if [ "$old_count" -ne "$desired_replicas" ]; then
printf '%s\n' "expected $desired_replicas running API containers, found $old_count" >&2
exit 1
fi
surge_replicas=$((desired_replicas + 1))
printf '%s\n' "building replacement API image"
compose build api
printf '%s\n' "starting one surge replica"
compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
wait_for_healthy "$surge_replicas"
sleep "$dns_settle_delay"
for old_container in $old_containers; do
old_instance=$(docker inspect --format '{{.Config.Hostname}}' "$old_container")
printf '%s\n' "draining and replacing API instance $old_instance"
docker stop --time "$stop_timeout" "$old_container" >/dev/null
docker rm "$old_container" >/dev/null
compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
wait_for_healthy "$surge_replicas"
sleep "$dns_settle_delay"
done
printf '%s\n' "removing surge capacity"
compose up -d --no-deps --scale api="$desired_replicas" --no-recreate api
wait_for_healthy "$desired_replicas"
sleep "$dns_settle_delay"
printf '%s\n' "rolling update completed with $desired_replicas healthy API replicas"