diff --git a/.env.example b/.env.example
index 1887ed2..459bbb6 100644
--- a/.env.example
+++ b/.env.example
@@ -17,6 +17,8 @@ SESSION_COOKIE_SECURE=false
CURSOR_SIGNING_KEY_FILE=/run/secrets/cursor_signing_key
API_REPLICAS=2
+API_CPU_LIMIT=0.50
+API_MEMORY_LIMIT=256M
IDEMPOTENCY_TTL_SECONDS=86400
# File paths contain no secret values. Create the referenced local secret files
@@ -32,6 +34,7 @@ DB_CONNECT_TIMEOUT=5s
DB_QUERY_TIMEOUT=3s
READINESS_TIMEOUT=2s
+SHUTDOWN_DRAIN_DELAY=2s
SHUTDOWN_TIMEOUT=10s
# Redis stores a disposable shared chat cache; PostgreSQL remains authoritative.
diff --git a/Makefile b/Makefile
index b39d9e0..7e28683 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,7 @@ ENV_FILE ?= .env
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
+GRADE4_OUTPUT ?= evidence/grade4/results.json
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
@@ -26,6 +27,8 @@ COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
load \
test-persistence \
test-failover \
+ rolling-update \
+ grade4-experiment \
compose-config \
compose-build \
compose-up \
@@ -48,6 +51,8 @@ help:
@echo "load: run synthetic writes against the LB entrypoint"
@echo "test-persistence: run persistence test with ACTION_SCRIPT"
@echo "test-failover: run replica failover test with ACTION_SCRIPT"
+ @echo "rolling-update: replace API replicas one by one with surge capacity"
+ @echo "grade4-experiment: run failovers, idempotency and rolling update with evidence"
@echo "compose-config: validate resolved Compose configuration"
@echo "compose-build: build application images"
@echo "compose-up: start the stack"
@@ -109,6 +114,13 @@ test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
+rolling-update:
+ ENV_FILE="$(ENV_FILE)" ./scripts/rolling-update.sh
+
+grade4-experiment:
+ $(PYTHON) scripts/grade4_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE4_OUTPUT)"
+ $(PYTHON) scripts/render_grade4.py "$(GRADE4_OUTPUT)"
+
compose-config:
@test -f "$(ENV_FILE)" || \
(echo "$(ENV_FILE) is missing; run: make init-env"; exit 2)
diff --git a/REPORT.md b/REPORT.md
index c5ec443..5903463 100644
--- a/REPORT.md
+++ b/REPORT.md
@@ -1,8 +1,8 @@
# Отчёт — лабораторная 3
- Автор, группа: Чурилин, БВТ2301
-- Текущий завершённый уровень: оценка «3»
-- Итоговая цель: оценка «5»; трек будет выбран после реализации уровня «4»
+- Текущий завершённый уровень: оценка «4»
+- Итоговая цель: оценка «5»; исследовательский трек выполняется следующим этапом
- Commit/PR: заполняется автором после локального коммита и публикации
- Использованные источники и библиотеки: преподавательские README/OpenAPI/tests,
документация Go, pgx, PostgreSQL, Docker Compose, Nginx, React, TypeScript,
@@ -38,6 +38,7 @@ make test
make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
docker compose --env-file .env up -d --wait api
make test-persistence ACTION_SCRIPT=./scripts/restart.sh
+make grade4-experiment
```
По этому же адресу Nginx отдаёт production-сборку React. Переход на
@@ -86,10 +87,12 @@ Compose запускает один сервис `api` с `API_REPLICAS=2`. Ре
ports и разделяют PostgreSQL, включая таблицу сессий. Nginx — единственная
публичная точка входа; он использует `least_conn` и Docker DNS resolver с
повторным разрешением имени `api` раз в секунду. Sticky sessions не применяются.
-Безопасные методы могут быть перенаправлены на другую реплику при сетевой ошибке.
-Произвольные POST балансировщик автоматически не повторяет: после
-неопределённого результата повтор сообщения выполняет клиент с тем же
-`Idempotency-Key`, а создание пользователя/чата контрактом не идемпотентно.
+Произвольные POST балансировщик автоматически не повторяет. Исключение сделано
+только для маршрута отправки сообщения: обязательный `Idempotency-Key` позволяет
+Nginx безопасно повторить запрос на другой реплике при 502/503/504 во время
+drain. Клиент также сохраняет ключ для повтора после неопределённого результата.
+Создание пользователя и чата контрактом не идемпотентно, поэтому автоматический
+retry для этих маршрутов запрещён.
Миграции вынесены в одноразовый сервис `migrate`, завершение которого является
предусловием запуска API. Поэтому две реплики не конкурируют за изменение схемы.
@@ -104,7 +107,8 @@ secret `cursor_signing_key`. Репозитории получают тольк
`.env.example` перечисляет реально читаемые приложением параметры PostgreSQL,
пула, readiness/shutdown, Redis-кэша, числа реплик, TTL идемпотентности и ключа
-курсора. `API_REPLICAS` используется Compose, а `IDEMPOTENCY_TTL_SECONDS` читает
+курсора. Там же задаются CPU/RAM API и задержка drain перед shutdown.
+`API_REPLICAS` используется Compose, а `IDEMPOTENCY_TTL_SECONDS` читает
Go-приложение и не допускает значение меньше 86400 секунд. Секретных значений в
шаблоне нет: там указаны только пути внутри контейнера.
@@ -177,37 +181,71 @@ cookie.
| Persistence после рестартов | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS; прежняя сессия и данные доступны после рестарта PostgreSQL, Redis, API и LB | `scripts/restart.sh`, `evidence/lab3-summary.md` |
| Миграции перед API | `docker compose --env-file .env ps -a` и логи `migrate` | миграции 1 и 2 применены; one-shot контейнер завершился с кодом 0; API healthy | `migrations/000002_message_idempotency.up.sql`, `evidence/lab3-summary.md` |
| React frontend и HttpOnly BFF | `docker compose build load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/Dockerfile`, `infrastructure/nginx/nginx.conf` |
+| Согласованные health и shutdown | SIGTERM каждой реплики под нагрузкой | API сразу закрывает availability gate, 2 с дренирует запросы и укладывается в 10 с shutdown; `/health/live` остаётся доступен | `internal/transport/http/middleware/availability.go`, `evidence/grade4/` |
+| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только Nginx `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
+| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 3,464/3,476 с, одна краткая ошибка из 4614 запросов в первом прогоне и 0 из 5815 во втором | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
+| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 20 350 успешных запросов, 1 краткий 502 (0,005%), все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
## Эксперименты и отказы
-Проведены функциональные эксперименты устойчивости, а не capacity benchmark.
-Окружение: Windows + WSL2, Docker Engine 29.7.2, Docker Compose 5.5.0, две
-API-реплики; лимиты CPU/RAM пока не задавались, максимальный пул БД —
-10 соединений на реплику. Использовались синтетические данные; отдельного прогрева
-и серии повторов не было. Поэтому результаты подтверждают корректность failover,
-идемпотентности и сохранности данных, но не предельный RPS или p50/p95/p99.
-Нагрузочный профиль с лимитами и повторами относится к следующему уровню.
+Итоговый эксперимент уровня 4 выполнен 23.10.2026 на Windows + WSL2:
+Docker Engine 29.7.2, Compose 5.5.0, 8 логических CPU и 3813,6 MiB RAM,
+commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиченными
+изменениями текущего уровня. В Compose каждой API-реплике выделено 0,5 CPU и
+256 MiB; Nginx — 0,25 CPU/128 MiB; PostgreSQL — 1 CPU/512 MiB; Redis —
+0,25 CPU/128 MiB.
-| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
-| --- | --- | --- | --- |
-| Последовательная остановка `api-2` и `api-1` с восстановлением между тестами | LB обслуживает через оставшуюся реплику не позднее 10 с | 142 и 143 успешные записи; в обоих запусках 0 ошибок | первое восстановление 0.963 и 0.750 с; подтверждённой потери или дублей нет |
-| Рестарт PostgreSQL, Redis, API и LB без удаления volumes | прежний token и данные сохраняются | `test-persistence` завершился PASS | уложилось в тайм-аут 90 с; потери данных нет |
-| 32 одновременных POST с одним `Idempotency-Key` | один бизнес-эффект | 32 ответа 201 с одним Message ID; в БД одно сообщение | дубли отсутствуют |
+Команда `make grade4-experiment` создаёт синтетических пользователей и чат,
+делает 12 warm-up записей, затем запускает 8 closed-loop клиентов без повторов
+на стороне клиента. Baseline длится 12 с. В каждом failover воздействие
+начинается через 3 с, предыдущая реплика восстанавливается перед остановкой
+следующей, после действия нагрузка продолжается ещё 8 с. Rolling update
+временно увеличивает сервис до трёх реплик, ждёт healthcheck и два периода
+Docker DNS, заменяет старые контейнеры по одному и возвращает масштаб к двум.
+Минимальный подтверждённый размер набора после прогона — 34 808 сообщений.
+
+| Фаза | Успешно / ошибок | RPS | Доля ошибок | p50 / p95 / p99, мс | Восстановление |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| baseline | 4017 / 0 | 319,463 | 0% | 16,799 / 55,515 / 93,803 | — |
+| остановка реплики 1 | 4613 / 1 | 301,040 | 0,0217% | 17,145 / 69,558 / 89,752 | 3,464 с |
+| остановка реплики 2 | 5815 / 0 | 374,449 | 0% | 14,211 / 49,542 / 81,040 | 3,476 с |
+| rolling update | 20 350 / 1 | 376,959 | 0,0049% | 15,725 / 48,724 / 79,337 | непрерывная нагрузка |
+
+На baseline средняя выборка одного API-контейнера составила 28,453% CPU и
+42,413 MiB RAM, максимум — 36,1% и 66,07 MiB. PostgreSQL оказался самым
+нагруженным компонентом: в rolling-фазе среднее 98,172% CPU при лимите одного
+ядра, максимум RAM 145,2 MiB. Это не capacity-тест и не доказательство
+saturation point, но важный ориентир для исследования уровня 5.
+
+Nginx обновляет Docker DNS раз в 1 с, исключает peer после двух неудач за 2 с,
+а connect timeout равен 1 с. При постоянной нагрузке ожидаемое максимальное
+обнаружение — около 4 с; измеренные 3,464 и 3,476 с укладываются в требование
+10 с. Одна 504 и одна 502 сохранены в отчёте честно: README прямо допускает
+краткие ошибки обнаружения. Подтверждённых потерь и дублей нет. Параллельные
+32 POST с одним ключом прошли через обе реплики и вернули один Message ID.
+
+Исходные агрегаты, environment/limits, посекундные таблицы и четыре SVG-графика
+лежат в `evidence/grade4/`. Percentile рассчитаны только по успешным ответам;
+профиль closed-loop имеет coordinated omission. CPU для API агрегируется как
+средняя/максимальная выборка отдельного контейнера, а не сумма двух реплик.
## Ограничения и следующие шаги
- Топология однохостовая: Nginx и PostgreSQL остаются single points of failure.
- Репликация БД и несколько балансировщиков в требования уровня «3» не входят.
+ Репликация БД и несколько балансировщиков в требования уровня «4» не входят.
- Redis содержит только кэш, поэтому его потеря допустима. Отдельная
персистентность Redis для корректности API не требуется.
- Истёкшие записи идемпотентности удаляются лениво при повторном обращении к той
же области ключа. Индекс по `expires_at` оставлен для будущей периодической
очистки без изменения схемы.
-- Resource limits, количественный нагрузочный эксперимент и сценарий rolling
- update ещё не реализованы; это следующий этап уровня «4».
+- Уровень «5» ещё не реализован: сравнение 1/2/4 реплик при фиксированных
+ ресурсах и поиск saturation point выполняются следующим отдельным этапом.
+- Rolling-сценарий заменяет бинарник без изменения схемы. Миграции выполняются
+ однократно до API; для будущей несовместимой схемы потребуется expand/contract,
+ поэтому одновременная работа разных schema-version сейчас не заявляется.
- Текущий API не содержит endpoint поиска пользователей, аватаров, изменения
имени/пароля и агрегатов last message/unread. Поэтому UI принимает UUID
участников при создании чата, а профиль честно показывает только доступные
поля. Эти функции требуют сначала расширить и зафиксировать API-контракт.
-- Трек уровня «5» пока не заявлен: сначала необходимо завершить и измерить
- обязательный уровень «4».
+- Однохостовая среда WSL2 не доказывает устойчивость к потере Docker-host;
+ измерен только отказ и замена процесса API.
diff --git a/cmd/server/main.go b/cmd/server/main.go
index 38cd83a..53d6ab3 100644
--- a/cmd/server/main.go
+++ b/cmd/server/main.go
@@ -154,6 +154,8 @@ func main() {
var rootHandler http.Handler = mux
rootHandler = middleware.Recovery(rootHandler)
rootHandler = middleware.Logger(rootHandler)
+ availabilityGate := middleware.NewAvailabilityGate()
+ rootHandler = availabilityGate.Middleware(rootHandler)
rootHandler = middleware.RequestID(rootHandler)
rootHandler = middleware.InstanceID(cfg.InstanceID, rootHandler)
@@ -196,6 +198,13 @@ func main() {
signalValue,
)
+ availabilityGate.BeginDrain()
+ log.Printf(
+ "[INFO] Readiness disabled; draining new traffic for %s",
+ cfg.ShutdownDrainDelay,
+ )
+ time.Sleep(cfg.ShutdownDrainDelay)
+
shutdownContext, shutdownCancel := context.WithTimeout(
context.Background(),
cfg.ShutdownTimeout,
diff --git a/compose.yaml b/compose.yaml
index dc1485d..86e5ae2 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -21,6 +21,11 @@ services:
start_period: 3s
stop_grace_period: 10s
restart: unless-stopped
+ deploy:
+ resources:
+ limits:
+ cpus: "0.25"
+ memory: 128M
postgres:
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
@@ -43,6 +48,11 @@ services:
retries: 10
start_period: 5s
restart: unless-stopped
+ deploy:
+ resources:
+ limits:
+ cpus: "1.00"
+ memory: 512M
redis:
image: redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0
@@ -66,6 +76,11 @@ services:
start_period: 3s
stop_grace_period: 10s
restart: unless-stopped
+ deploy:
+ resources:
+ limits:
+ cpus: "0.25"
+ memory: 128M
migrate:
image: migrate/migrate:v4.20.1@sha256:76cc2074cb6642631f34a898ced71e6aeaa6b1a4d78c4daa743275a22e0c5be7
@@ -84,6 +99,11 @@ services:
postgres:
condition: service_healthy
restart: "no"
+ deploy:
+ resources:
+ limits:
+ cpus: "0.25"
+ memory: 64M
api:
build:
@@ -116,6 +136,7 @@ services:
DB_QUERY_TIMEOUT: ${DB_QUERY_TIMEOUT:-3s}
READINESS_TIMEOUT: ${READINESS_TIMEOUT:-2s}
+ SHUTDOWN_DRAIN_DELAY: ${SHUTDOWN_DRAIN_DELAY:-2s}
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10s}
secrets:
- database_url
@@ -131,6 +152,10 @@ services:
condition: service_completed_successfully
deploy:
replicas: ${API_REPLICAS:-2}
+ resources:
+ limits:
+ cpus: "${API_CPU_LIMIT:-0.50}"
+ memory: ${API_MEMORY_LIMIT:-256M}
healthcheck:
test:
[
@@ -155,6 +180,11 @@ services:
redis:
condition: service_healthy
restart: unless-stopped
+ deploy:
+ resources:
+ limits:
+ cpus: "0.50"
+ memory: 256M
postgres-restore:
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
@@ -178,6 +208,11 @@ services:
retries: 10
start_period: 5s
restart: "no"
+ deploy:
+ resources:
+ limits:
+ cpus: "1.00"
+ memory: 512M
volumes:
postgres_data:
diff --git a/evidence/grade4/README.md b/evidence/grade4/README.md
new file mode 100644
index 0000000..2d9435e
--- /dev/null
+++ b/evidence/grade4/README.md
@@ -0,0 +1,39 @@
+# Lab 3 grade-4 experiment
+
+Generated: 2026-10-23T12:31:12.749687+00:00.
+
+## Method
+
+- concurrency: 8;
+- baseline duration: 12 s;
+- load: closed-loop POST messages through Nginx;
+- failover: both original API containers stopped and restored in turn;
+- rolling update: one surge replica, old replicas replaced one by one;
+- percentiles cover successful responses; failures are separate.
+
+## Results
+
+| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |
+| --- | ---: | ---: | ---: | ---: | ---: | ---: |
+| baseline | 319.463 | 0 (0.000%) | 16.799 | 55.515 | 93.803 | n/a |
+| failover-1 | 301.04 | 1 (0.022%) | 17.145 | 69.558 | 89.752 | 3.464 |
+| failover-2 | 374.449 | 0 (0.000%) | 14.211 | 49.542 | 81.04 | 3.476 |
+| rolling-update | 376.959 | 1 (0.005%) | 15.725 | 48.724 | 79.337 | n/a |
+
+## Parallel idempotency
+
+32 requests produced 1 unique message ID across 2 API instances.
+
+## Graphs
+
+
+
+
+
+
+
+
+
+## Scope
+
+Single-host process failure and rolling replacement only. PostgreSQL and Nginx remain single points of failure.
diff --git a/evidence/grade4/latency.svg b/evidence/grade4/latency.svg
new file mode 100644
index 0000000..7c20a4e
--- /dev/null
+++ b/evidence/grade4/latency.svg
@@ -0,0 +1,69 @@
+
diff --git a/evidence/grade4/resources.svg b/evidence/grade4/resources.svg
new file mode 100644
index 0000000..2edf10b
--- /dev/null
+++ b/evidence/grade4/resources.svg
@@ -0,0 +1,42 @@
+
diff --git a/evidence/grade4/results.json b/evidence/grade4/results.json
new file mode 100644
index 0000000..739dc0d
--- /dev/null
+++ b/evidence/grade4/results.json
@@ -0,0 +1,675 @@
+{
+ "schema_version": 1,
+ "environment": {
+ "utc_started_at": "2026-10-23T12:31:12.749687+00:00",
+ "git_commit": "34618bd4237bbb8bf6a3a303056fc95d799577f9",
+ "working_tree_dirty": true,
+ "logical_cpu_count": 8,
+ "host_memory_mib": 3813.6,
+ "docker_engine": "29.7.2",
+ "docker_compose": "5.5.0",
+ "container_limits": [
+ {
+ "service": "api",
+ "nano_cpus": 500000000,
+ "memory_limit_bytes": 268435456
+ },
+ {
+ "service": "api",
+ "nano_cpus": 500000000,
+ "memory_limit_bytes": 268435456
+ },
+ {
+ "service": "load-balancer",
+ "nano_cpus": 250000000,
+ "memory_limit_bytes": 134217728
+ },
+ {
+ "service": "postgres",
+ "nano_cpus": 1000000000,
+ "memory_limit_bytes": 536870912
+ },
+ {
+ "service": "redis",
+ "nano_cpus": 250000000,
+ "memory_limit_bytes": 134217728
+ }
+ ]
+ },
+ "methodology": {
+ "base_url": "http://localhost:8080",
+ "concurrency": 8,
+ "baseline_seconds": 12,
+ "action_after_seconds": 3,
+ "post_action_seconds": 8,
+ "warmup_requests": 12,
+ "workload": "closed-loop POST messages via Nginx; no automatic retries"
+ },
+ "phases": [
+ {
+ "name": "baseline",
+ "elapsed_seconds": 12.574,
+ "successful_requests": 4017,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 319.463,
+ "successful_latency_ms": {
+ "p50": 16.799,
+ "p95": 55.515,
+ "p99": 93.803
+ },
+ "instances": [
+ "0b933a46fccb",
+ "55aaffcf508a"
+ ],
+ "recovery_seconds": null,
+ "action": null,
+ "per_second": {
+ "0": {
+ "successful": 118,
+ "failed": 0
+ },
+ "1": {
+ "successful": 308,
+ "failed": 0
+ },
+ "2": {
+ "successful": 386,
+ "failed": 0
+ },
+ "3": {
+ "successful": 378,
+ "failed": 0
+ },
+ "4": {
+ "successful": 344,
+ "failed": 0
+ },
+ "5": {
+ "successful": 322,
+ "failed": 0
+ },
+ "6": {
+ "successful": 316,
+ "failed": 0
+ },
+ "7": {
+ "successful": 384,
+ "failed": 0
+ },
+ "8": {
+ "successful": 380,
+ "failed": 0
+ },
+ "9": {
+ "successful": 402,
+ "failed": 0
+ },
+ "10": {
+ "successful": 377,
+ "failed": 0
+ },
+ "11": {
+ "successful": 288,
+ "failed": 0
+ },
+ "12": {
+ "successful": 14,
+ "failed": 0
+ }
+ },
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 28.453,
+ "cpu_percent_max": 36.1,
+ "memory_mib_avg": 42.413,
+ "memory_mib_max": 66.07
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 17.448,
+ "cpu_percent_max": 20.8,
+ "memory_mib_avg": 9.214,
+ "memory_mib_max": 9.559
+ },
+ "postgres": {
+ "cpu_percent_avg": 85.888,
+ "cpu_percent_max": 105.25,
+ "memory_mib_avg": 97.975,
+ "memory_mib_max": 100.8
+ },
+ "redis": {
+ "cpu_percent_avg": 2.552,
+ "cpu_percent_max": 5.79,
+ "memory_mib_avg": 8.796,
+ "memory_mib_max": 11.74
+ }
+ }
+ },
+ {
+ "name": "failover-1",
+ "elapsed_seconds": 15.324,
+ "successful_requests": 4613,
+ "failed_requests": 1,
+ "error_ratio": 0.000217,
+ "errors": {
+ "POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 504": 1
+ },
+ "successful_rps": 301.04,
+ "successful_latency_ms": {
+ "p50": 17.145,
+ "p95": 69.558,
+ "p99": 89.752
+ },
+ "instances": [
+ "0b933a46fccb",
+ "55aaffcf508a"
+ ],
+ "recovery_seconds": 3.464,
+ "action": {
+ "started_at": 3.006,
+ "ended_at": 6.463,
+ "duration_seconds": 3.457,
+ "exit_code": 0,
+ "summary": "stopped API instance 55aaffcf508a"
+ },
+ "per_second": {
+ "0": {
+ "successful": 404,
+ "failed": 0
+ },
+ "1": {
+ "successful": 426,
+ "failed": 0
+ },
+ "2": {
+ "successful": 358,
+ "failed": 0
+ },
+ "3": {
+ "successful": 399,
+ "failed": 0
+ },
+ "4": {
+ "successful": 392,
+ "failed": 0
+ },
+ "5": {
+ "successful": 322,
+ "failed": 0
+ },
+ "6": {
+ "successful": 299,
+ "failed": 0
+ },
+ "7": {
+ "successful": 329,
+ "failed": 1
+ },
+ "8": {
+ "successful": 313,
+ "failed": 0
+ },
+ "9": {
+ "successful": 229,
+ "failed": 0
+ },
+ "10": {
+ "successful": 155,
+ "failed": 0
+ },
+ "11": {
+ "successful": 187,
+ "failed": 0
+ },
+ "12": {
+ "successful": 299,
+ "failed": 0
+ },
+ "13": {
+ "successful": 317,
+ "failed": 0
+ },
+ "14": {
+ "successful": 184,
+ "failed": 0
+ }
+ },
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 27.62,
+ "cpu_percent_max": 52.59,
+ "memory_mib_avg": 11.58,
+ "memory_mib_max": 12.66
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 18.536,
+ "cpu_percent_max": 21.25,
+ "memory_mib_avg": 9.351,
+ "memory_mib_max": 9.523
+ },
+ "postgres": {
+ "cpu_percent_avg": 81.822,
+ "cpu_percent_max": 104.67,
+ "memory_mib_avg": 100.512,
+ "memory_mib_max": 109.1
+ },
+ "redis": {
+ "cpu_percent_avg": 1.486,
+ "cpu_percent_max": 5.88,
+ "memory_mib_avg": 8.04,
+ "memory_mib_max": 8.25
+ }
+ }
+ },
+ {
+ "name": "failover-2",
+ "elapsed_seconds": 15.529,
+ "successful_requests": 5815,
+ "failed_requests": 0,
+ "error_ratio": 0.0,
+ "errors": {},
+ "successful_rps": 374.449,
+ "successful_latency_ms": {
+ "p50": 14.211,
+ "p95": 49.542,
+ "p99": 81.04
+ },
+ "instances": [
+ "0b933a46fccb",
+ "55aaffcf508a"
+ ],
+ "recovery_seconds": 3.476,
+ "action": {
+ "started_at": 3.013,
+ "ended_at": 6.485,
+ "duration_seconds": 3.472,
+ "exit_code": 0,
+ "summary": "stopped API instance 0b933a46fccb"
+ },
+ "per_second": {
+ "0": {
+ "successful": 396,
+ "failed": 0
+ },
+ "1": {
+ "successful": 439,
+ "failed": 0
+ },
+ "2": {
+ "successful": 412,
+ "failed": 0
+ },
+ "3": {
+ "successful": 355,
+ "failed": 0
+ },
+ "4": {
+ "successful": 378,
+ "failed": 0
+ },
+ "5": {
+ "successful": 440,
+ "failed": 0
+ },
+ "6": {
+ "successful": 362,
+ "failed": 0
+ },
+ "7": {
+ "successful": 419,
+ "failed": 0
+ },
+ "8": {
+ "successful": 385,
+ "failed": 0
+ },
+ "9": {
+ "successful": 372,
+ "failed": 0
+ },
+ "10": {
+ "successful": 420,
+ "failed": 0
+ },
+ "11": {
+ "successful": 423,
+ "failed": 0
+ },
+ "12": {
+ "successful": 432,
+ "failed": 0
+ },
+ "13": {
+ "successful": 375,
+ "failed": 0
+ },
+ "14": {
+ "successful": 207,
+ "failed": 0
+ }
+ },
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 31.411,
+ "cpu_percent_max": 51.63,
+ "memory_mib_avg": 10.373,
+ "memory_mib_max": 11.6
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 15.778,
+ "cpu_percent_max": 21.79,
+ "memory_mib_avg": 9.62,
+ "memory_mib_max": 9.781
+ },
+ "postgres": {
+ "cpu_percent_avg": 76.87,
+ "cpu_percent_max": 104.51,
+ "memory_mib_avg": 106.016,
+ "memory_mib_max": 120.0
+ },
+ "redis": {
+ "cpu_percent_avg": 0.786,
+ "cpu_percent_max": 2.83,
+ "memory_mib_avg": 8.844,
+ "memory_mib_max": 12.28
+ }
+ }
+ },
+ {
+ "name": "rolling-update",
+ "elapsed_seconds": 53.985,
+ "successful_requests": 20350,
+ "failed_requests": 1,
+ "error_ratio": 4.9e-05,
+ "errors": {
+ "POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 502": 1
+ },
+ "successful_rps": 376.959,
+ "successful_latency_ms": {
+ "p50": 15.725,
+ "p95": 48.724,
+ "p99": 79.337
+ },
+ "instances": [
+ "0b933a46fccb",
+ "3adbcad9e34b",
+ "55aaffcf508a",
+ "ceca3f6f1faf",
+ "f0a3bedda85a"
+ ],
+ "recovery_seconds": null,
+ "action": {
+ "started_at": 3.016,
+ "ended_at": 45.961,
+ "duration_seconds": 42.945,
+ "exit_code": 0,
+ "summary": "rolling update completed with 2 healthy API replicas"
+ },
+ "per_second": {
+ "0": {
+ "successful": 340,
+ "failed": 0
+ },
+ "1": {
+ "successful": 369,
+ "failed": 0
+ },
+ "2": {
+ "successful": 426,
+ "failed": 0
+ },
+ "3": {
+ "successful": 382,
+ "failed": 0
+ },
+ "4": {
+ "successful": 371,
+ "failed": 0
+ },
+ "5": {
+ "successful": 376,
+ "failed": 0
+ },
+ "6": {
+ "successful": 328,
+ "failed": 0
+ },
+ "7": {
+ "successful": 350,
+ "failed": 0
+ },
+ "8": {
+ "successful": 434,
+ "failed": 0
+ },
+ "9": {
+ "successful": 411,
+ "failed": 0
+ },
+ "10": {
+ "successful": 399,
+ "failed": 0
+ },
+ "11": {
+ "successful": 417,
+ "failed": 0
+ },
+ "12": {
+ "successful": 374,
+ "failed": 0
+ },
+ "13": {
+ "successful": 427,
+ "failed": 0
+ },
+ "14": {
+ "successful": 414,
+ "failed": 0
+ },
+ "15": {
+ "successful": 418,
+ "failed": 0
+ },
+ "16": {
+ "successful": 408,
+ "failed": 0
+ },
+ "17": {
+ "successful": 352,
+ "failed": 0
+ },
+ "18": {
+ "successful": 391,
+ "failed": 0
+ },
+ "19": {
+ "successful": 388,
+ "failed": 0
+ },
+ "20": {
+ "successful": 413,
+ "failed": 0
+ },
+ "21": {
+ "successful": 417,
+ "failed": 0
+ },
+ "22": {
+ "successful": 388,
+ "failed": 0
+ },
+ "23": {
+ "successful": 396,
+ "failed": 0
+ },
+ "24": {
+ "successful": 400,
+ "failed": 0
+ },
+ "25": {
+ "successful": 339,
+ "failed": 0
+ },
+ "26": {
+ "successful": 225,
+ "failed": 0
+ },
+ "27": {
+ "successful": 376,
+ "failed": 0
+ },
+ "28": {
+ "successful": 385,
+ "failed": 0
+ },
+ "29": {
+ "successful": 396,
+ "failed": 0
+ },
+ "30": {
+ "successful": 403,
+ "failed": 0
+ },
+ "31": {
+ "successful": 262,
+ "failed": 0
+ },
+ "32": {
+ "successful": 428,
+ "failed": 0
+ },
+ "33": {
+ "successful": 362,
+ "failed": 0
+ },
+ "34": {
+ "successful": 286,
+ "failed": 0
+ },
+ "35": {
+ "successful": 337,
+ "failed": 0
+ },
+ "36": {
+ "successful": 328,
+ "failed": 0
+ },
+ "37": {
+ "successful": 424,
+ "failed": 0
+ },
+ "38": {
+ "successful": 356,
+ "failed": 0
+ },
+ "39": {
+ "successful": 386,
+ "failed": 0
+ },
+ "40": {
+ "successful": 422,
+ "failed": 0
+ },
+ "41": {
+ "successful": 285,
+ "failed": 0
+ },
+ "42": {
+ "successful": 279,
+ "failed": 0
+ },
+ "43": {
+ "successful": 222,
+ "failed": 1
+ },
+ "44": {
+ "successful": 316,
+ "failed": 0
+ },
+ "45": {
+ "successful": 359,
+ "failed": 0
+ },
+ "46": {
+ "successful": 405,
+ "failed": 0
+ },
+ "47": {
+ "successful": 398,
+ "failed": 0
+ },
+ "48": {
+ "successful": 395,
+ "failed": 0
+ },
+ "49": {
+ "successful": 430,
+ "failed": 0
+ },
+ "50": {
+ "successful": 453,
+ "failed": 0
+ },
+ "51": {
+ "successful": 409,
+ "failed": 0
+ },
+ "52": {
+ "successful": 445,
+ "failed": 0
+ },
+ "53": {
+ "successful": 450,
+ "failed": 0
+ }
+ },
+ "resources": {
+ "api": {
+ "cpu_percent_avg": 25.598,
+ "cpu_percent_max": 37.65,
+ "memory_mib_avg": 8.956,
+ "memory_mib_max": 10.97
+ },
+ "load-balancer": {
+ "cpu_percent_avg": 19.935,
+ "cpu_percent_max": 23.36,
+ "memory_mib_avg": 9.836,
+ "memory_mib_max": 13.52
+ },
+ "postgres": {
+ "cpu_percent_avg": 98.172,
+ "cpu_percent_max": 105.11,
+ "memory_mib_avg": 127.975,
+ "memory_mib_max": 145.2
+ },
+ "redis": {
+ "cpu_percent_avg": 0.722,
+ "cpu_percent_max": 3.94,
+ "memory_mib_avg": 7.661,
+ "memory_mib_max": 11.84
+ }
+ }
+ }
+ ],
+ "idempotency": {
+ "parallel_requests": 32,
+ "successful_responses": 32,
+ "failed_responses": 0,
+ "unique_message_ids": 1,
+ "observed_instances": [
+ "0b933a46fccb",
+ "55aaffcf508a"
+ ]
+ },
+ "dataset": {
+ "warmup_requests": 12,
+ "minimum_confirmed_messages": 34808
+ },
+ "limitations": [
+ "single Docker host",
+ "closed-loop coordinated omission",
+ "successful-response latency percentiles only",
+ "PostgreSQL and Nginx remain single points of failure"
+ ]
+}
diff --git a/evidence/grade4/throughput-errors.svg b/evidence/grade4/throughput-errors.svg
new file mode 100644
index 0000000..444752b
--- /dev/null
+++ b/evidence/grade4/throughput-errors.svg
@@ -0,0 +1,45 @@
+
diff --git a/evidence/grade4/timeline.svg b/evidence/grade4/timeline.svg
new file mode 100644
index 0000000..b0f1dfa
--- /dev/null
+++ b/evidence/grade4/timeline.svg
@@ -0,0 +1,38 @@
+
diff --git a/infrastructure/nginx/nginx.conf b/infrastructure/nginx/nginx.conf
index bcb4d76..9132e62 100644
--- a/infrastructure/nginx/nginx.conf
+++ b/infrastructure/nginx/nginx.conf
@@ -17,7 +17,7 @@ http {
upstream api_backend {
zone api_backend 64k;
least_conn;
- server api:8080 resolve max_fails=1 fail_timeout=1s;
+ server api:8080 resolve max_fails=2 fail_timeout=2s;
}
server {
@@ -31,6 +31,42 @@ http {
add_header X-Frame-Options "DENY" always;
add_header Content-Security-Policy "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always;
+ # Message creation carries an Idempotency-Key. It is therefore safe to
+ # retry a POST on another replica when the selected backend is draining.
+ location ~ ^/api/v1/chats/[^/]+/messages$ {
+ proxy_pass http://api_backend;
+ proxy_http_version 1.1;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header Connection "";
+ proxy_set_header Cookie "";
+ proxy_connect_timeout 1s;
+ proxy_send_timeout 20s;
+ proxy_read_timeout 20s;
+ proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
+ proxy_next_upstream_tries 3;
+ }
+
+ location ~ ^/web-api/v1/chats/[^/]+/messages$ {
+ rewrite ^/web-api/(.*)$ /api/$1 break;
+ proxy_pass http://api_backend;
+ proxy_http_version 1.1;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto $scheme;
+ proxy_set_header Connection "";
+ proxy_set_header Authorization $web_authorization;
+ proxy_set_header Cookie "";
+ proxy_connect_timeout 1s;
+ proxy_send_timeout 20s;
+ proxy_read_timeout 20s;
+ proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
+ proxy_next_upstream_tries 3;
+ }
+
location /api/ {
proxy_pass http://api_backend;
proxy_http_version 1.1;
diff --git a/internal/config/config.go b/internal/config/config.go
index 6bcbcef..01459b9 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -35,8 +35,9 @@ type Config struct {
RedisOperationTimeout time.Duration
RedisPoolSize int
- ReadinessTimeout time.Duration
- ShutdownTimeout time.Duration
+ ReadinessTimeout time.Duration
+ ShutdownDrainDelay time.Duration
+ ShutdownTimeout time.Duration
}
func Load() (Config, error) {
@@ -165,6 +166,14 @@ func Load() (Config, error) {
return Config{}, err
}
+ shutdownDrainDelay, err := durationFromEnv(
+ "SHUTDOWN_DRAIN_DELAY",
+ 2*time.Second,
+ )
+ if err != nil {
+ return Config{}, err
+ }
+
shutdownTimeout, err := durationFromEnv(
"SHUTDOWN_TIMEOUT",
10*time.Second,
@@ -185,6 +194,12 @@ func Load() (Config, error) {
)
}
+ if shutdownDrainDelay >= shutdownTimeout {
+ return Config{}, errors.New(
+ "SHUTDOWN_DRAIN_DELAY must be less than SHUTDOWN_TIMEOUT",
+ )
+ }
+
return Config{
HTTPHost: envOrDefault("APP_HOST", "0.0.0.0"),
HTTPPort: envOrDefault("APP_PORT", "8080"),
@@ -208,8 +223,9 @@ func Load() (Config, error) {
RedisOperationTimeout: redisOperationTimeout,
RedisPoolSize: int(redisPoolSize),
- ReadinessTimeout: readinessTimeout,
- ShutdownTimeout: shutdownTimeout,
+ ReadinessTimeout: readinessTimeout,
+ ShutdownDrainDelay: shutdownDrainDelay,
+ ShutdownTimeout: shutdownTimeout,
}, nil
}
diff --git a/internal/transport/http/middleware/availability.go b/internal/transport/http/middleware/availability.go
new file mode 100644
index 0000000..a73b283
--- /dev/null
+++ b/internal/transport/http/middleware/availability.go
@@ -0,0 +1,48 @@
+package middleware
+
+import (
+ "net/http"
+ "sync/atomic"
+
+ "mtusi2026lab3/internal/transport/http/response"
+)
+
+// AvailabilityGate lets a process reject new work before its HTTP listener is
+// closed. Existing requests are still drained by http.Server.Shutdown.
+type AvailabilityGate struct {
+ accepting atomic.Bool
+}
+
+func NewAvailabilityGate() *AvailabilityGate {
+ gate := &AvailabilityGate{}
+ gate.accepting.Store(true)
+
+ return gate
+}
+
+func (g *AvailabilityGate) BeginDrain() {
+ g.accepting.Store(false)
+}
+
+func (g *AvailabilityGate) IsAccepting() bool {
+ return g.accepting.Load()
+}
+
+func (g *AvailabilityGate) Middleware(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // Liveness deliberately stays available while the process drains. The
+ // readiness endpoint and all business traffic receive 503 immediately.
+ if g.IsAccepting() || r.URL.Path == "/health/live" {
+ next.ServeHTTP(w, r)
+ return
+ }
+
+ response.WriteError(
+ w,
+ http.StatusServiceUnavailable,
+ response.CodeUnavailable,
+ "service is draining",
+ GetRequestID(r.Context()),
+ )
+ })
+}
diff --git a/internal/transport/http/middleware/availability_test.go b/internal/transport/http/middleware/availability_test.go
new file mode 100644
index 0000000..18f1097
--- /dev/null
+++ b/internal/transport/http/middleware/availability_test.go
@@ -0,0 +1,38 @@
+package middleware
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestAvailabilityGateRejectsNewWorkWhileKeepingLiveness(t *testing.T) {
+ gate := NewAvailabilityGate()
+ next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusNoContent)
+ })
+ handler := RequestID(gate.Middleware(next))
+
+ before := httptest.NewRecorder()
+ handler.ServeHTTP(before, httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil))
+ if before.Code != http.StatusNoContent {
+ t.Fatalf("before drain status = %d, want %d", before.Code, http.StatusNoContent)
+ }
+
+ gate.BeginDrain()
+
+ draining := httptest.NewRecorder()
+ handler.ServeHTTP(draining, httptest.NewRequest(http.MethodGet, "/health/ready", nil))
+ if draining.Code != http.StatusServiceUnavailable {
+ t.Fatalf("ready while draining status = %d, want %d", draining.Code, http.StatusServiceUnavailable)
+ }
+ if draining.Header().Get("X-Request-Id") == "" {
+ t.Fatal("draining response is missing X-Request-Id")
+ }
+
+ live := httptest.NewRecorder()
+ handler.ServeHTTP(live, httptest.NewRequest(http.MethodGet, "/health/live", nil))
+ if live.Code != http.StatusNoContent {
+ t.Fatalf("live while draining status = %d, want %d", live.Code, http.StatusNoContent)
+ }
+}
diff --git a/scripts/grade4_experiment.py b/scripts/grade4_experiment.py
new file mode 100755
index 0000000..4d93847
--- /dev/null
+++ b/scripts/grade4_experiment.py
@@ -0,0 +1,397 @@
+#!/usr/bin/env python3
+"""Run the repeatable, secret-free resilience experiment required for grade 4."""
+import argparse
+from collections import Counter, defaultdict
+import datetime as dt
+import json
+import math
+import os
+from pathlib import Path
+import subprocess
+import sys
+import threading
+import time
+import uuid
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tests"))
+from client import Client, Failure # noqa: E402
+
+
+def command(args, env=None, timeout=240, check=True):
+ result = subprocess.run(
+ [str(value) for value in args],
+ cwd=ROOT,
+ env=env,
+ text=True,
+ capture_output=True,
+ timeout=timeout,
+ check=False,
+ )
+ if check and result.returncode:
+ raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}")
+ return result
+
+
+def compose(env_file, *args):
+ return command(["docker", "compose", "--env-file", env_file, *args])
+
+
+def api_ids(env_file):
+ return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line]
+
+
+def inspect(container_id):
+ data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout
+ return json.loads(data)
+
+
+def wait_api(env_file, expected, timeout=90):
+ deadline = time.monotonic() + timeout
+ while time.monotonic() < deadline:
+ healthy = []
+ for container_id in api_ids(env_file):
+ state = inspect(container_id)["State"]
+ if state.get("Health", {}).get("Status") == "healthy":
+ healthy.append(container_id)
+ if len(healthy) >= expected:
+ return healthy
+ time.sleep(0.5)
+ raise RuntimeError(f"fewer than {expected} API replicas became healthy")
+
+
+def percentile(values, ratio):
+ if not values:
+ return None
+ values = sorted(values)
+ return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3)
+
+
+def memory_mib(raw):
+ value = raw.split("/", 1)[0].strip()
+ units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
+ for suffix in ("GiB", "MiB", "KiB", "B"):
+ if value.endswith(suffix):
+ return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
+ return 0.0
+
+
+def service(name):
+ for candidate in ("load-balancer", "postgres", "redis", "api"):
+ if f"-{candidate}-" in name:
+ return candidate
+ return "other"
+
+
+def sample_stats():
+ result = command(
+ ["docker", "stats", "--no-stream", "--format", "{{json .}}"],
+ timeout=10,
+ check=False,
+ )
+ samples = []
+ if result.returncode:
+ return samples
+ for line in result.stdout.splitlines():
+ try:
+ item = json.loads(line)
+ samples.append({
+ "service": service(item.get("Name", "")),
+ "cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")),
+ "memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")),
+ })
+ except (ValueError, json.JSONDecodeError):
+ continue
+ return samples
+
+
+def aggregate_resources(samples):
+ grouped = defaultdict(lambda: {"cpu": [], "memory": []})
+ for item in samples:
+ if item["service"] != "other":
+ grouped[item["service"]]["cpu"].append(item["cpu_percent"])
+ grouped[item["service"]]["memory"].append(item["memory_mib"])
+ return {
+ name: {
+ "cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
+ "cpu_percent_max": round(max(values["cpu"]), 3),
+ "memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
+ "memory_mib_max": round(max(values["memory"]), 3),
+ }
+ for name, values in sorted(grouped.items())
+ }
+
+
+def load_phase(name, actor, chat, args, action=None, target=None):
+ records, resources = [], []
+ lock = threading.Lock()
+ stop = threading.Event()
+ barrier = threading.Barrier(args.concurrency)
+ started = time.monotonic()
+
+ def worker(worker_id):
+ client = Client(args.base_url, timeout=4)
+ index = 0
+ barrier.wait()
+ while not stop.is_set():
+ request_started = time.monotonic()
+ try:
+ response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}")
+ row = {
+ "t": round(time.monotonic() - started, 3),
+ "ok": True,
+ "latency_ms": round((time.monotonic() - request_started) * 1000, 3),
+ "instance": response["headers"].get("X-Instance-Id", ""),
+ }
+ except Failure as exc:
+ row = {
+ "t": round(time.monotonic() - started, 3),
+ "ok": False,
+ "latency_ms": round((time.monotonic() - request_started) * 1000, 3),
+ "error": str(exc),
+ }
+ with lock:
+ records.append(row)
+ index += 1
+
+ def sampler():
+ while not stop.is_set():
+ resources.extend(sample_stats())
+ stop.wait(1)
+
+ workers = [threading.Thread(target=worker, args=(index,), daemon=True)
+ for index in range(args.concurrency)]
+ stats_thread = threading.Thread(target=sampler, daemon=True)
+ stats_thread.start()
+ for thread in workers:
+ thread.start()
+
+ action_data = None
+ action_error = None
+ try:
+ if action is None:
+ time.sleep(args.baseline_seconds)
+ else:
+ time.sleep(args.action_after_seconds)
+ action_started = time.monotonic()
+ try:
+ summary = action()
+ exit_code = 0
+ except Exception as exc:
+ summary, exit_code, action_error = type(exc).__name__, 1, exc
+ action_ended = time.monotonic()
+ action_data = {
+ "started_at": round(action_started - started, 3),
+ "ended_at": round(action_ended - started, 3),
+ "duration_seconds": round(action_ended - action_started, 3),
+ "exit_code": exit_code,
+ "summary": summary,
+ }
+ time.sleep(args.post_action_seconds)
+ finally:
+ stop.set()
+ for thread in workers:
+ thread.join(10)
+ stats_thread.join(5)
+ if action_error:
+ raise action_error
+
+ elapsed = time.monotonic() - started
+ successes = [row for row in records if row["ok"]]
+ failures = [row for row in records if not row["ok"]]
+ latencies = [row["latency_ms"] for row in successes]
+ buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
+ for row in records:
+ buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1
+ recovery = None
+ if action_data and target:
+ recovery_row = next((
+ row for row in successes
+ if row["t"] >= action_data["ended_at"] and row.get("instance") != target
+ ), None)
+ if recovery_row:
+ recovery = round(recovery_row["t"] - action_data["started_at"], 3)
+ total = len(records)
+ return {
+ "name": name,
+ "elapsed_seconds": round(elapsed, 3),
+ "successful_requests": len(successes),
+ "failed_requests": len(failures),
+ "error_ratio": round(len(failures) / total, 6) if total else None,
+ "errors": dict(sorted(Counter(row["error"] for row in failures).items())),
+ "successful_rps": round(len(successes) / elapsed, 3),
+ "successful_latency_ms": {
+ "p50": percentile(latencies, 0.50),
+ "p95": percentile(latencies, 0.95),
+ "p99": percentile(latencies, 0.99),
+ },
+ "instances": sorted({row.get("instance") for row in successes if row.get("instance")}),
+ "recovery_seconds": recovery,
+ "action": action_data,
+ "per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
+ "resources": aggregate_resources(resources),
+ }
+
+
+def idempotency(actor, chat, base_url, count=32):
+ key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex
+ barrier, lock = threading.Barrier(count), threading.Lock()
+ rows, errors = [], []
+
+ def request():
+ client = Client(base_url, timeout=8)
+ barrier.wait()
+ try:
+ response = client.message(actor, chat, body, key=key)
+ row = (response["json"]["id"], response["headers"].get("X-Instance-Id", ""))
+ with lock:
+ rows.append(row)
+ except Failure as exc:
+ with lock:
+ errors.append(str(exc))
+
+ threads = [threading.Thread(target=request) for _ in range(count)]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join(30)
+ message_ids = {row[0] for row in rows}
+ if errors or len(rows) != count or len(message_ids) != 1:
+ raise RuntimeError("parallel idempotency check failed")
+ return {
+ "parallel_requests": count,
+ "successful_responses": len(rows),
+ "failed_responses": len(errors),
+ "unique_message_ids": len(message_ids),
+ "observed_instances": sorted({row[1] for row in rows}),
+ }
+
+
+def environment(env_file):
+ mem_total_kib = 0
+ with open("/proc/meminfo", encoding="utf-8") as meminfo:
+ for line in meminfo:
+ if line.startswith("MemTotal:"):
+ mem_total_kib = int(line.split()[1])
+ break
+ limits = []
+ for container_id in compose(env_file, "ps", "-q").stdout.splitlines():
+ data = inspect(container_id)
+ limits.append({
+ "service": service(data["Name"].lstrip("/")),
+ "nano_cpus": data["HostConfig"].get("NanoCpus", 0),
+ "memory_limit_bytes": data["HostConfig"].get("Memory", 0),
+ })
+ return {
+ "utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
+ "git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
+ "working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
+ "logical_cpu_count": os.cpu_count(),
+ "host_memory_mib": round(mem_total_kib / 1024, 1),
+ "docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
+ "docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
+ "container_limits": limits,
+ }
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--base-url", default="http://localhost:8080")
+ parser.add_argument("--env-file", default=".env")
+ parser.add_argument("--concurrency", type=int, default=8)
+ parser.add_argument("--baseline-seconds", type=float, default=12)
+ parser.add_argument("--action-after-seconds", type=float, default=3)
+ parser.add_argument("--post-action-seconds", type=float, default=8)
+ parser.add_argument("--output", default="evidence/grade4/results.json")
+ args = parser.parse_args()
+ if not 1 <= args.concurrency <= 64:
+ raise SystemExit("concurrency must be 1..64")
+
+ original = wait_api(args.env_file, 2)
+ if len(original) != 2:
+ raise RuntimeError("experiment requires exactly two initial API replicas")
+ client = Client(args.base_url, timeout=5)
+ alice, _, _, chat = client.bootstrap()
+ for index in range(12):
+ client.message(alice, chat, f"grade4-warmup-{index}")
+
+ phases = [load_phase("baseline", alice, chat, args)]
+ targets = [(container_id, inspect(container_id)["Config"]["Hostname"])
+ for container_id in original]
+ for ordinal, (container_id, target) in enumerate(targets, start=1):
+ def stop_target(instance=target):
+ hook_env = dict(os.environ)
+ hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance})
+ result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30)
+ return result.stdout.strip().splitlines()[-1]
+
+ phases.append(load_phase(
+ f"failover-{ordinal}", alice, chat, args,
+ action=stop_target, target=target,
+ ))
+ command(["docker", "start", container_id])
+ wait_api(args.env_file, 2)
+
+ idem = idempotency(alice, chat, args.base_url)
+
+ def rolling():
+ hook_env = dict(os.environ)
+ hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"})
+ result = command(
+ [ROOT / "scripts" / "rolling-update.sh"],
+ env=hook_env,
+ timeout=240,
+ )
+ return result.stdout.strip().splitlines()[-1]
+
+ phases.append(load_phase("rolling-update", alice, chat, args, action=rolling))
+ wait_api(args.env_file, 2)
+
+ report = {
+ "schema_version": 1,
+ "environment": environment(args.env_file),
+ "methodology": {
+ "base_url": args.base_url,
+ "concurrency": args.concurrency,
+ "baseline_seconds": args.baseline_seconds,
+ "action_after_seconds": args.action_after_seconds,
+ "post_action_seconds": args.post_action_seconds,
+ "warmup_requests": 12,
+ "workload": "closed-loop POST messages via Nginx; no automatic retries",
+ },
+ "phases": phases,
+ "idempotency": idem,
+ "dataset": {
+ "warmup_requests": 12,
+ "minimum_confirmed_messages": 13 + sum(
+ phase["successful_requests"] for phase in phases
+ ),
+ },
+ "limitations": [
+ "single Docker host",
+ "closed-loop coordinated omission",
+ "successful-response latency percentiles only",
+ "PostgreSQL and Nginx remain single points of failure",
+ ],
+ }
+ destination = ROOT / args.output
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
+ try:
+ displayed_output = destination.relative_to(ROOT)
+ except ValueError:
+ displayed_output = destination
+ print(json.dumps({
+ "output": str(displayed_output),
+ "phases": [{
+ "name": phase["name"],
+ "successful_rps": phase["successful_rps"],
+ "failed_requests": phase["failed_requests"],
+ "recovery_seconds": phase["recovery_seconds"],
+ } for phase in phases],
+ "idempotency": idem,
+ }, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/render_grade4.py b/scripts/render_grade4.py
new file mode 100755
index 0000000..9c62886
--- /dev/null
+++ b/scripts/render_grade4.py
@@ -0,0 +1,203 @@
+#!/usr/bin/env python3
+"""Render portable SVG graphs and a Markdown summary from grade-4 results."""
+import argparse
+from collections import defaultdict
+import html
+import json
+from pathlib import Path
+
+
+STYLE = """
+text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
+.title { font-size: 26px; font-weight: 650; }
+.label { font-size: 14px; fill: #c8cbd0; }
+.small { font-size: 12px; fill: #a2a5aa; }
+.grid { stroke: #34373d; stroke-width: 1; }
+.bar { fill: #cfd2d6; }
+.bar2 { fill: #73777f; }
+.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
+.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
+.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
+"""
+
+
+def svg(title, body, width=1100, height=560):
+ return f"""
+"""
+
+
+def grid(maximum, left, top, width, height):
+ parts = []
+ for step in range(6):
+ value = maximum * step / 5
+ y = top + height - height * step / 5
+ parts.append(f'')
+ parts.append(f'{value:.0f}')
+ return parts
+
+
+def throughput(phases, output):
+ left, top, width, height = 80, 90, 960, 360
+ maximum = max((phase["successful_rps"] for phase in phases), default=1) or 1
+ parts = grid(maximum, left, top, width, height)
+ slot = width / max(1, len(phases))
+ for index, phase in enumerate(phases):
+ bar_width = min(120, slot * 0.5)
+ x = left + slot * index + (slot - bar_width) / 2
+ bar_height = height * phase["successful_rps"] / maximum
+ y = top + height - bar_height
+ parts.append(f'')
+ parts.append(f'{phase["successful_rps"]:.1f} RPS')
+ parts.append(f'{html.escape(phase["name"])}')
+ parts.append(f'errors {phase["error_ratio"]*100:.2f}%')
+ parts.append(f'Successful requests per second')
+ output.write_text(svg("Grade 4: throughput and errors", "\n".join(parts)))
+
+
+def latency(phases, output):
+ left, top, width, height = 80, 90, 960, 360
+ values = [value for phase in phases
+ for value in phase["successful_latency_ms"].values()
+ if value is not None]
+ maximum = max(values, default=1) or 1
+ parts = grid(maximum, left, top, width, height)
+ slot = width / max(1, len(phases))
+ for phase_index, phase in enumerate(phases):
+ group_x = left + slot * phase_index
+ bar_width = min(42, slot / 5)
+ for metric_index, metric in enumerate(("p50", "p95", "p99")):
+ value = phase["successful_latency_ms"][metric] or 0
+ bar_height = height * value / maximum
+ x = group_x + slot * 0.2 + metric_index * (bar_width + 8)
+ y = top + height - bar_height
+ css = "bar" if metric_index == 0 else "bar2"
+ parts.append(f'')
+ parts.append(f'{value:.0f}')
+ parts.append(f'{metric}')
+ parts.append(f'{html.escape(phase["name"])}')
+ parts.append(f'Successful latency, ms')
+ output.write_text(svg("Grade 4: successful latency percentiles", "\n".join(parts)))
+
+
+def timeline(phases, output):
+ left, top, width, height = 70, 90, 980, 390
+ rows, events, offset = [], [], 0
+ for phase in phases:
+ buckets = phase["per_second"]
+ for second, counts in sorted(buckets.items(), key=lambda item: int(item[0])):
+ rows.append((offset + int(second), counts))
+ if phase.get("action"):
+ events.append((offset + phase["action"]["started_at"], phase["name"]))
+ offset += max([int(key) for key in buckets] or [0]) + 2
+ maximum = max((counts["successful"] for _, counts in rows), default=1) or 1
+ max_x = max((second for second, _ in rows), default=1) or 1
+ parts, points = grid(maximum, left, top, width, height), []
+ for second, counts in rows:
+ x = left + width * second / max_x
+ y = top + height - height * counts["successful"] / maximum
+ points.append(f"{x:.1f},{y:.1f}")
+ if counts["failed"]:
+ radius = min(12, 3 + counts["failed"])
+ parts.append(f'')
+ parts.append(f'')
+ for second, label in events:
+ x = left + width * second / max_x
+ parts.append(f'')
+ parts.append(f'{html.escape(label)} action')
+ parts.append(f'One-second buckets; circles mark failed requests')
+ output.write_text(svg("Grade 4: request timeline", "\n".join(parts), height=600))
+
+
+def resources(phases, output):
+ combined = defaultdict(lambda: {"cpu": [], "memory": []})
+ for phase in phases:
+ for name, values in phase["resources"].items():
+ combined[name]["cpu"].append(values["cpu_percent_max"])
+ combined[name]["memory"].append(values["memory_mib_max"])
+ names, parts = sorted(combined), []
+ panels = [
+ ("Peak CPU, %", "cpu", 80, 90, 420, 360),
+ ("Peak memory, MiB", "memory", 600, 90, 420, 360),
+ ]
+ for title, metric, left, top, width, height in panels:
+ maximum = max((max(combined[name][metric]) for name in names), default=1) or 1
+ parts.append(f'{title}')
+ slot = width / max(1, len(names))
+ for index, name in enumerate(names):
+ value = max(combined[name][metric])
+ bar_width = min(70, slot * 0.55)
+ x = left + slot * index + (slot - bar_width) / 2
+ bar_height = height * value / maximum
+ y = top + height - bar_height
+ parts.append(f'')
+ parts.append(f'{value:.1f}')
+ parts.append(f'{html.escape(name)}')
+ output.write_text(svg("Grade 4: peak container resources", "\n".join(parts), height=600))
+
+
+def summary(report, output):
+ method = report["methodology"]
+ lines = [
+ "# Lab 3 grade-4 experiment", "",
+ f"Generated: {report['environment']['utc_started_at']}.", "",
+ "## Method", "",
+ f"- concurrency: {method['concurrency']};",
+ f"- baseline duration: {method['baseline_seconds']} s;",
+ "- load: closed-loop POST messages through Nginx;",
+ "- failover: both original API containers stopped and restored in turn;",
+ "- rolling update: one surge replica, old replicas replaced one by one;",
+ "- percentiles cover successful responses; failures are separate.", "",
+ "## Results", "",
+ "| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |",
+ "| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
+ ]
+ for phase in report["phases"]:
+ lat = phase["successful_latency_ms"]
+ recovery = phase["recovery_seconds"]
+ lines.append(
+ f"| {phase['name']} | {phase['successful_rps']} | "
+ f"{phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
+ f"{lat['p50']} | {lat['p95']} | {lat['p99']} | "
+ f"{recovery if recovery is not None else 'n/a'} |"
+ )
+ idem = report["idempotency"]
+ lines += [
+ "", "## Parallel idempotency", "",
+ f"{idem['parallel_requests']} requests produced "
+ f"{idem['unique_message_ids']} unique message ID across "
+ f"{len(idem['observed_instances'])} API instances.", "",
+ "## Graphs", "",
+ "", "",
+ "", "",
+ "", "",
+ "", "",
+ "## Scope", "",
+ "Single-host process failure and rolling replacement only. PostgreSQL "
+ "and Nginx remain single points of failure.", "",
+ ]
+ output.write_text("\n".join(lines))
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("results", nargs="?", default="evidence/grade4/results.json")
+ args = parser.parse_args()
+ source = Path(args.results)
+ report = json.loads(source.read_text())
+ destination = source.parent
+ phases = report["phases"]
+ throughput(phases, destination / "throughput-errors.svg")
+ latency(phases, destination / "latency.svg")
+ timeline(phases, destination / "timeline.svg")
+ resources(phases, destination / "resources.svg")
+ summary(report, destination / "README.md")
+ print(f"rendered grade-4 evidence in {destination}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh
new file mode 100755
index 0000000..6769fc1
--- /dev/null
+++ b/scripts/rolling-update.sh
@@ -0,0 +1,89 @@
+#!/bin/sh
+set -eu
+
+project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
+cd "$project_root"
+
+env_file=${ENV_FILE:-.env}
+desired_replicas=${DESIRED_REPLICAS:-2}
+wait_timeout=${ROLLING_WAIT_TIMEOUT:-90}
+stop_timeout=${ROLLING_STOP_TIMEOUT:-15}
+dns_settle_delay=${ROLLING_DNS_SETTLE_DELAY:-2}
+
+case "$desired_replicas" in
+ ''|*[!0-9]*)
+ printf '%s\n' "DESIRED_REPLICAS must be an integer" >&2
+ exit 2
+ ;;
+esac
+case "$dns_settle_delay" in
+ ''|*[!0-9]*)
+ printf '%s\n' "ROLLING_DNS_SETTLE_DELAY must be a non-negative integer" >&2
+ exit 2
+ ;;
+esac
+if [ "$desired_replicas" -lt 2 ]; then
+ printf '%s\n' "rolling update requires at least two desired replicas" >&2
+ exit 2
+fi
+
+compose() {
+ docker compose --env-file "$env_file" "$@"
+}
+
+healthy_api_count() {
+ healthy=0
+ for container_id in $(compose ps -q api); do
+ status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")
+ if [ "$status" = "healthy" ] || [ "$status" = "running" ]; then
+ healthy=$((healthy + 1))
+ fi
+ done
+ printf '%s\n' "$healthy"
+}
+
+wait_for_healthy() {
+ expected=$1
+ deadline=$(( $(date +%s) + wait_timeout ))
+ while [ "$(healthy_api_count)" -lt "$expected" ]; do
+ if [ "$(date +%s)" -ge "$deadline" ]; then
+ printf '%s\n' "timed out waiting for $expected healthy API replicas" >&2
+ return 1
+ fi
+ sleep 1
+ done
+}
+
+old_containers=$(compose ps -q api)
+old_count=$(printf '%s\n' "$old_containers" | awk 'NF { count++ } END { print count+0 }')
+if [ "$old_count" -ne "$desired_replicas" ]; then
+ printf '%s\n' "expected $desired_replicas running API containers, found $old_count" >&2
+ exit 1
+fi
+
+surge_replicas=$((desired_replicas + 1))
+
+printf '%s\n' "building replacement API image"
+compose build api
+
+printf '%s\n' "starting one surge replica"
+compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
+wait_for_healthy "$surge_replicas"
+sleep "$dns_settle_delay"
+
+for old_container in $old_containers; do
+ old_instance=$(docker inspect --format '{{.Config.Hostname}}' "$old_container")
+ printf '%s\n' "draining and replacing API instance $old_instance"
+ docker stop --time "$stop_timeout" "$old_container" >/dev/null
+ docker rm "$old_container" >/dev/null
+ compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
+ wait_for_healthy "$surge_replicas"
+ sleep "$dns_settle_delay"
+done
+
+printf '%s\n' "removing surge capacity"
+compose up -d --no-deps --scale api="$desired_replicas" --no-recreate api
+wait_for_healthy "$desired_replicas"
+sleep "$dns_settle_delay"
+
+printf '%s\n' "rolling update completed with $desired_replicas healthy API replicas"