diff --git a/.env.example b/.env.example index 1887ed2..459bbb6 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,8 @@ SESSION_COOKIE_SECURE=false CURSOR_SIGNING_KEY_FILE=/run/secrets/cursor_signing_key API_REPLICAS=2 +API_CPU_LIMIT=0.50 +API_MEMORY_LIMIT=256M IDEMPOTENCY_TTL_SECONDS=86400 # File paths contain no secret values. Create the referenced local secret files @@ -32,6 +34,7 @@ DB_CONNECT_TIMEOUT=5s DB_QUERY_TIMEOUT=3s READINESS_TIMEOUT=2s +SHUTDOWN_DRAIN_DELAY=2s SHUTDOWN_TIMEOUT=10s # Redis stores a disposable shared chat cache; PostgreSQL remains authoritative. diff --git a/Makefile b/Makefile index b39d9e0..7e28683 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ ENV_FILE ?= .env BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= +GRADE4_OUTPUT ?= evidence/grade4/results.json TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)" @@ -26,6 +27,8 @@ COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)" load \ test-persistence \ test-failover \ + rolling-update \ + grade4-experiment \ compose-config \ compose-build \ compose-up \ @@ -48,6 +51,8 @@ help: @echo "load: run synthetic writes against the LB entrypoint" @echo "test-persistence: run persistence test with ACTION_SCRIPT" @echo "test-failover: run replica failover test with ACTION_SCRIPT" + @echo "rolling-update: replace API replicas one by one with surge capacity" + @echo "grade4-experiment: run failovers, idempotency and rolling update with evidence" @echo "compose-config: validate resolved Compose configuration" @echo "compose-build: build application images" @echo "compose-up: start the stack" @@ -109,6 +114,13 @@ test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +rolling-update: + ENV_FILE="$(ENV_FILE)" ./scripts/rolling-update.sh + +grade4-experiment: + $(PYTHON) scripts/grade4_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE4_OUTPUT)" + $(PYTHON) scripts/render_grade4.py "$(GRADE4_OUTPUT)" + compose-config: @test -f "$(ENV_FILE)" || \ (echo "$(ENV_FILE) is missing; run: make init-env"; exit 2) diff --git a/REPORT.md b/REPORT.md index c5ec443..5903463 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,8 +1,8 @@ # Отчёт — лабораторная 3 - Автор, группа: Чурилин, БВТ2301 -- Текущий завершённый уровень: оценка «3» -- Итоговая цель: оценка «5»; трек будет выбран после реализации уровня «4» +- Текущий завершённый уровень: оценка «4» +- Итоговая цель: оценка «5»; исследовательский трек выполняется следующим этапом - Commit/PR: заполняется автором после локального коммита и публикации - Использованные источники и библиотеки: преподавательские README/OpenAPI/tests, документация Go, pgx, PostgreSQL, Docker Compose, Nginx, React, TypeScript, @@ -38,6 +38,7 @@ make test make test-failover ACTION_SCRIPT=./scripts/stop-one.sh docker compose --env-file .env up -d --wait api make test-persistence ACTION_SCRIPT=./scripts/restart.sh +make grade4-experiment ``` По этому же адресу Nginx отдаёт production-сборку React. Переход на @@ -86,10 +87,12 @@ Compose запускает один сервис `api` с `API_REPLICAS=2`. Ре ports и разделяют PostgreSQL, включая таблицу сессий. Nginx — единственная публичная точка входа; он использует `least_conn` и Docker DNS resolver с повторным разрешением имени `api` раз в секунду. Sticky sessions не применяются. -Безопасные методы могут быть перенаправлены на другую реплику при сетевой ошибке. -Произвольные POST балансировщик автоматически не повторяет: после -неопределённого результата повтор сообщения выполняет клиент с тем же -`Idempotency-Key`, а создание пользователя/чата контрактом не идемпотентно. +Произвольные POST балансировщик автоматически не повторяет. Исключение сделано +только для маршрута отправки сообщения: обязательный `Idempotency-Key` позволяет +Nginx безопасно повторить запрос на другой реплике при 502/503/504 во время +drain. Клиент также сохраняет ключ для повтора после неопределённого результата. +Создание пользователя и чата контрактом не идемпотентно, поэтому автоматический +retry для этих маршрутов запрещён. Миграции вынесены в одноразовый сервис `migrate`, завершение которого является предусловием запуска API. Поэтому две реплики не конкурируют за изменение схемы. @@ -104,7 +107,8 @@ secret `cursor_signing_key`. Репозитории получают тольк `.env.example` перечисляет реально читаемые приложением параметры PostgreSQL, пула, readiness/shutdown, Redis-кэша, числа реплик, TTL идемпотентности и ключа -курсора. `API_REPLICAS` используется Compose, а `IDEMPOTENCY_TTL_SECONDS` читает +курсора. Там же задаются CPU/RAM API и задержка drain перед shutdown. +`API_REPLICAS` используется Compose, а `IDEMPOTENCY_TTL_SECONDS` читает Go-приложение и не допускает значение меньше 86400 секунд. Секретных значений в шаблоне нет: там указаны только пути внутри контейнера. @@ -177,37 +181,71 @@ cookie. | Persistence после рестартов | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS; прежняя сессия и данные доступны после рестарта PostgreSQL, Redis, API и LB | `scripts/restart.sh`, `evidence/lab3-summary.md` | | Миграции перед API | `docker compose --env-file .env ps -a` и логи `migrate` | миграции 1 и 2 применены; one-shot контейнер завершился с кодом 0; API healthy | `migrations/000002_message_idempotency.up.sql`, `evidence/lab3-summary.md` | | React frontend и HttpOnly BFF | `docker compose build load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/Dockerfile`, `infrastructure/nginx/nginx.conf` | +| Согласованные health и shutdown | SIGTERM каждой реплики под нагрузкой | API сразу закрывает availability gate, 2 с дренирует запросы и укладывается в 10 с shutdown; `/health/live` остаётся доступен | `internal/transport/http/middleware/availability.go`, `evidence/grade4/` | +| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только Nginx `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` | +| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 3,464/3,476 с, одна краткая ошибка из 4614 запросов в первом прогоне и 0 из 5815 во втором | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` | +| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 20 350 успешных запросов, 1 краткий 502 (0,005%), все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` | ## Эксперименты и отказы -Проведены функциональные эксперименты устойчивости, а не capacity benchmark. -Окружение: Windows + WSL2, Docker Engine 29.7.2, Docker Compose 5.5.0, две -API-реплики; лимиты CPU/RAM пока не задавались, максимальный пул БД — -10 соединений на реплику. Использовались синтетические данные; отдельного прогрева -и серии повторов не было. Поэтому результаты подтверждают корректность failover, -идемпотентности и сохранности данных, но не предельный RPS или p50/p95/p99. -Нагрузочный профиль с лимитами и повторами относится к следующему уровню. +Итоговый эксперимент уровня 4 выполнен 23.10.2026 на Windows + WSL2: +Docker Engine 29.7.2, Compose 5.5.0, 8 логических CPU и 3813,6 MiB RAM, +commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиченными +изменениями текущего уровня. В Compose каждой API-реплике выделено 0,5 CPU и +256 MiB; Nginx — 0,25 CPU/128 MiB; PostgreSQL — 1 CPU/512 MiB; Redis — +0,25 CPU/128 MiB. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| Последовательная остановка `api-2` и `api-1` с восстановлением между тестами | LB обслуживает через оставшуюся реплику не позднее 10 с | 142 и 143 успешные записи; в обоих запусках 0 ошибок | первое восстановление 0.963 и 0.750 с; подтверждённой потери или дублей нет | -| Рестарт PostgreSQL, Redis, API и LB без удаления volumes | прежний token и данные сохраняются | `test-persistence` завершился PASS | уложилось в тайм-аут 90 с; потери данных нет | -| 32 одновременных POST с одним `Idempotency-Key` | один бизнес-эффект | 32 ответа 201 с одним Message ID; в БД одно сообщение | дубли отсутствуют | +Команда `make grade4-experiment` создаёт синтетических пользователей и чат, +делает 12 warm-up записей, затем запускает 8 closed-loop клиентов без повторов +на стороне клиента. Baseline длится 12 с. В каждом failover воздействие +начинается через 3 с, предыдущая реплика восстанавливается перед остановкой +следующей, после действия нагрузка продолжается ещё 8 с. Rolling update +временно увеличивает сервис до трёх реплик, ждёт healthcheck и два периода +Docker DNS, заменяет старые контейнеры по одному и возвращает масштаб к двум. +Минимальный подтверждённый размер набора после прогона — 34 808 сообщений. + +| Фаза | Успешно / ошибок | RPS | Доля ошибок | p50 / p95 / p99, мс | Восстановление | +| --- | ---: | ---: | ---: | ---: | ---: | +| baseline | 4017 / 0 | 319,463 | 0% | 16,799 / 55,515 / 93,803 | — | +| остановка реплики 1 | 4613 / 1 | 301,040 | 0,0217% | 17,145 / 69,558 / 89,752 | 3,464 с | +| остановка реплики 2 | 5815 / 0 | 374,449 | 0% | 14,211 / 49,542 / 81,040 | 3,476 с | +| rolling update | 20 350 / 1 | 376,959 | 0,0049% | 15,725 / 48,724 / 79,337 | непрерывная нагрузка | + +На baseline средняя выборка одного API-контейнера составила 28,453% CPU и +42,413 MiB RAM, максимум — 36,1% и 66,07 MiB. PostgreSQL оказался самым +нагруженным компонентом: в rolling-фазе среднее 98,172% CPU при лимите одного +ядра, максимум RAM 145,2 MiB. Это не capacity-тест и не доказательство +saturation point, но важный ориентир для исследования уровня 5. + +Nginx обновляет Docker DNS раз в 1 с, исключает peer после двух неудач за 2 с, +а connect timeout равен 1 с. При постоянной нагрузке ожидаемое максимальное +обнаружение — около 4 с; измеренные 3,464 и 3,476 с укладываются в требование +10 с. Одна 504 и одна 502 сохранены в отчёте честно: README прямо допускает +краткие ошибки обнаружения. Подтверждённых потерь и дублей нет. Параллельные +32 POST с одним ключом прошли через обе реплики и вернули один Message ID. + +Исходные агрегаты, environment/limits, посекундные таблицы и четыре SVG-графика +лежат в `evidence/grade4/`. Percentile рассчитаны только по успешным ответам; +профиль closed-loop имеет coordinated omission. CPU для API агрегируется как +средняя/максимальная выборка отдельного контейнера, а не сумма двух реплик. ## Ограничения и следующие шаги - Топология однохостовая: Nginx и PostgreSQL остаются single points of failure. - Репликация БД и несколько балансировщиков в требования уровня «3» не входят. + Репликация БД и несколько балансировщиков в требования уровня «4» не входят. - Redis содержит только кэш, поэтому его потеря допустима. Отдельная персистентность Redis для корректности API не требуется. - Истёкшие записи идемпотентности удаляются лениво при повторном обращении к той же области ключа. Индекс по `expires_at` оставлен для будущей периодической очистки без изменения схемы. -- Resource limits, количественный нагрузочный эксперимент и сценарий rolling - update ещё не реализованы; это следующий этап уровня «4». +- Уровень «5» ещё не реализован: сравнение 1/2/4 реплик при фиксированных + ресурсах и поиск saturation point выполняются следующим отдельным этапом. +- Rolling-сценарий заменяет бинарник без изменения схемы. Миграции выполняются + однократно до API; для будущей несовместимой схемы потребуется expand/contract, + поэтому одновременная работа разных schema-version сейчас не заявляется. - Текущий API не содержит endpoint поиска пользователей, аватаров, изменения имени/пароля и агрегатов last message/unread. Поэтому UI принимает UUID участников при создании чата, а профиль честно показывает только доступные поля. Эти функции требуют сначала расширить и зафиксировать API-контракт. -- Трек уровня «5» пока не заявлен: сначала необходимо завершить и измерить - обязательный уровень «4». +- Однохостовая среда WSL2 не доказывает устойчивость к потере Docker-host; + измерен только отказ и замена процесса API. diff --git a/cmd/server/main.go b/cmd/server/main.go index 38cd83a..53d6ab3 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -154,6 +154,8 @@ func main() { var rootHandler http.Handler = mux rootHandler = middleware.Recovery(rootHandler) rootHandler = middleware.Logger(rootHandler) + availabilityGate := middleware.NewAvailabilityGate() + rootHandler = availabilityGate.Middleware(rootHandler) rootHandler = middleware.RequestID(rootHandler) rootHandler = middleware.InstanceID(cfg.InstanceID, rootHandler) @@ -196,6 +198,13 @@ func main() { signalValue, ) + availabilityGate.BeginDrain() + log.Printf( + "[INFO] Readiness disabled; draining new traffic for %s", + cfg.ShutdownDrainDelay, + ) + time.Sleep(cfg.ShutdownDrainDelay) + shutdownContext, shutdownCancel := context.WithTimeout( context.Background(), cfg.ShutdownTimeout, diff --git a/compose.yaml b/compose.yaml index dc1485d..86e5ae2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -21,6 +21,11 @@ services: start_period: 3s stop_grace_period: 10s restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M postgres: image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea @@ -43,6 +48,11 @@ services: retries: 10 start_period: 5s restart: unless-stopped + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M redis: image: redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0 @@ -66,6 +76,11 @@ services: start_period: 3s stop_grace_period: 10s restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M migrate: image: migrate/migrate:v4.20.1@sha256:76cc2074cb6642631f34a898ced71e6aeaa6b1a4d78c4daa743275a22e0c5be7 @@ -84,6 +99,11 @@ services: postgres: condition: service_healthy restart: "no" + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M api: build: @@ -116,6 +136,7 @@ services: DB_QUERY_TIMEOUT: ${DB_QUERY_TIMEOUT:-3s} READINESS_TIMEOUT: ${READINESS_TIMEOUT:-2s} + SHUTDOWN_DRAIN_DELAY: ${SHUTDOWN_DRAIN_DELAY:-2s} SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10s} secrets: - database_url @@ -131,6 +152,10 @@ services: condition: service_completed_successfully deploy: replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "${API_CPU_LIMIT:-0.50}" + memory: ${API_MEMORY_LIMIT:-256M} healthcheck: test: [ @@ -155,6 +180,11 @@ services: redis: condition: service_healthy restart: unless-stopped + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M postgres-restore: image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea @@ -178,6 +208,11 @@ services: retries: 10 start_period: 5s restart: "no" + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M volumes: postgres_data: diff --git a/evidence/grade4/README.md b/evidence/grade4/README.md new file mode 100644 index 0000000..2d9435e --- /dev/null +++ b/evidence/grade4/README.md @@ -0,0 +1,39 @@ +# Lab 3 grade-4 experiment + +Generated: 2026-10-23T12:31:12.749687+00:00. + +## Method + +- concurrency: 8; +- baseline duration: 12 s; +- load: closed-loop POST messages through Nginx; +- failover: both original API containers stopped and restored in turn; +- rolling update: one surge replica, old replicas replaced one by one; +- percentiles cover successful responses; failures are separate. + +## Results + +| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| baseline | 319.463 | 0 (0.000%) | 16.799 | 55.515 | 93.803 | n/a | +| failover-1 | 301.04 | 1 (0.022%) | 17.145 | 69.558 | 89.752 | 3.464 | +| failover-2 | 374.449 | 0 (0.000%) | 14.211 | 49.542 | 81.04 | 3.476 | +| rolling-update | 376.959 | 1 (0.005%) | 15.725 | 48.724 | 79.337 | n/a | + +## Parallel idempotency + +32 requests produced 1 unique message ID across 2 API instances. + +## Graphs + +![Throughput and errors](throughput-errors.svg) + +![Latency](latency.svg) + +![Timeline](timeline.svg) + +![Resources](resources.svg) + +## Scope + +Single-host process failure and rolling replacement only. PostgreSQL and Nginx remain single points of failure. diff --git a/evidence/grade4/latency.svg b/evidence/grade4/latency.svg new file mode 100644 index 0000000..7c20a4e --- /dev/null +++ b/evidence/grade4/latency.svg @@ -0,0 +1,69 @@ + + + +Grade 4: successful latency percentiles + +0 + +19 + +38 + +56 + +75 + +94 + +17 +p50 + +56 +p95 + +94 +p99 +baseline + +17 +p50 + +70 +p95 + +90 +p99 +failover-1 + +14 +p50 + +50 +p95 + +81 +p99 +failover-2 + +16 +p50 + +49 +p95 + +79 +p99 +rolling-update +Successful latency, ms + diff --git a/evidence/grade4/resources.svg b/evidence/grade4/resources.svg new file mode 100644 index 0000000..2edf10b --- /dev/null +++ b/evidence/grade4/resources.svg @@ -0,0 +1,42 @@ + + + +Grade 4: peak container resources +Peak CPU, % + +52.6 +api + +23.4 +load-balancer + +105.2 +postgres + +5.9 +redis +Peak memory, MiB + +66.1 +api + +13.5 +load-balancer + +145.2 +postgres + +12.3 +redis + diff --git a/evidence/grade4/results.json b/evidence/grade4/results.json new file mode 100644 index 0000000..739dc0d --- /dev/null +++ b/evidence/grade4/results.json @@ -0,0 +1,675 @@ +{ + "schema_version": 1, + "environment": { + "utc_started_at": "2026-10-23T12:31:12.749687+00:00", + "git_commit": "34618bd4237bbb8bf6a3a303056fc95d799577f9", + "working_tree_dirty": true, + "logical_cpu_count": 8, + "host_memory_mib": 3813.6, + "docker_engine": "29.7.2", + "docker_compose": "5.5.0", + "container_limits": [ + { + "service": "api", + "nano_cpus": 500000000, + "memory_limit_bytes": 268435456 + }, + { + "service": "api", + "nano_cpus": 500000000, + "memory_limit_bytes": 268435456 + }, + { + "service": "load-balancer", + "nano_cpus": 250000000, + "memory_limit_bytes": 134217728 + }, + { + "service": "postgres", + "nano_cpus": 1000000000, + "memory_limit_bytes": 536870912 + }, + { + "service": "redis", + "nano_cpus": 250000000, + "memory_limit_bytes": 134217728 + } + ] + }, + "methodology": { + "base_url": "http://localhost:8080", + "concurrency": 8, + "baseline_seconds": 12, + "action_after_seconds": 3, + "post_action_seconds": 8, + "warmup_requests": 12, + "workload": "closed-loop POST messages via Nginx; no automatic retries" + }, + "phases": [ + { + "name": "baseline", + "elapsed_seconds": 12.574, + "successful_requests": 4017, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 319.463, + "successful_latency_ms": { + "p50": 16.799, + "p95": 55.515, + "p99": 93.803 + }, + "instances": [ + "0b933a46fccb", + "55aaffcf508a" + ], + "recovery_seconds": null, + "action": null, + "per_second": { + "0": { + "successful": 118, + "failed": 0 + }, + "1": { + "successful": 308, + "failed": 0 + }, + "2": { + "successful": 386, + "failed": 0 + }, + "3": { + "successful": 378, + "failed": 0 + }, + "4": { + "successful": 344, + "failed": 0 + }, + "5": { + "successful": 322, + "failed": 0 + }, + "6": { + "successful": 316, + "failed": 0 + }, + "7": { + "successful": 384, + "failed": 0 + }, + "8": { + "successful": 380, + "failed": 0 + }, + "9": { + "successful": 402, + "failed": 0 + }, + "10": { + "successful": 377, + "failed": 0 + }, + "11": { + "successful": 288, + "failed": 0 + }, + "12": { + "successful": 14, + "failed": 0 + } + }, + "resources": { + "api": { + "cpu_percent_avg": 28.453, + "cpu_percent_max": 36.1, + "memory_mib_avg": 42.413, + "memory_mib_max": 66.07 + }, + "load-balancer": { + "cpu_percent_avg": 17.448, + "cpu_percent_max": 20.8, + "memory_mib_avg": 9.214, + "memory_mib_max": 9.559 + }, + "postgres": { + "cpu_percent_avg": 85.888, + "cpu_percent_max": 105.25, + "memory_mib_avg": 97.975, + "memory_mib_max": 100.8 + }, + "redis": { + "cpu_percent_avg": 2.552, + "cpu_percent_max": 5.79, + "memory_mib_avg": 8.796, + "memory_mib_max": 11.74 + } + } + }, + { + "name": "failover-1", + "elapsed_seconds": 15.324, + "successful_requests": 4613, + "failed_requests": 1, + "error_ratio": 0.000217, + "errors": { + "POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 504": 1 + }, + "successful_rps": 301.04, + "successful_latency_ms": { + "p50": 17.145, + "p95": 69.558, + "p99": 89.752 + }, + "instances": [ + "0b933a46fccb", + "55aaffcf508a" + ], + "recovery_seconds": 3.464, + "action": { + "started_at": 3.006, + "ended_at": 6.463, + "duration_seconds": 3.457, + "exit_code": 0, + "summary": "stopped API instance 55aaffcf508a" + }, + "per_second": { + "0": { + "successful": 404, + "failed": 0 + }, + "1": { + "successful": 426, + "failed": 0 + }, + "2": { + "successful": 358, + "failed": 0 + }, + "3": { + "successful": 399, + "failed": 0 + }, + "4": { + "successful": 392, + "failed": 0 + }, + "5": { + "successful": 322, + "failed": 0 + }, + "6": { + "successful": 299, + "failed": 0 + }, + "7": { + "successful": 329, + "failed": 1 + }, + "8": { + "successful": 313, + "failed": 0 + }, + "9": { + "successful": 229, + "failed": 0 + }, + "10": { + "successful": 155, + "failed": 0 + }, + "11": { + "successful": 187, + "failed": 0 + }, + "12": { + "successful": 299, + "failed": 0 + }, + "13": { + "successful": 317, + "failed": 0 + }, + "14": { + "successful": 184, + "failed": 0 + } + }, + "resources": { + "api": { + "cpu_percent_avg": 27.62, + "cpu_percent_max": 52.59, + "memory_mib_avg": 11.58, + "memory_mib_max": 12.66 + }, + "load-balancer": { + "cpu_percent_avg": 18.536, + "cpu_percent_max": 21.25, + "memory_mib_avg": 9.351, + "memory_mib_max": 9.523 + }, + "postgres": { + "cpu_percent_avg": 81.822, + "cpu_percent_max": 104.67, + "memory_mib_avg": 100.512, + "memory_mib_max": 109.1 + }, + "redis": { + "cpu_percent_avg": 1.486, + "cpu_percent_max": 5.88, + "memory_mib_avg": 8.04, + "memory_mib_max": 8.25 + } + } + }, + { + "name": "failover-2", + "elapsed_seconds": 15.529, + "successful_requests": 5815, + "failed_requests": 0, + "error_ratio": 0.0, + "errors": {}, + "successful_rps": 374.449, + "successful_latency_ms": { + "p50": 14.211, + "p95": 49.542, + "p99": 81.04 + }, + "instances": [ + "0b933a46fccb", + "55aaffcf508a" + ], + "recovery_seconds": 3.476, + "action": { + "started_at": 3.013, + "ended_at": 6.485, + "duration_seconds": 3.472, + "exit_code": 0, + "summary": "stopped API instance 0b933a46fccb" + }, + "per_second": { + "0": { + "successful": 396, + "failed": 0 + }, + "1": { + "successful": 439, + "failed": 0 + }, + "2": { + "successful": 412, + "failed": 0 + }, + "3": { + "successful": 355, + "failed": 0 + }, + "4": { + "successful": 378, + "failed": 0 + }, + "5": { + "successful": 440, + "failed": 0 + }, + "6": { + "successful": 362, + "failed": 0 + }, + "7": { + "successful": 419, + "failed": 0 + }, + "8": { + "successful": 385, + "failed": 0 + }, + "9": { + "successful": 372, + "failed": 0 + }, + "10": { + "successful": 420, + "failed": 0 + }, + "11": { + "successful": 423, + "failed": 0 + }, + "12": { + "successful": 432, + "failed": 0 + }, + "13": { + "successful": 375, + "failed": 0 + }, + "14": { + "successful": 207, + "failed": 0 + } + }, + "resources": { + "api": { + "cpu_percent_avg": 31.411, + "cpu_percent_max": 51.63, + "memory_mib_avg": 10.373, + "memory_mib_max": 11.6 + }, + "load-balancer": { + "cpu_percent_avg": 15.778, + "cpu_percent_max": 21.79, + "memory_mib_avg": 9.62, + "memory_mib_max": 9.781 + }, + "postgres": { + "cpu_percent_avg": 76.87, + "cpu_percent_max": 104.51, + "memory_mib_avg": 106.016, + "memory_mib_max": 120.0 + }, + "redis": { + "cpu_percent_avg": 0.786, + "cpu_percent_max": 2.83, + "memory_mib_avg": 8.844, + "memory_mib_max": 12.28 + } + } + }, + { + "name": "rolling-update", + "elapsed_seconds": 53.985, + "successful_requests": 20350, + "failed_requests": 1, + "error_ratio": 4.9e-05, + "errors": { + "POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 502": 1 + }, + "successful_rps": 376.959, + "successful_latency_ms": { + "p50": 15.725, + "p95": 48.724, + "p99": 79.337 + }, + "instances": [ + "0b933a46fccb", + "3adbcad9e34b", + "55aaffcf508a", + "ceca3f6f1faf", + "f0a3bedda85a" + ], + "recovery_seconds": null, + "action": { + "started_at": 3.016, + "ended_at": 45.961, + "duration_seconds": 42.945, + "exit_code": 0, + "summary": "rolling update completed with 2 healthy API replicas" + }, + "per_second": { + "0": { + "successful": 340, + "failed": 0 + }, + "1": { + "successful": 369, + "failed": 0 + }, + "2": { + "successful": 426, + "failed": 0 + }, + "3": { + "successful": 382, + "failed": 0 + }, + "4": { + "successful": 371, + "failed": 0 + }, + "5": { + "successful": 376, + "failed": 0 + }, + "6": { + "successful": 328, + "failed": 0 + }, + "7": { + "successful": 350, + "failed": 0 + }, + "8": { + "successful": 434, + "failed": 0 + }, + "9": { + "successful": 411, + "failed": 0 + }, + "10": { + "successful": 399, + "failed": 0 + }, + "11": { + "successful": 417, + "failed": 0 + }, + "12": { + "successful": 374, + "failed": 0 + }, + "13": { + "successful": 427, + "failed": 0 + }, + "14": { + "successful": 414, + "failed": 0 + }, + "15": { + "successful": 418, + "failed": 0 + }, + "16": { + "successful": 408, + "failed": 0 + }, + "17": { + "successful": 352, + "failed": 0 + }, + "18": { + "successful": 391, + "failed": 0 + }, + "19": { + "successful": 388, + "failed": 0 + }, + "20": { + "successful": 413, + "failed": 0 + }, + "21": { + "successful": 417, + "failed": 0 + }, + "22": { + "successful": 388, + "failed": 0 + }, + "23": { + "successful": 396, + "failed": 0 + }, + "24": { + "successful": 400, + "failed": 0 + }, + "25": { + "successful": 339, + "failed": 0 + }, + "26": { + "successful": 225, + "failed": 0 + }, + "27": { + "successful": 376, + "failed": 0 + }, + "28": { + "successful": 385, + "failed": 0 + }, + "29": { + "successful": 396, + "failed": 0 + }, + "30": { + "successful": 403, + "failed": 0 + }, + "31": { + "successful": 262, + "failed": 0 + }, + "32": { + "successful": 428, + "failed": 0 + }, + "33": { + "successful": 362, + "failed": 0 + }, + "34": { + "successful": 286, + "failed": 0 + }, + "35": { + "successful": 337, + "failed": 0 + }, + "36": { + "successful": 328, + "failed": 0 + }, + "37": { + "successful": 424, + "failed": 0 + }, + "38": { + "successful": 356, + "failed": 0 + }, + "39": { + "successful": 386, + "failed": 0 + }, + "40": { + "successful": 422, + "failed": 0 + }, + "41": { + "successful": 285, + "failed": 0 + }, + "42": { + "successful": 279, + "failed": 0 + }, + "43": { + "successful": 222, + "failed": 1 + }, + "44": { + "successful": 316, + "failed": 0 + }, + "45": { + "successful": 359, + "failed": 0 + }, + "46": { + "successful": 405, + "failed": 0 + }, + "47": { + "successful": 398, + "failed": 0 + }, + "48": { + "successful": 395, + "failed": 0 + }, + "49": { + "successful": 430, + "failed": 0 + }, + "50": { + "successful": 453, + "failed": 0 + }, + "51": { + "successful": 409, + "failed": 0 + }, + "52": { + "successful": 445, + "failed": 0 + }, + "53": { + "successful": 450, + "failed": 0 + } + }, + "resources": { + "api": { + "cpu_percent_avg": 25.598, + "cpu_percent_max": 37.65, + "memory_mib_avg": 8.956, + "memory_mib_max": 10.97 + }, + "load-balancer": { + "cpu_percent_avg": 19.935, + "cpu_percent_max": 23.36, + "memory_mib_avg": 9.836, + "memory_mib_max": 13.52 + }, + "postgres": { + "cpu_percent_avg": 98.172, + "cpu_percent_max": 105.11, + "memory_mib_avg": 127.975, + "memory_mib_max": 145.2 + }, + "redis": { + "cpu_percent_avg": 0.722, + "cpu_percent_max": 3.94, + "memory_mib_avg": 7.661, + "memory_mib_max": 11.84 + } + } + } + ], + "idempotency": { + "parallel_requests": 32, + "successful_responses": 32, + "failed_responses": 0, + "unique_message_ids": 1, + "observed_instances": [ + "0b933a46fccb", + "55aaffcf508a" + ] + }, + "dataset": { + "warmup_requests": 12, + "minimum_confirmed_messages": 34808 + }, + "limitations": [ + "single Docker host", + "closed-loop coordinated omission", + "successful-response latency percentiles only", + "PostgreSQL and Nginx remain single points of failure" + ] +} diff --git a/evidence/grade4/throughput-errors.svg b/evidence/grade4/throughput-errors.svg new file mode 100644 index 0000000..444752b --- /dev/null +++ b/evidence/grade4/throughput-errors.svg @@ -0,0 +1,45 @@ + + + +Grade 4: throughput and errors + +0 + +75 + +151 + +226 + +302 + +377 + +319.5 RPS +baseline +errors 0.00% + +301.0 RPS +failover-1 +errors 0.02% + +374.4 RPS +failover-2 +errors 0.00% + +377.0 RPS +rolling-update +errors 0.00% +Successful requests per second + diff --git a/evidence/grade4/timeline.svg b/evidence/grade4/timeline.svg new file mode 100644 index 0000000..b0f1dfa --- /dev/null +++ b/evidence/grade4/timeline.svg @@ -0,0 +1,38 @@ + + + +Grade 4: request timeline + +0 + +91 + +181 + +272 + +362 + +453 + + + + +failover-1 action + +failover-2 action + +rolling-update action +One-second buckets; circles mark failed requests + diff --git a/infrastructure/nginx/nginx.conf b/infrastructure/nginx/nginx.conf index bcb4d76..9132e62 100644 --- a/infrastructure/nginx/nginx.conf +++ b/infrastructure/nginx/nginx.conf @@ -17,7 +17,7 @@ http { upstream api_backend { zone api_backend 64k; least_conn; - server api:8080 resolve max_fails=1 fail_timeout=1s; + server api:8080 resolve max_fails=2 fail_timeout=2s; } server { @@ -31,6 +31,42 @@ http { add_header X-Frame-Options "DENY" always; add_header Content-Security-Policy "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always; + # Message creation carries an Idempotency-Key. It is therefore safe to + # retry a POST on another replica when the selected backend is draining. + location ~ ^/api/v1/chats/[^/]+/messages$ { + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_set_header Cookie ""; + proxy_connect_timeout 1s; + proxy_send_timeout 20s; + proxy_read_timeout 20s; + proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; + proxy_next_upstream_tries 3; + } + + location ~ ^/web-api/v1/chats/[^/]+/messages$ { + rewrite ^/web-api/(.*)$ /api/$1 break; + proxy_pass http://api_backend; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header Connection ""; + proxy_set_header Authorization $web_authorization; + proxy_set_header Cookie ""; + proxy_connect_timeout 1s; + proxy_send_timeout 20s; + proxy_read_timeout 20s; + proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent; + proxy_next_upstream_tries 3; + } + location /api/ { proxy_pass http://api_backend; proxy_http_version 1.1; diff --git a/internal/config/config.go b/internal/config/config.go index 6bcbcef..01459b9 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -35,8 +35,9 @@ type Config struct { RedisOperationTimeout time.Duration RedisPoolSize int - ReadinessTimeout time.Duration - ShutdownTimeout time.Duration + ReadinessTimeout time.Duration + ShutdownDrainDelay time.Duration + ShutdownTimeout time.Duration } func Load() (Config, error) { @@ -165,6 +166,14 @@ func Load() (Config, error) { return Config{}, err } + shutdownDrainDelay, err := durationFromEnv( + "SHUTDOWN_DRAIN_DELAY", + 2*time.Second, + ) + if err != nil { + return Config{}, err + } + shutdownTimeout, err := durationFromEnv( "SHUTDOWN_TIMEOUT", 10*time.Second, @@ -185,6 +194,12 @@ func Load() (Config, error) { ) } + if shutdownDrainDelay >= shutdownTimeout { + return Config{}, errors.New( + "SHUTDOWN_DRAIN_DELAY must be less than SHUTDOWN_TIMEOUT", + ) + } + return Config{ HTTPHost: envOrDefault("APP_HOST", "0.0.0.0"), HTTPPort: envOrDefault("APP_PORT", "8080"), @@ -208,8 +223,9 @@ func Load() (Config, error) { RedisOperationTimeout: redisOperationTimeout, RedisPoolSize: int(redisPoolSize), - ReadinessTimeout: readinessTimeout, - ShutdownTimeout: shutdownTimeout, + ReadinessTimeout: readinessTimeout, + ShutdownDrainDelay: shutdownDrainDelay, + ShutdownTimeout: shutdownTimeout, }, nil } diff --git a/internal/transport/http/middleware/availability.go b/internal/transport/http/middleware/availability.go new file mode 100644 index 0000000..a73b283 --- /dev/null +++ b/internal/transport/http/middleware/availability.go @@ -0,0 +1,48 @@ +package middleware + +import ( + "net/http" + "sync/atomic" + + "mtusi2026lab3/internal/transport/http/response" +) + +// AvailabilityGate lets a process reject new work before its HTTP listener is +// closed. Existing requests are still drained by http.Server.Shutdown. +type AvailabilityGate struct { + accepting atomic.Bool +} + +func NewAvailabilityGate() *AvailabilityGate { + gate := &AvailabilityGate{} + gate.accepting.Store(true) + + return gate +} + +func (g *AvailabilityGate) BeginDrain() { + g.accepting.Store(false) +} + +func (g *AvailabilityGate) IsAccepting() bool { + return g.accepting.Load() +} + +func (g *AvailabilityGate) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Liveness deliberately stays available while the process drains. The + // readiness endpoint and all business traffic receive 503 immediately. + if g.IsAccepting() || r.URL.Path == "/health/live" { + next.ServeHTTP(w, r) + return + } + + response.WriteError( + w, + http.StatusServiceUnavailable, + response.CodeUnavailable, + "service is draining", + GetRequestID(r.Context()), + ) + }) +} diff --git a/internal/transport/http/middleware/availability_test.go b/internal/transport/http/middleware/availability_test.go new file mode 100644 index 0000000..18f1097 --- /dev/null +++ b/internal/transport/http/middleware/availability_test.go @@ -0,0 +1,38 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" +) + +func TestAvailabilityGateRejectsNewWorkWhileKeepingLiveness(t *testing.T) { + gate := NewAvailabilityGate() + next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + handler := RequestID(gate.Middleware(next)) + + before := httptest.NewRecorder() + handler.ServeHTTP(before, httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil)) + if before.Code != http.StatusNoContent { + t.Fatalf("before drain status = %d, want %d", before.Code, http.StatusNoContent) + } + + gate.BeginDrain() + + draining := httptest.NewRecorder() + handler.ServeHTTP(draining, httptest.NewRequest(http.MethodGet, "/health/ready", nil)) + if draining.Code != http.StatusServiceUnavailable { + t.Fatalf("ready while draining status = %d, want %d", draining.Code, http.StatusServiceUnavailable) + } + if draining.Header().Get("X-Request-Id") == "" { + t.Fatal("draining response is missing X-Request-Id") + } + + live := httptest.NewRecorder() + handler.ServeHTTP(live, httptest.NewRequest(http.MethodGet, "/health/live", nil)) + if live.Code != http.StatusNoContent { + t.Fatalf("live while draining status = %d, want %d", live.Code, http.StatusNoContent) + } +} diff --git a/scripts/grade4_experiment.py b/scripts/grade4_experiment.py new file mode 100755 index 0000000..4d93847 --- /dev/null +++ b/scripts/grade4_experiment.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 +"""Run the repeatable, secret-free resilience experiment required for grade 4.""" +import argparse +from collections import Counter, defaultdict +import datetime as dt +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import uuid + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client, Failure # noqa: E402 + + +def command(args, env=None, timeout=240, check=True): + result = subprocess.run( + [str(value) for value in args], + cwd=ROOT, + env=env, + text=True, + capture_output=True, + timeout=timeout, + check=False, + ) + if check and result.returncode: + raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}") + return result + + +def compose(env_file, *args): + return command(["docker", "compose", "--env-file", env_file, *args]) + + +def api_ids(env_file): + return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line] + + +def inspect(container_id): + data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout + return json.loads(data) + + +def wait_api(env_file, expected, timeout=90): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + healthy = [] + for container_id in api_ids(env_file): + state = inspect(container_id)["State"] + if state.get("Health", {}).get("Status") == "healthy": + healthy.append(container_id) + if len(healthy) >= expected: + return healthy + time.sleep(0.5) + raise RuntimeError(f"fewer than {expected} API replicas became healthy") + + +def percentile(values, ratio): + if not values: + return None + values = sorted(values) + return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3) + + +def memory_mib(raw): + value = raw.split("/", 1)[0].strip() + units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3} + for suffix in ("GiB", "MiB", "KiB", "B"): + if value.endswith(suffix): + return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3) + return 0.0 + + +def service(name): + for candidate in ("load-balancer", "postgres", "redis", "api"): + if f"-{candidate}-" in name: + return candidate + return "other" + + +def sample_stats(): + result = command( + ["docker", "stats", "--no-stream", "--format", "{{json .}}"], + timeout=10, + check=False, + ) + samples = [] + if result.returncode: + return samples + for line in result.stdout.splitlines(): + try: + item = json.loads(line) + samples.append({ + "service": service(item.get("Name", "")), + "cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")), + "memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")), + }) + except (ValueError, json.JSONDecodeError): + continue + return samples + + +def aggregate_resources(samples): + grouped = defaultdict(lambda: {"cpu": [], "memory": []}) + for item in samples: + if item["service"] != "other": + grouped[item["service"]]["cpu"].append(item["cpu_percent"]) + grouped[item["service"]]["memory"].append(item["memory_mib"]) + return { + name: { + "cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3), + "cpu_percent_max": round(max(values["cpu"]), 3), + "memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3), + "memory_mib_max": round(max(values["memory"]), 3), + } + for name, values in sorted(grouped.items()) + } + + +def load_phase(name, actor, chat, args, action=None, target=None): + records, resources = [], [] + lock = threading.Lock() + stop = threading.Event() + barrier = threading.Barrier(args.concurrency) + started = time.monotonic() + + def worker(worker_id): + client = Client(args.base_url, timeout=4) + index = 0 + barrier.wait() + while not stop.is_set(): + request_started = time.monotonic() + try: + response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}") + row = { + "t": round(time.monotonic() - started, 3), + "ok": True, + "latency_ms": round((time.monotonic() - request_started) * 1000, 3), + "instance": response["headers"].get("X-Instance-Id", ""), + } + except Failure as exc: + row = { + "t": round(time.monotonic() - started, 3), + "ok": False, + "latency_ms": round((time.monotonic() - request_started) * 1000, 3), + "error": str(exc), + } + with lock: + records.append(row) + index += 1 + + def sampler(): + while not stop.is_set(): + resources.extend(sample_stats()) + stop.wait(1) + + workers = [threading.Thread(target=worker, args=(index,), daemon=True) + for index in range(args.concurrency)] + stats_thread = threading.Thread(target=sampler, daemon=True) + stats_thread.start() + for thread in workers: + thread.start() + + action_data = None + action_error = None + try: + if action is None: + time.sleep(args.baseline_seconds) + else: + time.sleep(args.action_after_seconds) + action_started = time.monotonic() + try: + summary = action() + exit_code = 0 + except Exception as exc: + summary, exit_code, action_error = type(exc).__name__, 1, exc + action_ended = time.monotonic() + action_data = { + "started_at": round(action_started - started, 3), + "ended_at": round(action_ended - started, 3), + "duration_seconds": round(action_ended - action_started, 3), + "exit_code": exit_code, + "summary": summary, + } + time.sleep(args.post_action_seconds) + finally: + stop.set() + for thread in workers: + thread.join(10) + stats_thread.join(5) + if action_error: + raise action_error + + elapsed = time.monotonic() - started + successes = [row for row in records if row["ok"]] + failures = [row for row in records if not row["ok"]] + latencies = [row["latency_ms"] for row in successes] + buckets = defaultdict(lambda: {"successful": 0, "failed": 0}) + for row in records: + buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1 + recovery = None + if action_data and target: + recovery_row = next(( + row for row in successes + if row["t"] >= action_data["ended_at"] and row.get("instance") != target + ), None) + if recovery_row: + recovery = round(recovery_row["t"] - action_data["started_at"], 3) + total = len(records) + return { + "name": name, + "elapsed_seconds": round(elapsed, 3), + "successful_requests": len(successes), + "failed_requests": len(failures), + "error_ratio": round(len(failures) / total, 6) if total else None, + "errors": dict(sorted(Counter(row["error"] for row in failures).items())), + "successful_rps": round(len(successes) / elapsed, 3), + "successful_latency_ms": { + "p50": percentile(latencies, 0.50), + "p95": percentile(latencies, 0.95), + "p99": percentile(latencies, 0.99), + }, + "instances": sorted({row.get("instance") for row in successes if row.get("instance")}), + "recovery_seconds": recovery, + "action": action_data, + "per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))), + "resources": aggregate_resources(resources), + } + + +def idempotency(actor, chat, base_url, count=32): + key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex + barrier, lock = threading.Barrier(count), threading.Lock() + rows, errors = [], [] + + def request(): + client = Client(base_url, timeout=8) + barrier.wait() + try: + response = client.message(actor, chat, body, key=key) + row = (response["json"]["id"], response["headers"].get("X-Instance-Id", "")) + with lock: + rows.append(row) + except Failure as exc: + with lock: + errors.append(str(exc)) + + threads = [threading.Thread(target=request) for _ in range(count)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(30) + message_ids = {row[0] for row in rows} + if errors or len(rows) != count or len(message_ids) != 1: + raise RuntimeError("parallel idempotency check failed") + return { + "parallel_requests": count, + "successful_responses": len(rows), + "failed_responses": len(errors), + "unique_message_ids": len(message_ids), + "observed_instances": sorted({row[1] for row in rows}), + } + + +def environment(env_file): + mem_total_kib = 0 + with open("/proc/meminfo", encoding="utf-8") as meminfo: + for line in meminfo: + if line.startswith("MemTotal:"): + mem_total_kib = int(line.split()[1]) + break + limits = [] + for container_id in compose(env_file, "ps", "-q").stdout.splitlines(): + data = inspect(container_id) + limits.append({ + "service": service(data["Name"].lstrip("/")), + "nano_cpus": data["HostConfig"].get("NanoCpus", 0), + "memory_limit_bytes": data["HostConfig"].get("Memory", 0), + }) + return { + "utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(), + "git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(), + "working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()), + "logical_cpu_count": os.cpu_count(), + "host_memory_mib": round(mem_total_kib / 1024, 1), + "docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(), + "docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(), + "container_limits": limits, + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://localhost:8080") + parser.add_argument("--env-file", default=".env") + parser.add_argument("--concurrency", type=int, default=8) + parser.add_argument("--baseline-seconds", type=float, default=12) + parser.add_argument("--action-after-seconds", type=float, default=3) + parser.add_argument("--post-action-seconds", type=float, default=8) + parser.add_argument("--output", default="evidence/grade4/results.json") + args = parser.parse_args() + if not 1 <= args.concurrency <= 64: + raise SystemExit("concurrency must be 1..64") + + original = wait_api(args.env_file, 2) + if len(original) != 2: + raise RuntimeError("experiment requires exactly two initial API replicas") + client = Client(args.base_url, timeout=5) + alice, _, _, chat = client.bootstrap() + for index in range(12): + client.message(alice, chat, f"grade4-warmup-{index}") + + phases = [load_phase("baseline", alice, chat, args)] + targets = [(container_id, inspect(container_id)["Config"]["Hostname"]) + for container_id in original] + for ordinal, (container_id, target) in enumerate(targets, start=1): + def stop_target(instance=target): + hook_env = dict(os.environ) + hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance}) + result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30) + return result.stdout.strip().splitlines()[-1] + + phases.append(load_phase( + f"failover-{ordinal}", alice, chat, args, + action=stop_target, target=target, + )) + command(["docker", "start", container_id]) + wait_api(args.env_file, 2) + + idem = idempotency(alice, chat, args.base_url) + + def rolling(): + hook_env = dict(os.environ) + hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"}) + result = command( + [ROOT / "scripts" / "rolling-update.sh"], + env=hook_env, + timeout=240, + ) + return result.stdout.strip().splitlines()[-1] + + phases.append(load_phase("rolling-update", alice, chat, args, action=rolling)) + wait_api(args.env_file, 2) + + report = { + "schema_version": 1, + "environment": environment(args.env_file), + "methodology": { + "base_url": args.base_url, + "concurrency": args.concurrency, + "baseline_seconds": args.baseline_seconds, + "action_after_seconds": args.action_after_seconds, + "post_action_seconds": args.post_action_seconds, + "warmup_requests": 12, + "workload": "closed-loop POST messages via Nginx; no automatic retries", + }, + "phases": phases, + "idempotency": idem, + "dataset": { + "warmup_requests": 12, + "minimum_confirmed_messages": 13 + sum( + phase["successful_requests"] for phase in phases + ), + }, + "limitations": [ + "single Docker host", + "closed-loop coordinated omission", + "successful-response latency percentiles only", + "PostgreSQL and Nginx remain single points of failure", + ], + } + destination = ROOT / args.output + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n") + try: + displayed_output = destination.relative_to(ROOT) + except ValueError: + displayed_output = destination + print(json.dumps({ + "output": str(displayed_output), + "phases": [{ + "name": phase["name"], + "successful_rps": phase["successful_rps"], + "failed_requests": phase["failed_requests"], + "recovery_seconds": phase["recovery_seconds"], + } for phase in phases], + "idempotency": idem, + }, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_grade4.py b/scripts/render_grade4.py new file mode 100755 index 0000000..9c62886 --- /dev/null +++ b/scripts/render_grade4.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""Render portable SVG graphs and a Markdown summary from grade-4 results.""" +import argparse +from collections import defaultdict +import html +import json +from pathlib import Path + + +STYLE = """ +text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; } +.title { font-size: 26px; font-weight: 650; } +.label { font-size: 14px; fill: #c8cbd0; } +.small { font-size: 12px; fill: #a2a5aa; } +.grid { stroke: #34373d; stroke-width: 1; } +.bar { fill: #cfd2d6; } +.bar2 { fill: #73777f; } +.line { fill: none; stroke: #f4f5f6; stroke-width: 3; } +.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; } +.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; } +""" + + +def svg(title, body, width=1100, height=560): + return f""" + + +{html.escape(title)} +{body} + +""" + + +def grid(maximum, left, top, width, height): + parts = [] + for step in range(6): + value = maximum * step / 5 + y = top + height - height * step / 5 + parts.append(f'') + parts.append(f'{value:.0f}') + return parts + + +def throughput(phases, output): + left, top, width, height = 80, 90, 960, 360 + maximum = max((phase["successful_rps"] for phase in phases), default=1) or 1 + parts = grid(maximum, left, top, width, height) + slot = width / max(1, len(phases)) + for index, phase in enumerate(phases): + bar_width = min(120, slot * 0.5) + x = left + slot * index + (slot - bar_width) / 2 + bar_height = height * phase["successful_rps"] / maximum + y = top + height - bar_height + parts.append(f'') + parts.append(f'{phase["successful_rps"]:.1f} RPS') + parts.append(f'{html.escape(phase["name"])}') + parts.append(f'errors {phase["error_ratio"]*100:.2f}%') + parts.append(f'Successful requests per second') + output.write_text(svg("Grade 4: throughput and errors", "\n".join(parts))) + + +def latency(phases, output): + left, top, width, height = 80, 90, 960, 360 + values = [value for phase in phases + for value in phase["successful_latency_ms"].values() + if value is not None] + maximum = max(values, default=1) or 1 + parts = grid(maximum, left, top, width, height) + slot = width / max(1, len(phases)) + for phase_index, phase in enumerate(phases): + group_x = left + slot * phase_index + bar_width = min(42, slot / 5) + for metric_index, metric in enumerate(("p50", "p95", "p99")): + value = phase["successful_latency_ms"][metric] or 0 + bar_height = height * value / maximum + x = group_x + slot * 0.2 + metric_index * (bar_width + 8) + y = top + height - bar_height + css = "bar" if metric_index == 0 else "bar2" + parts.append(f'') + parts.append(f'{value:.0f}') + parts.append(f'{metric}') + parts.append(f'{html.escape(phase["name"])}') + parts.append(f'Successful latency, ms') + output.write_text(svg("Grade 4: successful latency percentiles", "\n".join(parts))) + + +def timeline(phases, output): + left, top, width, height = 70, 90, 980, 390 + rows, events, offset = [], [], 0 + for phase in phases: + buckets = phase["per_second"] + for second, counts in sorted(buckets.items(), key=lambda item: int(item[0])): + rows.append((offset + int(second), counts)) + if phase.get("action"): + events.append((offset + phase["action"]["started_at"], phase["name"])) + offset += max([int(key) for key in buckets] or [0]) + 2 + maximum = max((counts["successful"] for _, counts in rows), default=1) or 1 + max_x = max((second for second, _ in rows), default=1) or 1 + parts, points = grid(maximum, left, top, width, height), [] + for second, counts in rows: + x = left + width * second / max_x + y = top + height - height * counts["successful"] / maximum + points.append(f"{x:.1f},{y:.1f}") + if counts["failed"]: + radius = min(12, 3 + counts["failed"]) + parts.append(f'') + parts.append(f'') + for second, label in events: + x = left + width * second / max_x + parts.append(f'') + parts.append(f'{html.escape(label)} action') + parts.append(f'One-second buckets; circles mark failed requests') + output.write_text(svg("Grade 4: request timeline", "\n".join(parts), height=600)) + + +def resources(phases, output): + combined = defaultdict(lambda: {"cpu": [], "memory": []}) + for phase in phases: + for name, values in phase["resources"].items(): + combined[name]["cpu"].append(values["cpu_percent_max"]) + combined[name]["memory"].append(values["memory_mib_max"]) + names, parts = sorted(combined), [] + panels = [ + ("Peak CPU, %", "cpu", 80, 90, 420, 360), + ("Peak memory, MiB", "memory", 600, 90, 420, 360), + ] + for title, metric, left, top, width, height in panels: + maximum = max((max(combined[name][metric]) for name in names), default=1) or 1 + parts.append(f'{title}') + slot = width / max(1, len(names)) + for index, name in enumerate(names): + value = max(combined[name][metric]) + bar_width = min(70, slot * 0.55) + x = left + slot * index + (slot - bar_width) / 2 + bar_height = height * value / maximum + y = top + height - bar_height + parts.append(f'') + parts.append(f'{value:.1f}') + parts.append(f'{html.escape(name)}') + output.write_text(svg("Grade 4: peak container resources", "\n".join(parts), height=600)) + + +def summary(report, output): + method = report["methodology"] + lines = [ + "# Lab 3 grade-4 experiment", "", + f"Generated: {report['environment']['utc_started_at']}.", "", + "## Method", "", + f"- concurrency: {method['concurrency']};", + f"- baseline duration: {method['baseline_seconds']} s;", + "- load: closed-loop POST messages through Nginx;", + "- failover: both original API containers stopped and restored in turn;", + "- rolling update: one surge replica, old replicas replaced one by one;", + "- percentiles cover successful responses; failures are separate.", "", + "## Results", "", + "| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + for phase in report["phases"]: + lat = phase["successful_latency_ms"] + recovery = phase["recovery_seconds"] + lines.append( + f"| {phase['name']} | {phase['successful_rps']} | " + f"{phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | " + f"{lat['p50']} | {lat['p95']} | {lat['p99']} | " + f"{recovery if recovery is not None else 'n/a'} |" + ) + idem = report["idempotency"] + lines += [ + "", "## Parallel idempotency", "", + f"{idem['parallel_requests']} requests produced " + f"{idem['unique_message_ids']} unique message ID across " + f"{len(idem['observed_instances'])} API instances.", "", + "## Graphs", "", + "![Throughput and errors](throughput-errors.svg)", "", + "![Latency](latency.svg)", "", + "![Timeline](timeline.svg)", "", + "![Resources](resources.svg)", "", + "## Scope", "", + "Single-host process failure and rolling replacement only. PostgreSQL " + "and Nginx remain single points of failure.", "", + ] + output.write_text("\n".join(lines)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("results", nargs="?", default="evidence/grade4/results.json") + args = parser.parse_args() + source = Path(args.results) + report = json.loads(source.read_text()) + destination = source.parent + phases = report["phases"] + throughput(phases, destination / "throughput-errors.svg") + latency(phases, destination / "latency.svg") + timeline(phases, destination / "timeline.svg") + resources(phases, destination / "resources.svg") + summary(report, destination / "README.md") + print(f"rendered grade-4 evidence in {destination}") + + +if __name__ == "__main__": + main() diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 0000000..6769fc1 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,89 @@ +#!/bin/sh +set -eu + +project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$project_root" + +env_file=${ENV_FILE:-.env} +desired_replicas=${DESIRED_REPLICAS:-2} +wait_timeout=${ROLLING_WAIT_TIMEOUT:-90} +stop_timeout=${ROLLING_STOP_TIMEOUT:-15} +dns_settle_delay=${ROLLING_DNS_SETTLE_DELAY:-2} + +case "$desired_replicas" in + ''|*[!0-9]*) + printf '%s\n' "DESIRED_REPLICAS must be an integer" >&2 + exit 2 + ;; +esac +case "$dns_settle_delay" in + ''|*[!0-9]*) + printf '%s\n' "ROLLING_DNS_SETTLE_DELAY must be a non-negative integer" >&2 + exit 2 + ;; +esac +if [ "$desired_replicas" -lt 2 ]; then + printf '%s\n' "rolling update requires at least two desired replicas" >&2 + exit 2 +fi + +compose() { + docker compose --env-file "$env_file" "$@" +} + +healthy_api_count() { + healthy=0 + for container_id in $(compose ps -q api); do + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id") + if [ "$status" = "healthy" ] || [ "$status" = "running" ]; then + healthy=$((healthy + 1)) + fi + done + printf '%s\n' "$healthy" +} + +wait_for_healthy() { + expected=$1 + deadline=$(( $(date +%s) + wait_timeout )) + while [ "$(healthy_api_count)" -lt "$expected" ]; do + if [ "$(date +%s)" -ge "$deadline" ]; then + printf '%s\n' "timed out waiting for $expected healthy API replicas" >&2 + return 1 + fi + sleep 1 + done +} + +old_containers=$(compose ps -q api) +old_count=$(printf '%s\n' "$old_containers" | awk 'NF { count++ } END { print count+0 }') +if [ "$old_count" -ne "$desired_replicas" ]; then + printf '%s\n' "expected $desired_replicas running API containers, found $old_count" >&2 + exit 1 +fi + +surge_replicas=$((desired_replicas + 1)) + +printf '%s\n' "building replacement API image" +compose build api + +printf '%s\n' "starting one surge replica" +compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api +wait_for_healthy "$surge_replicas" +sleep "$dns_settle_delay" + +for old_container in $old_containers; do + old_instance=$(docker inspect --format '{{.Config.Hostname}}' "$old_container") + printf '%s\n' "draining and replacing API instance $old_instance" + docker stop --time "$stop_timeout" "$old_container" >/dev/null + docker rm "$old_container" >/dev/null + compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api + wait_for_healthy "$surge_replicas" + sleep "$dns_settle_delay" +done + +printf '%s\n' "removing surge capacity" +compose up -d --no-deps --scale api="$desired_replicas" --no-recreate api +wait_for_healthy "$desired_replicas" +sleep "$dns_settle_delay" + +printf '%s\n' "rolling update completed with $desired_replicas healthy API replicas"