оценка 4
This commit is contained in:
@@ -17,6 +17,8 @@ SESSION_COOKIE_SECURE=false
|
||||
CURSOR_SIGNING_KEY_FILE=/run/secrets/cursor_signing_key
|
||||
|
||||
API_REPLICAS=2
|
||||
API_CPU_LIMIT=0.50
|
||||
API_MEMORY_LIMIT=256M
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
# File paths contain no secret values. Create the referenced local secret files
|
||||
@@ -32,6 +34,7 @@ DB_CONNECT_TIMEOUT=5s
|
||||
DB_QUERY_TIMEOUT=3s
|
||||
|
||||
READINESS_TIMEOUT=2s
|
||||
SHUTDOWN_DRAIN_DELAY=2s
|
||||
SHUTDOWN_TIMEOUT=10s
|
||||
|
||||
# Redis stores a disposable shared chat cache; PostgreSQL remains authoritative.
|
||||
|
||||
@@ -6,6 +6,7 @@ ENV_FILE ?= .env
|
||||
BASE_URL ?= http://localhost:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
GRADE4_OUTPUT ?= evidence/grade4/results.json
|
||||
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
|
||||
@@ -26,6 +27,8 @@ COMPOSE_CMD = $(COMPOSE) --env-file "$(ENV_FILE)"
|
||||
load \
|
||||
test-persistence \
|
||||
test-failover \
|
||||
rolling-update \
|
||||
grade4-experiment \
|
||||
compose-config \
|
||||
compose-build \
|
||||
compose-up \
|
||||
@@ -48,6 +51,8 @@ help:
|
||||
@echo "load: run synthetic writes against the LB entrypoint"
|
||||
@echo "test-persistence: run persistence test with ACTION_SCRIPT"
|
||||
@echo "test-failover: run replica failover test with ACTION_SCRIPT"
|
||||
@echo "rolling-update: replace API replicas one by one with surge capacity"
|
||||
@echo "grade4-experiment: run failovers, idempotency and rolling update with evidence"
|
||||
@echo "compose-config: validate resolved Compose configuration"
|
||||
@echo "compose-build: build application images"
|
||||
@echo "compose-up: start the stack"
|
||||
@@ -109,6 +114,13 @@ test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
rolling-update:
|
||||
ENV_FILE="$(ENV_FILE)" ./scripts/rolling-update.sh
|
||||
|
||||
grade4-experiment:
|
||||
$(PYTHON) scripts/grade4_experiment.py --base-url "$(BASE_URL)" --env-file "$(ENV_FILE)" --output "$(GRADE4_OUTPUT)"
|
||||
$(PYTHON) scripts/render_grade4.py "$(GRADE4_OUTPUT)"
|
||||
|
||||
compose-config:
|
||||
@test -f "$(ENV_FILE)" || \
|
||||
(echo "$(ENV_FILE) is missing; run: make init-env"; exit 2)
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: Чурилин, БВТ2301
|
||||
- Текущий завершённый уровень: оценка «3»
|
||||
- Итоговая цель: оценка «5»; трек будет выбран после реализации уровня «4»
|
||||
- Текущий завершённый уровень: оценка «4»
|
||||
- Итоговая цель: оценка «5»; исследовательский трек выполняется следующим этапом
|
||||
- Commit/PR: заполняется автором после локального коммита и публикации
|
||||
- Использованные источники и библиотеки: преподавательские README/OpenAPI/tests,
|
||||
документация Go, pgx, PostgreSQL, Docker Compose, Nginx, React, TypeScript,
|
||||
@@ -38,6 +38,7 @@ make test
|
||||
make test-failover ACTION_SCRIPT=./scripts/stop-one.sh
|
||||
docker compose --env-file .env up -d --wait api
|
||||
make test-persistence ACTION_SCRIPT=./scripts/restart.sh
|
||||
make grade4-experiment
|
||||
```
|
||||
|
||||
По этому же адресу Nginx отдаёт production-сборку React. Переход на
|
||||
@@ -86,10 +87,12 @@ Compose запускает один сервис `api` с `API_REPLICAS=2`. Ре
|
||||
ports и разделяют PostgreSQL, включая таблицу сессий. Nginx — единственная
|
||||
публичная точка входа; он использует `least_conn` и Docker DNS resolver с
|
||||
повторным разрешением имени `api` раз в секунду. Sticky sessions не применяются.
|
||||
Безопасные методы могут быть перенаправлены на другую реплику при сетевой ошибке.
|
||||
Произвольные POST балансировщик автоматически не повторяет: после
|
||||
неопределённого результата повтор сообщения выполняет клиент с тем же
|
||||
`Idempotency-Key`, а создание пользователя/чата контрактом не идемпотентно.
|
||||
Произвольные POST балансировщик автоматически не повторяет. Исключение сделано
|
||||
только для маршрута отправки сообщения: обязательный `Idempotency-Key` позволяет
|
||||
Nginx безопасно повторить запрос на другой реплике при 502/503/504 во время
|
||||
drain. Клиент также сохраняет ключ для повтора после неопределённого результата.
|
||||
Создание пользователя и чата контрактом не идемпотентно, поэтому автоматический
|
||||
retry для этих маршрутов запрещён.
|
||||
|
||||
Миграции вынесены в одноразовый сервис `migrate`, завершение которого является
|
||||
предусловием запуска API. Поэтому две реплики не конкурируют за изменение схемы.
|
||||
@@ -104,7 +107,8 @@ secret `cursor_signing_key`. Репозитории получают тольк
|
||||
|
||||
`.env.example` перечисляет реально читаемые приложением параметры PostgreSQL,
|
||||
пула, readiness/shutdown, Redis-кэша, числа реплик, TTL идемпотентности и ключа
|
||||
курсора. `API_REPLICAS` используется Compose, а `IDEMPOTENCY_TTL_SECONDS` читает
|
||||
курсора. Там же задаются CPU/RAM API и задержка drain перед shutdown.
|
||||
`API_REPLICAS` используется Compose, а `IDEMPOTENCY_TTL_SECONDS` читает
|
||||
Go-приложение и не допускает значение меньше 86400 секунд. Секретных значений в
|
||||
шаблоне нет: там указаны только пути внутри контейнера.
|
||||
|
||||
@@ -177,37 +181,71 @@ cookie.
|
||||
| Persistence после рестартов | `make test-persistence ACTION_SCRIPT=./scripts/restart.sh` | PASS; прежняя сессия и данные доступны после рестарта PostgreSQL, Redis, API и LB | `scripts/restart.sh`, `evidence/lab3-summary.md` |
|
||||
| Миграции перед API | `docker compose --env-file .env ps -a` и логи `migrate` | миграции 1 и 2 применены; one-shot контейнер завершился с кодом 0; API healthy | `migrations/000002_message_idempotency.up.sql`, `evidence/lab3-summary.md` |
|
||||
| React frontend и HttpOnly BFF | `docker compose build load-balancer`; browser QA; cookie-only curl-сценарий | Vitest/TypeScript/Vite build PASS; register/login 201; прямой `/api` 401; `/web-api` 200; logout 204, после logout 401; browser console без warning/error | `frontend/`, `infrastructure/nginx/Dockerfile`, `infrastructure/nginx/nginx.conf` |
|
||||
| Согласованные health и shutdown | SIGTERM каждой реплики под нагрузкой | API сразу закрывает availability gate, 2 с дренирует запросы и укладывается в 10 с shutdown; `/health/live` остаётся доступен | `internal/transport/http/middleware/availability.go`, `evidence/grade4/` |
|
||||
| CPU/RAM и изоляция портов | `docker inspect`; `docker compose ps` | API: 0,5 CPU/256 MiB на реплику; наружу опубликован только Nginx `127.0.0.1:8080`; `container_name` отсутствует | `compose.yaml`, `evidence/grade4/results.json` |
|
||||
| Последовательный failover уровня 4 | `make grade4-experiment` | обе исходные реплики остановлены по очереди и восстановлены; recovery 3,464/3,476 с, одна краткая ошибка из 4614 запросов в первом прогоне и 0 из 5815 во втором | `evidence/grade4/README.md`, `throughput-errors.svg`, `timeline.svg` |
|
||||
| Rolling update под нагрузкой | surge до 3 реплик, замена двух старых по одной, возврат к 2 | 20 350 успешных запросов, 1 краткий 502 (0,005%), все новые реплики проходят healthcheck и DNS-settle | `scripts/rolling-update.sh`, `evidence/grade4/` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
Проведены функциональные эксперименты устойчивости, а не capacity benchmark.
|
||||
Окружение: Windows + WSL2, Docker Engine 29.7.2, Docker Compose 5.5.0, две
|
||||
API-реплики; лимиты CPU/RAM пока не задавались, максимальный пул БД —
|
||||
10 соединений на реплику. Использовались синтетические данные; отдельного прогрева
|
||||
и серии повторов не было. Поэтому результаты подтверждают корректность failover,
|
||||
идемпотентности и сохранности данных, но не предельный RPS или p50/p95/p99.
|
||||
Нагрузочный профиль с лимитами и повторами относится к следующему уровню.
|
||||
Итоговый эксперимент уровня 4 выполнен 23.10.2026 на Windows + WSL2:
|
||||
Docker Engine 29.7.2, Compose 5.5.0, 8 логических CPU и 3813,6 MiB RAM,
|
||||
commit-base `34618bd4237bbb8bf6a3a303056fc95d799577f9` с незакоммиченными
|
||||
изменениями текущего уровня. В Compose каждой API-реплике выделено 0,5 CPU и
|
||||
256 MiB; Nginx — 0,25 CPU/128 MiB; PostgreSQL — 1 CPU/512 MiB; Redis —
|
||||
0,25 CPU/128 MiB.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| Последовательная остановка `api-2` и `api-1` с восстановлением между тестами | LB обслуживает через оставшуюся реплику не позднее 10 с | 142 и 143 успешные записи; в обоих запусках 0 ошибок | первое восстановление 0.963 и 0.750 с; подтверждённой потери или дублей нет |
|
||||
| Рестарт PostgreSQL, Redis, API и LB без удаления volumes | прежний token и данные сохраняются | `test-persistence` завершился PASS | уложилось в тайм-аут 90 с; потери данных нет |
|
||||
| 32 одновременных POST с одним `Idempotency-Key` | один бизнес-эффект | 32 ответа 201 с одним Message ID; в БД одно сообщение | дубли отсутствуют |
|
||||
Команда `make grade4-experiment` создаёт синтетических пользователей и чат,
|
||||
делает 12 warm-up записей, затем запускает 8 closed-loop клиентов без повторов
|
||||
на стороне клиента. Baseline длится 12 с. В каждом failover воздействие
|
||||
начинается через 3 с, предыдущая реплика восстанавливается перед остановкой
|
||||
следующей, после действия нагрузка продолжается ещё 8 с. Rolling update
|
||||
временно увеличивает сервис до трёх реплик, ждёт healthcheck и два периода
|
||||
Docker DNS, заменяет старые контейнеры по одному и возвращает масштаб к двум.
|
||||
Минимальный подтверждённый размер набора после прогона — 34 808 сообщений.
|
||||
|
||||
| Фаза | Успешно / ошибок | RPS | Доля ошибок | p50 / p95 / p99, мс | Восстановление |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: |
|
||||
| baseline | 4017 / 0 | 319,463 | 0% | 16,799 / 55,515 / 93,803 | — |
|
||||
| остановка реплики 1 | 4613 / 1 | 301,040 | 0,0217% | 17,145 / 69,558 / 89,752 | 3,464 с |
|
||||
| остановка реплики 2 | 5815 / 0 | 374,449 | 0% | 14,211 / 49,542 / 81,040 | 3,476 с |
|
||||
| rolling update | 20 350 / 1 | 376,959 | 0,0049% | 15,725 / 48,724 / 79,337 | непрерывная нагрузка |
|
||||
|
||||
На baseline средняя выборка одного API-контейнера составила 28,453% CPU и
|
||||
42,413 MiB RAM, максимум — 36,1% и 66,07 MiB. PostgreSQL оказался самым
|
||||
нагруженным компонентом: в rolling-фазе среднее 98,172% CPU при лимите одного
|
||||
ядра, максимум RAM 145,2 MiB. Это не capacity-тест и не доказательство
|
||||
saturation point, но важный ориентир для исследования уровня 5.
|
||||
|
||||
Nginx обновляет Docker DNS раз в 1 с, исключает peer после двух неудач за 2 с,
|
||||
а connect timeout равен 1 с. При постоянной нагрузке ожидаемое максимальное
|
||||
обнаружение — около 4 с; измеренные 3,464 и 3,476 с укладываются в требование
|
||||
10 с. Одна 504 и одна 502 сохранены в отчёте честно: README прямо допускает
|
||||
краткие ошибки обнаружения. Подтверждённых потерь и дублей нет. Параллельные
|
||||
32 POST с одним ключом прошли через обе реплики и вернули один Message ID.
|
||||
|
||||
Исходные агрегаты, environment/limits, посекундные таблицы и четыре SVG-графика
|
||||
лежат в `evidence/grade4/`. Percentile рассчитаны только по успешным ответам;
|
||||
профиль closed-loop имеет coordinated omission. CPU для API агрегируется как
|
||||
средняя/максимальная выборка отдельного контейнера, а не сумма двух реплик.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
- Топология однохостовая: Nginx и PostgreSQL остаются single points of failure.
|
||||
Репликация БД и несколько балансировщиков в требования уровня «3» не входят.
|
||||
Репликация БД и несколько балансировщиков в требования уровня «4» не входят.
|
||||
- Redis содержит только кэш, поэтому его потеря допустима. Отдельная
|
||||
персистентность Redis для корректности API не требуется.
|
||||
- Истёкшие записи идемпотентности удаляются лениво при повторном обращении к той
|
||||
же области ключа. Индекс по `expires_at` оставлен для будущей периодической
|
||||
очистки без изменения схемы.
|
||||
- Resource limits, количественный нагрузочный эксперимент и сценарий rolling
|
||||
update ещё не реализованы; это следующий этап уровня «4».
|
||||
- Уровень «5» ещё не реализован: сравнение 1/2/4 реплик при фиксированных
|
||||
ресурсах и поиск saturation point выполняются следующим отдельным этапом.
|
||||
- Rolling-сценарий заменяет бинарник без изменения схемы. Миграции выполняются
|
||||
однократно до API; для будущей несовместимой схемы потребуется expand/contract,
|
||||
поэтому одновременная работа разных schema-version сейчас не заявляется.
|
||||
- Текущий API не содержит endpoint поиска пользователей, аватаров, изменения
|
||||
имени/пароля и агрегатов last message/unread. Поэтому UI принимает UUID
|
||||
участников при создании чата, а профиль честно показывает только доступные
|
||||
поля. Эти функции требуют сначала расширить и зафиксировать API-контракт.
|
||||
- Трек уровня «5» пока не заявлен: сначала необходимо завершить и измерить
|
||||
обязательный уровень «4».
|
||||
- Однохостовая среда WSL2 не доказывает устойчивость к потере Docker-host;
|
||||
измерен только отказ и замена процесса API.
|
||||
|
||||
@@ -154,6 +154,8 @@ func main() {
|
||||
var rootHandler http.Handler = mux
|
||||
rootHandler = middleware.Recovery(rootHandler)
|
||||
rootHandler = middleware.Logger(rootHandler)
|
||||
availabilityGate := middleware.NewAvailabilityGate()
|
||||
rootHandler = availabilityGate.Middleware(rootHandler)
|
||||
rootHandler = middleware.RequestID(rootHandler)
|
||||
rootHandler = middleware.InstanceID(cfg.InstanceID, rootHandler)
|
||||
|
||||
@@ -196,6 +198,13 @@ func main() {
|
||||
signalValue,
|
||||
)
|
||||
|
||||
availabilityGate.BeginDrain()
|
||||
log.Printf(
|
||||
"[INFO] Readiness disabled; draining new traffic for %s",
|
||||
cfg.ShutdownDrainDelay,
|
||||
)
|
||||
time.Sleep(cfg.ShutdownDrainDelay)
|
||||
|
||||
shutdownContext, shutdownCancel := context.WithTimeout(
|
||||
context.Background(),
|
||||
cfg.ShutdownTimeout,
|
||||
|
||||
@@ -21,6 +21,11 @@ services:
|
||||
start_period: 3s
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
postgres:
|
||||
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
|
||||
@@ -43,6 +48,11 @@ services:
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
|
||||
redis:
|
||||
image: redis:8.10.2-alpine3.23@sha256:3811787313eba226a2ef38658c6ccb91cd5e110edc89c37767de373120a0e5a0
|
||||
@@ -66,6 +76,11 @@ services:
|
||||
start_period: 3s
|
||||
stop_grace_period: 10s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
|
||||
migrate:
|
||||
image: migrate/migrate:v4.20.1@sha256:76cc2074cb6642631f34a898ced71e6aeaa6b1a4d78c4daa743275a22e0c5be7
|
||||
@@ -84,6 +99,11 @@ services:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 64M
|
||||
|
||||
api:
|
||||
build:
|
||||
@@ -116,6 +136,7 @@ services:
|
||||
DB_QUERY_TIMEOUT: ${DB_QUERY_TIMEOUT:-3s}
|
||||
|
||||
READINESS_TIMEOUT: ${READINESS_TIMEOUT:-2s}
|
||||
SHUTDOWN_DRAIN_DELAY: ${SHUTDOWN_DRAIN_DELAY:-2s}
|
||||
SHUTDOWN_TIMEOUT: ${SHUTDOWN_TIMEOUT:-10s}
|
||||
secrets:
|
||||
- database_url
|
||||
@@ -131,6 +152,10 @@ services:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPU_LIMIT:-0.50}"
|
||||
memory: ${API_MEMORY_LIMIT:-256M}
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
@@ -155,6 +180,11 @@ services:
|
||||
redis:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 256M
|
||||
|
||||
postgres-restore:
|
||||
image: postgres:16.15-alpine3.24@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea
|
||||
@@ -178,6 +208,11 @@ services:
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
restart: "no"
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Lab 3 grade-4 experiment
|
||||
|
||||
Generated: 2026-10-23T12:31:12.749687+00:00.
|
||||
|
||||
## Method
|
||||
|
||||
- concurrency: 8;
|
||||
- baseline duration: 12 s;
|
||||
- load: closed-loop POST messages through Nginx;
|
||||
- failover: both original API containers stopped and restored in turn;
|
||||
- rolling update: one surge replica, old replicas replaced one by one;
|
||||
- percentiles cover successful responses; failures are separate.
|
||||
|
||||
## Results
|
||||
|
||||
| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| baseline | 319.463 | 0 (0.000%) | 16.799 | 55.515 | 93.803 | n/a |
|
||||
| failover-1 | 301.04 | 1 (0.022%) | 17.145 | 69.558 | 89.752 | 3.464 |
|
||||
| failover-2 | 374.449 | 0 (0.000%) | 14.211 | 49.542 | 81.04 | 3.476 |
|
||||
| rolling-update | 376.959 | 1 (0.005%) | 15.725 | 48.724 | 79.337 | n/a |
|
||||
|
||||
## Parallel idempotency
|
||||
|
||||
32 requests produced 1 unique message ID across 2 API instances.
|
||||
|
||||
## Graphs
|
||||
|
||||

|
||||
|
||||

|
||||
|
||||

|
||||
|
||||

|
||||
|
||||
## Scope
|
||||
|
||||
Single-host process failure and rolling replacement only. PostgreSQL and Nginx remain single points of failure.
|
||||
@@ -0,0 +1,69 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="560" viewBox="0 0 1100 560">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.bar { fill: #cfd2d6; }
|
||||
.bar2 { fill: #73777f; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
|
||||
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
|
||||
</style>
|
||||
<text x="48" y="42" class="title">Grade 4: successful latency percentiles</text>
|
||||
<line x1="80" y1="450.0" x2="1040" y2="450.0" class="grid"/>
|
||||
<text x="68" y="454.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="80" y1="378.0" x2="1040" y2="378.0" class="grid"/>
|
||||
<text x="68" y="382.0" text-anchor="end" class="small">19</text>
|
||||
<line x1="80" y1="306.0" x2="1040" y2="306.0" class="grid"/>
|
||||
<text x="68" y="310.0" text-anchor="end" class="small">38</text>
|
||||
<line x1="80" y1="234.0" x2="1040" y2="234.0" class="grid"/>
|
||||
<text x="68" y="238.0" text-anchor="end" class="small">56</text>
|
||||
<line x1="80" y1="162.0" x2="1040" y2="162.0" class="grid"/>
|
||||
<text x="68" y="166.0" text-anchor="end" class="small">75</text>
|
||||
<line x1="80" y1="90.0" x2="1040" y2="90.0" class="grid"/>
|
||||
<text x="68" y="94.0" text-anchor="end" class="small">94</text>
|
||||
<rect x="128.0" y="385.5" width="42.0" height="64.5" rx="7" class="bar"/>
|
||||
<text x="149.0" y="378.5" text-anchor="middle" class="small">17</text>
|
||||
<text x="149.0" y="470" text-anchor="middle" class="small">p50</text>
|
||||
<rect x="178.0" y="236.9" width="42.0" height="213.1" rx="7" class="bar2"/>
|
||||
<text x="199.0" y="229.9" text-anchor="middle" class="small">56</text>
|
||||
<text x="199.0" y="470" text-anchor="middle" class="small">p95</text>
|
||||
<rect x="228.0" y="90.0" width="42.0" height="360.0" rx="7" class="bar2"/>
|
||||
<text x="249.0" y="83.0" text-anchor="middle" class="small">94</text>
|
||||
<text x="249.0" y="470" text-anchor="middle" class="small">p99</text>
|
||||
<text x="200.0" y="498" text-anchor="middle" class="label">baseline</text>
|
||||
<rect x="368.0" y="384.2" width="42.0" height="65.8" rx="7" class="bar"/>
|
||||
<text x="389.0" y="377.2" text-anchor="middle" class="small">17</text>
|
||||
<text x="389.0" y="470" text-anchor="middle" class="small">p50</text>
|
||||
<rect x="418.0" y="183.0" width="42.0" height="267.0" rx="7" class="bar2"/>
|
||||
<text x="439.0" y="176.0" text-anchor="middle" class="small">70</text>
|
||||
<text x="439.0" y="470" text-anchor="middle" class="small">p95</text>
|
||||
<rect x="468.0" y="105.5" width="42.0" height="344.5" rx="7" class="bar2"/>
|
||||
<text x="489.0" y="98.5" text-anchor="middle" class="small">90</text>
|
||||
<text x="489.0" y="470" text-anchor="middle" class="small">p99</text>
|
||||
<text x="440.0" y="498" text-anchor="middle" class="label">failover-1</text>
|
||||
<rect x="608.0" y="395.5" width="42.0" height="54.5" rx="7" class="bar"/>
|
||||
<text x="629.0" y="388.5" text-anchor="middle" class="small">14</text>
|
||||
<text x="629.0" y="470" text-anchor="middle" class="small">p50</text>
|
||||
<rect x="658.0" y="259.9" width="42.0" height="190.1" rx="7" class="bar2"/>
|
||||
<text x="679.0" y="252.9" text-anchor="middle" class="small">50</text>
|
||||
<text x="679.0" y="470" text-anchor="middle" class="small">p95</text>
|
||||
<rect x="708.0" y="139.0" width="42.0" height="311.0" rx="7" class="bar2"/>
|
||||
<text x="729.0" y="132.0" text-anchor="middle" class="small">81</text>
|
||||
<text x="729.0" y="470" text-anchor="middle" class="small">p99</text>
|
||||
<text x="680.0" y="498" text-anchor="middle" class="label">failover-2</text>
|
||||
<rect x="848.0" y="389.7" width="42.0" height="60.3" rx="7" class="bar"/>
|
||||
<text x="869.0" y="382.7" text-anchor="middle" class="small">16</text>
|
||||
<text x="869.0" y="470" text-anchor="middle" class="small">p50</text>
|
||||
<rect x="898.0" y="263.0" width="42.0" height="187.0" rx="7" class="bar2"/>
|
||||
<text x="919.0" y="256.0" text-anchor="middle" class="small">49</text>
|
||||
<text x="919.0" y="470" text-anchor="middle" class="small">p95</text>
|
||||
<rect x="948.0" y="145.5" width="42.0" height="304.5" rx="7" class="bar2"/>
|
||||
<text x="969.0" y="138.5" text-anchor="middle" class="small">79</text>
|
||||
<text x="969.0" y="470" text-anchor="middle" class="small">p99</text>
|
||||
<text x="920.0" y="498" text-anchor="middle" class="label">rolling-update</text>
|
||||
<text x="24" y="270.0" transform="rotate(-90 24 270.0)" class="label">Successful latency, ms</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 4.4 KiB |
@@ -0,0 +1,42 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="600" viewBox="0 0 1100 600">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.bar { fill: #cfd2d6; }
|
||||
.bar2 { fill: #73777f; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
|
||||
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
|
||||
</style>
|
||||
<text x="48" y="42" class="title">Grade 4: peak container resources</text>
|
||||
<text x="80" y="72" class="label">Peak CPU, %</text>
|
||||
<rect x="103.6" y="270.1" width="57.8" height="179.9" rx="9" class="bar"/>
|
||||
<text x="132.5" y="262.1" text-anchor="middle" class="small">52.6</text>
|
||||
<text x="132.5" y="475" text-anchor="middle" class="small">api</text>
|
||||
<rect x="208.6" y="370.1" width="57.8" height="79.9" rx="9" class="bar"/>
|
||||
<text x="237.5" y="362.1" text-anchor="middle" class="small">23.4</text>
|
||||
<text x="237.5" y="475" text-anchor="middle" class="small">load-balancer</text>
|
||||
<rect x="313.6" y="90.0" width="57.8" height="360.0" rx="9" class="bar"/>
|
||||
<text x="342.5" y="82.0" text-anchor="middle" class="small">105.2</text>
|
||||
<text x="342.5" y="475" text-anchor="middle" class="small">postgres</text>
|
||||
<rect x="418.6" y="429.9" width="57.8" height="20.1" rx="9" class="bar"/>
|
||||
<text x="447.5" y="421.9" text-anchor="middle" class="small">5.9</text>
|
||||
<text x="447.5" y="475" text-anchor="middle" class="small">redis</text>
|
||||
<text x="600" y="72" class="label">Peak memory, MiB</text>
|
||||
<rect x="623.6" y="286.2" width="57.8" height="163.8" rx="9" class="bar"/>
|
||||
<text x="652.5" y="278.2" text-anchor="middle" class="small">66.1</text>
|
||||
<text x="652.5" y="475" text-anchor="middle" class="small">api</text>
|
||||
<rect x="728.6" y="416.5" width="57.8" height="33.5" rx="9" class="bar"/>
|
||||
<text x="757.5" y="408.5" text-anchor="middle" class="small">13.5</text>
|
||||
<text x="757.5" y="475" text-anchor="middle" class="small">load-balancer</text>
|
||||
<rect x="833.6" y="90.0" width="57.8" height="360.0" rx="9" class="bar"/>
|
||||
<text x="862.5" y="82.0" text-anchor="middle" class="small">145.2</text>
|
||||
<text x="862.5" y="475" text-anchor="middle" class="small">postgres</text>
|
||||
<rect x="938.6" y="419.6" width="57.8" height="30.4" rx="9" class="bar"/>
|
||||
<text x="967.5" y="411.6" text-anchor="middle" class="small">12.3</text>
|
||||
<text x="967.5" y="475" text-anchor="middle" class="small">redis</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.5 KiB |
@@ -0,0 +1,675 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"environment": {
|
||||
"utc_started_at": "2026-10-23T12:31:12.749687+00:00",
|
||||
"git_commit": "34618bd4237bbb8bf6a3a303056fc95d799577f9",
|
||||
"working_tree_dirty": true,
|
||||
"logical_cpu_count": 8,
|
||||
"host_memory_mib": 3813.6,
|
||||
"docker_engine": "29.7.2",
|
||||
"docker_compose": "5.5.0",
|
||||
"container_limits": [
|
||||
{
|
||||
"service": "api",
|
||||
"nano_cpus": 500000000,
|
||||
"memory_limit_bytes": 268435456
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"nano_cpus": 500000000,
|
||||
"memory_limit_bytes": 268435456
|
||||
},
|
||||
{
|
||||
"service": "load-balancer",
|
||||
"nano_cpus": 250000000,
|
||||
"memory_limit_bytes": 134217728
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"nano_cpus": 1000000000,
|
||||
"memory_limit_bytes": 536870912
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"nano_cpus": 250000000,
|
||||
"memory_limit_bytes": 134217728
|
||||
}
|
||||
]
|
||||
},
|
||||
"methodology": {
|
||||
"base_url": "http://localhost:8080",
|
||||
"concurrency": 8,
|
||||
"baseline_seconds": 12,
|
||||
"action_after_seconds": 3,
|
||||
"post_action_seconds": 8,
|
||||
"warmup_requests": 12,
|
||||
"workload": "closed-loop POST messages via Nginx; no automatic retries"
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"name": "baseline",
|
||||
"elapsed_seconds": 12.574,
|
||||
"successful_requests": 4017,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"errors": {},
|
||||
"successful_rps": 319.463,
|
||||
"successful_latency_ms": {
|
||||
"p50": 16.799,
|
||||
"p95": 55.515,
|
||||
"p99": 93.803
|
||||
},
|
||||
"instances": [
|
||||
"0b933a46fccb",
|
||||
"55aaffcf508a"
|
||||
],
|
||||
"recovery_seconds": null,
|
||||
"action": null,
|
||||
"per_second": {
|
||||
"0": {
|
||||
"successful": 118,
|
||||
"failed": 0
|
||||
},
|
||||
"1": {
|
||||
"successful": 308,
|
||||
"failed": 0
|
||||
},
|
||||
"2": {
|
||||
"successful": 386,
|
||||
"failed": 0
|
||||
},
|
||||
"3": {
|
||||
"successful": 378,
|
||||
"failed": 0
|
||||
},
|
||||
"4": {
|
||||
"successful": 344,
|
||||
"failed": 0
|
||||
},
|
||||
"5": {
|
||||
"successful": 322,
|
||||
"failed": 0
|
||||
},
|
||||
"6": {
|
||||
"successful": 316,
|
||||
"failed": 0
|
||||
},
|
||||
"7": {
|
||||
"successful": 384,
|
||||
"failed": 0
|
||||
},
|
||||
"8": {
|
||||
"successful": 380,
|
||||
"failed": 0
|
||||
},
|
||||
"9": {
|
||||
"successful": 402,
|
||||
"failed": 0
|
||||
},
|
||||
"10": {
|
||||
"successful": 377,
|
||||
"failed": 0
|
||||
},
|
||||
"11": {
|
||||
"successful": 288,
|
||||
"failed": 0
|
||||
},
|
||||
"12": {
|
||||
"successful": 14,
|
||||
"failed": 0
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"cpu_percent_avg": 28.453,
|
||||
"cpu_percent_max": 36.1,
|
||||
"memory_mib_avg": 42.413,
|
||||
"memory_mib_max": 66.07
|
||||
},
|
||||
"load-balancer": {
|
||||
"cpu_percent_avg": 17.448,
|
||||
"cpu_percent_max": 20.8,
|
||||
"memory_mib_avg": 9.214,
|
||||
"memory_mib_max": 9.559
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_percent_avg": 85.888,
|
||||
"cpu_percent_max": 105.25,
|
||||
"memory_mib_avg": 97.975,
|
||||
"memory_mib_max": 100.8
|
||||
},
|
||||
"redis": {
|
||||
"cpu_percent_avg": 2.552,
|
||||
"cpu_percent_max": 5.79,
|
||||
"memory_mib_avg": 8.796,
|
||||
"memory_mib_max": 11.74
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "failover-1",
|
||||
"elapsed_seconds": 15.324,
|
||||
"successful_requests": 4613,
|
||||
"failed_requests": 1,
|
||||
"error_ratio": 0.000217,
|
||||
"errors": {
|
||||
"POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 504": 1
|
||||
},
|
||||
"successful_rps": 301.04,
|
||||
"successful_latency_ms": {
|
||||
"p50": 17.145,
|
||||
"p95": 69.558,
|
||||
"p99": 89.752
|
||||
},
|
||||
"instances": [
|
||||
"0b933a46fccb",
|
||||
"55aaffcf508a"
|
||||
],
|
||||
"recovery_seconds": 3.464,
|
||||
"action": {
|
||||
"started_at": 3.006,
|
||||
"ended_at": 6.463,
|
||||
"duration_seconds": 3.457,
|
||||
"exit_code": 0,
|
||||
"summary": "stopped API instance 55aaffcf508a"
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"successful": 404,
|
||||
"failed": 0
|
||||
},
|
||||
"1": {
|
||||
"successful": 426,
|
||||
"failed": 0
|
||||
},
|
||||
"2": {
|
||||
"successful": 358,
|
||||
"failed": 0
|
||||
},
|
||||
"3": {
|
||||
"successful": 399,
|
||||
"failed": 0
|
||||
},
|
||||
"4": {
|
||||
"successful": 392,
|
||||
"failed": 0
|
||||
},
|
||||
"5": {
|
||||
"successful": 322,
|
||||
"failed": 0
|
||||
},
|
||||
"6": {
|
||||
"successful": 299,
|
||||
"failed": 0
|
||||
},
|
||||
"7": {
|
||||
"successful": 329,
|
||||
"failed": 1
|
||||
},
|
||||
"8": {
|
||||
"successful": 313,
|
||||
"failed": 0
|
||||
},
|
||||
"9": {
|
||||
"successful": 229,
|
||||
"failed": 0
|
||||
},
|
||||
"10": {
|
||||
"successful": 155,
|
||||
"failed": 0
|
||||
},
|
||||
"11": {
|
||||
"successful": 187,
|
||||
"failed": 0
|
||||
},
|
||||
"12": {
|
||||
"successful": 299,
|
||||
"failed": 0
|
||||
},
|
||||
"13": {
|
||||
"successful": 317,
|
||||
"failed": 0
|
||||
},
|
||||
"14": {
|
||||
"successful": 184,
|
||||
"failed": 0
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"cpu_percent_avg": 27.62,
|
||||
"cpu_percent_max": 52.59,
|
||||
"memory_mib_avg": 11.58,
|
||||
"memory_mib_max": 12.66
|
||||
},
|
||||
"load-balancer": {
|
||||
"cpu_percent_avg": 18.536,
|
||||
"cpu_percent_max": 21.25,
|
||||
"memory_mib_avg": 9.351,
|
||||
"memory_mib_max": 9.523
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_percent_avg": 81.822,
|
||||
"cpu_percent_max": 104.67,
|
||||
"memory_mib_avg": 100.512,
|
||||
"memory_mib_max": 109.1
|
||||
},
|
||||
"redis": {
|
||||
"cpu_percent_avg": 1.486,
|
||||
"cpu_percent_max": 5.88,
|
||||
"memory_mib_avg": 8.04,
|
||||
"memory_mib_max": 8.25
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "failover-2",
|
||||
"elapsed_seconds": 15.529,
|
||||
"successful_requests": 5815,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"errors": {},
|
||||
"successful_rps": 374.449,
|
||||
"successful_latency_ms": {
|
||||
"p50": 14.211,
|
||||
"p95": 49.542,
|
||||
"p99": 81.04
|
||||
},
|
||||
"instances": [
|
||||
"0b933a46fccb",
|
||||
"55aaffcf508a"
|
||||
],
|
||||
"recovery_seconds": 3.476,
|
||||
"action": {
|
||||
"started_at": 3.013,
|
||||
"ended_at": 6.485,
|
||||
"duration_seconds": 3.472,
|
||||
"exit_code": 0,
|
||||
"summary": "stopped API instance 0b933a46fccb"
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"successful": 396,
|
||||
"failed": 0
|
||||
},
|
||||
"1": {
|
||||
"successful": 439,
|
||||
"failed": 0
|
||||
},
|
||||
"2": {
|
||||
"successful": 412,
|
||||
"failed": 0
|
||||
},
|
||||
"3": {
|
||||
"successful": 355,
|
||||
"failed": 0
|
||||
},
|
||||
"4": {
|
||||
"successful": 378,
|
||||
"failed": 0
|
||||
},
|
||||
"5": {
|
||||
"successful": 440,
|
||||
"failed": 0
|
||||
},
|
||||
"6": {
|
||||
"successful": 362,
|
||||
"failed": 0
|
||||
},
|
||||
"7": {
|
||||
"successful": 419,
|
||||
"failed": 0
|
||||
},
|
||||
"8": {
|
||||
"successful": 385,
|
||||
"failed": 0
|
||||
},
|
||||
"9": {
|
||||
"successful": 372,
|
||||
"failed": 0
|
||||
},
|
||||
"10": {
|
||||
"successful": 420,
|
||||
"failed": 0
|
||||
},
|
||||
"11": {
|
||||
"successful": 423,
|
||||
"failed": 0
|
||||
},
|
||||
"12": {
|
||||
"successful": 432,
|
||||
"failed": 0
|
||||
},
|
||||
"13": {
|
||||
"successful": 375,
|
||||
"failed": 0
|
||||
},
|
||||
"14": {
|
||||
"successful": 207,
|
||||
"failed": 0
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"cpu_percent_avg": 31.411,
|
||||
"cpu_percent_max": 51.63,
|
||||
"memory_mib_avg": 10.373,
|
||||
"memory_mib_max": 11.6
|
||||
},
|
||||
"load-balancer": {
|
||||
"cpu_percent_avg": 15.778,
|
||||
"cpu_percent_max": 21.79,
|
||||
"memory_mib_avg": 9.62,
|
||||
"memory_mib_max": 9.781
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_percent_avg": 76.87,
|
||||
"cpu_percent_max": 104.51,
|
||||
"memory_mib_avg": 106.016,
|
||||
"memory_mib_max": 120.0
|
||||
},
|
||||
"redis": {
|
||||
"cpu_percent_avg": 0.786,
|
||||
"cpu_percent_max": 2.83,
|
||||
"memory_mib_avg": 8.844,
|
||||
"memory_mib_max": 12.28
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "rolling-update",
|
||||
"elapsed_seconds": 53.985,
|
||||
"successful_requests": 20350,
|
||||
"failed_requests": 1,
|
||||
"error_ratio": 4.9e-05,
|
||||
"errors": {
|
||||
"POST /api/v1/chats/60f7f146-7d10-4711-8a99-2098bc1c2496/messages: expected [201], received 502": 1
|
||||
},
|
||||
"successful_rps": 376.959,
|
||||
"successful_latency_ms": {
|
||||
"p50": 15.725,
|
||||
"p95": 48.724,
|
||||
"p99": 79.337
|
||||
},
|
||||
"instances": [
|
||||
"0b933a46fccb",
|
||||
"3adbcad9e34b",
|
||||
"55aaffcf508a",
|
||||
"ceca3f6f1faf",
|
||||
"f0a3bedda85a"
|
||||
],
|
||||
"recovery_seconds": null,
|
||||
"action": {
|
||||
"started_at": 3.016,
|
||||
"ended_at": 45.961,
|
||||
"duration_seconds": 42.945,
|
||||
"exit_code": 0,
|
||||
"summary": "rolling update completed with 2 healthy API replicas"
|
||||
},
|
||||
"per_second": {
|
||||
"0": {
|
||||
"successful": 340,
|
||||
"failed": 0
|
||||
},
|
||||
"1": {
|
||||
"successful": 369,
|
||||
"failed": 0
|
||||
},
|
||||
"2": {
|
||||
"successful": 426,
|
||||
"failed": 0
|
||||
},
|
||||
"3": {
|
||||
"successful": 382,
|
||||
"failed": 0
|
||||
},
|
||||
"4": {
|
||||
"successful": 371,
|
||||
"failed": 0
|
||||
},
|
||||
"5": {
|
||||
"successful": 376,
|
||||
"failed": 0
|
||||
},
|
||||
"6": {
|
||||
"successful": 328,
|
||||
"failed": 0
|
||||
},
|
||||
"7": {
|
||||
"successful": 350,
|
||||
"failed": 0
|
||||
},
|
||||
"8": {
|
||||
"successful": 434,
|
||||
"failed": 0
|
||||
},
|
||||
"9": {
|
||||
"successful": 411,
|
||||
"failed": 0
|
||||
},
|
||||
"10": {
|
||||
"successful": 399,
|
||||
"failed": 0
|
||||
},
|
||||
"11": {
|
||||
"successful": 417,
|
||||
"failed": 0
|
||||
},
|
||||
"12": {
|
||||
"successful": 374,
|
||||
"failed": 0
|
||||
},
|
||||
"13": {
|
||||
"successful": 427,
|
||||
"failed": 0
|
||||
},
|
||||
"14": {
|
||||
"successful": 414,
|
||||
"failed": 0
|
||||
},
|
||||
"15": {
|
||||
"successful": 418,
|
||||
"failed": 0
|
||||
},
|
||||
"16": {
|
||||
"successful": 408,
|
||||
"failed": 0
|
||||
},
|
||||
"17": {
|
||||
"successful": 352,
|
||||
"failed": 0
|
||||
},
|
||||
"18": {
|
||||
"successful": 391,
|
||||
"failed": 0
|
||||
},
|
||||
"19": {
|
||||
"successful": 388,
|
||||
"failed": 0
|
||||
},
|
||||
"20": {
|
||||
"successful": 413,
|
||||
"failed": 0
|
||||
},
|
||||
"21": {
|
||||
"successful": 417,
|
||||
"failed": 0
|
||||
},
|
||||
"22": {
|
||||
"successful": 388,
|
||||
"failed": 0
|
||||
},
|
||||
"23": {
|
||||
"successful": 396,
|
||||
"failed": 0
|
||||
},
|
||||
"24": {
|
||||
"successful": 400,
|
||||
"failed": 0
|
||||
},
|
||||
"25": {
|
||||
"successful": 339,
|
||||
"failed": 0
|
||||
},
|
||||
"26": {
|
||||
"successful": 225,
|
||||
"failed": 0
|
||||
},
|
||||
"27": {
|
||||
"successful": 376,
|
||||
"failed": 0
|
||||
},
|
||||
"28": {
|
||||
"successful": 385,
|
||||
"failed": 0
|
||||
},
|
||||
"29": {
|
||||
"successful": 396,
|
||||
"failed": 0
|
||||
},
|
||||
"30": {
|
||||
"successful": 403,
|
||||
"failed": 0
|
||||
},
|
||||
"31": {
|
||||
"successful": 262,
|
||||
"failed": 0
|
||||
},
|
||||
"32": {
|
||||
"successful": 428,
|
||||
"failed": 0
|
||||
},
|
||||
"33": {
|
||||
"successful": 362,
|
||||
"failed": 0
|
||||
},
|
||||
"34": {
|
||||
"successful": 286,
|
||||
"failed": 0
|
||||
},
|
||||
"35": {
|
||||
"successful": 337,
|
||||
"failed": 0
|
||||
},
|
||||
"36": {
|
||||
"successful": 328,
|
||||
"failed": 0
|
||||
},
|
||||
"37": {
|
||||
"successful": 424,
|
||||
"failed": 0
|
||||
},
|
||||
"38": {
|
||||
"successful": 356,
|
||||
"failed": 0
|
||||
},
|
||||
"39": {
|
||||
"successful": 386,
|
||||
"failed": 0
|
||||
},
|
||||
"40": {
|
||||
"successful": 422,
|
||||
"failed": 0
|
||||
},
|
||||
"41": {
|
||||
"successful": 285,
|
||||
"failed": 0
|
||||
},
|
||||
"42": {
|
||||
"successful": 279,
|
||||
"failed": 0
|
||||
},
|
||||
"43": {
|
||||
"successful": 222,
|
||||
"failed": 1
|
||||
},
|
||||
"44": {
|
||||
"successful": 316,
|
||||
"failed": 0
|
||||
},
|
||||
"45": {
|
||||
"successful": 359,
|
||||
"failed": 0
|
||||
},
|
||||
"46": {
|
||||
"successful": 405,
|
||||
"failed": 0
|
||||
},
|
||||
"47": {
|
||||
"successful": 398,
|
||||
"failed": 0
|
||||
},
|
||||
"48": {
|
||||
"successful": 395,
|
||||
"failed": 0
|
||||
},
|
||||
"49": {
|
||||
"successful": 430,
|
||||
"failed": 0
|
||||
},
|
||||
"50": {
|
||||
"successful": 453,
|
||||
"failed": 0
|
||||
},
|
||||
"51": {
|
||||
"successful": 409,
|
||||
"failed": 0
|
||||
},
|
||||
"52": {
|
||||
"successful": 445,
|
||||
"failed": 0
|
||||
},
|
||||
"53": {
|
||||
"successful": 450,
|
||||
"failed": 0
|
||||
}
|
||||
},
|
||||
"resources": {
|
||||
"api": {
|
||||
"cpu_percent_avg": 25.598,
|
||||
"cpu_percent_max": 37.65,
|
||||
"memory_mib_avg": 8.956,
|
||||
"memory_mib_max": 10.97
|
||||
},
|
||||
"load-balancer": {
|
||||
"cpu_percent_avg": 19.935,
|
||||
"cpu_percent_max": 23.36,
|
||||
"memory_mib_avg": 9.836,
|
||||
"memory_mib_max": 13.52
|
||||
},
|
||||
"postgres": {
|
||||
"cpu_percent_avg": 98.172,
|
||||
"cpu_percent_max": 105.11,
|
||||
"memory_mib_avg": 127.975,
|
||||
"memory_mib_max": 145.2
|
||||
},
|
||||
"redis": {
|
||||
"cpu_percent_avg": 0.722,
|
||||
"cpu_percent_max": 3.94,
|
||||
"memory_mib_avg": 7.661,
|
||||
"memory_mib_max": 11.84
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"idempotency": {
|
||||
"parallel_requests": 32,
|
||||
"successful_responses": 32,
|
||||
"failed_responses": 0,
|
||||
"unique_message_ids": 1,
|
||||
"observed_instances": [
|
||||
"0b933a46fccb",
|
||||
"55aaffcf508a"
|
||||
]
|
||||
},
|
||||
"dataset": {
|
||||
"warmup_requests": 12,
|
||||
"minimum_confirmed_messages": 34808
|
||||
},
|
||||
"limitations": [
|
||||
"single Docker host",
|
||||
"closed-loop coordinated omission",
|
||||
"successful-response latency percentiles only",
|
||||
"PostgreSQL and Nginx remain single points of failure"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="560" viewBox="0 0 1100 560">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.bar { fill: #cfd2d6; }
|
||||
.bar2 { fill: #73777f; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
|
||||
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
|
||||
</style>
|
||||
<text x="48" y="42" class="title">Grade 4: throughput and errors</text>
|
||||
<line x1="80" y1="450.0" x2="1040" y2="450.0" class="grid"/>
|
||||
<text x="68" y="454.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="80" y1="378.0" x2="1040" y2="378.0" class="grid"/>
|
||||
<text x="68" y="382.0" text-anchor="end" class="small">75</text>
|
||||
<line x1="80" y1="306.0" x2="1040" y2="306.0" class="grid"/>
|
||||
<text x="68" y="310.0" text-anchor="end" class="small">151</text>
|
||||
<line x1="80" y1="234.0" x2="1040" y2="234.0" class="grid"/>
|
||||
<text x="68" y="238.0" text-anchor="end" class="small">226</text>
|
||||
<line x1="80" y1="162.0" x2="1040" y2="162.0" class="grid"/>
|
||||
<text x="68" y="166.0" text-anchor="end" class="small">302</text>
|
||||
<line x1="80" y1="90.0" x2="1040" y2="90.0" class="grid"/>
|
||||
<text x="68" y="94.0" text-anchor="end" class="small">377</text>
|
||||
<rect x="140.0" y="144.9" width="120.0" height="305.1" rx="10" class="bar"/>
|
||||
<text x="200.0" y="134.9" text-anchor="middle" class="label">319.5 RPS</text>
|
||||
<text x="200.0" y="478" text-anchor="middle" class="label">baseline</text>
|
||||
<text x="200.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
|
||||
<rect x="380.0" y="162.5" width="120.0" height="287.5" rx="10" class="bar"/>
|
||||
<text x="440.0" y="152.5" text-anchor="middle" class="label">301.0 RPS</text>
|
||||
<text x="440.0" y="478" text-anchor="middle" class="label">failover-1</text>
|
||||
<text x="440.0" y="498" text-anchor="middle" class="small">errors 0.02%</text>
|
||||
<rect x="620.0" y="92.4" width="120.0" height="357.6" rx="10" class="bar"/>
|
||||
<text x="680.0" y="82.4" text-anchor="middle" class="label">374.4 RPS</text>
|
||||
<text x="680.0" y="478" text-anchor="middle" class="label">failover-2</text>
|
||||
<text x="680.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
|
||||
<rect x="860.0" y="90.0" width="120.0" height="360.0" rx="10" class="bar"/>
|
||||
<text x="920.0" y="80.0" text-anchor="middle" class="label">377.0 RPS</text>
|
||||
<text x="920.0" y="478" text-anchor="middle" class="label">rolling-update</text>
|
||||
<text x="920.0" y="498" text-anchor="middle" class="small">errors 0.00%</text>
|
||||
<text x="24" y="270.0" transform="rotate(-90 24 270.0)" class="label">Successful requests per second</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.8 KiB |
@@ -0,0 +1,38 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="1100" height="600" viewBox="0 0 1100 600">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.bar { fill: #cfd2d6; }
|
||||
.bar2 { fill: #73777f; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
|
||||
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
|
||||
</style>
|
||||
<text x="48" y="42" class="title">Grade 4: request timeline</text>
|
||||
<line x1="70" y1="480.0" x2="1050" y2="480.0" class="grid"/>
|
||||
<text x="58" y="484.0" text-anchor="end" class="small">0</text>
|
||||
<line x1="70" y1="402.0" x2="1050" y2="402.0" class="grid"/>
|
||||
<text x="58" y="406.0" text-anchor="end" class="small">91</text>
|
||||
<line x1="70" y1="324.0" x2="1050" y2="324.0" class="grid"/>
|
||||
<text x="58" y="328.0" text-anchor="end" class="small">181</text>
|
||||
<line x1="70" y1="246.0" x2="1050" y2="246.0" class="grid"/>
|
||||
<text x="58" y="250.0" text-anchor="end" class="small">272</text>
|
||||
<line x1="70" y1="168.0" x2="1050" y2="168.0" class="grid"/>
|
||||
<text x="58" y="172.0" text-anchor="end" class="small">362</text>
|
||||
<line x1="70" y1="90.0" x2="1050" y2="90.0" class="grid"/>
|
||||
<text x="58" y="94.0" text-anchor="end" class="small">453</text>
|
||||
<circle cx="277.9" cy="472" r="4" class="error"/>
|
||||
<circle cx="951.0" cy="472" r="4" class="error"/>
|
||||
<polyline points="70.0,378.4 79.9,214.8 89.8,147.7 99.7,154.6 109.6,183.8 119.5,202.8 129.4,207.9 139.3,149.4 149.2,152.8 159.1,133.9 169.0,155.4 178.9,232.1 188.8,467.9 208.6,132.2 218.5,113.2 228.4,171.8 238.3,136.5 248.2,142.5 258.1,202.8 268.0,222.6 277.9,196.8 287.8,210.5 297.7,282.8 307.6,346.6 317.5,319.0 327.4,222.6 337.3,207.1 347.2,321.6 367.0,139.1 376.9,102.1 386.8,125.3 396.7,174.4 406.6,154.6 416.5,101.2 426.4,168.3 436.3,119.3 446.2,148.5 456.1,159.7 466.0,118.4 475.9,115.8 485.8,108.1 495.7,157.2 505.6,301.8 525.4,187.3 535.3,162.3 545.2,113.2 555.1,151.1 564.9,160.6 574.8,156.3 584.7,197.6 594.6,178.7 604.5,106.4 614.4,126.2 624.3,136.5 634.2,121.0 644.1,158.0 654.0,112.4 663.9,123.6 673.8,120.1 683.7,128.7 693.6,177.0 703.5,143.4 713.4,146.0 723.3,124.4 733.2,121.0 743.1,146.0 753.0,139.1 762.9,135.6 772.8,188.1 782.7,286.3 792.6,156.3 802.5,148.5 812.4,139.1 822.3,133.0 832.2,254.4 842.1,111.5 852.0,168.3 861.9,233.8 871.8,189.9 881.7,197.6 891.6,115.0 901.5,173.5 911.4,147.7 921.3,116.7 931.2,234.6 941.1,239.8 951.0,288.9 960.9,207.9 970.8,170.9 980.7,131.3 990.6,137.4 1000.5,139.9 1010.4,109.8 1020.3,90.0 1030.2,127.9 1040.1,96.9 1050.0,92.6" class="line"/>
|
||||
<line x1="238.3" y1="90" x2="238.3" y2="480" class="event"/>
|
||||
<text x="244.3" y="108" class="small">failover-1 action</text>
|
||||
<line x1="396.8" y1="90" x2="396.8" y2="480" class="event"/>
|
||||
<text x="402.8" y="108" class="small">failover-2 action</text>
|
||||
<line x1="555.2" y1="90" x2="555.2" y2="480" class="event"/>
|
||||
<text x="561.2" y="108" class="small">rolling-update action</text>
|
||||
<text x="70" y="512" class="label">One-second buckets; circles mark failed requests</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.2 KiB |
@@ -17,7 +17,7 @@ http {
|
||||
upstream api_backend {
|
||||
zone api_backend 64k;
|
||||
least_conn;
|
||||
server api:8080 resolve max_fails=1 fail_timeout=1s;
|
||||
server api:8080 resolve max_fails=2 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
@@ -31,6 +31,42 @@ http {
|
||||
add_header X-Frame-Options "DENY" always;
|
||||
add_header Content-Security-Policy "default-src 'self'; img-src 'self' data:; style-src 'self'; script-src 'self'; connect-src 'self'; frame-ancestors 'none'; base-uri 'self'; form-action 'self'" always;
|
||||
|
||||
# Message creation carries an Idempotency-Key. It is therefore safe to
|
||||
# retry a POST on another replica when the selected backend is draining.
|
||||
location ~ ^/api/v1/chats/[^/]+/messages$ {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Cookie "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
|
||||
proxy_next_upstream_tries 3;
|
||||
}
|
||||
|
||||
location ~ ^/web-api/v1/chats/[^/]+/messages$ {
|
||||
rewrite ^/web-api/(.*)$ /api/$1 break;
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Authorization $web_authorization;
|
||||
proxy_set_header Cookie "";
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_send_timeout 20s;
|
||||
proxy_read_timeout 20s;
|
||||
proxy_next_upstream error timeout http_502 http_503 http_504 non_idempotent;
|
||||
proxy_next_upstream_tries 3;
|
||||
}
|
||||
|
||||
location /api/ {
|
||||
proxy_pass http://api_backend;
|
||||
proxy_http_version 1.1;
|
||||
|
||||
@@ -35,8 +35,9 @@ type Config struct {
|
||||
RedisOperationTimeout time.Duration
|
||||
RedisPoolSize int
|
||||
|
||||
ReadinessTimeout time.Duration
|
||||
ShutdownTimeout time.Duration
|
||||
ReadinessTimeout time.Duration
|
||||
ShutdownDrainDelay time.Duration
|
||||
ShutdownTimeout time.Duration
|
||||
}
|
||||
|
||||
func Load() (Config, error) {
|
||||
@@ -165,6 +166,14 @@ func Load() (Config, error) {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
shutdownDrainDelay, err := durationFromEnv(
|
||||
"SHUTDOWN_DRAIN_DELAY",
|
||||
2*time.Second,
|
||||
)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
shutdownTimeout, err := durationFromEnv(
|
||||
"SHUTDOWN_TIMEOUT",
|
||||
10*time.Second,
|
||||
@@ -185,6 +194,12 @@ func Load() (Config, error) {
|
||||
)
|
||||
}
|
||||
|
||||
if shutdownDrainDelay >= shutdownTimeout {
|
||||
return Config{}, errors.New(
|
||||
"SHUTDOWN_DRAIN_DELAY must be less than SHUTDOWN_TIMEOUT",
|
||||
)
|
||||
}
|
||||
|
||||
return Config{
|
||||
HTTPHost: envOrDefault("APP_HOST", "0.0.0.0"),
|
||||
HTTPPort: envOrDefault("APP_PORT", "8080"),
|
||||
@@ -208,8 +223,9 @@ func Load() (Config, error) {
|
||||
RedisOperationTimeout: redisOperationTimeout,
|
||||
RedisPoolSize: int(redisPoolSize),
|
||||
|
||||
ReadinessTimeout: readinessTimeout,
|
||||
ShutdownTimeout: shutdownTimeout,
|
||||
ReadinessTimeout: readinessTimeout,
|
||||
ShutdownDrainDelay: shutdownDrainDelay,
|
||||
ShutdownTimeout: shutdownTimeout,
|
||||
}, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
|
||||
"mtusi2026lab3/internal/transport/http/response"
|
||||
)
|
||||
|
||||
// AvailabilityGate lets a process reject new work before its HTTP listener is
|
||||
// closed. Existing requests are still drained by http.Server.Shutdown.
|
||||
type AvailabilityGate struct {
|
||||
accepting atomic.Bool
|
||||
}
|
||||
|
||||
func NewAvailabilityGate() *AvailabilityGate {
|
||||
gate := &AvailabilityGate{}
|
||||
gate.accepting.Store(true)
|
||||
|
||||
return gate
|
||||
}
|
||||
|
||||
func (g *AvailabilityGate) BeginDrain() {
|
||||
g.accepting.Store(false)
|
||||
}
|
||||
|
||||
func (g *AvailabilityGate) IsAccepting() bool {
|
||||
return g.accepting.Load()
|
||||
}
|
||||
|
||||
func (g *AvailabilityGate) Middleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// Liveness deliberately stays available while the process drains. The
|
||||
// readiness endpoint and all business traffic receive 503 immediately.
|
||||
if g.IsAccepting() || r.URL.Path == "/health/live" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
response.WriteError(
|
||||
w,
|
||||
http.StatusServiceUnavailable,
|
||||
response.CodeUnavailable,
|
||||
"service is draining",
|
||||
GetRequestID(r.Context()),
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAvailabilityGateRejectsNewWorkWhileKeepingLiveness(t *testing.T) {
|
||||
gate := NewAvailabilityGate()
|
||||
next := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
})
|
||||
handler := RequestID(gate.Middleware(next))
|
||||
|
||||
before := httptest.NewRecorder()
|
||||
handler.ServeHTTP(before, httptest.NewRequest(http.MethodGet, "/api/v1/users/me", nil))
|
||||
if before.Code != http.StatusNoContent {
|
||||
t.Fatalf("before drain status = %d, want %d", before.Code, http.StatusNoContent)
|
||||
}
|
||||
|
||||
gate.BeginDrain()
|
||||
|
||||
draining := httptest.NewRecorder()
|
||||
handler.ServeHTTP(draining, httptest.NewRequest(http.MethodGet, "/health/ready", nil))
|
||||
if draining.Code != http.StatusServiceUnavailable {
|
||||
t.Fatalf("ready while draining status = %d, want %d", draining.Code, http.StatusServiceUnavailable)
|
||||
}
|
||||
if draining.Header().Get("X-Request-Id") == "" {
|
||||
t.Fatal("draining response is missing X-Request-Id")
|
||||
}
|
||||
|
||||
live := httptest.NewRecorder()
|
||||
handler.ServeHTTP(live, httptest.NewRequest(http.MethodGet, "/health/live", nil))
|
||||
if live.Code != http.StatusNoContent {
|
||||
t.Fatalf("live while draining status = %d, want %d", live.Code, http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
Executable
+397
@@ -0,0 +1,397 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run the repeatable, secret-free resilience experiment required for grade 4."""
|
||||
import argparse
|
||||
from collections import Counter, defaultdict
|
||||
import datetime as dt
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
from client import Client, Failure # noqa: E402
|
||||
|
||||
|
||||
def command(args, env=None, timeout=240, check=True):
|
||||
result = subprocess.run(
|
||||
[str(value) for value in args],
|
||||
cwd=ROOT,
|
||||
env=env,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if check and result.returncode:
|
||||
raise RuntimeError(f"{' '.join(map(str, args[:3]))} exited {result.returncode}")
|
||||
return result
|
||||
|
||||
|
||||
def compose(env_file, *args):
|
||||
return command(["docker", "compose", "--env-file", env_file, *args])
|
||||
|
||||
|
||||
def api_ids(env_file):
|
||||
return [line for line in compose(env_file, "ps", "-q", "api").stdout.splitlines() if line]
|
||||
|
||||
|
||||
def inspect(container_id):
|
||||
data = command(["docker", "inspect", "--format", "{{json .}}", container_id]).stdout
|
||||
return json.loads(data)
|
||||
|
||||
|
||||
def wait_api(env_file, expected, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
healthy = []
|
||||
for container_id in api_ids(env_file):
|
||||
state = inspect(container_id)["State"]
|
||||
if state.get("Health", {}).get("Status") == "healthy":
|
||||
healthy.append(container_id)
|
||||
if len(healthy) >= expected:
|
||||
return healthy
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f"fewer than {expected} API replicas became healthy")
|
||||
|
||||
|
||||
def percentile(values, ratio):
|
||||
if not values:
|
||||
return None
|
||||
values = sorted(values)
|
||||
return round(values[max(0, math.ceil(ratio * len(values)) - 1)], 3)
|
||||
|
||||
|
||||
def memory_mib(raw):
|
||||
value = raw.split("/", 1)[0].strip()
|
||||
units = {"B": 1, "KiB": 1024, "MiB": 1024**2, "GiB": 1024**3}
|
||||
for suffix in ("GiB", "MiB", "KiB", "B"):
|
||||
if value.endswith(suffix):
|
||||
return round(float(value[:-len(suffix)]) * units[suffix] / 1024**2, 3)
|
||||
return 0.0
|
||||
|
||||
|
||||
def service(name):
|
||||
for candidate in ("load-balancer", "postgres", "redis", "api"):
|
||||
if f"-{candidate}-" in name:
|
||||
return candidate
|
||||
return "other"
|
||||
|
||||
|
||||
def sample_stats():
|
||||
result = command(
|
||||
["docker", "stats", "--no-stream", "--format", "{{json .}}"],
|
||||
timeout=10,
|
||||
check=False,
|
||||
)
|
||||
samples = []
|
||||
if result.returncode:
|
||||
return samples
|
||||
for line in result.stdout.splitlines():
|
||||
try:
|
||||
item = json.loads(line)
|
||||
samples.append({
|
||||
"service": service(item.get("Name", "")),
|
||||
"cpu_percent": float(item.get("CPUPerc", "0%").rstrip("%")),
|
||||
"memory_mib": memory_mib(item.get("MemUsage", "0B / 0B")),
|
||||
})
|
||||
except (ValueError, json.JSONDecodeError):
|
||||
continue
|
||||
return samples
|
||||
|
||||
|
||||
def aggregate_resources(samples):
|
||||
grouped = defaultdict(lambda: {"cpu": [], "memory": []})
|
||||
for item in samples:
|
||||
if item["service"] != "other":
|
||||
grouped[item["service"]]["cpu"].append(item["cpu_percent"])
|
||||
grouped[item["service"]]["memory"].append(item["memory_mib"])
|
||||
return {
|
||||
name: {
|
||||
"cpu_percent_avg": round(sum(values["cpu"]) / len(values["cpu"]), 3),
|
||||
"cpu_percent_max": round(max(values["cpu"]), 3),
|
||||
"memory_mib_avg": round(sum(values["memory"]) / len(values["memory"]), 3),
|
||||
"memory_mib_max": round(max(values["memory"]), 3),
|
||||
}
|
||||
for name, values in sorted(grouped.items())
|
||||
}
|
||||
|
||||
|
||||
def load_phase(name, actor, chat, args, action=None, target=None):
|
||||
records, resources = [], []
|
||||
lock = threading.Lock()
|
||||
stop = threading.Event()
|
||||
barrier = threading.Barrier(args.concurrency)
|
||||
started = time.monotonic()
|
||||
|
||||
def worker(worker_id):
|
||||
client = Client(args.base_url, timeout=4)
|
||||
index = 0
|
||||
barrier.wait()
|
||||
while not stop.is_set():
|
||||
request_started = time.monotonic()
|
||||
try:
|
||||
response = client.message(actor, chat, f"grade4-{name}-{worker_id}-{index}")
|
||||
row = {
|
||||
"t": round(time.monotonic() - started, 3),
|
||||
"ok": True,
|
||||
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
||||
"instance": response["headers"].get("X-Instance-Id", ""),
|
||||
}
|
||||
except Failure as exc:
|
||||
row = {
|
||||
"t": round(time.monotonic() - started, 3),
|
||||
"ok": False,
|
||||
"latency_ms": round((time.monotonic() - request_started) * 1000, 3),
|
||||
"error": str(exc),
|
||||
}
|
||||
with lock:
|
||||
records.append(row)
|
||||
index += 1
|
||||
|
||||
def sampler():
|
||||
while not stop.is_set():
|
||||
resources.extend(sample_stats())
|
||||
stop.wait(1)
|
||||
|
||||
workers = [threading.Thread(target=worker, args=(index,), daemon=True)
|
||||
for index in range(args.concurrency)]
|
||||
stats_thread = threading.Thread(target=sampler, daemon=True)
|
||||
stats_thread.start()
|
||||
for thread in workers:
|
||||
thread.start()
|
||||
|
||||
action_data = None
|
||||
action_error = None
|
||||
try:
|
||||
if action is None:
|
||||
time.sleep(args.baseline_seconds)
|
||||
else:
|
||||
time.sleep(args.action_after_seconds)
|
||||
action_started = time.monotonic()
|
||||
try:
|
||||
summary = action()
|
||||
exit_code = 0
|
||||
except Exception as exc:
|
||||
summary, exit_code, action_error = type(exc).__name__, 1, exc
|
||||
action_ended = time.monotonic()
|
||||
action_data = {
|
||||
"started_at": round(action_started - started, 3),
|
||||
"ended_at": round(action_ended - started, 3),
|
||||
"duration_seconds": round(action_ended - action_started, 3),
|
||||
"exit_code": exit_code,
|
||||
"summary": summary,
|
||||
}
|
||||
time.sleep(args.post_action_seconds)
|
||||
finally:
|
||||
stop.set()
|
||||
for thread in workers:
|
||||
thread.join(10)
|
||||
stats_thread.join(5)
|
||||
if action_error:
|
||||
raise action_error
|
||||
|
||||
elapsed = time.monotonic() - started
|
||||
successes = [row for row in records if row["ok"]]
|
||||
failures = [row for row in records if not row["ok"]]
|
||||
latencies = [row["latency_ms"] for row in successes]
|
||||
buckets = defaultdict(lambda: {"successful": 0, "failed": 0})
|
||||
for row in records:
|
||||
buckets[str(int(row["t"]))]["successful" if row["ok"] else "failed"] += 1
|
||||
recovery = None
|
||||
if action_data and target:
|
||||
recovery_row = next((
|
||||
row for row in successes
|
||||
if row["t"] >= action_data["ended_at"] and row.get("instance") != target
|
||||
), None)
|
||||
if recovery_row:
|
||||
recovery = round(recovery_row["t"] - action_data["started_at"], 3)
|
||||
total = len(records)
|
||||
return {
|
||||
"name": name,
|
||||
"elapsed_seconds": round(elapsed, 3),
|
||||
"successful_requests": len(successes),
|
||||
"failed_requests": len(failures),
|
||||
"error_ratio": round(len(failures) / total, 6) if total else None,
|
||||
"errors": dict(sorted(Counter(row["error"] for row in failures).items())),
|
||||
"successful_rps": round(len(successes) / elapsed, 3),
|
||||
"successful_latency_ms": {
|
||||
"p50": percentile(latencies, 0.50),
|
||||
"p95": percentile(latencies, 0.95),
|
||||
"p99": percentile(latencies, 0.99),
|
||||
},
|
||||
"instances": sorted({row.get("instance") for row in successes if row.get("instance")}),
|
||||
"recovery_seconds": recovery,
|
||||
"action": action_data,
|
||||
"per_second": dict(sorted(buckets.items(), key=lambda item: int(item[0]))),
|
||||
"resources": aggregate_resources(resources),
|
||||
}
|
||||
|
||||
|
||||
def idempotency(actor, chat, base_url, count=32):
|
||||
key, body = uuid.uuid4().hex, "grade4-idempotency-" + uuid.uuid4().hex
|
||||
barrier, lock = threading.Barrier(count), threading.Lock()
|
||||
rows, errors = [], []
|
||||
|
||||
def request():
|
||||
client = Client(base_url, timeout=8)
|
||||
barrier.wait()
|
||||
try:
|
||||
response = client.message(actor, chat, body, key=key)
|
||||
row = (response["json"]["id"], response["headers"].get("X-Instance-Id", ""))
|
||||
with lock:
|
||||
rows.append(row)
|
||||
except Failure as exc:
|
||||
with lock:
|
||||
errors.append(str(exc))
|
||||
|
||||
threads = [threading.Thread(target=request) for _ in range(count)]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
thread.join(30)
|
||||
message_ids = {row[0] for row in rows}
|
||||
if errors or len(rows) != count or len(message_ids) != 1:
|
||||
raise RuntimeError("parallel idempotency check failed")
|
||||
return {
|
||||
"parallel_requests": count,
|
||||
"successful_responses": len(rows),
|
||||
"failed_responses": len(errors),
|
||||
"unique_message_ids": len(message_ids),
|
||||
"observed_instances": sorted({row[1] for row in rows}),
|
||||
}
|
||||
|
||||
|
||||
def environment(env_file):
|
||||
mem_total_kib = 0
|
||||
with open("/proc/meminfo", encoding="utf-8") as meminfo:
|
||||
for line in meminfo:
|
||||
if line.startswith("MemTotal:"):
|
||||
mem_total_kib = int(line.split()[1])
|
||||
break
|
||||
limits = []
|
||||
for container_id in compose(env_file, "ps", "-q").stdout.splitlines():
|
||||
data = inspect(container_id)
|
||||
limits.append({
|
||||
"service": service(data["Name"].lstrip("/")),
|
||||
"nano_cpus": data["HostConfig"].get("NanoCpus", 0),
|
||||
"memory_limit_bytes": data["HostConfig"].get("Memory", 0),
|
||||
})
|
||||
return {
|
||||
"utc_started_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"git_commit": command(["git", "rev-parse", "HEAD"]).stdout.strip(),
|
||||
"working_tree_dirty": bool(command(["git", "status", "--porcelain"]).stdout.strip()),
|
||||
"logical_cpu_count": os.cpu_count(),
|
||||
"host_memory_mib": round(mem_total_kib / 1024, 1),
|
||||
"docker_engine": command(["docker", "version", "--format", "{{.Server.Version}}"]).stdout.strip(),
|
||||
"docker_compose": command(["docker", "compose", "version", "--short"]).stdout.strip(),
|
||||
"container_limits": limits,
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--base-url", default="http://localhost:8080")
|
||||
parser.add_argument("--env-file", default=".env")
|
||||
parser.add_argument("--concurrency", type=int, default=8)
|
||||
parser.add_argument("--baseline-seconds", type=float, default=12)
|
||||
parser.add_argument("--action-after-seconds", type=float, default=3)
|
||||
parser.add_argument("--post-action-seconds", type=float, default=8)
|
||||
parser.add_argument("--output", default="evidence/grade4/results.json")
|
||||
args = parser.parse_args()
|
||||
if not 1 <= args.concurrency <= 64:
|
||||
raise SystemExit("concurrency must be 1..64")
|
||||
|
||||
original = wait_api(args.env_file, 2)
|
||||
if len(original) != 2:
|
||||
raise RuntimeError("experiment requires exactly two initial API replicas")
|
||||
client = Client(args.base_url, timeout=5)
|
||||
alice, _, _, chat = client.bootstrap()
|
||||
for index in range(12):
|
||||
client.message(alice, chat, f"grade4-warmup-{index}")
|
||||
|
||||
phases = [load_phase("baseline", alice, chat, args)]
|
||||
targets = [(container_id, inspect(container_id)["Config"]["Hostname"])
|
||||
for container_id in original]
|
||||
for ordinal, (container_id, target) in enumerate(targets, start=1):
|
||||
def stop_target(instance=target):
|
||||
hook_env = dict(os.environ)
|
||||
hook_env.update({"ENV_FILE": args.env_file, "TARGET_INSTANCE_ID": instance})
|
||||
result = command([ROOT / "scripts" / "stop-one.sh"], env=hook_env, timeout=30)
|
||||
return result.stdout.strip().splitlines()[-1]
|
||||
|
||||
phases.append(load_phase(
|
||||
f"failover-{ordinal}", alice, chat, args,
|
||||
action=stop_target, target=target,
|
||||
))
|
||||
command(["docker", "start", container_id])
|
||||
wait_api(args.env_file, 2)
|
||||
|
||||
idem = idempotency(alice, chat, args.base_url)
|
||||
|
||||
def rolling():
|
||||
hook_env = dict(os.environ)
|
||||
hook_env.update({"ENV_FILE": args.env_file, "DESIRED_REPLICAS": "2"})
|
||||
result = command(
|
||||
[ROOT / "scripts" / "rolling-update.sh"],
|
||||
env=hook_env,
|
||||
timeout=240,
|
||||
)
|
||||
return result.stdout.strip().splitlines()[-1]
|
||||
|
||||
phases.append(load_phase("rolling-update", alice, chat, args, action=rolling))
|
||||
wait_api(args.env_file, 2)
|
||||
|
||||
report = {
|
||||
"schema_version": 1,
|
||||
"environment": environment(args.env_file),
|
||||
"methodology": {
|
||||
"base_url": args.base_url,
|
||||
"concurrency": args.concurrency,
|
||||
"baseline_seconds": args.baseline_seconds,
|
||||
"action_after_seconds": args.action_after_seconds,
|
||||
"post_action_seconds": args.post_action_seconds,
|
||||
"warmup_requests": 12,
|
||||
"workload": "closed-loop POST messages via Nginx; no automatic retries",
|
||||
},
|
||||
"phases": phases,
|
||||
"idempotency": idem,
|
||||
"dataset": {
|
||||
"warmup_requests": 12,
|
||||
"minimum_confirmed_messages": 13 + sum(
|
||||
phase["successful_requests"] for phase in phases
|
||||
),
|
||||
},
|
||||
"limitations": [
|
||||
"single Docker host",
|
||||
"closed-loop coordinated omission",
|
||||
"successful-response latency percentiles only",
|
||||
"PostgreSQL and Nginx remain single points of failure",
|
||||
],
|
||||
}
|
||||
destination = ROOT / args.output
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n")
|
||||
try:
|
||||
displayed_output = destination.relative_to(ROOT)
|
||||
except ValueError:
|
||||
displayed_output = destination
|
||||
print(json.dumps({
|
||||
"output": str(displayed_output),
|
||||
"phases": [{
|
||||
"name": phase["name"],
|
||||
"successful_rps": phase["successful_rps"],
|
||||
"failed_requests": phase["failed_requests"],
|
||||
"recovery_seconds": phase["recovery_seconds"],
|
||||
} for phase in phases],
|
||||
"idempotency": idem,
|
||||
}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+203
@@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Render portable SVG graphs and a Markdown summary from grade-4 results."""
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
STYLE = """
|
||||
text { font-family: Inter, Segoe UI, sans-serif; fill: #eceef0; }
|
||||
.title { font-size: 26px; font-weight: 650; }
|
||||
.label { font-size: 14px; fill: #c8cbd0; }
|
||||
.small { font-size: 12px; fill: #a2a5aa; }
|
||||
.grid { stroke: #34373d; stroke-width: 1; }
|
||||
.bar { fill: #cfd2d6; }
|
||||
.bar2 { fill: #73777f; }
|
||||
.line { fill: none; stroke: #f4f5f6; stroke-width: 3; }
|
||||
.error { fill: #8f949c; stroke: #f4f5f6; stroke-width: 1; }
|
||||
.event { stroke: #a2a5aa; stroke-width: 2; stroke-dasharray: 6 5; }
|
||||
"""
|
||||
|
||||
|
||||
def svg(title, body, width=1100, height=560):
|
||||
return f"""<svg xmlns="http://www.w3.org/2000/svg" width="{width}" height="{height}" viewBox="0 0 {width} {height}">
|
||||
<rect width="100%" height="100%" fill="#0e1013"/>
|
||||
<style>{STYLE}</style>
|
||||
<text x="48" y="42" class="title">{html.escape(title)}</text>
|
||||
{body}
|
||||
</svg>
|
||||
"""
|
||||
|
||||
|
||||
def grid(maximum, left, top, width, height):
|
||||
parts = []
|
||||
for step in range(6):
|
||||
value = maximum * step / 5
|
||||
y = top + height - height * step / 5
|
||||
parts.append(f'<line x1="{left}" y1="{y:.1f}" x2="{left+width}" y2="{y:.1f}" class="grid"/>')
|
||||
parts.append(f'<text x="{left-12}" y="{y+4:.1f}" text-anchor="end" class="small">{value:.0f}</text>')
|
||||
return parts
|
||||
|
||||
|
||||
def throughput(phases, output):
|
||||
left, top, width, height = 80, 90, 960, 360
|
||||
maximum = max((phase["successful_rps"] for phase in phases), default=1) or 1
|
||||
parts = grid(maximum, left, top, width, height)
|
||||
slot = width / max(1, len(phases))
|
||||
for index, phase in enumerate(phases):
|
||||
bar_width = min(120, slot * 0.5)
|
||||
x = left + slot * index + (slot - bar_width) / 2
|
||||
bar_height = height * phase["successful_rps"] / maximum
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="10" class="bar"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-10:.1f}" text-anchor="middle" class="label">{phase["successful_rps"]:.1f} RPS</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+28}" text-anchor="middle" class="label">{html.escape(phase["name"])}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+48}" text-anchor="middle" class="small">errors {phase["error_ratio"]*100:.2f}%</text>')
|
||||
parts.append(f'<text x="24" y="{top+height/2}" transform="rotate(-90 24 {top+height/2})" class="label">Successful requests per second</text>')
|
||||
output.write_text(svg("Grade 4: throughput and errors", "\n".join(parts)))
|
||||
|
||||
|
||||
def latency(phases, output):
|
||||
left, top, width, height = 80, 90, 960, 360
|
||||
values = [value for phase in phases
|
||||
for value in phase["successful_latency_ms"].values()
|
||||
if value is not None]
|
||||
maximum = max(values, default=1) or 1
|
||||
parts = grid(maximum, left, top, width, height)
|
||||
slot = width / max(1, len(phases))
|
||||
for phase_index, phase in enumerate(phases):
|
||||
group_x = left + slot * phase_index
|
||||
bar_width = min(42, slot / 5)
|
||||
for metric_index, metric in enumerate(("p50", "p95", "p99")):
|
||||
value = phase["successful_latency_ms"][metric] or 0
|
||||
bar_height = height * value / maximum
|
||||
x = group_x + slot * 0.2 + metric_index * (bar_width + 8)
|
||||
y = top + height - bar_height
|
||||
css = "bar" if metric_index == 0 else "bar2"
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="7" class="{css}"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-7:.1f}" text-anchor="middle" class="small">{value:.0f}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+20}" text-anchor="middle" class="small">{metric}</text>')
|
||||
parts.append(f'<text x="{group_x+slot/2:.1f}" y="{top+height+48}" text-anchor="middle" class="label">{html.escape(phase["name"])}</text>')
|
||||
parts.append(f'<text x="24" y="{top+height/2}" transform="rotate(-90 24 {top+height/2})" class="label">Successful latency, ms</text>')
|
||||
output.write_text(svg("Grade 4: successful latency percentiles", "\n".join(parts)))
|
||||
|
||||
|
||||
def timeline(phases, output):
|
||||
left, top, width, height = 70, 90, 980, 390
|
||||
rows, events, offset = [], [], 0
|
||||
for phase in phases:
|
||||
buckets = phase["per_second"]
|
||||
for second, counts in sorted(buckets.items(), key=lambda item: int(item[0])):
|
||||
rows.append((offset + int(second), counts))
|
||||
if phase.get("action"):
|
||||
events.append((offset + phase["action"]["started_at"], phase["name"]))
|
||||
offset += max([int(key) for key in buckets] or [0]) + 2
|
||||
maximum = max((counts["successful"] for _, counts in rows), default=1) or 1
|
||||
max_x = max((second for second, _ in rows), default=1) or 1
|
||||
parts, points = grid(maximum, left, top, width, height), []
|
||||
for second, counts in rows:
|
||||
x = left + width * second / max_x
|
||||
y = top + height - height * counts["successful"] / maximum
|
||||
points.append(f"{x:.1f},{y:.1f}")
|
||||
if counts["failed"]:
|
||||
radius = min(12, 3 + counts["failed"])
|
||||
parts.append(f'<circle cx="{x:.1f}" cy="{top+height-8}" r="{radius}" class="error"/>')
|
||||
parts.append(f'<polyline points="{" ".join(points)}" class="line"/>')
|
||||
for second, label in events:
|
||||
x = left + width * second / max_x
|
||||
parts.append(f'<line x1="{x:.1f}" y1="{top}" x2="{x:.1f}" y2="{top+height}" class="event"/>')
|
||||
parts.append(f'<text x="{x+6:.1f}" y="{top+18}" class="small">{html.escape(label)} action</text>')
|
||||
parts.append(f'<text x="{left}" y="{top+height+32}" class="label">One-second buckets; circles mark failed requests</text>')
|
||||
output.write_text(svg("Grade 4: request timeline", "\n".join(parts), height=600))
|
||||
|
||||
|
||||
def resources(phases, output):
|
||||
combined = defaultdict(lambda: {"cpu": [], "memory": []})
|
||||
for phase in phases:
|
||||
for name, values in phase["resources"].items():
|
||||
combined[name]["cpu"].append(values["cpu_percent_max"])
|
||||
combined[name]["memory"].append(values["memory_mib_max"])
|
||||
names, parts = sorted(combined), []
|
||||
panels = [
|
||||
("Peak CPU, %", "cpu", 80, 90, 420, 360),
|
||||
("Peak memory, MiB", "memory", 600, 90, 420, 360),
|
||||
]
|
||||
for title, metric, left, top, width, height in panels:
|
||||
maximum = max((max(combined[name][metric]) for name in names), default=1) or 1
|
||||
parts.append(f'<text x="{left}" y="{top-18}" class="label">{title}</text>')
|
||||
slot = width / max(1, len(names))
|
||||
for index, name in enumerate(names):
|
||||
value = max(combined[name][metric])
|
||||
bar_width = min(70, slot * 0.55)
|
||||
x = left + slot * index + (slot - bar_width) / 2
|
||||
bar_height = height * value / maximum
|
||||
y = top + height - bar_height
|
||||
parts.append(f'<rect x="{x:.1f}" y="{y:.1f}" width="{bar_width:.1f}" height="{bar_height:.1f}" rx="9" class="bar"/>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{y-8:.1f}" text-anchor="middle" class="small">{value:.1f}</text>')
|
||||
parts.append(f'<text x="{x+bar_width/2:.1f}" y="{top+height+25}" text-anchor="middle" class="small">{html.escape(name)}</text>')
|
||||
output.write_text(svg("Grade 4: peak container resources", "\n".join(parts), height=600))
|
||||
|
||||
|
||||
def summary(report, output):
|
||||
method = report["methodology"]
|
||||
lines = [
|
||||
"# Lab 3 grade-4 experiment", "",
|
||||
f"Generated: {report['environment']['utc_started_at']}.", "",
|
||||
"## Method", "",
|
||||
f"- concurrency: {method['concurrency']};",
|
||||
f"- baseline duration: {method['baseline_seconds']} s;",
|
||||
"- load: closed-loop POST messages through Nginx;",
|
||||
"- failover: both original API containers stopped and restored in turn;",
|
||||
"- rolling update: one surge replica, old replicas replaced one by one;",
|
||||
"- percentiles cover successful responses; failures are separate.", "",
|
||||
"## Results", "",
|
||||
"| Phase | RPS | Errors | p50 ms | p95 ms | p99 ms | Recovery s |",
|
||||
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
|
||||
]
|
||||
for phase in report["phases"]:
|
||||
lat = phase["successful_latency_ms"]
|
||||
recovery = phase["recovery_seconds"]
|
||||
lines.append(
|
||||
f"| {phase['name']} | {phase['successful_rps']} | "
|
||||
f"{phase['failed_requests']} ({phase['error_ratio']*100:.3f}%) | "
|
||||
f"{lat['p50']} | {lat['p95']} | {lat['p99']} | "
|
||||
f"{recovery if recovery is not None else 'n/a'} |"
|
||||
)
|
||||
idem = report["idempotency"]
|
||||
lines += [
|
||||
"", "## Parallel idempotency", "",
|
||||
f"{idem['parallel_requests']} requests produced "
|
||||
f"{idem['unique_message_ids']} unique message ID across "
|
||||
f"{len(idem['observed_instances'])} API instances.", "",
|
||||
"## Graphs", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"", "",
|
||||
"## Scope", "",
|
||||
"Single-host process failure and rolling replacement only. PostgreSQL "
|
||||
"and Nginx remain single points of failure.", "",
|
||||
]
|
||||
output.write_text("\n".join(lines))
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("results", nargs="?", default="evidence/grade4/results.json")
|
||||
args = parser.parse_args()
|
||||
source = Path(args.results)
|
||||
report = json.loads(source.read_text())
|
||||
destination = source.parent
|
||||
phases = report["phases"]
|
||||
throughput(phases, destination / "throughput-errors.svg")
|
||||
latency(phases, destination / "latency.svg")
|
||||
timeline(phases, destination / "timeline.svg")
|
||||
resources(phases, destination / "resources.svg")
|
||||
summary(report, destination / "README.md")
|
||||
print(f"rendered grade-4 evidence in {destination}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
project_root=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd)
|
||||
cd "$project_root"
|
||||
|
||||
env_file=${ENV_FILE:-.env}
|
||||
desired_replicas=${DESIRED_REPLICAS:-2}
|
||||
wait_timeout=${ROLLING_WAIT_TIMEOUT:-90}
|
||||
stop_timeout=${ROLLING_STOP_TIMEOUT:-15}
|
||||
dns_settle_delay=${ROLLING_DNS_SETTLE_DELAY:-2}
|
||||
|
||||
case "$desired_replicas" in
|
||||
''|*[!0-9]*)
|
||||
printf '%s\n' "DESIRED_REPLICAS must be an integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
case "$dns_settle_delay" in
|
||||
''|*[!0-9]*)
|
||||
printf '%s\n' "ROLLING_DNS_SETTLE_DELAY must be a non-negative integer" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
if [ "$desired_replicas" -lt 2 ]; then
|
||||
printf '%s\n' "rolling update requires at least two desired replicas" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
compose() {
|
||||
docker compose --env-file "$env_file" "$@"
|
||||
}
|
||||
|
||||
healthy_api_count() {
|
||||
healthy=0
|
||||
for container_id in $(compose ps -q api); do
|
||||
status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id")
|
||||
if [ "$status" = "healthy" ] || [ "$status" = "running" ]; then
|
||||
healthy=$((healthy + 1))
|
||||
fi
|
||||
done
|
||||
printf '%s\n' "$healthy"
|
||||
}
|
||||
|
||||
wait_for_healthy() {
|
||||
expected=$1
|
||||
deadline=$(( $(date +%s) + wait_timeout ))
|
||||
while [ "$(healthy_api_count)" -lt "$expected" ]; do
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
printf '%s\n' "timed out waiting for $expected healthy API replicas" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
old_containers=$(compose ps -q api)
|
||||
old_count=$(printf '%s\n' "$old_containers" | awk 'NF { count++ } END { print count+0 }')
|
||||
if [ "$old_count" -ne "$desired_replicas" ]; then
|
||||
printf '%s\n' "expected $desired_replicas running API containers, found $old_count" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
surge_replicas=$((desired_replicas + 1))
|
||||
|
||||
printf '%s\n' "building replacement API image"
|
||||
compose build api
|
||||
|
||||
printf '%s\n' "starting one surge replica"
|
||||
compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
|
||||
wait_for_healthy "$surge_replicas"
|
||||
sleep "$dns_settle_delay"
|
||||
|
||||
for old_container in $old_containers; do
|
||||
old_instance=$(docker inspect --format '{{.Config.Hostname}}' "$old_container")
|
||||
printf '%s\n' "draining and replacing API instance $old_instance"
|
||||
docker stop --time "$stop_timeout" "$old_container" >/dev/null
|
||||
docker rm "$old_container" >/dev/null
|
||||
compose up -d --no-deps --scale api="$surge_replicas" --no-recreate api
|
||||
wait_for_healthy "$surge_replicas"
|
||||
sleep "$dns_settle_delay"
|
||||
done
|
||||
|
||||
printf '%s\n' "removing surge capacity"
|
||||
compose up -d --no-deps --scale api="$desired_replicas" --no-recreate api
|
||||
wait_for_healthy "$desired_replicas"
|
||||
sleep "$dns_settle_delay"
|
||||
|
||||
printf '%s\n' "rolling update completed with $desired_replicas healthy API replicas"
|
||||
Reference in New Issue
Block a user