From c8fc32e8b18c8dbdf8a72b505a225f5a09b598f9 Mon Sep 17 00:00:00 2001 From: da1zy3r Date: Sat, 24 Oct 2026 00:43:12 +0300 Subject: [PATCH] Fix nginx routing --- .github/pull_request_template.md | 1 + REPORT.md | 20 +++--- compose.yaml | 5 ++ evidence/failover-1.json | 72 +++++++++---------- evidence/failover-2.json | 70 +++++++++---------- evidence/final-containers.json | 38 +++++----- evidence/grade4-summary.json | 90 ++++++++++++------------ evidence/idempotency.json | 4 +- evidence/public-failover-1.txt | 2 +- evidence/public-failover-2.txt | 2 +- evidence/readiness.json | 74 ++++++++++++++++++++ evidence/restart.txt | 20 +++--- evidence/rolling-load.json | 116 +++++++++++++++---------------- evidence/unit-tests.txt | 2 +- infra/nginx.conf | 4 +- infra/readiness.sh | 32 +++++++++ scripts/ops.py | 2 +- scripts/verify_readiness.py | 109 +++++++++++++++++++++++++++++ 18 files changed, 442 insertions(+), 221 deletions(-) create mode 100644 evidence/readiness.json create mode 100644 infra/readiness.sh create mode 100644 scripts/verify_readiness.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index d4f63e5..022c6aa 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -11,6 +11,7 @@ cp .env.example .env # задать POSTGRES_PASSWORD docker compose up -d --build --wait --wait-timeout 120 python -X utf8 scripts/verify_lab3.py +python -X utf8 scripts/verify_readiness.py docker compose exec -T api python -m unittest discover -s student-tests -v ``` diff --git a/REPORT.md b/REPORT.md index acdbf22..9943e78 100644 --- a/REPORT.md +++ b/REPORT.md @@ -2,7 +2,7 @@ - Автор, группа: Ромашко Владимир, БВТ2301 - Целевая оценка; выбранный трек: **5**, трек 1 "Цена масштабирования" -- Commit/PR: база уровня 4 — `1d85938`; коммит/PR трека 1 — после публикации +- Commit/PR: уровень 4 — `1d85938`, трек 1 — `3417e22`; исправление readiness — после публикации - Источники: контракт задания, документация Docker Compose / nginx / PostgreSQL. Библиотеки — `requirements.txt`; графики — matplotlib. Использован Codex ## Запуск из чистого clone @@ -15,6 +15,7 @@ Copy-Item .env.example .env docker compose config --quiet docker compose up -d --build --wait --wait-timeout 120 python -X utf8 scripts/verify_lab3.py +python -X utf8 scripts/verify_readiness.py docker compose exec -T api python -m unittest discover -s student-tests -v ``` @@ -42,7 +43,7 @@ docker compose -p mtusi2026lab3-track1 -f compose.yaml -f infra/scaling.compose. PostgreSQL — источник истины. Сообщение, версия истории и ключ идемпотентности — одна транзакция с блокировкой строки чата. Повтор ключа → то же сообщение; другое тело → 409. Сессии/кэш общие; Redis — AOF. Миграция 003; смешанное обновление lab2→lab3 не поддерживается. -Healthcheck API: 2 с / timeout 2 с / 2 неудачи / start 5 с. Nginx: connect 1 с, read 6 с, `max_fails=1`, exclude 2 с; DNS 2 с. SIGTERM: до 5 с, Compose ждёт 10 с. Таймауты БД: connect/pool/lock 2 с, statement 3 с; Redis 1 с. +Healthcheck API: 2 с / timeout 2 с / 2 неудачи / start 5 с. Скрипт `infra/readiness.sh` при nginx получает все IPv4-адреса API и проверяет `/health/ready` (timeout 1 с на реплику, пауза между циклами 2 с). В upstream включаются только готовые API; при изменении списка выполняется reload. Для двух API обнаружение занимает до ~5 с с учётом DNS и проверок; текущие запросы завершаются при graceful reload. Когда готовых API нет, nginx возвращает 502. Docker-статус сам по себе маршрутизацией не управляет. Nginx: connect 1 с, read 6 с; пассивное исключение остаётся дополнительной защитой. SIGTERM: до 5 с, Compose ждёт 10 с. Таймауты БД: connect/pool/lock 2 с, statement 3 с; Redis 1 с. Лимиты: API 1 CPU/512 MiB каждая, PostgreSQL 1/512, nginx 0,5/128, Redis 0,5/256. Наружу только nginx; `container_name` нет. В `.env` — БД, TTL, лимиты запросов; `APP_PORT` — порт nginx. Хосты PostgreSQL/Redis задаёт Compose. @@ -54,25 +55,26 @@ Healthcheck API: 2 с / timeout 2 с / 2 неудачи / start 5 с. Nginx: con | Общая сессия, разные X-Instance-Id | smoke через nginx | один пользователь на двух API | [smoke](evidence/final-smoke.txt) | | Идемпотентность через разные API | конкурентные повторы в `verify_lab3.py` | 24 запроса / 8 одновременно → 1 сообщение; конфликт тела → 409 | [idempotency](evidence/idempotency.json) | | Контракт, smoke, persistence | `verify_lab3.py`: check, smoke, restart | PASS; перезапущены API, nginx, PostgreSQL, Redis | [check](evidence/check.txt), [persistence](evidence/persistence.txt), [restart](evidence/restart.txt), [idempotency-restart](evidence/idempotency-restart.json) | -| Отказ любой API, восстановление ≤10 с | поочерёдный stop | обе проверки PASS, по 124 записи без ошибок | [failover-1](evidence/public-failover-1.txt), [failover-2](evidence/public-failover-2.txt) | +| Отказ любой API, восстановление ≤10 с | поочерёдный stop | обе проверки PASS, 117/118 записей без ошибок | [failover-1](evidence/public-failover-1.txt), [failover-2](evidence/public-failover-2.txt) | | Health/DNS/timeouts/SIGTERM | stop и замена под нагрузкой | недоступные API исключены | [nginx](evidence/nginx-check.txt), [failover-1](evidence/failover-1.json), [failover-2](evidence/failover-2.json) | +| Живая, но неготовая API | `python -X utf8 scripts/verify_readiness.py` | При неверном REDIS_URL: live=200, ready=503; 24 регистрации идут здоровой API. После восстановления используются обе; без готовых API — 502 | [readiness](evidence/readiness.json) | | Лимиты CPU/RAM, порты | inspect после запуска | лимиты есть, API без портов наружу | [containers](evidence/final-containers.json) | | Rolling update, отдельные миграции | `verify_lab3.py` rolling; unittest | обе API заменены без потерь; 15 тестов OK | [rolling](evidence/rolling-load.json), [unit-tests](evidence/unit-tests.txt) | | Трек 1: scaling / saturation / improvement | `scaling_suite.py` ×3 | 30 прогонов; см. ниже | [summary.csv](evidence/track1/summary.csv), [track1/](evidence/track1/) | ## Эксперименты и отказы -Дата: 23.10.2026. Docker Desktop Linux x86_64, VM 8 CPU / 7,76 GiB RAM. Отказы: 2 клиента, POST+GET+пауза 20 мс, прогрев 3 с, нагрузка ≥20 с, 1 прогон на воздействие. +Проверка отказов после исправления: 24.10.2026. Docker Desktop Linux x86_64, VM 8 CPU / 7,76 GiB RAM. Отказы: 2 клиента, POST+GET+пауза 20 мс, прогрев 3 с, нагрузка ≥20 с, 1 прогон на воздействие. | Воздействие | Успешных HTTP/с | Ошибки | p50 / p95 / p99, мс | Восстановление / потеря данных | | --- | ---: | ---: | --- | --- | -| Stop API `6fc32ef53bc9` | 61,90 | 0 | 18,00 / 52,71 / 71,25 | чтение 2,466 с, запись 2,458 с; потерь нет | -| Stop API `a6ee3d7e3779` | 65,84 | 0 | 16,38 / 48,29 / 70,66 | чтение 2,409 с, запись 2,397 с; потерь нет | -| Поочерёдная замена API | 62,91 | 0 | 16,51 / 48,68 / 75,14 | запросов не теряли; потерь данных нет | +| Stop API `ac72a6939d0d` | 53,79 | 0 | 22,01 / 64,40 / 78,39 | чтение 2,393 с, запись 2,369 с; потерь нет | +| Stop API `2891905bd75d` | 53,76 | 0 | 22,09 / 60,96 / 72,63 | чтение 2,505 с, запись 2,522 с; потерь нет | +| Поочерёдная замена API | 50,63 | 0 | 23,70 / 66,04 / 84,77 | запросов не теряли; потерь данных нет | -Время восстановления включает Docker CLI. Замена: 29,127 с; макс. пауза между успешными R/W: 0,216 / 0,231 с. Сырые сводки: [grade4-summary.json](evidence/grade4-summary.json). +Время восстановления включает Docker CLI. Замена: 29,611 с; макс. пауза между успешными R/W: 0,195 / 0,182 с. Сырые сводки: [grade4-summary.json](evidence/grade4-summary.json). -**Трек 1.** Гипотеза: доп. API помогают до насыщения общей БД; при фиксированном бюджете дробление CPU может ухудшить результат. Профиль: 1 пользователь/чат, 100 стартовых сообщений, POST 64 символа + уникальный ключ. Сравнение: 8 клиентов без пауз, прогрев 3 с + 15 с, 3 повтора с чередованием порядка; БД очищается перед прогоном. +**Трек 1.** Измерения ниже относятся к коммиту 3417e22 до исправления readiness; код приложения не изменён. Гипотеза: доп. API помогают до насыщения общей БД; при фиксированном бюджете дробление CPU может ухудшить результат. Профиль: 1 пользователь/чат, 100 стартовых сообщений, POST 64 символа + уникальный ключ. Сравнение: 8 клиентов без пауз, прогрев 3 с + 15 с, 3 повтора с чередованием порядка; БД очищается перед прогоном. Фиксированный бюджет API: 2 CPU / 1024 MiB на 1/2/4 реплики. Растущий: по 1 CPU / 512 MiB на реплику. Остальные сервисы — как в стенде. Генератор отдельно: 2 CPU / 512 MiB. В 18 прогонах сравнения ошибок и потерь нет. diff --git a/compose.yaml b/compose.yaml index a9d141a..dea1b7c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -50,10 +50,15 @@ services: nginx: image: nginx:1.28.0-alpine + entrypoint: [sh, /etc/nginx/readiness.sh] + dns_opt: + - timeout:1 + - attempts:1 ports: - '127.0.0.1:${APP_PORT:-8080}:8080' volumes: - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/readiness.sh:/etc/nginx/readiness.sh:ro networks: [edge, backend] depends_on: api: diff --git a/evidence/failover-1.json b/evidence/failover-1.json index 95dfab2..6d4644e 100644 --- a/evidence/failover-1.json +++ b/evidence/failover-1.json @@ -1,56 +1,56 @@ { "summary": { - "duration_seconds": 20.032, + "duration_seconds": 20.041, "concurrency": 2, - "successful_http_rps": 61.9, - "successful_write_rps": 30.95, - "successful_requests": 1240, + "successful_http_rps": 53.79, + "successful_write_rps": 26.89, + "successful_requests": 1078, "errors": 0, "action_started_seconds": 3.001, - "action_finished_seconds": 5.454, + "action_finished_seconds": 5.364, "first_read_write_after_action_seconds_from_start": { - "read": 2.466, - "write": 2.458 + "read": 2.393, + "write": 2.369 }, "max_success_gap_seconds": { - "read": 0.12, - "write": 0.116 + "read": 0.198, + "write": 0.143 }, "latency_ms": { - "50": 18.0, - "95": 52.71, - "99": 71.25 + "50": 22.01, + "95": 64.4, + "99": 78.39 }, "confirmed_messages_preserved": true, "duplicate_messages": 0, "instances": [ - "6fc32ef53bc9", - "a6ee3d7e3779" + "2891905bd75d", + "ac72a6939d0d" ], - "target": "6fc32ef53bc9" + "target": "ac72a6939d0d" }, "containers": { - "target": "6fc32ef53bc9", + "target": "ac72a6939d0d", "before": [ { - "id": "6fc32ef53bc923ae2b9300d22d99d833c1b59a0689e6cbbb78ca3dab0a1654d6", - "name": "/mtusi2026lab3-api-3", - "instance": "6fc32ef53bc9", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "ac72a6939d0d1eb94252d534981bb5e40aa424bd42a96b6e3a76bdbaf9213805", + "name": "/mtusi2026lab3-api-5", + "instance": "ac72a6939d0d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:19:29.450305334Z", + "started": "2026-10-23T21:34:50.055092146Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "a6ee3d7e3779d847efe497f22021f9942f805202ade85495734e1f93418aeb5a", - "name": "/mtusi2026lab3-api-4", - "instance": "a6ee3d7e3779", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "2891905bd75d391242dd54891f53ba42163183d556b740f396e557efab08a682", + "name": "/mtusi2026lab3-api-6", + "instance": "2891905bd75d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:20:24.84845663Z", + "started": "2026-10-23T21:35:45.544522094Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, @@ -59,24 +59,24 @@ ], "after": [ { - "id": "6fc32ef53bc923ae2b9300d22d99d833c1b59a0689e6cbbb78ca3dab0a1654d6", - "name": "/mtusi2026lab3-api-3", - "instance": "6fc32ef53bc9", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "ac72a6939d0d1eb94252d534981bb5e40aa424bd42a96b6e3a76bdbaf9213805", + "name": "/mtusi2026lab3-api-5", + "instance": "ac72a6939d0d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "exited", - "started": "2026-10-23T12:19:29.450305334Z", + "started": "2026-10-23T21:34:50.055092146Z", "health": "unhealthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "a6ee3d7e3779d847efe497f22021f9942f805202ade85495734e1f93418aeb5a", - "name": "/mtusi2026lab3-api-4", - "instance": "a6ee3d7e3779", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "2891905bd75d391242dd54891f53ba42163183d556b740f396e557efab08a682", + "name": "/mtusi2026lab3-api-6", + "instance": "2891905bd75d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:20:24.84845663Z", + "started": "2026-10-23T21:35:45.544522094Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, diff --git a/evidence/failover-2.json b/evidence/failover-2.json index 34fbcc6..ca80da5 100644 --- a/evidence/failover-2.json +++ b/evidence/failover-2.json @@ -2,55 +2,55 @@ "summary": { "duration_seconds": 20.05, "concurrency": 2, - "successful_http_rps": 65.84, - "successful_write_rps": 32.92, - "successful_requests": 1320, + "successful_http_rps": 53.76, + "successful_write_rps": 26.88, + "successful_requests": 1078, "errors": 0, "action_started_seconds": 3.001, - "action_finished_seconds": 5.363, + "action_finished_seconds": 5.498, "first_read_write_after_action_seconds_from_start": { - "read": 2.409, - "write": 2.397 + "read": 2.505, + "write": 2.522 }, "max_success_gap_seconds": { - "read": 0.114, - "write": 0.102 + "read": 0.157, + "write": 0.161 }, "latency_ms": { - "50": 16.38, - "95": 48.29, - "99": 70.66 + "50": 22.09, + "95": 60.96, + "99": 72.63 }, "confirmed_messages_preserved": true, "duplicate_messages": 0, "instances": [ - "6fc32ef53bc9", - "a6ee3d7e3779" + "2891905bd75d", + "ac72a6939d0d" ], - "target": "a6ee3d7e3779" + "target": "2891905bd75d" }, "containers": { - "target": "a6ee3d7e3779", + "target": "2891905bd75d", "before": [ { - "id": "6fc32ef53bc923ae2b9300d22d99d833c1b59a0689e6cbbb78ca3dab0a1654d6", - "name": "/mtusi2026lab3-api-3", - "instance": "6fc32ef53bc9", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "ac72a6939d0d1eb94252d534981bb5e40aa424bd42a96b6e3a76bdbaf9213805", + "name": "/mtusi2026lab3-api-5", + "instance": "ac72a6939d0d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:20:57.481834724Z", + "started": "2026-10-23T21:36:18.801951904Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "a6ee3d7e3779d847efe497f22021f9942f805202ade85495734e1f93418aeb5a", - "name": "/mtusi2026lab3-api-4", - "instance": "a6ee3d7e3779", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "2891905bd75d391242dd54891f53ba42163183d556b740f396e557efab08a682", + "name": "/mtusi2026lab3-api-6", + "instance": "2891905bd75d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:20:24.84845663Z", + "started": "2026-10-23T21:35:45.544522094Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, @@ -59,24 +59,24 @@ ], "after": [ { - "id": "6fc32ef53bc923ae2b9300d22d99d833c1b59a0689e6cbbb78ca3dab0a1654d6", - "name": "/mtusi2026lab3-api-3", - "instance": "6fc32ef53bc9", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "ac72a6939d0d1eb94252d534981bb5e40aa424bd42a96b6e3a76bdbaf9213805", + "name": "/mtusi2026lab3-api-5", + "instance": "ac72a6939d0d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:20:57.481834724Z", + "started": "2026-10-23T21:36:18.801951904Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "a6ee3d7e3779d847efe497f22021f9942f805202ade85495734e1f93418aeb5a", - "name": "/mtusi2026lab3-api-4", - "instance": "a6ee3d7e3779", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "2891905bd75d391242dd54891f53ba42163183d556b740f396e557efab08a682", + "name": "/mtusi2026lab3-api-6", + "instance": "2891905bd75d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "exited", - "started": "2026-10-23T12:20:24.84845663Z", + "started": "2026-10-23T21:35:45.544522094Z", "health": "unhealthy", "cpu": 1000000000, "memory": 536870912, diff --git a/evidence/final-containers.json b/evidence/final-containers.json index a9d568b..a062089 100644 --- a/evidence/final-containers.json +++ b/evidence/final-containers.json @@ -1,47 +1,47 @@ [ { - "id": "52a2db0bf074fa8d0c8ac02253549dabd0fb214d4c2ab35eaa64174c43687f52", - "name": "/mtusi2026lab3-api-5", - "instance": "52a2db0bf074", - "image": "sha256:24f3d4879fd558d1741b8c68e8d6dd7ad1c3c2622d344e21d1f261c44ad6a168", + "id": "c22f03375149f347fcde4825cffb8f1a9f5bc81be5e40aab0124ce6f2481afcf", + "name": "/mtusi2026lab3-api-7", + "instance": "c22f03375149", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", "state": "running", - "started": "2026-10-23T12:34:15.338654375Z", + "started": "2026-10-23T21:37:20.306882221Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "c5211dc7b5256fc33563384f55525ce3f39839ad27c16205aad0a6f33364ae4a", - "name": "/mtusi2026lab3-api-6", - "instance": "c5211dc7b525", - "image": "sha256:24f3d4879fd558d1741b8c68e8d6dd7ad1c3c2622d344e21d1f261c44ad6a168", + "id": "42a599593c3be9d2ed03970716323a5dae60817b3b47eb2b6f8630a57563ac91", + "name": "/mtusi2026lab3-api-8", + "instance": "42a599593c3b", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", "state": "running", - "started": "2026-10-23T12:34:15.606253965Z", + "started": "2026-10-23T21:37:33.136215003Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "0fec18666f11eaf3b139f22852b71383ec3cf06b9cf171053950ca887a15119a", + "id": "d146d247a99436e9b6821270ecc4ae9b75170376e8c9bdba74168e01ecebfac7", "name": "/mtusi2026lab3-migrate-1", - "instance": "0fec18666f11", - "image": "sha256:24f3d4879fd558d1741b8c68e8d6dd7ad1c3c2622d344e21d1f261c44ad6a168", + "instance": "d146d247a994", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "exited", - "started": "2026-10-23T12:34:13.577380317Z", + "started": "2026-10-23T21:29:31.993889235Z", "health": null, "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "cb856359deb9d61c5130f7d9374aa7cecbe39999902e8f87560f4d5fc1a62f79", + "id": "1a760f69967b8d689bad4d578fefecb10de116955ead1ee38660d7f29ec61147", "name": "/mtusi2026lab3-nginx-1", - "instance": "cb856359deb9", + "instance": "1a760f69967b", "image": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", "state": "running", - "started": "2026-10-23T12:19:29.173913476Z", + "started": "2026-10-23T21:34:49.926357642Z", "health": "healthy", "cpu": 500000000, "memory": 134217728, @@ -60,7 +60,7 @@ "instance": "a1b2b187e907", "image": "sha256:304ab813518754228f9f792f79d6da36359b82d8ecf418096c636725f8c930ad", "state": "running", - "started": "2026-10-23T12:19:28.899601583Z", + "started": "2026-10-23T21:34:49.579202856Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, @@ -72,7 +72,7 @@ "instance": "57f3df8ca109", "image": "sha256:fbdbaea47b9ae4ecc2082ecdb4e1cea81e32176ffb1dcf643d422ad07427e5d9", "state": "running", - "started": "2026-10-23T12:19:28.754756013Z", + "started": "2026-10-23T21:34:49.490854509Z", "health": "healthy", "cpu": 500000000, "memory": 268435456, diff --git a/evidence/grade4-summary.json b/evidence/grade4-summary.json index d6f8410..7822e72 100644 --- a/evidence/grade4-summary.json +++ b/evidence/grade4-summary.json @@ -1,93 +1,93 @@ { "failover-1": { - "duration_seconds": 20.032, + "duration_seconds": 20.041, "concurrency": 2, - "successful_http_rps": 61.9, - "successful_write_rps": 30.95, - "successful_requests": 1240, + "successful_http_rps": 53.79, + "successful_write_rps": 26.89, + "successful_requests": 1078, "errors": 0, "action_started_seconds": 3.001, - "action_finished_seconds": 5.454, + "action_finished_seconds": 5.364, "first_read_write_after_action_seconds_from_start": { - "read": 2.466, - "write": 2.458 + "read": 2.393, + "write": 2.369 }, "max_success_gap_seconds": { - "read": 0.12, - "write": 0.116 + "read": 0.198, + "write": 0.143 }, "latency_ms": { - "50": 18.0, - "95": 52.71, - "99": 71.25 + "50": 22.01, + "95": 64.4, + "99": 78.39 }, "confirmed_messages_preserved": true, "duplicate_messages": 0, "instances": [ - "6fc32ef53bc9", - "a6ee3d7e3779" + "2891905bd75d", + "ac72a6939d0d" ], - "target": "6fc32ef53bc9" + "target": "ac72a6939d0d" }, "failover-2": { "duration_seconds": 20.05, "concurrency": 2, - "successful_http_rps": 65.84, - "successful_write_rps": 32.92, - "successful_requests": 1320, + "successful_http_rps": 53.76, + "successful_write_rps": 26.88, + "successful_requests": 1078, "errors": 0, "action_started_seconds": 3.001, - "action_finished_seconds": 5.363, + "action_finished_seconds": 5.498, "first_read_write_after_action_seconds_from_start": { - "read": 2.409, - "write": 2.397 + "read": 2.505, + "write": 2.522 }, "max_success_gap_seconds": { - "read": 0.114, - "write": 0.102 + "read": 0.157, + "write": 0.161 }, "latency_ms": { - "50": 16.38, - "95": 48.29, - "99": 70.66 + "50": 22.09, + "95": 60.96, + "99": 72.63 }, "confirmed_messages_preserved": true, "duplicate_messages": 0, "instances": [ - "6fc32ef53bc9", - "a6ee3d7e3779" + "2891905bd75d", + "ac72a6939d0d" ], - "target": "a6ee3d7e3779" + "target": "2891905bd75d" }, "rolling": { - "duration_seconds": 37.166, + "duration_seconds": 37.648, "concurrency": 2, - "successful_http_rps": 62.91, - "successful_write_rps": 31.45, - "successful_requests": 2338, + "successful_http_rps": 50.63, + "successful_write_rps": 25.31, + "successful_requests": 1906, "errors": 0, "action_started_seconds": 3.001, - "action_finished_seconds": 32.128, + "action_finished_seconds": 32.612, "first_read_write_after_action_seconds_from_start": { - "read": 29.135, - "write": 29.137 + "read": 29.611, + "write": 29.641 }, "max_success_gap_seconds": { - "read": 0.216, - "write": 0.231 + "read": 0.195, + "write": 0.182 }, "latency_ms": { - "50": 16.51, - "95": 48.68, - "99": 75.14 + "50": 23.7, + "95": 66.04, + "99": 84.77 }, "confirmed_messages_preserved": true, "duplicate_messages": 0, "instances": [ - "6fc32ef53bc9", - "8c864b1beb6a", - "a6ee3d7e3779", - "bb8669ea27d1" + "2891905bd75d", + "42a599593c3b", + "ac72a6939d0d", + "c22f03375149" ], "target": null } diff --git a/evidence/idempotency.json b/evidence/idempotency.json index 12a3216..610d150 100644 --- a/evidence/idempotency.json +++ b/evidence/idempotency.json @@ -2,8 +2,8 @@ "requests": 24, "concurrency": 8, "instances": [ - "6fc32ef53bc9", - "a6ee3d7e3779" + "2891905bd75d", + "ac72a6939d0d" ], "unique_messages": 1, "same_response": true, diff --git a/evidence/public-failover-1.txt b/evidence/public-failover-1.txt index 51c081c..24fa0ff 100644 --- a/evidence/public-failover-1.txt +++ b/evidence/public-failover-1.txt @@ -1,3 +1,3 @@ Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 2.204} +{"observed_instances_before": 2, "successful_writes": 117, "failed_attempts": 0, "first_recovery_seconds": 2.341} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-failover-2.txt b/evidence/public-failover-2.txt index 9c90634..755a8eb 100644 --- a/evidence/public-failover-2.txt +++ b/evidence/public-failover-2.txt @@ -1,3 +1,3 @@ Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 124, "failed_attempts": 0, "first_recovery_seconds": 2.233} +{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 0, "first_recovery_seconds": 2.634} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/readiness.json b/evidence/readiness.json new file mode 100644 index 0000000..859ed78 --- /dev/null +++ b/evidence/readiness.json @@ -0,0 +1,74 @@ +{ + "before": [ + { + "id": "c22f03375149f347fcde4825cffb8f1a9f5bc81be5e40aab0124ce6f2481afcf", + "name": "/mtusi2026lab3-api-7", + "instance": "c22f03375149", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", + "state": "running", + "started": "2026-10-23T21:37:20.306882221Z", + "health": "healthy", + "cpu": 1000000000, + "memory": 536870912, + "ports": {} + }, + { + "id": "42a599593c3be9d2ed03970716323a5dae60817b3b47eb2b6f8630a57563ac91", + "name": "/mtusi2026lab3-api-8", + "instance": "42a599593c3b", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", + "state": "running", + "started": "2026-10-23T21:37:33.136215003Z", + "health": "healthy", + "cpu": 1000000000, + "memory": 536870912, + "ports": {} + } + ], + "unready": { + "id": "5d07bde92a73fe7c4d83ee7b9ba04fb6ed42c6d54daecbd944ee9550c65a3e1b", + "instance": "5d07bde92a73", + "health": "unhealthy", + "ip": "172.22.0.6" + }, + "health_http": { + "live": 200, + "ready": 503 + }, + "upstream_when_unready": "server 172.22.0.5:8080 max_fails=1 fail_timeout=2s;", + "registrations_with_unready_peer": { + "c22f03375149": 24 + }, + "no_ready_backend_status": 502, + "registrations_after_recovery": { + "42a599593c3b": 15, + "c22f03375149": 9 + }, + "after": [ + { + "id": "c22f03375149f347fcde4825cffb8f1a9f5bc81be5e40aab0124ce6f2481afcf", + "name": "/mtusi2026lab3-api-7", + "instance": "c22f03375149", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", + "state": "running", + "started": "2026-10-23T21:39:42.239886207Z", + "health": "healthy", + "cpu": 1000000000, + "memory": 536870912, + "ports": {} + }, + { + "id": "42a599593c3be9d2ed03970716323a5dae60817b3b47eb2b6f8630a57563ac91", + "name": "/mtusi2026lab3-api-8", + "instance": "42a599593c3b", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", + "state": "running", + "started": "2026-10-23T21:39:41.935894867Z", + "health": "healthy", + "cpu": 1000000000, + "memory": 536870912, + "ports": {} + } + ], + "result": "PASS" +} diff --git a/evidence/restart.txt b/evidence/restart.txt index c840750..bd7390c 100644 --- a/evidence/restart.txt +++ b/evidence/restart.txt @@ -1,13 +1,13 @@ Command: docker compose restart api postgres redis nginx Before (UTC): -/mtusi2026lab3-api-3 2026-10-23T12:18:00.276504145Z -/mtusi2026lab3-api-4 2026-10-23T12:18:00.50467995Z -/mtusi2026lab3-nginx-1 2026-10-23T12:18:06.464884093Z -/mtusi2026lab3-postgres-1 2026-10-23T12:17:55.596165683Z -/mtusi2026lab3-redis-1 2026-10-23T12:17:55.593753417Z +/mtusi2026lab3-api-5 2026-10-23T21:32:42.391273254Z +/mtusi2026lab3-api-6 2026-10-23T21:32:42.144906592Z +/mtusi2026lab3-nginx-1 2026-10-23T21:30:17.969912705Z +/mtusi2026lab3-postgres-1 2026-10-23T21:29:29.040381628Z +/mtusi2026lab3-redis-1 2026-10-23T21:29:29.042717056Z After (UTC): -/mtusi2026lab3-api-3 2026-10-23T12:19:29.450305334Z -/mtusi2026lab3-api-4 2026-10-23T12:19:29.302397436Z -/mtusi2026lab3-nginx-1 2026-10-23T12:19:29.173913476Z -/mtusi2026lab3-postgres-1 2026-10-23T12:19:28.899601583Z -/mtusi2026lab3-redis-1 2026-10-23T12:19:28.754756013Z +/mtusi2026lab3-api-5 2026-10-23T21:34:50.055092146Z +/mtusi2026lab3-api-6 2026-10-23T21:34:50.12635016Z +/mtusi2026lab3-nginx-1 2026-10-23T21:34:49.926357642Z +/mtusi2026lab3-postgres-1 2026-10-23T21:34:49.579202856Z +/mtusi2026lab3-redis-1 2026-10-23T21:34:49.490854509Z diff --git a/evidence/rolling-load.json b/evidence/rolling-load.json index ac5fdf0..785b9ff 100644 --- a/evidence/rolling-load.json +++ b/evidence/rolling-load.json @@ -1,57 +1,57 @@ { "summary": { - "duration_seconds": 37.166, + "duration_seconds": 37.648, "concurrency": 2, - "successful_http_rps": 62.91, - "successful_write_rps": 31.45, - "successful_requests": 2338, + "successful_http_rps": 50.63, + "successful_write_rps": 25.31, + "successful_requests": 1906, "errors": 0, "action_started_seconds": 3.001, - "action_finished_seconds": 32.128, + "action_finished_seconds": 32.612, "first_read_write_after_action_seconds_from_start": { - "read": 29.135, - "write": 29.137 + "read": 29.611, + "write": 29.641 }, "max_success_gap_seconds": { - "read": 0.216, - "write": 0.231 + "read": 0.195, + "write": 0.182 }, "latency_ms": { - "50": 16.51, - "95": 48.68, - "99": 75.14 + "50": 23.7, + "95": 66.04, + "99": 84.77 }, "confirmed_messages_preserved": true, "duplicate_messages": 0, "instances": [ - "6fc32ef53bc9", - "8c864b1beb6a", - "a6ee3d7e3779", - "bb8669ea27d1" + "2891905bd75d", + "42a599593c3b", + "ac72a6939d0d", + "c22f03375149" ], "target": null }, "containers": { "before": [ { - "id": "6fc32ef53bc923ae2b9300d22d99d833c1b59a0689e6cbbb78ca3dab0a1654d6", - "name": "/mtusi2026lab3-api-3", - "instance": "6fc32ef53bc9", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "ac72a6939d0d1eb94252d534981bb5e40aa424bd42a96b6e3a76bdbaf9213805", + "name": "/mtusi2026lab3-api-5", + "instance": "ac72a6939d0d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:20:57.481834724Z", + "started": "2026-10-23T21:36:18.801951904Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "a6ee3d7e3779d847efe497f22021f9942f805202ade85495734e1f93418aeb5a", - "name": "/mtusi2026lab3-api-4", - "instance": "a6ee3d7e3779", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "2891905bd75d391242dd54891f53ba42163183d556b740f396e557efab08a682", + "name": "/mtusi2026lab3-api-6", + "instance": "2891905bd75d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:21:29.455170279Z", + "started": "2026-10-23T21:36:51.544606095Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, @@ -60,27 +60,27 @@ ], "steps": [ { - "replaced": "6fc32ef53bc923ae2b9300d22d99d833c1b59a0689e6cbbb78ca3dab0a1654d6", + "replaced": "ac72a6939d0d1eb94252d534981bb5e40aa424bd42a96b6e3a76bdbaf9213805", "ready": [ { - "id": "a6ee3d7e3779d847efe497f22021f9942f805202ade85495734e1f93418aeb5a", - "name": "/mtusi2026lab3-api-4", - "instance": "a6ee3d7e3779", - "image": "sha256:cc3df05d4a5e2283a9dd616442a821fa2a8a4a81a67e4d407a7395c3fcb9f082", + "id": "2891905bd75d391242dd54891f53ba42163183d556b740f396e557efab08a682", + "name": "/mtusi2026lab3-api-6", + "instance": "2891905bd75d", + "image": "sha256:2c2255ef772766b8212fe811686f04ebcaa8dde98615d6f111509a29bf5c8793", "state": "running", - "started": "2026-10-23T12:21:29.455170279Z", + "started": "2026-10-23T21:36:51.544606095Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "8c864b1beb6a090a0eff2397fe989ab21ebd0190291f164e6df7dacf7285e82a", - "name": "/mtusi2026lab3-api-5", - "instance": "8c864b1beb6a", - "image": "sha256:7eed22c328d1b387371156ba7c4a30a8017bd48be3acb0b94b63c561925b43ea", + "id": "c22f03375149f347fcde4825cffb8f1a9f5bc81be5e40aab0124ce6f2481afcf", + "name": "/mtusi2026lab3-api-7", + "instance": "c22f03375149", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", "state": "running", - "started": "2026-10-23T12:21:55.548004137Z", + "started": "2026-10-23T21:37:20.306882221Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, @@ -89,27 +89,27 @@ ] }, { - "replaced": "a6ee3d7e3779d847efe497f22021f9942f805202ade85495734e1f93418aeb5a", + "replaced": "2891905bd75d391242dd54891f53ba42163183d556b740f396e557efab08a682", "ready": [ { - "id": "8c864b1beb6a090a0eff2397fe989ab21ebd0190291f164e6df7dacf7285e82a", - "name": "/mtusi2026lab3-api-5", - "instance": "8c864b1beb6a", - "image": "sha256:7eed22c328d1b387371156ba7c4a30a8017bd48be3acb0b94b63c561925b43ea", + "id": "c22f03375149f347fcde4825cffb8f1a9f5bc81be5e40aab0124ce6f2481afcf", + "name": "/mtusi2026lab3-api-7", + "instance": "c22f03375149", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", "state": "running", - "started": "2026-10-23T12:21:55.548004137Z", + "started": "2026-10-23T21:37:20.306882221Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "bb8669ea27d1bd1fa5b6442a1d1ebcc689aa0ec3991dffdfaa5cb96fb61fe5da", - "name": "/mtusi2026lab3-api-6", - "instance": "bb8669ea27d1", - "image": "sha256:7eed22c328d1b387371156ba7c4a30a8017bd48be3acb0b94b63c561925b43ea", + "id": "42a599593c3be9d2ed03970716323a5dae60817b3b47eb2b6f8630a57563ac91", + "name": "/mtusi2026lab3-api-8", + "instance": "42a599593c3b", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", "state": "running", - "started": "2026-10-23T12:22:08.293929923Z", + "started": "2026-10-23T21:37:33.136215003Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, @@ -120,24 +120,24 @@ ], "after": [ { - "id": "8c864b1beb6a090a0eff2397fe989ab21ebd0190291f164e6df7dacf7285e82a", - "name": "/mtusi2026lab3-api-5", - "instance": "8c864b1beb6a", - "image": "sha256:7eed22c328d1b387371156ba7c4a30a8017bd48be3acb0b94b63c561925b43ea", + "id": "c22f03375149f347fcde4825cffb8f1a9f5bc81be5e40aab0124ce6f2481afcf", + "name": "/mtusi2026lab3-api-7", + "instance": "c22f03375149", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", "state": "running", - "started": "2026-10-23T12:21:55.548004137Z", + "started": "2026-10-23T21:37:20.306882221Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, "ports": {} }, { - "id": "bb8669ea27d1bd1fa5b6442a1d1ebcc689aa0ec3991dffdfaa5cb96fb61fe5da", - "name": "/mtusi2026lab3-api-6", - "instance": "bb8669ea27d1", - "image": "sha256:7eed22c328d1b387371156ba7c4a30a8017bd48be3acb0b94b63c561925b43ea", + "id": "42a599593c3be9d2ed03970716323a5dae60817b3b47eb2b6f8630a57563ac91", + "name": "/mtusi2026lab3-api-8", + "instance": "42a599593c3b", + "image": "sha256:1aa913a2e803822a97ffe52e1b0dcf3ff204758de3abeeb2bd23912defb9b885", "state": "running", - "started": "2026-10-23T12:22:08.293929923Z", + "started": "2026-10-23T21:37:33.136215003Z", "health": "healthy", "cpu": 1000000000, "memory": 536870912, diff --git a/evidence/unit-tests.txt b/evidence/unit-tests.txt index 0af7205..2c0c2bc 100644 --- a/evidence/unit-tests.txt +++ b/evidence/unit-tests.txt @@ -15,6 +15,6 @@ test_registration_race (test_lab2.Lab2Tests.test_registration_race) ... ok test_unicode_password (test_lab2.Lab2Tests.test_unicode_password) ... ok ---------------------------------------------------------------------- -Ran 15 tests in 19.204s +Ran 15 tests in 19.590s OK diff --git a/infra/nginx.conf b/infra/nginx.conf index fc7ecdb..4dee29a 100644 --- a/infra/nginx.conf +++ b/infra/nginx.conf @@ -2,11 +2,9 @@ worker_processes 1; worker_shutdown_timeout 8s; events { worker_connections 1024; } http { - resolver 127.0.0.11 valid=2s ipv6=off; - resolver_timeout 1s; upstream messenger { zone messenger 64k; - server api:8080 resolve max_fails=1 fail_timeout=2s; + include /etc/nginx/ready-backends.conf; } log_format upstream '$request_method $uri $status upstream=$upstream_addr ' 'upstream_status=$upstream_status time=$request_time'; diff --git a/infra/readiness.sh b/infra/readiness.sh new file mode 100644 index 0000000..5bb2563 --- /dev/null +++ b/infra/readiness.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Only ready API addresses enter the upstream, including at nginx startup. +update_backends() { + next=/etc/nginx/ready-backends.conf.next + : > "$next" + for address in $(getent ahostsv4 api | awk '{print $1}' | sort -u); do + if wget -q -T 1 -O /dev/null "http://$address:8080/health/ready"; then + echo "server $address:8080 max_fails=1 fail_timeout=2s;" >> "$next" + fi + done + # An empty upstream is invalid; a down placeholder returns 502 without + # sending business requests to an unready API. + if [ ! -s "$next" ]; then + echo 'server 127.0.0.1:9 down;' > "$next" + fi + if ! cmp -s "$next" /etc/nginx/ready-backends.conf; then + mv "$next" /etc/nginx/ready-backends.conf + if [ "${1:-}" = reload ] && [ -f /var/run/nginx.pid ]; then + nginx -s reload + fi + else + rm -f "$next" + fi +} + +update_backends +( + while sleep 2; do + update_backends reload + done +) & +exec nginx -g 'daemon off;' diff --git a/scripts/ops.py b/scripts/ops.py index 9e86362..40567c4 100644 --- a/scripts/ops.py +++ b/scripts/ops.py @@ -34,7 +34,7 @@ def wait_api(count=2, timeout=60): while time.monotonic() < deadline: rows = snapshot('api') if len(rows) == count and all(r['health'] == 'healthy' and r['state'] == 'running' for r in rows): - time.sleep(3) # Docker DNS validity 2s, plus resolver budget 1s. + time.sleep(3) # Allow the nginx readiness poll (2s) and reload. return rows time.sleep(0.5) raise RuntimeError('API replicas did not become healthy') diff --git a/scripts/verify_readiness.py b/scripts/verify_readiness.py new file mode 100644 index 0000000..08e09a8 --- /dev/null +++ b/scripts/verify_readiness.py @@ -0,0 +1,109 @@ +"""Reproduce an alive but unready API, then check exclusion and recovery.""" +import json +import sys +import time +import urllib.error +import urllib.request +from uuid import uuid4 + +from ops import ROOT, compose, docker, restore, save, snapshot, wait_api + +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check + + +def wait_for(probe, timeout=30): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if probe(): + return + time.sleep(0.5) + raise AssertionError('Readiness transition timed out') + + +def state(container): + return json.loads(docker('inspect', '--format', + '{"id":{{json .Id}},"instance":{{json .Config.Hostname}},' + '"health":{{json .State.Health.Status}},' + '"ip":{{range .NetworkSettings.Networks}}{{json .IPAddress}}{{end}}}', container)) + + +def backends(): + return compose('exec', '-T', 'nginx', 'cat', '/etc/nginx/ready-backends.conf') + + +def registrations(): + instances = {} + client = Client() + for _ in range(24): + response = client.call('POST', '/api/v1/users', 201, body={ + 'username': 'ready_' + uuid4().hex[:16], + 'display_name': 'Readiness test', 'password': 'Readiness-test-password'}, schema='User') + instance = response['headers']['X-Instance-Id'] + instances[instance] = instances.get(instance, 0) + 1 + return instances + + +def main(): + original = wait_api() + check(len(original) == 2, 'Run with exactly two API replicas') + peer, replaced = original + temporary = None + result = {'before': original} + try: + docker('stop', '--time', '8', replaced['id']) + temporary = compose('run', '-d', '--no-deps', '--use-aliases', + '-e', 'REDIS_URL=redis://127.0.0.1:6399/0', 'api').splitlines()[-1] + wait_for(lambda: state(temporary)['health'] == 'unhealthy') + bad = state(temporary) + result['unready'] = bad + # Call each endpoint inside this exact container, bypassing the LB. + probe = ('import urllib.request, urllib.error; ' + 'opener = urllib.request.build_opener(urllib.request.ProxyHandler({})); ' + 'response = opener.open("http://127.0.0.1:8080/health/live"); ' + 'print(response.status)') + live = docker('exec', temporary, 'python', '-c', probe) + ready_probe = ('import urllib.request, urllib.error\n' + 'try:\n' + ' print(urllib.request.urlopen("http://127.0.0.1:8080/health/ready").status)\n' + 'except urllib.error.HTTPError as error:\n' + ' print(error.code)') + ready = docker('exec', temporary, 'python', '-c', ready_probe) + check((live, ready) == ('200', '503'), 'Expected live=200, ready=503') + result['health_http'] = {'live': int(live), 'ready': int(ready)} + result['upstream_when_unready'] = backends() + check(bad['ip'] not in result['upstream_when_unready'], + 'Unhealthy backend is still in nginx upstream') + result['registrations_with_unready_peer'] = registrations() + check(result['registrations_with_unready_peer'] == {peer['instance']: 24}, + 'Registration reached an unready API') + + # No ready API must mean an error, not a fallback to the unready one. + docker('stop', '--time', '8', peer['id']) + wait_for(lambda: '127.0.0.1:9 down' in backends(), timeout=10) + try: + urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=5) + raise AssertionError('LB routed to an unready API') + except urllib.error.HTTPError as error: + check(error.code == 502, 'Expected 502 with no ready backend') + result['no_ready_backend_status'] = error.code + + docker('rm', '-f', temporary) + temporary = None + restore() + wait_for(lambda: backends().count('max_fails=1') == 2, timeout=10) + result['registrations_after_recovery'] = registrations() + check(set(result['registrations_after_recovery']) == {r['instance'] for r in original}, + 'Recovered API did not return to load balancing') + result['after'] = snapshot('api') + result['result'] = 'PASS' + save('readiness.json', result) + print(json.dumps(result, indent=2)) + finally: + if temporary: + docker('rm', '-f', temporary) + restore() + + +if __name__ == '__main__': + main()