From b48e71f4334937e88a0e59bfc1cee082637095c4 Mon Sep 17 00:00:00 2001 From: myDan Date: Fri, 6 Nov 2026 05:12:28 +0300 Subject: [PATCH] feat: implement estim 4 --- Dockerfile | 2 + Makefile | 4 +- REPORT.md | 117 ++++++++++++++++++++++++++++++---- cmd/server/main.go | 13 +++- compose.yaml | 51 +++++++++++++-- evidence/dataset.txt | 8 +++ evidence/drain.txt | 13 ++++ evidence/env.txt | 12 ++++ evidence/failover-1.txt | 43 +++++++++++-- evidence/failover-2.txt | 47 +++++++++++--- evidence/failover-load-1.json | 17 +++++ evidence/failover-load-2.json | 17 +++++ evidence/idempotency-race.txt | 2 +- evidence/instances.txt | 22 +++---- evidence/limits.txt | 23 +++++++ evidence/migrate.txt | 23 ++++--- evidence/persistence.txt | 26 ++++---- evidence/rolling-load.json | 17 +++++ evidence/rolling.txt | 45 +++++++++++++ evidence/test-go.txt | 37 +++++------ infra/haproxy/haproxy.cfg | 2 + internal/handler/handler.go | 6 ++ internal/handler/health.go | 4 ++ scripts/explain.sh | 10 --- scripts/explain.sql | 66 ------------------- scripts/outage-demo.sh | 27 -------- scripts/restart.sh | 3 +- scripts/rolling-update.sh | 37 +++++++++++ scripts/stop-one.sh | 4 +- 29 files changed, 504 insertions(+), 194 deletions(-) create mode 100644 evidence/dataset.txt create mode 100644 evidence/drain.txt create mode 100644 evidence/env.txt create mode 100644 evidence/failover-load-1.json create mode 100644 evidence/failover-load-2.json create mode 100644 evidence/limits.txt create mode 100644 evidence/rolling-load.json create mode 100644 evidence/rolling.txt delete mode 100755 scripts/explain.sh delete mode 100644 scripts/explain.sql delete mode 100755 scripts/outage-demo.sh create mode 100755 scripts/rolling-update.sh diff --git a/Dockerfile b/Dockerfile index c6ef4eea..6de4ddab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,6 +7,8 @@ COPY internal ./internal RUN CGO_ENABLED=0 go build -trimpath -o /out/server ./cmd/server FROM alpine:3.24.1 +ARG BUILD_ID=dev +LABEL org.opencontainers.image.revision=$BUILD_ID RUN adduser -D -H -u 10001 app COPY --from=build /out/server /usr/local/bin/server USER app diff --git a/Makefile b/Makefile index 09297c3c..1a60e773 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,8 @@ PYTHON ?= python3 BASE_URL ?= http://localhost:8080 CA_FILE ?= ACTION_SCRIPT ?= -COMPOSE ?= podman compose +ENGINE ?= $(shell command -v podman >/dev/null 2>&1 && echo podman || echo docker) +COMPOSE ?= $(if $(filter podman,$(ENGINE)),podman-compose,docker compose) TEST_DATABASE_URL ?= $(shell sed 's/@postgres:5432/@127.0.0.1:55432/' secrets/database_url 2>/dev/null) TEST_REDIS_URL ?= redis://127.0.0.1:56379/0 TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) @@ -44,3 +45,4 @@ test-idempotency: test-go: @TEST_DATABASE_URL="$(TEST_DATABASE_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" go test -count=1 -v ./... + diff --git a/REPORT.md b/REPORT.md index c2679005..2c261424 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,126 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Спесивцев Даниил БПИ2301 +- Целевая оценка 4; выбранный трек 5, если нужен: не заявлен - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: Go 1.27.1 (стандартная библиотека). Для консультации, реализации некоторых фич, написания отчета использовались LLM. Библиотеки и компоненты: + - [go-chi/chi v5.3.2](https://github.com/go-chi/chi) — HTTP-роутинг; + - [jackc/pgx v5.11.0](https://github.com/jackc/pgx) — драйвер PostgreSQL и пул `pgxpool`, транзакция идемпотентной отправки; + - [redis/go-redis v9.23.0](https://github.com/redis/go-redis) — клиент Redis для общих сессий; + - [golang.org/x/crypto v0.58.0](https://pkg.go.dev/golang.org/x/crypto/argon2) — Argon2id для паролей; + - образ [migrate/migrate v4.20.1](https://github.com/golang-migrate/migrate) — однократные миграции отдельным контейнером; + - образ [HAProxy 3.4.7](https://www.haproxy.org/) — балансировщик. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Стенд: i5-11400H (12 потоков), 16 GB RAM, Linux 7.1.8, podman 6.1.0 + podman-compose 1.6.0 (`evidence/env.txt`). Образы: `postgres:18.6-alpine3.24`, `redis:8.10.3-alpine3.23`, `migrate/migrate:v4.20.1`, `haproxy:3.4.7-alpine`; API из `Dockerfile` (`golang:1.27.1-alpine3.24` → `alpine:3.24.1`). + +Движок выбирается автоматически в `Makefile` и скриптах: podman + `podman-compose`, если podman установлен, иначе docker + `docker compose`. Явно: `ENGINE=docker COMPOSE="docker compose"`. `compose.yaml` проходит `docker compose config --quiet` и `podman-compose config`. + +```sh +scripts/init-secrets.sh # локальные secrets в ./secrets, вне Git; повторный запуск не перезаписывает +make up # build; postgres, redis → migrate (однократно) → api ×2; lb +until curl -sf http://localhost:8080/health/ready; do sleep 1; done + +make check && make test && make test-go +make test-idempotency # один Idempotency-Key, 20 параллельных POST через LB +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +$COMPOSE start api # вернуть остановленную реплику и повторить для другой +scripts/rolling-update.sh # rolling update (под нагрузкой: параллельно python3 tests/load.py --duration 60) + +make down # остановка, volumes сохраняются +$COMPOSE down -v # намеренное удаление учебных данных +``` + +Число реплик — `API_REPLICAS` (по умолчанию 2, балансируется до 4: `server-template api 4`). Публичная точка входа — только `lb` на `:8080`. Bootstrap безопасно повторяется: `migrate` при повторном запуске отвечает `no change`, `init-secrets.sh` не трогает существующие файлы. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +client → lb (HAProxy :8080) ──edge──▶ api ×2 ──backend──▶ postgres (источник истины) + └────────▶ redis (сессии, AOF) + migrate (one-shot) ─backend─▶ postgres +``` + +| Роль | Service | Образ | Health / readiness | Volume | Сеть / порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | `lb` | `haproxy:3.4.7-alpine` | активная проверка `GET /health/ready` каждой реплики | — | edge, `0.0.0.0:8080` | нет | +| API | `api` ×2 | `localhost/messenger-api:lab3` | `/health/live`; `/health/ready` = PostgreSQL + Redis + не draining | — | edge + backend, порт не публикуется | secrets `database_url`, `cache_url` | +| Миграции | `migrate` | `migrate/migrate:v4.20.1` | exit 0 | SQL read-only | backend | secret `database_url` | +| БД | `postgres` | `postgres:18.6-alpine3.24` | `pg_isready` | `pgdata` | backend, `127.0.0.1:55432` для `make test-go` | secret `postgres_password` | +| Сессии | `redis` | `redis:8.10.3-alpine3.23` | `redis-cli ping` | `redisdata` (AOF everysec) | backend, `127.0.0.1:56379` для `make test-go` | нет | + +- **Перенесено из lab2:** API на Go, PostgreSQL, Bearer-сессии в Redis, Argon2id. Cursor истории не зависит от процесса, поэтому любая реплика обслуживает любой запрос без sticky sessions. +- **X-Instance-Id:** hostname контейнера (короткий ID), выставляется middleware на все ответы API. `scripts/stop-one.sh` находит контейнер с таким hostname по labels compose и останавливает только его. +- **Идемпотентность:** таблица `message_idempotency`, PK `(user_id, chat_id, key)`; метод и путь фиксированы для эндпоинта. В одной транзакции выполняются `INSERT ... ON CONFLICT DO NOTHING` ключа и `INSERT` сообщения. Конкурентный дубль ждёт на уникальном индексе; при совпадении `sha256(text)` возвращается то же сообщение с 201, иначе 409. При 5xx транзакция откатывается, и ключ не фиксируется. Повтор заново проверяет сессию и доступ к чату. +- **Миграции:** выполняются отдельным контейнером `migrate` до старта API (advisory lock golang-migrate); реплики миграций не запускают. Совместимость с работающей версией: допускаются только добавляющие изменения (expand/contract); `000002` лишь добавляет таблицу. Удаление или переименование колонок потребует двух релизов. `scripts/rolling-update.sh` выполняет `migrate` до замены реплик. +- **Rolling update** (`scripts/rolling-update.sh`): + 1. сборка нового образа с тем же тегом и новым label `BUILD_ID`; + 2. `$COMPOSE run --rm migrate`; + 3. для каждой старой реплики по очереди: `stop -t 10` (drain) → `rm` → `$COMPOSE up -d --no-deps --no-recreate api` → ожидание `healthy` новой реплики. +- **Graceful shutdown:** по SIGTERM `/health/ready` сразу отвечает 503, keep-alive отключается, 3 с drain, затем `Shutdown` с таймаутом 5 с. + +| Таймаут | Значение | +| --- | --- | +| PostgreSQL `statement_timeout` / connect | 5 s / 3 s | +| Readiness ping зависимостей | 1 s | +| HTTP API: ReadHeader / Read / Write / Idle | 5 / 15 / 15 / 60 s | +| HAProxy: connect / server / client / http-request / check | 2 / 20 / 30 / 10 / 1 s; `maxconn 2000` | +| Остановка: drain + shutdown | 3 + 5 = 8 s < `stop -t 10`, `stop_grace_period: 10s` | + +**Обнаружение отказа.** Период проверок HAProxy: `inter 1s`, `fall 2`, `rise 2`, `timeout check 1s`; DNS `hold valid 1s`. +- При SIGTERM реплика выводится из LB по 503 readiness за ≤ 2 s, до закрытия listener (измерено 2.0 s, `evidence/drain.txt`). +- При аварийной остановке отказ соединения сразу уходит на другую реплику (`retries 3`, `option redispatch`; повторяется только неудавшееся соединение, отправленный POST не повторяется). Максимальное время исключения backend — `inter × fall + timeout check` = 3 s. + +Соответствие `.env.example`: +- `SESSION_STORE=redis`; +- `DATABASE_URL_FILE`, `CACHE_URL_FILE` — файлы secrets; +- `API_REPLICAS` → `scale` сервиса `api`; +- `SESSION_TTL_SECONDS`, `APP_ENV` передаются в API; +- `IDEMPOTENCY_TTL_SECONDS` не используется: ключи хранятся бессрочно, что покрывает требование «не менее 24 ч». ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `make test` | PASS | `evidence/smoke.txt` | +| Стенд в compose.yaml: ≥2 API, одна точка входа, общие БД и сессии | `make up`, `ps` | 2 реплики `api`, порт публикует только `lb` | `evidence/instances.txt`, `evidence/limits.txt` | +| Разные реплики с одной сессией, X-Instance-Id | 20 × `GET /users/me` через LB одним токеном | по 10 ответов от двух ID; ID = hostname контейнера | `evidence/instances.txt` | +| Отказ любой одной реплики: ≤ 10 s, подтверждённые сообщения доступны | `make test-failover` для `api_1` и `api_2` | PASS, 0 ошибок, первая запись после hook за 3.3–3.4 s | `evidence/failover-1.txt`, `evidence/failover-2.txt` | +| Повтор с тем же Idempotency-Key без дубля, в т. ч. конкурентный и через разные реплики | `make test`, `make test-idempotency` | 20/20 ответов 201 с одним сообщением, обслужили обе реплики | `evidence/smoke.txt`, `evidence/idempotency-race.txt` | +| persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; перезапущены lb, api, redis, postgres с теми же ID; volumes сохранены | `evidence/persistence.txt` | +| Миграции отдельно, без гонки реплик | `make up` | `migrate` exit 0, в логах API миграций нет, версия схемы 2 | `evidence/migrate.txt` | +| Собственные Go-тесты | `make test-go` | 13 PASS | `evidence/test-go.txt` | +| 4: readiness, timeouts, SIGTERM, исключение backend; период и время обнаружения | `stop` реплики | ready 503 → LB DOWN за 2.0 s → listener закрыт через 3 s | `evidence/drain.txt`, разделы выше | +| 4: поочерёдная остановка каждой реплики, RPS / ошибки / восстановление | `make test-failover` под фоновым `tests/load.py` | см. таблицу ниже | `evidence/failover-{1,2}.txt`, `evidence/failover-load-{1,2}.json` | +| 4: общая идемпотентность параллельными запросами с одним ключом | `make test-idempotency` | одно сообщение на 20 параллельных запросов | `evidence/idempotency-race.txt` | +| 4: CPU/RAM-лимиты, API без портов, секреты и хранилища только где нужно, нет container_name | `inspect`, `ls /run/secrets`, `getent` из lb | лимиты у всех сервисов; lb не видит postgres и redis; секреты по ролям | `evidence/limits.txt` | +| 4: rolling update под нагрузкой; миграции однократны и совместимы | `scripts/rolling-update.sh` под фоновым `tests/load.py --duration 60` | обе реплики заменены новым образом за 21 s, 0 ошибок | `evidence/rolling.txt`, `evidence/rolling-load.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +- **Профиль:** публичный `tests/load.py`, closed-loop POST сообщений, concurrency 4, 30 s (rolling — 60 s), без автоматических повторов. Отказ запускается через `make test-failover` примерно на 4-й секунде нагрузки; тест сам повторяет неопределённый POST с тем же ключом и проверяет сохранность и отсутствие дублей. Прогрев отдельно не выделялся; один прогон на сценарий. +- **Лимиты:** api 1 CPU / 256 MiB на реплику, postgres 1 CPU / 512 MiB, redis 0.5 CPU / 128 MiB, lb 0.5 CPU / 64 MiB, migrate 0.5 CPU / 64 MiB. +- **Потребление во время нагрузки** (`stats --no-stream`): api 4–5 % CPU и 28–92 MiB, postgres 12 % CPU и 135 MiB, lb 3–5 % CPU и 28–32 MiB, redis < 1 % CPU и 7 MiB. +- **Набор данных после всех прогонов:** 294 274 сообщения, 292 пользователя, 84 чата, БД 167 MB (`evidence/dataset.txt`). + +| Прогон | Успешные RPS (`load.py`) | Ошибки `load.py` / теста | p50 / p95 / p99, ms | Восстановление записи (тест) | Записей теста за 15 s | +| --- | --- | --- | --- | --- | --- | +| stop `api_1` | 290.3 | 0 / 0 | 13.1 / 17.6 / 23.9 | 3.34 s от старта hook (hook ≈ 3.2 s, drain) | 131 | +| stop `api_2` | 316.8 | 0 / 0 | 12.1 / 16.1 / 22.4 | 3.37 s от старта hook | 132 | +| rolling update | 288.5 | 0 | 13.2 / 17.0 / 20.1 | замена 2 реплик за 21 s | — | | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `stop` реплики (SIGTERM) | вывод из LB до закрытия listener, без ошибок | DOWN по 503 за 2.0 s, 0 ошибок под нагрузкой | запись не прерывалась / потерь и дублей нет | +| Rolling update | поочерёдная замена без простоя | 0 ошибок за 60 s, новый image и revision у обеих реплик | — / потерь нет | +| Restart всего стенда | данные и токен сохраняются | PASS persistence | в пределах бюджета теста 90 s / потерь нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Потеря host, LB или PostgreSQL не переживается, это SPOF. Отказоустойчивость показана только для процесса API. +- Порты PostgreSQL и Redis опубликованы только на `127.0.0.1` для `make test-go`. Redis без пароля доступен только с локальной машины. +- `server-template api 4` при `API_REPLICAS` > 4 лишние реплики в балансировку не попадут. +- Нагрузка closed-loop с одного клиента ограничена клиентом, есть coordinated omission, один прогон на сценарий. Это не оценка предельной пропускной способности. +- Ключи идемпотентности не удаляются, таблица растёт. Для production понадобится фоновая очистка ключей старше 24 ч. diff --git a/cmd/server/main.go b/cmd/server/main.go index 71b84f30..676f8570 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -18,7 +18,10 @@ import ( "messenger/internal/service" ) -const shutdownTimeout = 5 * time.Second +const ( + drainDelay = 3 * time.Second + shutdownTimeout = 5 * time.Second +) func main() { log := slog.New(slog.NewJSONHandler(os.Stdout, nil)) @@ -59,9 +62,10 @@ func run(log *slog.Logger) error { messages := service.NewMessageService(chats, store) deps := []handler.Pinger{store.Ping, sessions.Ping} + h := handler.New(users, auth, chats, messages, deps, log) srv := &http.Server{ Addr: cfg.Addr(), - Handler: router.New(handler.New(users, auth, chats, messages, deps, log), instanceID), + Handler: router.New(h, instanceID), ReadHeaderTimeout: 5 * time.Second, ReadTimeout: 15 * time.Second, WriteTimeout: 15 * time.Second, @@ -84,7 +88,10 @@ func run(log *slog.Logger) error { return err case <-ctx.Done(): } - log.Info("shutdown signal received", "timeout", shutdownTimeout.String()) + log.Info("shutdown signal received, draining", "drain", drainDelay.String(), "timeout", shutdownTimeout.String()) + h.StartDraining() + srv.SetKeepAlivesEnabled(false) + time.Sleep(drainDelay) shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) defer cancel() diff --git a/compose.yaml b/compose.yaml index 40754f1b..95ece40f 100644 --- a/compose.yaml +++ b/compose.yaml @@ -11,6 +11,8 @@ services: - postgres_password volumes: - pgdata:/var/lib/postgresql + networks: + - backend ports: - "127.0.0.1:55432:5432" healthcheck: @@ -18,6 +20,11 @@ services: interval: 2s timeout: 3s retries: 30 + deploy: + resources: + limits: + cpus: "1.0" + memory: 512M restart: unless-stopped redis: @@ -25,6 +32,8 @@ services: command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] volumes: - redisdata:/data + networks: + - backend ports: - "127.0.0.1:56379:6379" healthcheck: @@ -32,6 +41,11 @@ services: interval: 2s timeout: 3s retries: 30 + deploy: + resources: + limits: + cpus: "0.5" + memory: 128M restart: unless-stopped migrate: @@ -42,9 +56,16 @@ services: - database_url volumes: - ./internal/repository/postgres/migrations:/migrations:ro + networks: + - backend depends_on: postgres: condition: service_healthy + deploy: + resources: + limits: + cpus: "0.5" + memory: 64M restart: "no" api: @@ -60,9 +81,22 @@ services: secrets: - database_url - cache_url + networks: + - edge + - backend + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"] + interval: 5s + timeout: 2s + retries: 3 + start_period: 5s + stop_grace_period: 10s + scale: ${API_REPLICAS:-2} deploy: - mode: replicated - replicas: ${API_REPLICAS:-2} + resources: + limits: + cpus: "1.0" + memory: 256M depends_on: migrate: condition: service_completed_successfully @@ -74,12 +108,21 @@ services: image: docker.io/library/haproxy:3.4.7-alpine volumes: - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + networks: + - edge ports: - "8080:8080" - depends_on: - - api + deploy: + resources: + limits: + cpus: "0.5" + memory: 64M restart: unless-stopped +networks: + edge: + backend: + secrets: postgres_password: file: ./secrets/postgres_password diff --git a/evidence/dataset.txt b/evidence/dataset.txt new file mode 100644 index 00000000..d07dec5d --- /dev/null +++ b/evidence/dataset.txt @@ -0,0 +1,8 @@ +$ row counts after all runs (date 2026-11-06T02:07:36Z) +chats 84 +users 292 +messages 294274 +message_idempotency 294274 + +$ database size +167 MB diff --git a/evidence/drain.txt b/evidence/drain.txt new file mode 100644 index 00000000..96f0ee1e --- /dev/null +++ b/evidence/drain.txt @@ -0,0 +1,13 @@ +$ date -u; podman stop -t 10 messenger-lab3_api_1; date -u +02:07:18.261 +02:07:21.454 + +$ podman logs messenger-lab3_api_1 (shutdown sequence, UTC) +02:07:18.345 shutdown signal received, draining 3s 5s +02:07:21.345 server stopped + +$ TZ=UTC podman logs -t messenger-lab3_lb_1 (backend state, UTC) +02:07:20.346 Server api/api2 is DOWN, reason: Layer7 wrong status, code: 503, info: "Service Unavailable", check duration: 0ms. 1 active and 0 backup + +SIGTERM 02:07:18.345 -> LB marks DOWN by 503 readiness 02:07:20.346 (2.0 s, <= inter 1s x fall 2) -> listener closed 02:07:21.345 (drain 3 s); hook returned in 3.2 s. +HAProxy slot names (api1..api4 from server-template) are assigned by DNS order and do not match container numbers. diff --git a/evidence/env.txt b/evidence/env.txt new file mode 100644 index 00000000..ec0a8a99 --- /dev/null +++ b/evidence/env.txt @@ -0,0 +1,12 @@ +date: 2026-11-06T01:54:08Z +host: i5-11400H, 12 threads, 15Gi RAM, Linux 7.1.8-arch1-3 +podman version 6.1.0 +podman-compose version 1.6.0 +go version go1.27.1 linux/amd64 +images: + image: docker.io/library/postgres:18.6-alpine3.24 + image: docker.io/library/redis:8.10.3-alpine3.23 + image: docker.io/migrate/migrate:v4.20.1 + image: localhost/messenger-api:lab3 + image: docker.io/library/haproxy:3.4.7-alpine +limits: see evidence/limits.txt diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt index 90cbe2ff..ce1caf67 100644 --- a/evidence/failover-1.txt +++ b/evidence/failover-1.txt @@ -1,14 +1,47 @@ $ api replicas before -messenger-lab3_api_1 hostname=0405cf8a4a54 Up 10 seconds -messenger-lab3_api_2 hostname=2751ae93e6d1 Up 10 seconds +messenger-lab3_api_2 hostname=5a9ffe83734a Up About a minute (healthy) +messenger-lab3_api_1 hostname=ab01920585fe Up About a minute (healthy) + +$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/failover-load-1.json & (background load during the test) +$ podman stats --no-stream (during load, before failure) +messenger-lab3_api_1 cpu=3.87% mem=88.06MB / 268.4MB +messenger-lab3_api_2 cpu=4.17% mem=92.08MB / 268.4MB +messenger-lab3_lb_1 cpu=2.81% mem=28.44MB / 67.11MB +messenger-lab3_postgres_1 cpu=12.21% mem=135.6MB / 536.9MB +messenger-lab3_redis_1 cpu=0.58% mem=6.857MB / 134.2MB $ make test-failover ACTION_SCRIPT=scripts/stop-one.sh python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 87, "failed_attempts": 1, "first_recovery_seconds": 2.52} +{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 3.339} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 +$ podman stats --no-stream (during load, one replica stopped) +messenger-lab3_api_2 cpu=5.01% mem=27.76MB / 268.4MB +messenger-lab3_lb_1 cpu=3.77% mem=29.78MB / 67.11MB +messenger-lab3_postgres_1 cpu=12.32% mem=133.1MB / 536.9MB +messenger-lab3_redis_1 cpu=0.59% mem=7.111MB / 134.2MB + +$ background tests/load.py result +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.008, + "concurrency": 4, + "successful_requests": 8712, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 290.322, + "successful_latency_ms": { + "p50": 13.117, + "p95": 17.627, + "p99": 23.947 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} + $ api replicas after (stopped one = TARGET_INSTANCE_ID) -messenger-lab3_api_1 hostname=0405cf8a4a54 Up 26 seconds -messenger-lab3_api_2 hostname=2751ae93e6d1 Exited (0) 15 seconds ago +messenger-lab3_api_2 hostname=5a9ffe83734a Up 2 minutes (healthy) +messenger-lab3_api_1 hostname=ab01920585fe Exited (0) 22 seconds ago (healthy) diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt index 9fa42b53..4a96b50a 100644 --- a/evidence/failover-2.txt +++ b/evidence/failover-2.txt @@ -1,18 +1,47 @@ -$ api replicas before (api_2 restored with: podman compose start api) -messenger-lab3_api_1 hostname=0405cf8a4a54 Up 2 minutes -messenger-lab3_api_2 hostname=2751ae93e6d1 Up 19 seconds - -$ curl -s localhost:8080/health/live (shift round-robin so the test write lands on the other replica) -{"status":"ok"} +$ api replicas before +messenger-lab3_api_2 hostname=5a9ffe83734a Up 2 minutes (healthy) +messenger-lab3_api_1 hostname=ab01920585fe Up 4 seconds (healthy) +$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/failover-load-2.json & (background load during the test) +$ podman stats --no-stream (during load, before failure) +messenger-lab3_api_1 cpu=4.21% mem=47.89MB / 268.4MB +messenger-lab3_api_2 cpu=5.90% mem=92.86MB / 268.4MB +messenger-lab3_lb_1 cpu=4.60% mem=31.47MB / 67.11MB +messenger-lab3_postgres_1 cpu=12.45% mem=148.5MB / 536.9MB +messenger-lab3_redis_1 cpu=0.59% mem=6.611MB / 134.2MB $ make test-failover ACTION_SCRIPT=scripts/stop-one.sh python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" Executing explicit failover hook; hook output suppressed to avoid leaking secrets. -{"observed_instances_before": 2, "successful_writes": 81, "failed_attempts": 1, "first_recovery_seconds": 2.424} +{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 3.37} PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 +$ podman stats --no-stream (during load, one replica stopped) +messenger-lab3_api_1 cpu=7.73% mem=33.08MB / 268.4MB +messenger-lab3_lb_1 cpu=4.84% mem=32.26MB / 67.11MB +messenger-lab3_postgres_1 cpu=12.54% mem=146.1MB / 536.9MB +messenger-lab3_redis_1 cpu=0.59% mem=7.373MB / 134.2MB + +$ background tests/load.py result +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.013, + "concurrency": 4, + "successful_requests": 9508, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 316.791, + "successful_latency_ms": { + "p50": 12.105, + "p95": 16.14, + "p99": 22.351 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} + $ api replicas after (stopped one = TARGET_INSTANCE_ID) -messenger-lab3_api_1 hostname=0405cf8a4a54 Exited (0) 15 seconds ago -messenger-lab3_api_2 hostname=2751ae93e6d1 Up 35 seconds +messenger-lab3_api_2 hostname=5a9ffe83734a Exited (0) 22 seconds ago (healthy) +messenger-lab3_api_1 hostname=ab01920585fe Up 35 seconds (healthy) diff --git a/evidence/failover-load-1.json b/evidence/failover-load-1.json new file mode 100644 index 00000000..d81cacfe --- /dev/null +++ b/evidence/failover-load-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.008, + "concurrency": 4, + "successful_requests": 8712, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 290.322, + "successful_latency_ms": { + "p50": 13.117, + "p95": 17.627, + "p99": 23.947 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/failover-load-2.json b/evidence/failover-load-2.json new file mode 100644 index 00000000..d30ed419 --- /dev/null +++ b/evidence/failover-load-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.013, + "concurrency": 4, + "successful_requests": 9508, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 316.791, + "successful_latency_ms": { + "p50": 12.105, + "p95": 16.14, + "p99": 22.351 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/idempotency-race.txt b/evidence/idempotency-race.txt index 6e077415..eb9a1a11 100644 --- a/evidence/idempotency-race.txt +++ b/evidence/idempotency-race.txt @@ -4,5 +4,5 @@ successful responses (201): 20 failed requests: 0 [] distinct message ids in responses: 1 messages stored in history: 1 -instances that served requests: ['0405cf8a4a54', '2751ae93e6d1'] +instances that served requests: ['56e8d4ca6c28', 'd63350b561c6'] OK: exactly one message created diff --git a/evidence/instances.txt b/evidence/instances.txt index 4051a758..e8c05422 100644 --- a/evidence/instances.txt +++ b/evidence/instances.txt @@ -1,18 +1,16 @@ 20 GET /api/v1/users/me through LB (localhost:8080) with ONE bearer session, no re-login -X-Instance-Id 0405cf8a4a54 responses 10 -X-Instance-Id 2751ae93e6d1 responses 10 +X-Instance-Id 56e8d4ca6c28 responses 10 +X-Instance-Id d63350b561c6 responses 10 -$ podman ps --filter label=com.docker.compose.service=api --format "{{.Names}} {{.ID}} {{.Status}}" -messenger-lab3_api_1 0405cf8a4a54 Up 54 seconds -messenger-lab3_api_2 2751ae93e6d1 Up 53 seconds +$ podman ps (api replicas: name, container ID = hostname = X-Instance-Id) +messenger-lab3_api_1 d63350b561c6 Up 41 seconds (healthy) +messenger-lab3_api_2 56e8d4ca6c28 Up About a minute (healthy) +messenger-lab3_api_1 hostname=d63350b561c6 +messenger-lab3_api_2 hostname=56e8d4ca6c28 -$ podman inspect -f "{{.Name}} hostname={{.Config.Hostname}} ports={{.NetworkSettings.Ports}}" api replicas -messenger-lab3_api_1 hostname=0405cf8a4a54 ports=map[8080/tcp:[]] -messenger-lab3_api_2 hostname=2751ae93e6d1 ports=map[8080/tcp:[]] - -$ podman ps --format "{{.Names}} {{.Ports}}" (only lb publishes 8080) -messenger-lab3_postgres_1 127.0.0.1:55432->5432/tcp -messenger-lab3_redis_1 127.0.0.1:56379->6379/tcp +$ podman ps (published ports: only lb exposes 8080) messenger-lab3_api_1 8080/tcp messenger-lab3_api_2 8080/tcp messenger-lab3_lb_1 0.0.0.0:8080->8080/tcp +messenger-lab3_postgres_1 127.0.0.1:55432->5432/tcp +messenger-lab3_redis_1 127.0.0.1:56379->6379/tcp diff --git a/evidence/limits.txt b/evidence/limits.txt new file mode 100644 index 00000000..dc3c4553 --- /dev/null +++ b/evidence/limits.txt @@ -0,0 +1,23 @@ +$ podman inspect: limits, networks, published ports +messenger-lab3_api_1 cpus=1000000000 memory=268435456 networks=messenger-lab3_backend messenger-lab3_edge ports= +messenger-lab3_api_2 cpus=1000000000 memory=268435456 networks=messenger-lab3_backend messenger-lab3_edge ports= +messenger-lab3_lb_1 cpus=500000000 memory=67108864 networks=messenger-lab3_edge ports=8080/tcp->0.0.0.0:8080 +messenger-lab3_migrate_1 cpus=500000000 memory=67108864 networks=messenger-lab3_backend ports= +messenger-lab3_postgres_1 cpus=1000000000 memory=536870912 networks=messenger-lab3_backend ports=5432/tcp->127.0.0.1:55432 +messenger-lab3_redis_1 cpus=500000000 memory=134217728 networks=messenger-lab3_backend ports=6379/tcp->127.0.0.1:56379 + +$ ls /run/secrets in each running container +messenger-lab3_api_1: cache_url database_url +messenger-lab3_api_2: cache_url database_url +messenger-lab3_lb_1: +messenger-lab3_postgres_1: postgres_password +messenger-lab3_redis_1: +migrate (one-shot): /run/secrets/database_url + +$ lb sees only the edge network +postgres: not resolvable from lb +redis: not resolvable from lb +10.89.4.28 api.dns.podman api.dns.podman api + +$ grep -c container_name compose.yaml +0 diff --git a/evidence/migrate.txt b/evidence/migrate.txt index 0eb3123a..212151f8 100644 --- a/evidence/migrate.txt +++ b/evidence/migrate.txt @@ -1,15 +1,22 @@ -$ podman ps -a --filter label=com.docker.compose.service=migrate -messenger-lab3_migrate_1 Exited (0) 53 seconds ago +$ podman ps -a (migrate is one-shot, exited 0; api replicas started after it) +messenger-lab3_migrate_1 Exited (0) About a minute ago -$ podman logs messenger-lab3_migrate_1 -1/u init (10.119477ms) -2/u message_idempotency (14.878187ms) +$ podman logs migrate (schema already at version 2 in the kept volume; repeated run is safe) no change no change no change -$ podman logs messenger-lab3_api_1 | grep -i migrat (API replicas do not run migrations) -0 +$ API replicas run no migrations (count of "migrat" in logs) +api_1: 0 +api_2: 0 -$ psql: SELECT version, dirty FROM schema_migrations +$ SELECT version, dirty FROM schema_migrations 2|f + +$ \dt +chat_members +chats +message_idempotency +messages +schema_migrations +users diff --git a/evidence/persistence.txt b/evidence/persistence.txt index e15ff852..b3fcafb8 100644 --- a/evidence/persistence.txt +++ b/evidence/persistence.txt @@ -1,10 +1,10 @@ $ containers before -messenger-lab3_postgres_1 8c509b4f5143 Up About a minute (healthy) -messenger-lab3_redis_1 b5db2a433051 Up About a minute (healthy) -messenger-lab3_migrate_1 9fa71320cdba Exited (0) About a minute ago -messenger-lab3_api_1 0405cf8a4a54 Up About a minute -messenger-lab3_api_2 2751ae93e6d1 Up About a minute -messenger-lab3_lb_1 d3a35fe2295d Up About a minute +messenger-lab3_api_1 d63350b561c6 Up 42 seconds (healthy) +messenger-lab3_api_2 56e8d4ca6c28 Up About a minute (healthy) +messenger-lab3_lb_1 d0ec4e5311dd Up 4 minutes +messenger-lab3_migrate_1 82c01bec10d9 Exited (0) 14 minutes ago +messenger-lab3_postgres_1 0d8a6c71fcaf Up 13 minutes (healthy) +messenger-lab3_redis_1 37f40fddaa42 Up 13 minutes (healthy) $ make test-persistence ACTION_SCRIPT=scripts/restart.sh python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" @@ -12,13 +12,13 @@ Executing explicit persistence hook; hook output suppressed to avoid leaking sec PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. exit=0 -$ containers after (same IDs, fresh Up time = restarted, volumes kept) -messenger-lab3_postgres_1 8c509b4f5143 Up 1 second (healthy) -messenger-lab3_redis_1 b5db2a433051 Up 1 second (healthy) -messenger-lab3_migrate_1 9fa71320cdba Exited (0) Less than a second ago -messenger-lab3_api_1 0405cf8a4a54 Up Less than a second -messenger-lab3_api_2 2751ae93e6d1 Up Less than a second -messenger-lab3_lb_1 d3a35fe2295d Up Less than a second +$ containers after (same IDs, restarted, volumes kept) +messenger-lab3_api_1 d63350b561c6 Up 2 seconds (healthy) +messenger-lab3_api_2 56e8d4ca6c28 Up 1 second (healthy) +messenger-lab3_lb_1 d0ec4e5311dd Up 2 seconds +messenger-lab3_migrate_1 82c01bec10d9 Exited (0) 14 minutes ago +messenger-lab3_postgres_1 0d8a6c71fcaf Up 2 seconds (healthy) +messenger-lab3_redis_1 37f40fddaa42 Up 1 second (healthy) $ podman volume ls messenger-lab3_pgdata diff --git a/evidence/rolling-load.json b/evidence/rolling-load.json new file mode 100644 index 00000000..9a8caf8e --- /dev/null +++ b/evidence/rolling-load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 60.0, + "elapsed_seconds": 60.013, + "concurrency": 4, + "successful_requests": 17311, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 288.453, + "successful_latency_ms": { + "p50": 13.202, + "p95": 17.032, + "p99": 20.075 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/rolling.txt b/evidence/rolling.txt new file mode 100644 index 00000000..f157c792 --- /dev/null +++ b/evidence/rolling.txt @@ -0,0 +1,45 @@ +$ api replicas before +messenger-lab3_api_1 container=ab01920585fe image=3fcba3e82597 revision=1793930539 health=healthy +messenger-lab3_api_2 container=5a9ffe83734a image=3fcba3e82597 revision=1793930539 health=healthy + +$ python3 tests/load.py --duration 60 --concurrency 4 --output evidence/rolling-load.json & (background load) +$ podman stats --no-stream (during load, before update) +messenger-lab3_api_1 cpu=5.87% mem=106.7MB / 268.4MB +messenger-lab3_api_2 cpu=2.47% mem=47.7MB / 268.4MB +messenger-lab3_lb_1 cpu=4.56% mem=31.62MB / 67.11MB +messenger-lab3_postgres_1 cpu=12.33% mem=156.8MB / 536.9MB +messenger-lab3_redis_1 cpu=0.59% mem=7.361MB / 134.2MB + +$ scripts/rolling-update.sh (build new image -> migrate once -> replace replicas one by one) +exit=0 duration=21s +migrate: no change + +$ podman stats --no-stream (during load, after update) +messenger-lab3_api_1 cpu=4.77% mem=12.05MB / 268.4MB +messenger-lab3_api_2 cpu=7.42% mem=11.21MB / 268.4MB +messenger-lab3_lb_1 cpu=4.78% mem=31.83MB / 67.11MB +messenger-lab3_postgres_1 cpu=12.49% mem=159.6MB / 536.9MB +messenger-lab3_redis_1 cpu=0.59% mem=7.119MB / 134.2MB + +$ background tests/load.py result +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 60.0, + "elapsed_seconds": 60.013, + "concurrency": 4, + "successful_requests": 17311, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 288.453, + "successful_latency_ms": { + "p50": 13.202, + "p95": 17.032, + "p99": 20.075 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} + +$ api replicas after (new containers, new image and revision) +messenger-lab3_api_1 container=d63350b561c6 image=3ab3deda3163 revision=1793930771 health=healthy +messenger-lab3_api_2 container=56e8d4ca6c28 image=3ab3deda3163 revision=1793930771 health=healthy diff --git a/evidence/test-go.txt b/evidence/test-go.txt index 51721aa3..d2bac068 100644 --- a/evidence/test-go.txt +++ b/evidence/test-go.txt @@ -2,41 +2,42 @@ ? messenger/internal/config [no test files] ? messenger/internal/domain [no test files] === RUN TestSessionExpiry ---- PASS: TestSessionExpiry (3.09s) +--- PASS: TestSessionExpiry (3.11s) === RUN TestLogout ---- PASS: TestLogout (0.08s) +--- PASS: TestLogout (0.13s) === RUN TestWrongPassword ---- PASS: TestWrongPassword (0.10s) +--- PASS: TestWrongPassword (0.11s) === RUN TestRedisUnavailable -redis: 2026/11/06 03:16:52 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/06 03:16:52 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/06 03:16:53 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/06 03:16:53 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/06 03:16:54 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused -redis: 2026/11/06 03:16:54 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused ---- PASS: TestRedisUnavailable (2.89s) +redis: 2026/11/06 05:07:57 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/06 05:07:57 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/06 05:07:58 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/06 05:07:58 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/06 05:07:59 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +redis: 2026/11/06 05:07:59 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +--- PASS: TestRedisUnavailable (2.94s) === RUN TestDatabaseUnavailable ---- PASS: TestDatabaseUnavailable (0.17s) +--- PASS: TestDatabaseUnavailable (0.20s) === RUN TestInvalidJSON --- PASS: TestInvalidJSON (0.01s) === RUN TestForeignChat ---- PASS: TestForeignChat (0.17s) +redis: 2026/11/06 05:08:00 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused +--- PASS: TestForeignChat (0.23s) === RUN TestEmptyMessage --- PASS: TestEmptyMessage (0.08s) === RUN TestDuplicateUsername ---- PASS: TestDuplicateUsername (0.09s) +--- PASS: TestDuplicateUsername (0.10s) === RUN TestPaginationMultiplePages ---- PASS: TestPaginationMultiplePages (0.08s) +--- PASS: TestPaginationMultiplePages (0.09s) === RUN TestConcurrentMessages ---- PASS: TestConcurrentMessages (0.52s) +--- PASS: TestConcurrentMessages (0.57s) PASS -ok messenger/internal/handler 7.273s +ok messenger/internal/handler 7.578s === RUN TestRegisterUsernameRace ---- PASS: TestRegisterUsernameRace (0.02s) +--- PASS: TestRegisterUsernameRace (0.07s) === RUN TestCreateChatRollback --- PASS: TestCreateChatRollback (0.02s) PASS -ok messenger/internal/repository/postgres 0.043s +ok messenger/internal/repository/postgres 0.088s ? messenger/internal/repository/redis [no test files] ? messenger/internal/router [no test files] ? messenger/internal/service [no test files] diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg index 23fe37ae..d31efa8b 100644 --- a/infra/haproxy/haproxy.cfg +++ b/infra/haproxy/haproxy.cfg @@ -1,5 +1,6 @@ global log stdout format raw local0 info + maxconn 2000 defaults mode http @@ -9,6 +10,7 @@ defaults timeout client 30s timeout server 20s timeout http-request 10s + timeout check 1s retries 3 option redispatch diff --git a/internal/handler/handler.go b/internal/handler/handler.go index 0491ac47..818d35c0 100644 --- a/internal/handler/handler.go +++ b/internal/handler/handler.go @@ -4,6 +4,7 @@ import ( "context" "log/slog" "net/http" + "sync/atomic" "messenger/internal/service" ) @@ -17,6 +18,7 @@ type Handler struct { messages *service.MessageService deps []Pinger log *slog.Logger + draining atomic.Bool } func New(users *service.UserService, auth *service.AuthService, chats *service.ChatService, @@ -24,6 +26,10 @@ func New(users *service.UserService, auth *service.AuthService, chats *service.C return &Handler{users: users, auth: auth, chats: chats, messages: messages, deps: deps, log: log} } +func (h *Handler) StartDraining() { + h.draining.Store(true) +} + func (h *Handler) NotFound(w http.ResponseWriter, _ *http.Request) { writeError(w, http.StatusNotFound, "not_found", "resource not found") } diff --git a/internal/handler/health.go b/internal/handler/health.go index 7389335e..375ee0b8 100644 --- a/internal/handler/health.go +++ b/internal/handler/health.go @@ -16,6 +16,10 @@ func (h *Handler) Live(w http.ResponseWriter, _ *http.Request) { } func (h *Handler) Ready(w http.ResponseWriter, r *http.Request) { + if h.draining.Load() { + h.fail(w, r, fmt.Errorf("%w: draining", domain.ErrUnavailable)) + return + } ctx, cancel := context.WithTimeout(r.Context(), readyTimeout) defer cancel() for _, dep := range h.deps { diff --git a/scripts/explain.sh b/scripts/explain.sh deleted file mode 100755 index a6a5d4d2..00000000 --- a/scripts/explain.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -eu -cd "$(dirname "$0")/.." -COMPOSE=${COMPOSE:-docker compose} - -{ - echo "# scripts/explain.sh; date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" - $COMPOSE exec -T postgres psql -U messenger -d messenger -X -q < scripts/explain.sql -} > evidence/explain.txt -cat evidence/explain.txt diff --git a/scripts/explain.sql b/scripts/explain.sql deleted file mode 100644 index e8b02b5a..00000000 --- a/scripts/explain.sql +++ /dev/null @@ -1,66 +0,0 @@ -\set ON_ERROR_STOP on -\set big_chat '''00000000-0000-4000-8000-00000000e001''' -\set seed_user '''00000000-0000-4000-8000-00000000e000''' - -INSERT INTO users (id, username, display_name, password_hash, created_at) -VALUES (:seed_user, 'explain_seed', 'Explain seed', 'not-a-login', now()) -ON CONFLICT DO NOTHING; - -INSERT INTO chats (id, title, created_at) -SELECT CASE WHEN g = 0 THEN :big_chat::uuid - ELSE ('00000000-0000-4000-8000-' || lpad(to_hex(57345 + g), 12, '0'))::uuid END, - 'explain chat ' || g, now() -FROM generate_series(0, 20) AS g -ON CONFLICT DO NOTHING; - -INSERT INTO chat_members (chat_id, user_id, position) -SELECT id, :seed_user, 0 FROM chats WHERE title LIKE 'explain chat %' -ON CONFLICT DO NOTHING; - -INSERT INTO messages (id, chat_id, sender_id, text, created_at) -SELECT gen_random_uuid(), c.id, :seed_user, 'synthetic message ' || n, - timestamptz '2026-01-01' + n * interval '1 second' -FROM chats c -CROSS JOIN LATERAL generate_series(1, CASE WHEN c.id = :big_chat THEN 20000 ELSE 2000 END) AS n -WHERE c.title LIKE 'explain chat %' - AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id); - -ANALYZE users; -ANALYZE chats; -ANALYZE chat_members; -ANALYZE messages; - -SELECT count(*) AS messages_total, - count(*) FILTER (WHERE chat_id = :big_chat) AS messages_in_big_chat -FROM messages; - -SELECT created_at AS cur_ts, id AS cur_id -FROM messages WHERE chat_id = :big_chat -ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset - -\echo '=== 1. First page (limit=50 -> LIMIT 51), index messages_chat_created_id_idx' -EXPLAIN (ANALYZE, BUFFERS) -SELECT id, chat_id, sender_id, text, created_at -FROM messages -WHERE chat_id = :big_chat -ORDER BY created_at, id -LIMIT 51; - -\echo '=== 2. Page in the middle by keyset cursor (after 10 000 messages)' -EXPLAIN (ANALYZE, BUFFERS) -SELECT id, chat_id, sender_id, text, created_at -FROM messages -WHERE chat_id = :big_chat AND (created_at, id) > (:'cur_ts', :'cur_id') -ORDER BY created_at, id -LIMIT 51; - -\echo '=== 3. Same middle page WITHOUT the index (dropped inside a rolled-back transaction)' -BEGIN; -DROP INDEX messages_chat_created_id_idx; -EXPLAIN (ANALYZE, BUFFERS) -SELECT id, chat_id, sender_id, text, created_at -FROM messages -WHERE chat_id = :big_chat AND (created_at, id) > (:'cur_ts', :'cur_id') -ORDER BY created_at, id -LIMIT 51; -ROLLBACK; diff --git a/scripts/outage-demo.sh b/scripts/outage-demo.sh deleted file mode 100755 index 35676036..00000000 --- a/scripts/outage-demo.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -set -u -cd "$(dirname "$0")/.." -B=http://localhost:8080 -C=${COMPOSE:-docker compose} -u="outage_$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')"; p="Outage-demo-password" -code() { curl -s -o /dev/null -w '%{http_code}' "$@"; } -wait_ready() { for i in $(seq 1 60); do [ "$(code $B/health/ready)" = 200 ] && return; sleep 1; done; } -echo "# Dependency outage demo on the live stand; date: $(date -u +%Y-%m-%dT%H:%M:%SZ). Token is kept in a shell variable, never printed." -curl -s -o /dev/null -X POST $B/api/v1/users -H 'Content-Type: application/json' -d "{\"username\":\"$u\",\"display_name\":\"Demo\",\"password\":\"$p\"}" -T=$(curl -s -X POST $B/api/v1/auth/sessions -u "$u:$p" | python3 -c 'import sys,json;print(json.load(sys.stdin)["token"])') -echo "baseline: GET /users/me (Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")" -echo; echo "## $C stop redis"; $C stop redis >/dev/null 2>&1 -echo "GET /health/ready -> $(code $B/health/ready)" -echo "GET /users/me (valid Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")" -echo "GET /users/me (no auth) -> $(code $B/api/v1/users/me)" -echo "POST /auth/sessions (Basic) -> $(code -X POST $B/api/v1/auth/sessions -u "$u:$p")" -echo "body: $(curl -s $B/api/v1/users/me -H "Authorization: Bearer $T")" -$C start redis >/dev/null 2>&1; wait_ready -echo "## $C start redis -> ready; same token: GET /users/me -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")" -echo; echo "## $C stop postgres"; $C stop postgres >/dev/null 2>&1 -echo "GET /health/ready -> $(code $B/health/ready)" -echo "GET /users/me (valid Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")" -echo "GET /chats (valid Bearer) -> $(code $B/api/v1/chats -H "Authorization: Bearer $T")" -echo "body: $(curl -s $B/api/v1/chats -H "Authorization: Bearer $T")" -$C start postgres >/dev/null 2>&1; wait_ready -echo "## $C start postgres -> ready; same token: GET /users/me -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")" diff --git a/scripts/restart.sh b/scripts/restart.sh index 7887237b..d6444010 100755 --- a/scripts/restart.sh +++ b/scripts/restart.sh @@ -1,7 +1,8 @@ #!/bin/sh set -eu cd "$(dirname "$0")/.." -COMPOSE=${COMPOSE:-podman compose} +ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)} +COMPOSE=${COMPOSE:-$([ "$ENGINE" = podman ] && echo podman-compose || echo "docker compose")} $COMPOSE stop lb api redis postgres $COMPOSE start postgres redis api lb diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..78655886 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)} +COMPOSE=${COMPOSE:-$([ "$ENGINE" = podman ] && echo podman-compose || echo "docker compose")} +PROJECT=${PROJECT:-messenger-lab3} + +replicas() { + $ENGINE ps -aq --no-trunc --filter "label=com.docker.compose.project=$PROJECT" --filter "label=com.docker.compose.service=api" | sort +} + +wait_healthy() { + for _ in $(seq 1 60); do + if [ "$($ENGINE inspect -f '{{.State.Health.Status}}' "$1" 2>/dev/null)" = "healthy" ]; then + return 0 + fi + sleep 1 + done + echo "replica $1 did not become healthy" >&2 + exit 1 +} + +$ENGINE build --build-arg "BUILD_ID=${BUILD_ID:-$(date +%s)}" -t localhost/messenger-api:lab3 . +$COMPOSE run --rm migrate + +for old in $(replicas); do + before=$(replicas) + $ENGINE stop -t 10 "$old" + $ENGINE rm "$old" + $COMPOSE up -d --no-deps --no-recreate api + for id in $(replicas); do + if ! echo "$before" | grep -q "$id"; then + wait_healthy "$id" + fi + done + sleep 2 +done diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh index f305b729..96d49fa0 100755 --- a/scripts/stop-one.sh +++ b/scripts/stop-one.sh @@ -1,12 +1,12 @@ #!/bin/sh set -eu -ENGINE=${ENGINE:-podman} +ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)} PROJECT=${PROJECT:-messenger-lab3} : "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" for id in $($ENGINE ps -q --filter "label=com.docker.compose.project=$PROJECT" --filter "label=com.docker.compose.service=api"); do if [ "$($ENGINE inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then - $ENGINE stop -t 5 "$id" >/dev/null + $ENGINE stop -t 10 "$id" >/dev/null exit 0 fi done