feat: implement estim 4

This commit is contained in:
2026-11-06 05:12:28 +03:00
parent ec111aadfb
commit b48e71f433
29 changed files with 504 additions and 194 deletions
+2
View File
@@ -7,6 +7,8 @@ COPY internal ./internal
RUN CGO_ENABLED=0 go build -trimpath -o /out/server ./cmd/server
FROM alpine:3.24.1
ARG BUILD_ID=dev
LABEL org.opencontainers.image.revision=$BUILD_ID
RUN adduser -D -H -u 10001 app
COPY --from=build /out/server /usr/local/bin/server
USER app
+3 -1
View File
@@ -2,7 +2,8 @@ PYTHON ?= python3
BASE_URL ?= http://localhost:8080
CA_FILE ?=
ACTION_SCRIPT ?=
COMPOSE ?= podman compose
ENGINE ?= $(shell command -v podman >/dev/null 2>&1 && echo podman || echo docker)
COMPOSE ?= $(if $(filter podman,$(ENGINE)),podman-compose,docker compose)
TEST_DATABASE_URL ?= $(shell sed 's/@postgres:5432/@127.0.0.1:55432/' secrets/database_url 2>/dev/null)
TEST_REDIS_URL ?= redis://127.0.0.1:56379/0
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
@@ -44,3 +45,4 @@ test-idempotency:
test-go:
@TEST_DATABASE_URL="$(TEST_DATABASE_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" go test -count=1 -v ./...
+103 -14
View File
@@ -1,37 +1,126 @@
# Отчёт — лабораторная 3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Автор, группа: Спесивцев Даниил БПИ2301
- Целевая оценка 4; выбранный трек 5, если нужен: не заявлен
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Использованные источники, библиотеки и помощь ИИ: Go 1.27.1 (стандартная библиотека). Для консультации, реализации некоторых фич, написания отчета использовались LLM. Библиотеки и компоненты:
- [go-chi/chi v5.3.2](https://github.com/go-chi/chi) — HTTP-роутинг;
- [jackc/pgx v5.11.0](https://github.com/jackc/pgx) — драйвер PostgreSQL и пул `pgxpool`, транзакция идемпотентной отправки;
- [redis/go-redis v9.23.0](https://github.com/redis/go-redis) — клиент Redis для общих сессий;
- [golang.org/x/crypto v0.58.0](https://pkg.go.dev/golang.org/x/crypto/argon2) — Argon2id для паролей;
- образ [migrate/migrate v4.20.1](https://github.com/golang-migrate/migrate) — однократные миграции отдельным контейнером;
- образ [HAProxy 3.4.7](https://www.haproxy.org/) — балансировщик.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Стенд: i5-11400H (12 потоков), 16 GB RAM, Linux 7.1.8, podman 6.1.0 + podman-compose 1.6.0 (`evidence/env.txt`). Образы: `postgres:18.6-alpine3.24`, `redis:8.10.3-alpine3.23`, `migrate/migrate:v4.20.1`, `haproxy:3.4.7-alpine`; API из `Dockerfile` (`golang:1.27.1-alpine3.24` → `alpine:3.24.1`).
Движок выбирается автоматически в `Makefile` и скриптах: podman + `podman-compose`, если podman установлен, иначе docker + `docker compose`. Явно: `ENGINE=docker COMPOSE="docker compose"`. `compose.yaml` проходит `docker compose config --quiet` и `podman-compose config`.
```sh
scripts/init-secrets.sh # локальные secrets в ./secrets, вне Git; повторный запуск не перезаписывает
make up # build; postgres, redis → migrate (однократно) → api ×2; lb
until curl -sf http://localhost:8080/health/ready; do sleep 1; done
make check && make test && make test-go
make test-idempotency # один Idempotency-Key, 20 параллельных POST через LB
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
$COMPOSE start api # вернуть остановленную реплику и повторить для другой
scripts/rolling-update.sh # rolling update (под нагрузкой: параллельно python3 tests/load.py --duration 60)
make down # остановка, volumes сохраняются
$COMPOSE down -v # намеренное удаление учебных данных
```
Число реплик — `API_REPLICAS` (по умолчанию 2, балансируется до 4: `server-template api 4`). Публичная точка входа — только `lb` на `:8080`. Bootstrap безопасно повторяется: `migrate` при повторном запуске отвечает `no change`, `init-secrets.sh` не трогает существующие файлы.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
client → lb (HAProxy :8080) ──edge──▶ api ×2 ──backend──▶ postgres (источник истины)
└────────▶ redis (сессии, AOF)
migrate (one-shot) ─backend─▶ postgres
```
| Роль | Service | Образ | Health / readiness | Volume | Сеть / порт | Credentials |
| --- | --- | --- | --- | --- | --- | --- |
| LB | `lb` | `haproxy:3.4.7-alpine` | активная проверка `GET /health/ready` каждой реплики | — | edge, `0.0.0.0:8080` | нет |
| API | `api` ×2 | `localhost/messenger-api:lab3` | `/health/live`; `/health/ready` = PostgreSQL + Redis + не draining | — | edge + backend, порт не публикуется | secrets `database_url`, `cache_url` |
| Миграции | `migrate` | `migrate/migrate:v4.20.1` | exit 0 | SQL read-only | backend | secret `database_url` |
| БД | `postgres` | `postgres:18.6-alpine3.24` | `pg_isready` | `pgdata` | backend, `127.0.0.1:55432` для `make test-go` | secret `postgres_password` |
| Сессии | `redis` | `redis:8.10.3-alpine3.23` | `redis-cli ping` | `redisdata` (AOF everysec) | backend, `127.0.0.1:56379` для `make test-go` | нет |
- **Перенесено из lab2:** API на Go, PostgreSQL, Bearer-сессии в Redis, Argon2id. Cursor истории не зависит от процесса, поэтому любая реплика обслуживает любой запрос без sticky sessions.
- **X-Instance-Id:** hostname контейнера (короткий ID), выставляется middleware на все ответы API. `scripts/stop-one.sh` находит контейнер с таким hostname по labels compose и останавливает только его.
- **Идемпотентность:** таблица `message_idempotency`, PK `(user_id, chat_id, key)`; метод и путь фиксированы для эндпоинта. В одной транзакции выполняются `INSERT ... ON CONFLICT DO NOTHING` ключа и `INSERT` сообщения. Конкурентный дубль ждёт на уникальном индексе; при совпадении `sha256(text)` возвращается то же сообщение с 201, иначе 409. При 5xx транзакция откатывается, и ключ не фиксируется. Повтор заново проверяет сессию и доступ к чату.
- **Миграции:** выполняются отдельным контейнером `migrate` до старта API (advisory lock golang-migrate); реплики миграций не запускают. Совместимость с работающей версией: допускаются только добавляющие изменения (expand/contract); `000002` лишь добавляет таблицу. Удаление или переименование колонок потребует двух релизов. `scripts/rolling-update.sh` выполняет `migrate` до замены реплик.
- **Rolling update** (`scripts/rolling-update.sh`):
1. сборка нового образа с тем же тегом и новым label `BUILD_ID`;
2. `$COMPOSE run --rm migrate`;
3. для каждой старой реплики по очереди: `stop -t 10` (drain) → `rm` → `$COMPOSE up -d --no-deps --no-recreate api` → ожидание `healthy` новой реплики.
- **Graceful shutdown:** по SIGTERM `/health/ready` сразу отвечает 503, keep-alive отключается, 3 с drain, затем `Shutdown` с таймаутом 5 с.
| Таймаут | Значение |
| --- | --- |
| PostgreSQL `statement_timeout` / connect | 5 s / 3 s |
| Readiness ping зависимостей | 1 s |
| HTTP API: ReadHeader / Read / Write / Idle | 5 / 15 / 15 / 60 s |
| HAProxy: connect / server / client / http-request / check | 2 / 20 / 30 / 10 / 1 s; `maxconn 2000` |
| Остановка: drain + shutdown | 3 + 5 = 8 s < `stop -t 10`, `stop_grace_period: 10s` |
**Обнаружение отказа.** Период проверок HAProxy: `inter 1s`, `fall 2`, `rise 2`, `timeout check 1s`; DNS `hold valid 1s`.
- При SIGTERM реплика выводится из LB по 503 readiness за ≤ 2 s, до закрытия listener (измерено 2.0 s, `evidence/drain.txt`).
- При аварийной остановке отказ соединения сразу уходит на другую реплику (`retries 3`, `option redispatch`; повторяется только неудавшееся соединение, отправленный POST не повторяется). Максимальное время исключения backend — `inter × fall + timeout check` = 3 s.
Соответствие `.env.example`:
- `SESSION_STORE=redis`;
- `DATABASE_URL_FILE`, `CACHE_URL_FILE` — файлы secrets;
- `API_REPLICAS` → `scale` сервиса `api`;
- `SESSION_TTL_SECONDS`, `APP_ENV` передаются в API;
- `IDEMPOTENCY_TTL_SECONDS` не используется: ключи хранятся бессрочно, что покрывает требование «не менее 24 ч».
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Публичный smoke | `make test` | PASS | `evidence/smoke.txt` |
| Стенд в compose.yaml: ≥2 API, одна точка входа, общие БД и сессии | `make up`, `ps` | 2 реплики `api`, порт публикует только `lb` | `evidence/instances.txt`, `evidence/limits.txt` |
| Разные реплики с одной сессией, X-Instance-Id | 20 × `GET /users/me` через LB одним токеном | по 10 ответов от двух ID; ID = hostname контейнера | `evidence/instances.txt` |
| Отказ любой одной реплики: ≤ 10 s, подтверждённые сообщения доступны | `make test-failover` для `api_1` и `api_2` | PASS, 0 ошибок, первая запись после hook за 3.3–3.4 s | `evidence/failover-1.txt`, `evidence/failover-2.txt` |
| Повтор с тем же Idempotency-Key без дубля, в т. ч. конкурентный и через разные реплики | `make test`, `make test-idempotency` | 20/20 ответов 201 с одним сообщением, обслужили обе реплики | `evidence/smoke.txt`, `evidence/idempotency-race.txt` |
| persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; перезапущены lb, api, redis, postgres с теми же ID; volumes сохранены | `evidence/persistence.txt` |
| Миграции отдельно, без гонки реплик | `make up` | `migrate` exit 0, в логах API миграций нет, версия схемы 2 | `evidence/migrate.txt` |
| Собственные Go-тесты | `make test-go` | 13 PASS | `evidence/test-go.txt` |
| 4: readiness, timeouts, SIGTERM, исключение backend; период и время обнаружения | `stop` реплики | ready 503 → LB DOWN за 2.0 s → listener закрыт через 3 s | `evidence/drain.txt`, разделы выше |
| 4: поочерёдная остановка каждой реплики, RPS / ошибки / восстановление | `make test-failover` под фоновым `tests/load.py` | см. таблицу ниже | `evidence/failover-{1,2}.txt`, `evidence/failover-load-{1,2}.json` |
| 4: общая идемпотентность параллельными запросами с одним ключом | `make test-idempotency` | одно сообщение на 20 параллельных запросов | `evidence/idempotency-race.txt` |
| 4: CPU/RAM-лимиты, API без портов, секреты и хранилища только где нужно, нет container_name | `inspect`, `ls /run/secrets`, `getent` из lb | лимиты у всех сервисов; lb не видит postgres и redis; секреты по ролям | `evidence/limits.txt` |
| 4: rolling update под нагрузкой; миграции однократны и совместимы | `scripts/rolling-update.sh` под фоновым `tests/load.py --duration 60` | обе реплики заменены новым образом за 21 s, 0 ошибок | `evidence/rolling.txt`, `evidence/rolling-load.json` |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
- **Профиль:** публичный `tests/load.py`, closed-loop POST сообщений, concurrency 4, 30 s (rolling — 60 s), без автоматических повторов. Отказ запускается через `make test-failover` примерно на 4-й секунде нагрузки; тест сам повторяет неопределённый POST с тем же ключом и проверяет сохранность и отсутствие дублей. Прогрев отдельно не выделялся; один прогон на сценарий.
- **Лимиты:** api 1 CPU / 256 MiB на реплику, postgres 1 CPU / 512 MiB, redis 0.5 CPU / 128 MiB, lb 0.5 CPU / 64 MiB, migrate 0.5 CPU / 64 MiB.
- **Потребление во время нагрузки** (`stats --no-stream`): api 4–5 % CPU и 28–92 MiB, postgres 12 % CPU и 135 MiB, lb 3–5 % CPU и 28–32 MiB, redis < 1 % CPU и 7 MiB.
- **Набор данных после всех прогонов:** 294 274 сообщения, 292 пользователя, 84 чата, БД 167 MB (`evidence/dataset.txt`).
| Прогон | Успешные RPS (`load.py`) | Ошибки `load.py` / теста | p50 / p95 / p99, ms | Восстановление записи (тест) | Записей теста за 15 s |
| --- | --- | --- | --- | --- | --- |
| stop `api_1` | 290.3 | 0 / 0 | 13.1 / 17.6 / 23.9 | 3.34 s от старта hook (hook ≈ 3.2 s, drain) | 131 |
| stop `api_2` | 316.8 | 0 / 0 | 12.1 / 16.1 / 22.4 | 3.37 s от старта hook | 132 |
| rolling update | 288.5 | 0 | 13.2 / 17.0 / 20.1 | замена 2 реплик за 21 s | — |
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `stop` реплики (SIGTERM) | вывод из LB до закрытия listener, без ошибок | DOWN по 503 за 2.0 s, 0 ошибок под нагрузкой | запись не прерывалась / потерь и дублей нет |
| Rolling update | поочерёдная замена без простоя | 0 ошибок за 60 s, новый image и revision у обеих реплик | — / потерь нет |
| Restart всего стенда | данные и токен сохраняются | PASS persistence | в пределах бюджета теста 90 s / потерь нет |
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Потеря host, LB или PostgreSQL не переживается, это SPOF. Отказоустойчивость показана только для процесса API.
- Порты PostgreSQL и Redis опубликованы только на `127.0.0.1` для `make test-go`. Redis без пароля доступен только с локальной машины.
- `server-template api 4` при `API_REPLICAS` > 4 лишние реплики в балансировку не попадут.
- Нагрузка closed-loop с одного клиента ограничена клиентом, есть coordinated omission, один прогон на сценарий. Это не оценка предельной пропускной способности.
- Ключи идемпотентности не удаляются, таблица растёт. Для production понадобится фоновая очистка ключей старше 24 ч.
+10 -3
View File
@@ -18,7 +18,10 @@ import (
"messenger/internal/service"
)
const shutdownTimeout = 5 * time.Second
const (
drainDelay = 3 * time.Second
shutdownTimeout = 5 * time.Second
)
func main() {
log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
@@ -59,9 +62,10 @@ func run(log *slog.Logger) error {
messages := service.NewMessageService(chats, store)
deps := []handler.Pinger{store.Ping, sessions.Ping}
h := handler.New(users, auth, chats, messages, deps, log)
srv := &http.Server{
Addr: cfg.Addr(),
Handler: router.New(handler.New(users, auth, chats, messages, deps, log), instanceID),
Handler: router.New(h, instanceID),
ReadHeaderTimeout: 5 * time.Second,
ReadTimeout: 15 * time.Second,
WriteTimeout: 15 * time.Second,
@@ -84,7 +88,10 @@ func run(log *slog.Logger) error {
return err
case <-ctx.Done():
}
log.Info("shutdown signal received", "timeout", shutdownTimeout.String())
log.Info("shutdown signal received, draining", "drain", drainDelay.String(), "timeout", shutdownTimeout.String())
h.StartDraining()
srv.SetKeepAlivesEnabled(false)
time.Sleep(drainDelay)
shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
defer cancel()
+47 -4
View File
@@ -11,6 +11,8 @@ services:
- postgres_password
volumes:
- pgdata:/var/lib/postgresql
networks:
- backend
ports:
- "127.0.0.1:55432:5432"
healthcheck:
@@ -18,6 +20,11 @@ services:
interval: 2s
timeout: 3s
retries: 30
deploy:
resources:
limits:
cpus: "1.0"
memory: 512M
restart: unless-stopped
redis:
@@ -25,6 +32,8 @@ services:
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
volumes:
- redisdata:/data
networks:
- backend
ports:
- "127.0.0.1:56379:6379"
healthcheck:
@@ -32,6 +41,11 @@ services:
interval: 2s
timeout: 3s
retries: 30
deploy:
resources:
limits:
cpus: "0.5"
memory: 128M
restart: unless-stopped
migrate:
@@ -42,9 +56,16 @@ services:
- database_url
volumes:
- ./internal/repository/postgres/migrations:/migrations:ro
networks:
- backend
depends_on:
postgres:
condition: service_healthy
deploy:
resources:
limits:
cpus: "0.5"
memory: 64M
restart: "no"
api:
@@ -60,9 +81,22 @@ services:
secrets:
- database_url
- cache_url
networks:
- edge
- backend
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
interval: 5s
timeout: 2s
retries: 3
start_period: 5s
stop_grace_period: 10s
scale: ${API_REPLICAS:-2}
deploy:
mode: replicated
replicas: ${API_REPLICAS:-2}
resources:
limits:
cpus: "1.0"
memory: 256M
depends_on:
migrate:
condition: service_completed_successfully
@@ -74,12 +108,21 @@ services:
image: docker.io/library/haproxy:3.4.7-alpine
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
networks:
- edge
ports:
- "8080:8080"
depends_on:
- api
deploy:
resources:
limits:
cpus: "0.5"
memory: 64M
restart: unless-stopped
networks:
edge:
backend:
secrets:
postgres_password:
file: ./secrets/postgres_password
+8
View File
@@ -0,0 +1,8 @@
$ row counts after all runs (date 2026-11-06T02:07:36Z)
chats 84
users 292
messages 294274
message_idempotency 294274
$ database size
167 MB
+13
View File
@@ -0,0 +1,13 @@
$ date -u; podman stop -t 10 messenger-lab3_api_1; date -u
02:07:18.261
02:07:21.454
$ podman logs messenger-lab3_api_1 (shutdown sequence, UTC)
02:07:18.345 shutdown signal received, draining 3s 5s
02:07:21.345 server stopped
$ TZ=UTC podman logs -t messenger-lab3_lb_1 (backend state, UTC)
02:07:20.346 Server api/api2 is DOWN, reason: Layer7 wrong status, code: 503, info: "Service Unavailable", check duration: 0ms. 1 active and 0 backup
SIGTERM 02:07:18.345 -> LB marks DOWN by 503 readiness 02:07:20.346 (2.0 s, <= inter 1s x fall 2) -> listener closed 02:07:21.345 (drain 3 s); hook returned in 3.2 s.
HAProxy slot names (api1..api4 from server-template) are assigned by DNS order and do not match container numbers.
+12
View File
@@ -0,0 +1,12 @@
date: 2026-11-06T01:54:08Z
host: i5-11400H, 12 threads, 15Gi RAM, Linux 7.1.8-arch1-3
podman version 6.1.0
podman-compose version 1.6.0
go version go1.27.1 linux/amd64
images:
image: docker.io/library/postgres:18.6-alpine3.24
image: docker.io/library/redis:8.10.3-alpine3.23
image: docker.io/migrate/migrate:v4.20.1
image: localhost/messenger-api:lab3
image: docker.io/library/haproxy:3.4.7-alpine
limits: see evidence/limits.txt
+38 -5
View File
@@ -1,14 +1,47 @@
$ api replicas before
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 10 seconds
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 10 seconds
messenger-lab3_api_2 hostname=5a9ffe83734a Up About a minute (healthy)
messenger-lab3_api_1 hostname=ab01920585fe Up About a minute (healthy)
$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/failover-load-1.json & (background load during the test)
$ podman stats --no-stream (during load, before failure)
messenger-lab3_api_1 cpu=3.87% mem=88.06MB / 268.4MB
messenger-lab3_api_2 cpu=4.17% mem=92.08MB / 268.4MB
messenger-lab3_lb_1 cpu=2.81% mem=28.44MB / 67.11MB
messenger-lab3_postgres_1 cpu=12.21% mem=135.6MB / 536.9MB
messenger-lab3_redis_1 cpu=0.58% mem=6.857MB / 134.2MB
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 87, "failed_attempts": 1, "first_recovery_seconds": 2.52}
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 3.339}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ podman stats --no-stream (during load, one replica stopped)
messenger-lab3_api_2 cpu=5.01% mem=27.76MB / 268.4MB
messenger-lab3_lb_1 cpu=3.77% mem=29.78MB / 67.11MB
messenger-lab3_postgres_1 cpu=12.32% mem=133.1MB / 536.9MB
messenger-lab3_redis_1 cpu=0.59% mem=7.111MB / 134.2MB
$ background tests/load.py result
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.008,
"concurrency": 4,
"successful_requests": 8712,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 290.322,
"successful_latency_ms": {
"p50": 13.117,
"p95": 17.627,
"p99": 23.947
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
$ api replicas after (stopped one = TARGET_INSTANCE_ID)
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 26 seconds
messenger-lab3_api_2 hostname=2751ae93e6d1 Exited (0) 15 seconds ago
messenger-lab3_api_2 hostname=5a9ffe83734a Up 2 minutes (healthy)
messenger-lab3_api_1 hostname=ab01920585fe Exited (0) 22 seconds ago (healthy)
+38 -9
View File
@@ -1,18 +1,47 @@
$ api replicas before (api_2 restored with: podman compose start api)
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 2 minutes
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 19 seconds
$ curl -s localhost:8080/health/live (shift round-robin so the test write lands on the other replica)
{"status":"ok"}
$ api replicas before
messenger-lab3_api_2 hostname=5a9ffe83734a Up 2 minutes (healthy)
messenger-lab3_api_1 hostname=ab01920585fe Up 4 seconds (healthy)
$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/failover-load-2.json & (background load during the test)
$ podman stats --no-stream (during load, before failure)
messenger-lab3_api_1 cpu=4.21% mem=47.89MB / 268.4MB
messenger-lab3_api_2 cpu=5.90% mem=92.86MB / 268.4MB
messenger-lab3_lb_1 cpu=4.60% mem=31.47MB / 67.11MB
messenger-lab3_postgres_1 cpu=12.45% mem=148.5MB / 536.9MB
messenger-lab3_redis_1 cpu=0.59% mem=6.611MB / 134.2MB
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 81, "failed_attempts": 1, "first_recovery_seconds": 2.424}
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 3.37}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ podman stats --no-stream (during load, one replica stopped)
messenger-lab3_api_1 cpu=7.73% mem=33.08MB / 268.4MB
messenger-lab3_lb_1 cpu=4.84% mem=32.26MB / 67.11MB
messenger-lab3_postgres_1 cpu=12.54% mem=146.1MB / 536.9MB
messenger-lab3_redis_1 cpu=0.59% mem=7.373MB / 134.2MB
$ background tests/load.py result
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.013,
"concurrency": 4,
"successful_requests": 9508,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 316.791,
"successful_latency_ms": {
"p50": 12.105,
"p95": 16.14,
"p99": 22.351
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
$ api replicas after (stopped one = TARGET_INSTANCE_ID)
messenger-lab3_api_1 hostname=0405cf8a4a54 Exited (0) 15 seconds ago
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 35 seconds
messenger-lab3_api_2 hostname=5a9ffe83734a Exited (0) 22 seconds ago (healthy)
messenger-lab3_api_1 hostname=ab01920585fe Up 35 seconds (healthy)
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.008,
"concurrency": 4,
"successful_requests": 8712,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 290.322,
"successful_latency_ms": {
"p50": 13.117,
"p95": 17.627,
"p99": 23.947
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.013,
"concurrency": 4,
"successful_requests": 9508,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 316.791,
"successful_latency_ms": {
"p50": 12.105,
"p95": 16.14,
"p99": 22.351
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+1 -1
View File
@@ -4,5 +4,5 @@ successful responses (201): 20
failed requests: 0 []
distinct message ids in responses: 1
messages stored in history: 1
instances that served requests: ['0405cf8a4a54', '2751ae93e6d1']
instances that served requests: ['56e8d4ca6c28', 'd63350b561c6']
OK: exactly one message created
+10 -12
View File
@@ -1,18 +1,16 @@
20 GET /api/v1/users/me through LB (localhost:8080) with ONE bearer session, no re-login
X-Instance-Id 0405cf8a4a54 responses 10
X-Instance-Id 2751ae93e6d1 responses 10
X-Instance-Id 56e8d4ca6c28 responses 10
X-Instance-Id d63350b561c6 responses 10
$ podman ps --filter label=com.docker.compose.service=api --format "{{.Names}} {{.ID}} {{.Status}}"
messenger-lab3_api_1 0405cf8a4a54 Up 54 seconds
messenger-lab3_api_2 2751ae93e6d1 Up 53 seconds
$ podman ps (api replicas: name, container ID = hostname = X-Instance-Id)
messenger-lab3_api_1 d63350b561c6 Up 41 seconds (healthy)
messenger-lab3_api_2 56e8d4ca6c28 Up About a minute (healthy)
messenger-lab3_api_1 hostname=d63350b561c6
messenger-lab3_api_2 hostname=56e8d4ca6c28
$ podman inspect -f "{{.Name}} hostname={{.Config.Hostname}} ports={{.NetworkSettings.Ports}}" api replicas
messenger-lab3_api_1 hostname=0405cf8a4a54 ports=map[8080/tcp:[]]
messenger-lab3_api_2 hostname=2751ae93e6d1 ports=map[8080/tcp:[]]
$ podman ps --format "{{.Names}} {{.Ports}}" (only lb publishes 8080)
messenger-lab3_postgres_1 127.0.0.1:55432->5432/tcp
messenger-lab3_redis_1 127.0.0.1:56379->6379/tcp
$ podman ps (published ports: only lb exposes 8080)
messenger-lab3_api_1 8080/tcp
messenger-lab3_api_2 8080/tcp
messenger-lab3_lb_1 0.0.0.0:8080->8080/tcp
messenger-lab3_postgres_1 127.0.0.1:55432->5432/tcp
messenger-lab3_redis_1 127.0.0.1:56379->6379/tcp
+23
View File
@@ -0,0 +1,23 @@
$ podman inspect: limits, networks, published ports
messenger-lab3_api_1 cpus=1000000000 memory=268435456 networks=messenger-lab3_backend messenger-lab3_edge ports=
messenger-lab3_api_2 cpus=1000000000 memory=268435456 networks=messenger-lab3_backend messenger-lab3_edge ports=
messenger-lab3_lb_1 cpus=500000000 memory=67108864 networks=messenger-lab3_edge ports=8080/tcp->0.0.0.0:8080
messenger-lab3_migrate_1 cpus=500000000 memory=67108864 networks=messenger-lab3_backend ports=
messenger-lab3_postgres_1 cpus=1000000000 memory=536870912 networks=messenger-lab3_backend ports=5432/tcp->127.0.0.1:55432
messenger-lab3_redis_1 cpus=500000000 memory=134217728 networks=messenger-lab3_backend ports=6379/tcp->127.0.0.1:56379
$ ls /run/secrets in each running container
messenger-lab3_api_1: cache_url database_url
messenger-lab3_api_2: cache_url database_url
messenger-lab3_lb_1:
messenger-lab3_postgres_1: postgres_password
messenger-lab3_redis_1:
migrate (one-shot): /run/secrets/database_url
$ lb sees only the edge network
postgres: not resolvable from lb
redis: not resolvable from lb
10.89.4.28 api.dns.podman api.dns.podman api
$ grep -c container_name compose.yaml
0
+15 -8
View File
@@ -1,15 +1,22 @@
$ podman ps -a --filter label=com.docker.compose.service=migrate
messenger-lab3_migrate_1 Exited (0) 53 seconds ago
$ podman ps -a (migrate is one-shot, exited 0; api replicas started after it)
messenger-lab3_migrate_1 Exited (0) About a minute ago
$ podman logs messenger-lab3_migrate_1
1/u init (10.119477ms)
2/u message_idempotency (14.878187ms)
$ podman logs migrate (schema already at version 2 in the kept volume; repeated run is safe)
no change
no change
no change
$ podman logs messenger-lab3_api_1 | grep -i migrat (API replicas do not run migrations)
0
$ API replicas run no migrations (count of "migrat" in logs)
api_1: 0
api_2: 0
$ psql: SELECT version, dirty FROM schema_migrations
$ SELECT version, dirty FROM schema_migrations
2|f
$ \dt
chat_members
chats
message_idempotency
messages
schema_migrations
users
+13 -13
View File
@@ -1,10 +1,10 @@
$ containers before
messenger-lab3_postgres_1 8c509b4f5143 Up About a minute (healthy)
messenger-lab3_redis_1 b5db2a433051 Up About a minute (healthy)
messenger-lab3_migrate_1 9fa71320cdba Exited (0) About a minute ago
messenger-lab3_api_1 0405cf8a4a54 Up About a minute
messenger-lab3_api_2 2751ae93e6d1 Up About a minute
messenger-lab3_lb_1 d3a35fe2295d Up About a minute
messenger-lab3_api_1 d63350b561c6 Up 42 seconds (healthy)
messenger-lab3_api_2 56e8d4ca6c28 Up About a minute (healthy)
messenger-lab3_lb_1 d0ec4e5311dd Up 4 minutes
messenger-lab3_migrate_1 82c01bec10d9 Exited (0) 14 minutes ago
messenger-lab3_postgres_1 0d8a6c71fcaf Up 13 minutes (healthy)
messenger-lab3_redis_1 37f40fddaa42 Up 13 minutes (healthy)
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
@@ -12,13 +12,13 @@ Executing explicit persistence hook; hook output suppressed to avoid leaking sec
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
$ containers after (same IDs, fresh Up time = restarted, volumes kept)
messenger-lab3_postgres_1 8c509b4f5143 Up 1 second (healthy)
messenger-lab3_redis_1 b5db2a433051 Up 1 second (healthy)
messenger-lab3_migrate_1 9fa71320cdba Exited (0) Less than a second ago
messenger-lab3_api_1 0405cf8a4a54 Up Less than a second
messenger-lab3_api_2 2751ae93e6d1 Up Less than a second
messenger-lab3_lb_1 d3a35fe2295d Up Less than a second
$ containers after (same IDs, restarted, volumes kept)
messenger-lab3_api_1 d63350b561c6 Up 2 seconds (healthy)
messenger-lab3_api_2 56e8d4ca6c28 Up 1 second (healthy)
messenger-lab3_lb_1 d0ec4e5311dd Up 2 seconds
messenger-lab3_migrate_1 82c01bec10d9 Exited (0) 14 minutes ago
messenger-lab3_postgres_1 0d8a6c71fcaf Up 2 seconds (healthy)
messenger-lab3_redis_1 37f40fddaa42 Up 1 second (healthy)
$ podman volume ls
messenger-lab3_pgdata
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 60.0,
"elapsed_seconds": 60.013,
"concurrency": 4,
"successful_requests": 17311,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 288.453,
"successful_latency_ms": {
"p50": 13.202,
"p95": 17.032,
"p99": 20.075
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+45
View File
@@ -0,0 +1,45 @@
$ api replicas before
messenger-lab3_api_1 container=ab01920585fe image=3fcba3e82597 revision=1793930539 health=healthy
messenger-lab3_api_2 container=5a9ffe83734a image=3fcba3e82597 revision=1793930539 health=healthy
$ python3 tests/load.py --duration 60 --concurrency 4 --output evidence/rolling-load.json & (background load)
$ podman stats --no-stream (during load, before update)
messenger-lab3_api_1 cpu=5.87% mem=106.7MB / 268.4MB
messenger-lab3_api_2 cpu=2.47% mem=47.7MB / 268.4MB
messenger-lab3_lb_1 cpu=4.56% mem=31.62MB / 67.11MB
messenger-lab3_postgres_1 cpu=12.33% mem=156.8MB / 536.9MB
messenger-lab3_redis_1 cpu=0.59% mem=7.361MB / 134.2MB
$ scripts/rolling-update.sh (build new image -> migrate once -> replace replicas one by one)
exit=0 duration=21s
migrate: no change
$ podman stats --no-stream (during load, after update)
messenger-lab3_api_1 cpu=4.77% mem=12.05MB / 268.4MB
messenger-lab3_api_2 cpu=7.42% mem=11.21MB / 268.4MB
messenger-lab3_lb_1 cpu=4.78% mem=31.83MB / 67.11MB
messenger-lab3_postgres_1 cpu=12.49% mem=159.6MB / 536.9MB
messenger-lab3_redis_1 cpu=0.59% mem=7.119MB / 134.2MB
$ background tests/load.py result
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 60.0,
"elapsed_seconds": 60.013,
"concurrency": 4,
"successful_requests": 17311,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 288.453,
"successful_latency_ms": {
"p50": 13.202,
"p95": 17.032,
"p99": 20.075
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
$ api replicas after (new containers, new image and revision)
messenger-lab3_api_1 container=d63350b561c6 image=3ab3deda3163 revision=1793930771 health=healthy
messenger-lab3_api_2 container=56e8d4ca6c28 image=3ab3deda3163 revision=1793930771 health=healthy
+19 -18
View File
@@ -2,41 +2,42 @@
? messenger/internal/config [no test files]
? messenger/internal/domain [no test files]
=== RUN TestSessionExpiry
--- PASS: TestSessionExpiry (3.09s)
--- PASS: TestSessionExpiry (3.11s)
=== RUN TestLogout
--- PASS: TestLogout (0.08s)
--- PASS: TestLogout (0.13s)
=== RUN TestWrongPassword
--- PASS: TestWrongPassword (0.10s)
--- PASS: TestWrongPassword (0.11s)
=== RUN TestRedisUnavailable
redis: 2026/11/06 03:16:52 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:52 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:53 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:53 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:54 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 03:16:54 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
--- PASS: TestRedisUnavailable (2.89s)
redis: 2026/11/06 05:07:57 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 05:07:57 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 05:07:58 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 05:07:58 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 05:07:59 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
redis: 2026/11/06 05:07:59 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
--- PASS: TestRedisUnavailable (2.94s)
=== RUN TestDatabaseUnavailable
--- PASS: TestDatabaseUnavailable (0.17s)
--- PASS: TestDatabaseUnavailable (0.20s)
=== RUN TestInvalidJSON
--- PASS: TestInvalidJSON (0.01s)
=== RUN TestForeignChat
--- PASS: TestForeignChat (0.17s)
redis: 2026/11/06 05:08:00 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
--- PASS: TestForeignChat (0.23s)
=== RUN TestEmptyMessage
--- PASS: TestEmptyMessage (0.08s)
=== RUN TestDuplicateUsername
--- PASS: TestDuplicateUsername (0.09s)
--- PASS: TestDuplicateUsername (0.10s)
=== RUN TestPaginationMultiplePages
--- PASS: TestPaginationMultiplePages (0.08s)
--- PASS: TestPaginationMultiplePages (0.09s)
=== RUN TestConcurrentMessages
--- PASS: TestConcurrentMessages (0.52s)
--- PASS: TestConcurrentMessages (0.57s)
PASS
ok messenger/internal/handler 7.273s
ok messenger/internal/handler 7.578s
=== RUN TestRegisterUsernameRace
--- PASS: TestRegisterUsernameRace (0.02s)
--- PASS: TestRegisterUsernameRace (0.07s)
=== RUN TestCreateChatRollback
--- PASS: TestCreateChatRollback (0.02s)
PASS
ok messenger/internal/repository/postgres 0.043s
ok messenger/internal/repository/postgres 0.088s
? messenger/internal/repository/redis [no test files]
? messenger/internal/router [no test files]
? messenger/internal/service [no test files]
+2
View File
@@ -1,5 +1,6 @@
global
log stdout format raw local0 info
maxconn 2000
defaults
mode http
@@ -9,6 +10,7 @@ defaults
timeout client 30s
timeout server 20s
timeout http-request 10s
timeout check 1s
retries 3
option redispatch
+6
View File
@@ -4,6 +4,7 @@ import (
"context"
"log/slog"
"net/http"
"sync/atomic"
"messenger/internal/service"
)
@@ -17,6 +18,7 @@ type Handler struct {
messages *service.MessageService
deps []Pinger
log *slog.Logger
draining atomic.Bool
}
func New(users *service.UserService, auth *service.AuthService, chats *service.ChatService,
@@ -24,6 +26,10 @@ func New(users *service.UserService, auth *service.AuthService, chats *service.C
return &Handler{users: users, auth: auth, chats: chats, messages: messages, deps: deps, log: log}
}
func (h *Handler) StartDraining() {
h.draining.Store(true)
}
func (h *Handler) NotFound(w http.ResponseWriter, _ *http.Request) {
writeError(w, http.StatusNotFound, "not_found", "resource not found")
}
+4
View File
@@ -16,6 +16,10 @@ func (h *Handler) Live(w http.ResponseWriter, _ *http.Request) {
}
func (h *Handler) Ready(w http.ResponseWriter, r *http.Request) {
if h.draining.Load() {
h.fail(w, r, fmt.Errorf("%w: draining", domain.ErrUnavailable))
return
}
ctx, cancel := context.WithTimeout(r.Context(), readyTimeout)
defer cancel()
for _, dep := range h.deps {
-10
View File
@@ -1,10 +0,0 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
COMPOSE=${COMPOSE:-docker compose}
{
echo "# scripts/explain.sh; date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
$COMPOSE exec -T postgres psql -U messenger -d messenger -X -q < scripts/explain.sql
} > evidence/explain.txt
cat evidence/explain.txt
-66
View File
@@ -1,66 +0,0 @@
\set ON_ERROR_STOP on
\set big_chat '''00000000-0000-4000-8000-00000000e001'''
\set seed_user '''00000000-0000-4000-8000-00000000e000'''
INSERT INTO users (id, username, display_name, password_hash, created_at)
VALUES (:seed_user, 'explain_seed', 'Explain seed', 'not-a-login', now())
ON CONFLICT DO NOTHING;
INSERT INTO chats (id, title, created_at)
SELECT CASE WHEN g = 0 THEN :big_chat::uuid
ELSE ('00000000-0000-4000-8000-' || lpad(to_hex(57345 + g), 12, '0'))::uuid END,
'explain chat ' || g, now()
FROM generate_series(0, 20) AS g
ON CONFLICT DO NOTHING;
INSERT INTO chat_members (chat_id, user_id, position)
SELECT id, :seed_user, 0 FROM chats WHERE title LIKE 'explain chat %'
ON CONFLICT DO NOTHING;
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
SELECT gen_random_uuid(), c.id, :seed_user, 'synthetic message ' || n,
timestamptz '2026-01-01' + n * interval '1 second'
FROM chats c
CROSS JOIN LATERAL generate_series(1, CASE WHEN c.id = :big_chat THEN 20000 ELSE 2000 END) AS n
WHERE c.title LIKE 'explain chat %'
AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id);
ANALYZE users;
ANALYZE chats;
ANALYZE chat_members;
ANALYZE messages;
SELECT count(*) AS messages_total,
count(*) FILTER (WHERE chat_id = :big_chat) AS messages_in_big_chat
FROM messages;
SELECT created_at AS cur_ts, id AS cur_id
FROM messages WHERE chat_id = :big_chat
ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
\echo '=== 1. First page (limit=50 -> LIMIT 51), index messages_chat_created_id_idx'
EXPLAIN (ANALYZE, BUFFERS)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = :big_chat
ORDER BY created_at, id
LIMIT 51;
\echo '=== 2. Page in the middle by keyset cursor (after 10 000 messages)'
EXPLAIN (ANALYZE, BUFFERS)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = :big_chat AND (created_at, id) > (:'cur_ts', :'cur_id')
ORDER BY created_at, id
LIMIT 51;
\echo '=== 3. Same middle page WITHOUT the index (dropped inside a rolled-back transaction)'
BEGIN;
DROP INDEX messages_chat_created_id_idx;
EXPLAIN (ANALYZE, BUFFERS)
SELECT id, chat_id, sender_id, text, created_at
FROM messages
WHERE chat_id = :big_chat AND (created_at, id) > (:'cur_ts', :'cur_id')
ORDER BY created_at, id
LIMIT 51;
ROLLBACK;
-27
View File
@@ -1,27 +0,0 @@
#!/bin/sh
set -u
cd "$(dirname "$0")/.."
B=http://localhost:8080
C=${COMPOSE:-docker compose}
u="outage_$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')"; p="Outage-demo-password"
code() { curl -s -o /dev/null -w '%{http_code}' "$@"; }
wait_ready() { for i in $(seq 1 60); do [ "$(code $B/health/ready)" = 200 ] && return; sleep 1; done; }
echo "# Dependency outage demo on the live stand; date: $(date -u +%Y-%m-%dT%H:%M:%SZ). Token is kept in a shell variable, never printed."
curl -s -o /dev/null -X POST $B/api/v1/users -H 'Content-Type: application/json' -d "{\"username\":\"$u\",\"display_name\":\"Demo\",\"password\":\"$p\"}"
T=$(curl -s -X POST $B/api/v1/auth/sessions -u "$u:$p" | python3 -c 'import sys,json;print(json.load(sys.stdin)["token"])')
echo "baseline: GET /users/me (Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
echo; echo "## $C stop redis"; $C stop redis >/dev/null 2>&1
echo "GET /health/ready -> $(code $B/health/ready)"
echo "GET /users/me (valid Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
echo "GET /users/me (no auth) -> $(code $B/api/v1/users/me)"
echo "POST /auth/sessions (Basic) -> $(code -X POST $B/api/v1/auth/sessions -u "$u:$p")"
echo "body: $(curl -s $B/api/v1/users/me -H "Authorization: Bearer $T")"
$C start redis >/dev/null 2>&1; wait_ready
echo "## $C start redis -> ready; same token: GET /users/me -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
echo; echo "## $C stop postgres"; $C stop postgres >/dev/null 2>&1
echo "GET /health/ready -> $(code $B/health/ready)"
echo "GET /users/me (valid Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
echo "GET /chats (valid Bearer) -> $(code $B/api/v1/chats -H "Authorization: Bearer $T")"
echo "body: $(curl -s $B/api/v1/chats -H "Authorization: Bearer $T")"
$C start postgres >/dev/null 2>&1; wait_ready
echo "## $C start postgres -> ready; same token: GET /users/me -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
+2 -1
View File
@@ -1,7 +1,8 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
COMPOSE=${COMPOSE:-podman compose}
ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)}
COMPOSE=${COMPOSE:-$([ "$ENGINE" = podman ] && echo podman-compose || echo "docker compose")}
$COMPOSE stop lb api redis postgres
$COMPOSE start postgres redis api lb
+37
View File
@@ -0,0 +1,37 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)}
COMPOSE=${COMPOSE:-$([ "$ENGINE" = podman ] && echo podman-compose || echo "docker compose")}
PROJECT=${PROJECT:-messenger-lab3}
replicas() {
$ENGINE ps -aq --no-trunc --filter "label=com.docker.compose.project=$PROJECT" --filter "label=com.docker.compose.service=api" | sort
}
wait_healthy() {
for _ in $(seq 1 60); do
if [ "$($ENGINE inspect -f '{{.State.Health.Status}}' "$1" 2>/dev/null)" = "healthy" ]; then
return 0
fi
sleep 1
done
echo "replica $1 did not become healthy" >&2
exit 1
}
$ENGINE build --build-arg "BUILD_ID=${BUILD_ID:-$(date +%s)}" -t localhost/messenger-api:lab3 .
$COMPOSE run --rm migrate
for old in $(replicas); do
before=$(replicas)
$ENGINE stop -t 10 "$old"
$ENGINE rm "$old"
$COMPOSE up -d --no-deps --no-recreate api
for id in $(replicas); do
if ! echo "$before" | grep -q "$id"; then
wait_healthy "$id"
fi
done
sleep 2
done
+2 -2
View File
@@ -1,12 +1,12 @@
#!/bin/sh
set -eu
ENGINE=${ENGINE:-podman}
ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)}
PROJECT=${PROJECT:-messenger-lab3}
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
for id in $($ENGINE ps -q --filter "label=com.docker.compose.project=$PROJECT" --filter "label=com.docker.compose.service=api"); do
if [ "$($ENGINE inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
$ENGINE stop -t 5 "$id" >/dev/null
$ENGINE stop -t 10 "$id" >/dev/null
exit 0
fi
done