forked from mtusi2026/mtusi2026lab3
feat: implement estim 4
This commit is contained in:
@@ -7,6 +7,8 @@ COPY internal ./internal
|
||||
RUN CGO_ENABLED=0 go build -trimpath -o /out/server ./cmd/server
|
||||
|
||||
FROM alpine:3.24.1
|
||||
ARG BUILD_ID=dev
|
||||
LABEL org.opencontainers.image.revision=$BUILD_ID
|
||||
RUN adduser -D -H -u 10001 app
|
||||
COPY --from=build /out/server /usr/local/bin/server
|
||||
USER app
|
||||
|
||||
@@ -2,7 +2,8 @@ PYTHON ?= python3
|
||||
BASE_URL ?= http://localhost:8080
|
||||
CA_FILE ?=
|
||||
ACTION_SCRIPT ?=
|
||||
COMPOSE ?= podman compose
|
||||
ENGINE ?= $(shell command -v podman >/dev/null 2>&1 && echo podman || echo docker)
|
||||
COMPOSE ?= $(if $(filter podman,$(ENGINE)),podman-compose,docker compose)
|
||||
TEST_DATABASE_URL ?= $(shell sed 's/@postgres:5432/@127.0.0.1:55432/' secrets/database_url 2>/dev/null)
|
||||
TEST_REDIS_URL ?= redis://127.0.0.1:56379/0
|
||||
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
|
||||
@@ -44,3 +45,4 @@ test-idempotency:
|
||||
|
||||
test-go:
|
||||
@TEST_DATABASE_URL="$(TEST_DATABASE_URL)" TEST_REDIS_URL="$(TEST_REDIS_URL)" go test -count=1 -v ./...
|
||||
|
||||
|
||||
@@ -1,37 +1,126 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Автор, группа: Спесивцев Даниил БПИ2301
|
||||
- Целевая оценка 4; выбранный трек 5, если нужен: не заявлен
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Использованные источники, библиотеки и помощь ИИ: Go 1.27.1 (стандартная библиотека). Для консультации, реализации некоторых фич, написания отчета использовались LLM. Библиотеки и компоненты:
|
||||
- [go-chi/chi v5.3.2](https://github.com/go-chi/chi) — HTTP-роутинг;
|
||||
- [jackc/pgx v5.11.0](https://github.com/jackc/pgx) — драйвер PostgreSQL и пул `pgxpool`, транзакция идемпотентной отправки;
|
||||
- [redis/go-redis v9.23.0](https://github.com/redis/go-redis) — клиент Redis для общих сессий;
|
||||
- [golang.org/x/crypto v0.58.0](https://pkg.go.dev/golang.org/x/crypto/argon2) — Argon2id для паролей;
|
||||
- образ [migrate/migrate v4.20.1](https://github.com/golang-migrate/migrate) — однократные миграции отдельным контейнером;
|
||||
- образ [HAProxy 3.4.7](https://www.haproxy.org/) — балансировщик.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Стенд: i5-11400H (12 потоков), 16 GB RAM, Linux 7.1.8, podman 6.1.0 + podman-compose 1.6.0 (`evidence/env.txt`). Образы: `postgres:18.6-alpine3.24`, `redis:8.10.3-alpine3.23`, `migrate/migrate:v4.20.1`, `haproxy:3.4.7-alpine`; API из `Dockerfile` (`golang:1.27.1-alpine3.24` → `alpine:3.24.1`).
|
||||
|
||||
Движок выбирается автоматически в `Makefile` и скриптах: podman + `podman-compose`, если podman установлен, иначе docker + `docker compose`. Явно: `ENGINE=docker COMPOSE="docker compose"`. `compose.yaml` проходит `docker compose config --quiet` и `podman-compose config`.
|
||||
|
||||
```sh
|
||||
scripts/init-secrets.sh # локальные secrets в ./secrets, вне Git; повторный запуск не перезаписывает
|
||||
make up # build; postgres, redis → migrate (однократно) → api ×2; lb
|
||||
until curl -sf http://localhost:8080/health/ready; do sleep 1; done
|
||||
|
||||
make check && make test && make test-go
|
||||
make test-idempotency # один Idempotency-Key, 20 параллельных POST через LB
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
$COMPOSE start api # вернуть остановленную реплику и повторить для другой
|
||||
scripts/rolling-update.sh # rolling update (под нагрузкой: параллельно python3 tests/load.py --duration 60)
|
||||
|
||||
make down # остановка, volumes сохраняются
|
||||
$COMPOSE down -v # намеренное удаление учебных данных
|
||||
```
|
||||
|
||||
Число реплик — `API_REPLICAS` (по умолчанию 2, балансируется до 4: `server-template api 4`). Публичная точка входа — только `lb` на `:8080`. Bootstrap безопасно повторяется: `migrate` при повторном запуске отвечает `no change`, `init-secrets.sh` не трогает существующие файлы.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
client → lb (HAProxy :8080) ──edge──▶ api ×2 ──backend──▶ postgres (источник истины)
|
||||
└────────▶ redis (сессии, AOF)
|
||||
migrate (one-shot) ─backend─▶ postgres
|
||||
```
|
||||
|
||||
| Роль | Service | Образ | Health / readiness | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| LB | `lb` | `haproxy:3.4.7-alpine` | активная проверка `GET /health/ready` каждой реплики | — | edge, `0.0.0.0:8080` | нет |
|
||||
| API | `api` ×2 | `localhost/messenger-api:lab3` | `/health/live`; `/health/ready` = PostgreSQL + Redis + не draining | — | edge + backend, порт не публикуется | secrets `database_url`, `cache_url` |
|
||||
| Миграции | `migrate` | `migrate/migrate:v4.20.1` | exit 0 | SQL read-only | backend | secret `database_url` |
|
||||
| БД | `postgres` | `postgres:18.6-alpine3.24` | `pg_isready` | `pgdata` | backend, `127.0.0.1:55432` для `make test-go` | secret `postgres_password` |
|
||||
| Сессии | `redis` | `redis:8.10.3-alpine3.23` | `redis-cli ping` | `redisdata` (AOF everysec) | backend, `127.0.0.1:56379` для `make test-go` | нет |
|
||||
|
||||
- **Перенесено из lab2:** API на Go, PostgreSQL, Bearer-сессии в Redis, Argon2id. Cursor истории не зависит от процесса, поэтому любая реплика обслуживает любой запрос без sticky sessions.
|
||||
- **X-Instance-Id:** hostname контейнера (короткий ID), выставляется middleware на все ответы API. `scripts/stop-one.sh` находит контейнер с таким hostname по labels compose и останавливает только его.
|
||||
- **Идемпотентность:** таблица `message_idempotency`, PK `(user_id, chat_id, key)`; метод и путь фиксированы для эндпоинта. В одной транзакции выполняются `INSERT ... ON CONFLICT DO NOTHING` ключа и `INSERT` сообщения. Конкурентный дубль ждёт на уникальном индексе; при совпадении `sha256(text)` возвращается то же сообщение с 201, иначе 409. При 5xx транзакция откатывается, и ключ не фиксируется. Повтор заново проверяет сессию и доступ к чату.
|
||||
- **Миграции:** выполняются отдельным контейнером `migrate` до старта API (advisory lock golang-migrate); реплики миграций не запускают. Совместимость с работающей версией: допускаются только добавляющие изменения (expand/contract); `000002` лишь добавляет таблицу. Удаление или переименование колонок потребует двух релизов. `scripts/rolling-update.sh` выполняет `migrate` до замены реплик.
|
||||
- **Rolling update** (`scripts/rolling-update.sh`):
|
||||
1. сборка нового образа с тем же тегом и новым label `BUILD_ID`;
|
||||
2. `$COMPOSE run --rm migrate`;
|
||||
3. для каждой старой реплики по очереди: `stop -t 10` (drain) → `rm` → `$COMPOSE up -d --no-deps --no-recreate api` → ожидание `healthy` новой реплики.
|
||||
- **Graceful shutdown:** по SIGTERM `/health/ready` сразу отвечает 503, keep-alive отключается, 3 с drain, затем `Shutdown` с таймаутом 5 с.
|
||||
|
||||
| Таймаут | Значение |
|
||||
| --- | --- |
|
||||
| PostgreSQL `statement_timeout` / connect | 5 s / 3 s |
|
||||
| Readiness ping зависимостей | 1 s |
|
||||
| HTTP API: ReadHeader / Read / Write / Idle | 5 / 15 / 15 / 60 s |
|
||||
| HAProxy: connect / server / client / http-request / check | 2 / 20 / 30 / 10 / 1 s; `maxconn 2000` |
|
||||
| Остановка: drain + shutdown | 3 + 5 = 8 s < `stop -t 10`, `stop_grace_period: 10s` |
|
||||
|
||||
**Обнаружение отказа.** Период проверок HAProxy: `inter 1s`, `fall 2`, `rise 2`, `timeout check 1s`; DNS `hold valid 1s`.
|
||||
- При SIGTERM реплика выводится из LB по 503 readiness за ≤ 2 s, до закрытия listener (измерено 2.0 s, `evidence/drain.txt`).
|
||||
- При аварийной остановке отказ соединения сразу уходит на другую реплику (`retries 3`, `option redispatch`; повторяется только неудавшееся соединение, отправленный POST не повторяется). Максимальное время исключения backend — `inter × fall + timeout check` = 3 s.
|
||||
|
||||
Соответствие `.env.example`:
|
||||
- `SESSION_STORE=redis`;
|
||||
- `DATABASE_URL_FILE`, `CACHE_URL_FILE` — файлы secrets;
|
||||
- `API_REPLICAS` → `scale` сервиса `api`;
|
||||
- `SESSION_TTL_SECONDS`, `APP_ENV` передаются в API;
|
||||
- `IDEMPOTENCY_TTL_SECONDS` не используется: ключи хранятся бессрочно, что покрывает требование «не менее 24 ч».
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `make test` | PASS | `evidence/smoke.txt` |
|
||||
| Стенд в compose.yaml: ≥2 API, одна точка входа, общие БД и сессии | `make up`, `ps` | 2 реплики `api`, порт публикует только `lb` | `evidence/instances.txt`, `evidence/limits.txt` |
|
||||
| Разные реплики с одной сессией, X-Instance-Id | 20 × `GET /users/me` через LB одним токеном | по 10 ответов от двух ID; ID = hostname контейнера | `evidence/instances.txt` |
|
||||
| Отказ любой одной реплики: ≤ 10 s, подтверждённые сообщения доступны | `make test-failover` для `api_1` и `api_2` | PASS, 0 ошибок, первая запись после hook за 3.3–3.4 s | `evidence/failover-1.txt`, `evidence/failover-2.txt` |
|
||||
| Повтор с тем же Idempotency-Key без дубля, в т. ч. конкурентный и через разные реплики | `make test`, `make test-idempotency` | 20/20 ответов 201 с одним сообщением, обслужили обе реплики | `evidence/smoke.txt`, `evidence/idempotency-race.txt` |
|
||||
| persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS; перезапущены lb, api, redis, postgres с теми же ID; volumes сохранены | `evidence/persistence.txt` |
|
||||
| Миграции отдельно, без гонки реплик | `make up` | `migrate` exit 0, в логах API миграций нет, версия схемы 2 | `evidence/migrate.txt` |
|
||||
| Собственные Go-тесты | `make test-go` | 13 PASS | `evidence/test-go.txt` |
|
||||
| 4: readiness, timeouts, SIGTERM, исключение backend; период и время обнаружения | `stop` реплики | ready 503 → LB DOWN за 2.0 s → listener закрыт через 3 s | `evidence/drain.txt`, разделы выше |
|
||||
| 4: поочерёдная остановка каждой реплики, RPS / ошибки / восстановление | `make test-failover` под фоновым `tests/load.py` | см. таблицу ниже | `evidence/failover-{1,2}.txt`, `evidence/failover-load-{1,2}.json` |
|
||||
| 4: общая идемпотентность параллельными запросами с одним ключом | `make test-idempotency` | одно сообщение на 20 параллельных запросов | `evidence/idempotency-race.txt` |
|
||||
| 4: CPU/RAM-лимиты, API без портов, секреты и хранилища только где нужно, нет container_name | `inspect`, `ls /run/secrets`, `getent` из lb | лимиты у всех сервисов; lb не видит postgres и redis; секреты по ролям | `evidence/limits.txt` |
|
||||
| 4: rolling update под нагрузкой; миграции однократны и совместимы | `scripts/rolling-update.sh` под фоновым `tests/load.py --duration 60` | обе реплики заменены новым образом за 21 s, 0 ошибок | `evidence/rolling.txt`, `evidence/rolling-load.json` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
- **Профиль:** публичный `tests/load.py`, closed-loop POST сообщений, concurrency 4, 30 s (rolling — 60 s), без автоматических повторов. Отказ запускается через `make test-failover` примерно на 4-й секунде нагрузки; тест сам повторяет неопределённый POST с тем же ключом и проверяет сохранность и отсутствие дублей. Прогрев отдельно не выделялся; один прогон на сценарий.
|
||||
- **Лимиты:** api 1 CPU / 256 MiB на реплику, postgres 1 CPU / 512 MiB, redis 0.5 CPU / 128 MiB, lb 0.5 CPU / 64 MiB, migrate 0.5 CPU / 64 MiB.
|
||||
- **Потребление во время нагрузки** (`stats --no-stream`): api 4–5 % CPU и 28–92 MiB, postgres 12 % CPU и 135 MiB, lb 3–5 % CPU и 28–32 MiB, redis < 1 % CPU и 7 MiB.
|
||||
- **Набор данных после всех прогонов:** 294 274 сообщения, 292 пользователя, 84 чата, БД 167 MB (`evidence/dataset.txt`).
|
||||
|
||||
| Прогон | Успешные RPS (`load.py`) | Ошибки `load.py` / теста | p50 / p95 / p99, ms | Восстановление записи (тест) | Записей теста за 15 s |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| stop `api_1` | 290.3 | 0 / 0 | 13.1 / 17.6 / 23.9 | 3.34 s от старта hook (hook ≈ 3.2 s, drain) | 131 |
|
||||
| stop `api_2` | 316.8 | 0 / 0 | 12.1 / 16.1 / 22.4 | 3.37 s от старта hook | 132 |
|
||||
| rolling update | 288.5 | 0 | 13.2 / 17.0 / 20.1 | замена 2 реплик за 21 s | — |
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `stop` реплики (SIGTERM) | вывод из LB до закрытия listener, без ошибок | DOWN по 503 за 2.0 s, 0 ошибок под нагрузкой | запись не прерывалась / потерь и дублей нет |
|
||||
| Rolling update | поочерёдная замена без простоя | 0 ошибок за 60 s, новый image и revision у обеих реплик | — / потерь нет |
|
||||
| Restart всего стенда | данные и токен сохраняются | PASS persistence | в пределах бюджета теста 90 s / потерь нет |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Потеря host, LB или PostgreSQL не переживается, это SPOF. Отказоустойчивость показана только для процесса API.
|
||||
- Порты PostgreSQL и Redis опубликованы только на `127.0.0.1` для `make test-go`. Redis без пароля доступен только с локальной машины.
|
||||
- `server-template api 4` при `API_REPLICAS` > 4 лишние реплики в балансировку не попадут.
|
||||
- Нагрузка closed-loop с одного клиента ограничена клиентом, есть coordinated omission, один прогон на сценарий. Это не оценка предельной пропускной способности.
|
||||
- Ключи идемпотентности не удаляются, таблица растёт. Для production понадобится фоновая очистка ключей старше 24 ч.
|
||||
|
||||
+10
-3
@@ -18,7 +18,10 @@ import (
|
||||
"messenger/internal/service"
|
||||
)
|
||||
|
||||
const shutdownTimeout = 5 * time.Second
|
||||
const (
|
||||
drainDelay = 3 * time.Second
|
||||
shutdownTimeout = 5 * time.Second
|
||||
)
|
||||
|
||||
func main() {
|
||||
log := slog.New(slog.NewJSONHandler(os.Stdout, nil))
|
||||
@@ -59,9 +62,10 @@ func run(log *slog.Logger) error {
|
||||
messages := service.NewMessageService(chats, store)
|
||||
deps := []handler.Pinger{store.Ping, sessions.Ping}
|
||||
|
||||
h := handler.New(users, auth, chats, messages, deps, log)
|
||||
srv := &http.Server{
|
||||
Addr: cfg.Addr(),
|
||||
Handler: router.New(handler.New(users, auth, chats, messages, deps, log), instanceID),
|
||||
Handler: router.New(h, instanceID),
|
||||
ReadHeaderTimeout: 5 * time.Second,
|
||||
ReadTimeout: 15 * time.Second,
|
||||
WriteTimeout: 15 * time.Second,
|
||||
@@ -84,7 +88,10 @@ func run(log *slog.Logger) error {
|
||||
return err
|
||||
case <-ctx.Done():
|
||||
}
|
||||
log.Info("shutdown signal received", "timeout", shutdownTimeout.String())
|
||||
log.Info("shutdown signal received, draining", "drain", drainDelay.String(), "timeout", shutdownTimeout.String())
|
||||
h.StartDraining()
|
||||
srv.SetKeepAlivesEnabled(false)
|
||||
time.Sleep(drainDelay)
|
||||
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownTimeout)
|
||||
defer cancel()
|
||||
|
||||
+47
-4
@@ -11,6 +11,8 @@ services:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql
|
||||
networks:
|
||||
- backend
|
||||
ports:
|
||||
- "127.0.0.1:55432:5432"
|
||||
healthcheck:
|
||||
@@ -18,6 +20,11 @@ services:
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
redis:
|
||||
@@ -25,6 +32,8 @@ services:
|
||||
command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"]
|
||||
volumes:
|
||||
- redisdata:/data
|
||||
networks:
|
||||
- backend
|
||||
ports:
|
||||
- "127.0.0.1:56379:6379"
|
||||
healthcheck:
|
||||
@@ -32,6 +41,11 @@ services:
|
||||
interval: 2s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
@@ -42,9 +56,16 @@ services:
|
||||
- database_url
|
||||
volumes:
|
||||
- ./internal/repository/postgres/migrations:/migrations:ro
|
||||
networks:
|
||||
- backend
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 64M
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
@@ -60,9 +81,22 @@ services:
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
networks:
|
||||
- edge
|
||||
- backend
|
||||
healthcheck:
|
||||
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/health/ready"]
|
||||
interval: 5s
|
||||
timeout: 2s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
stop_grace_period: 10s
|
||||
scale: ${API_REPLICAS:-2}
|
||||
deploy:
|
||||
mode: replicated
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 256M
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
@@ -74,12 +108,21 @@ services:
|
||||
image: docker.io/library/haproxy:3.4.7-alpine
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
networks:
|
||||
- edge
|
||||
ports:
|
||||
- "8080:8080"
|
||||
depends_on:
|
||||
- api
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 64M
|
||||
restart: unless-stopped
|
||||
|
||||
networks:
|
||||
edge:
|
||||
backend:
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
$ row counts after all runs (date 2026-11-06T02:07:36Z)
|
||||
chats 84
|
||||
users 292
|
||||
messages 294274
|
||||
message_idempotency 294274
|
||||
|
||||
$ database size
|
||||
167 MB
|
||||
@@ -0,0 +1,13 @@
|
||||
$ date -u; podman stop -t 10 messenger-lab3_api_1; date -u
|
||||
02:07:18.261
|
||||
02:07:21.454
|
||||
|
||||
$ podman logs messenger-lab3_api_1 (shutdown sequence, UTC)
|
||||
02:07:18.345 shutdown signal received, draining 3s 5s
|
||||
02:07:21.345 server stopped
|
||||
|
||||
$ TZ=UTC podman logs -t messenger-lab3_lb_1 (backend state, UTC)
|
||||
02:07:20.346 Server api/api2 is DOWN, reason: Layer7 wrong status, code: 503, info: "Service Unavailable", check duration: 0ms. 1 active and 0 backup
|
||||
|
||||
SIGTERM 02:07:18.345 -> LB marks DOWN by 503 readiness 02:07:20.346 (2.0 s, <= inter 1s x fall 2) -> listener closed 02:07:21.345 (drain 3 s); hook returned in 3.2 s.
|
||||
HAProxy slot names (api1..api4 from server-template) are assigned by DNS order and do not match container numbers.
|
||||
@@ -0,0 +1,12 @@
|
||||
date: 2026-11-06T01:54:08Z
|
||||
host: i5-11400H, 12 threads, 15Gi RAM, Linux 7.1.8-arch1-3
|
||||
podman version 6.1.0
|
||||
podman-compose version 1.6.0
|
||||
go version go1.27.1 linux/amd64
|
||||
images:
|
||||
image: docker.io/library/postgres:18.6-alpine3.24
|
||||
image: docker.io/library/redis:8.10.3-alpine3.23
|
||||
image: docker.io/migrate/migrate:v4.20.1
|
||||
image: localhost/messenger-api:lab3
|
||||
image: docker.io/library/haproxy:3.4.7-alpine
|
||||
limits: see evidence/limits.txt
|
||||
+38
-5
@@ -1,14 +1,47 @@
|
||||
$ api replicas before
|
||||
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 10 seconds
|
||||
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 10 seconds
|
||||
messenger-lab3_api_2 hostname=5a9ffe83734a Up About a minute (healthy)
|
||||
messenger-lab3_api_1 hostname=ab01920585fe Up About a minute (healthy)
|
||||
|
||||
$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/failover-load-1.json & (background load during the test)
|
||||
$ podman stats --no-stream (during load, before failure)
|
||||
messenger-lab3_api_1 cpu=3.87% mem=88.06MB / 268.4MB
|
||||
messenger-lab3_api_2 cpu=4.17% mem=92.08MB / 268.4MB
|
||||
messenger-lab3_lb_1 cpu=2.81% mem=28.44MB / 67.11MB
|
||||
messenger-lab3_postgres_1 cpu=12.21% mem=135.6MB / 536.9MB
|
||||
messenger-lab3_redis_1 cpu=0.58% mem=6.857MB / 134.2MB
|
||||
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 87, "failed_attempts": 1, "first_recovery_seconds": 2.52}
|
||||
{"observed_instances_before": 2, "successful_writes": 131, "failed_attempts": 0, "first_recovery_seconds": 3.339}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ podman stats --no-stream (during load, one replica stopped)
|
||||
messenger-lab3_api_2 cpu=5.01% mem=27.76MB / 268.4MB
|
||||
messenger-lab3_lb_1 cpu=3.77% mem=29.78MB / 67.11MB
|
||||
messenger-lab3_postgres_1 cpu=12.32% mem=133.1MB / 536.9MB
|
||||
messenger-lab3_redis_1 cpu=0.59% mem=7.111MB / 134.2MB
|
||||
|
||||
$ background tests/load.py result
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.008,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 8712,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 290.322,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.117,
|
||||
"p95": 17.627,
|
||||
"p99": 23.947
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
|
||||
$ api replicas after (stopped one = TARGET_INSTANCE_ID)
|
||||
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 26 seconds
|
||||
messenger-lab3_api_2 hostname=2751ae93e6d1 Exited (0) 15 seconds ago
|
||||
messenger-lab3_api_2 hostname=5a9ffe83734a Up 2 minutes (healthy)
|
||||
messenger-lab3_api_1 hostname=ab01920585fe Exited (0) 22 seconds ago (healthy)
|
||||
|
||||
+38
-9
@@ -1,18 +1,47 @@
|
||||
$ api replicas before (api_2 restored with: podman compose start api)
|
||||
messenger-lab3_api_1 hostname=0405cf8a4a54 Up 2 minutes
|
||||
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 19 seconds
|
||||
|
||||
$ curl -s localhost:8080/health/live (shift round-robin so the test write lands on the other replica)
|
||||
{"status":"ok"}
|
||||
$ api replicas before
|
||||
messenger-lab3_api_2 hostname=5a9ffe83734a Up 2 minutes (healthy)
|
||||
messenger-lab3_api_1 hostname=ab01920585fe Up 4 seconds (healthy)
|
||||
|
||||
$ python3 tests/load.py --duration 30 --concurrency 4 --output evidence/failover-load-2.json & (background load during the test)
|
||||
$ podman stats --no-stream (during load, before failure)
|
||||
messenger-lab3_api_1 cpu=4.21% mem=47.89MB / 268.4MB
|
||||
messenger-lab3_api_2 cpu=5.90% mem=92.86MB / 268.4MB
|
||||
messenger-lab3_lb_1 cpu=4.60% mem=31.47MB / 67.11MB
|
||||
messenger-lab3_postgres_1 cpu=12.45% mem=148.5MB / 536.9MB
|
||||
messenger-lab3_redis_1 cpu=0.59% mem=6.611MB / 134.2MB
|
||||
|
||||
$ make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 81, "failed_attempts": 1, "first_recovery_seconds": 2.424}
|
||||
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 3.37}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ podman stats --no-stream (during load, one replica stopped)
|
||||
messenger-lab3_api_1 cpu=7.73% mem=33.08MB / 268.4MB
|
||||
messenger-lab3_lb_1 cpu=4.84% mem=32.26MB / 67.11MB
|
||||
messenger-lab3_postgres_1 cpu=12.54% mem=146.1MB / 536.9MB
|
||||
messenger-lab3_redis_1 cpu=0.59% mem=7.373MB / 134.2MB
|
||||
|
||||
$ background tests/load.py result
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.013,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 9508,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 316.791,
|
||||
"successful_latency_ms": {
|
||||
"p50": 12.105,
|
||||
"p95": 16.14,
|
||||
"p99": 22.351
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
|
||||
$ api replicas after (stopped one = TARGET_INSTANCE_ID)
|
||||
messenger-lab3_api_1 hostname=0405cf8a4a54 Exited (0) 15 seconds ago
|
||||
messenger-lab3_api_2 hostname=2751ae93e6d1 Up 35 seconds
|
||||
messenger-lab3_api_2 hostname=5a9ffe83734a Exited (0) 22 seconds ago (healthy)
|
||||
messenger-lab3_api_1 hostname=ab01920585fe Up 35 seconds (healthy)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.008,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 8712,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 290.322,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.117,
|
||||
"p95": 17.627,
|
||||
"p99": 23.947
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.013,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 9508,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 316.791,
|
||||
"successful_latency_ms": {
|
||||
"p50": 12.105,
|
||||
"p95": 16.14,
|
||||
"p99": 22.351
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -4,5 +4,5 @@ successful responses (201): 20
|
||||
failed requests: 0 []
|
||||
distinct message ids in responses: 1
|
||||
messages stored in history: 1
|
||||
instances that served requests: ['0405cf8a4a54', '2751ae93e6d1']
|
||||
instances that served requests: ['56e8d4ca6c28', 'd63350b561c6']
|
||||
OK: exactly one message created
|
||||
|
||||
+10
-12
@@ -1,18 +1,16 @@
|
||||
20 GET /api/v1/users/me through LB (localhost:8080) with ONE bearer session, no re-login
|
||||
X-Instance-Id 0405cf8a4a54 responses 10
|
||||
X-Instance-Id 2751ae93e6d1 responses 10
|
||||
X-Instance-Id 56e8d4ca6c28 responses 10
|
||||
X-Instance-Id d63350b561c6 responses 10
|
||||
|
||||
$ podman ps --filter label=com.docker.compose.service=api --format "{{.Names}} {{.ID}} {{.Status}}"
|
||||
messenger-lab3_api_1 0405cf8a4a54 Up 54 seconds
|
||||
messenger-lab3_api_2 2751ae93e6d1 Up 53 seconds
|
||||
$ podman ps (api replicas: name, container ID = hostname = X-Instance-Id)
|
||||
messenger-lab3_api_1 d63350b561c6 Up 41 seconds (healthy)
|
||||
messenger-lab3_api_2 56e8d4ca6c28 Up About a minute (healthy)
|
||||
messenger-lab3_api_1 hostname=d63350b561c6
|
||||
messenger-lab3_api_2 hostname=56e8d4ca6c28
|
||||
|
||||
$ podman inspect -f "{{.Name}} hostname={{.Config.Hostname}} ports={{.NetworkSettings.Ports}}" api replicas
|
||||
messenger-lab3_api_1 hostname=0405cf8a4a54 ports=map[8080/tcp:[]]
|
||||
messenger-lab3_api_2 hostname=2751ae93e6d1 ports=map[8080/tcp:[]]
|
||||
|
||||
$ podman ps --format "{{.Names}} {{.Ports}}" (only lb publishes 8080)
|
||||
messenger-lab3_postgres_1 127.0.0.1:55432->5432/tcp
|
||||
messenger-lab3_redis_1 127.0.0.1:56379->6379/tcp
|
||||
$ podman ps (published ports: only lb exposes 8080)
|
||||
messenger-lab3_api_1 8080/tcp
|
||||
messenger-lab3_api_2 8080/tcp
|
||||
messenger-lab3_lb_1 0.0.0.0:8080->8080/tcp
|
||||
messenger-lab3_postgres_1 127.0.0.1:55432->5432/tcp
|
||||
messenger-lab3_redis_1 127.0.0.1:56379->6379/tcp
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
$ podman inspect: limits, networks, published ports
|
||||
messenger-lab3_api_1 cpus=1000000000 memory=268435456 networks=messenger-lab3_backend messenger-lab3_edge ports=
|
||||
messenger-lab3_api_2 cpus=1000000000 memory=268435456 networks=messenger-lab3_backend messenger-lab3_edge ports=
|
||||
messenger-lab3_lb_1 cpus=500000000 memory=67108864 networks=messenger-lab3_edge ports=8080/tcp->0.0.0.0:8080
|
||||
messenger-lab3_migrate_1 cpus=500000000 memory=67108864 networks=messenger-lab3_backend ports=
|
||||
messenger-lab3_postgres_1 cpus=1000000000 memory=536870912 networks=messenger-lab3_backend ports=5432/tcp->127.0.0.1:55432
|
||||
messenger-lab3_redis_1 cpus=500000000 memory=134217728 networks=messenger-lab3_backend ports=6379/tcp->127.0.0.1:56379
|
||||
|
||||
$ ls /run/secrets in each running container
|
||||
messenger-lab3_api_1: cache_url database_url
|
||||
messenger-lab3_api_2: cache_url database_url
|
||||
messenger-lab3_lb_1:
|
||||
messenger-lab3_postgres_1: postgres_password
|
||||
messenger-lab3_redis_1:
|
||||
migrate (one-shot): /run/secrets/database_url
|
||||
|
||||
$ lb sees only the edge network
|
||||
postgres: not resolvable from lb
|
||||
redis: not resolvable from lb
|
||||
10.89.4.28 api.dns.podman api.dns.podman api
|
||||
|
||||
$ grep -c container_name compose.yaml
|
||||
0
|
||||
+15
-8
@@ -1,15 +1,22 @@
|
||||
$ podman ps -a --filter label=com.docker.compose.service=migrate
|
||||
messenger-lab3_migrate_1 Exited (0) 53 seconds ago
|
||||
$ podman ps -a (migrate is one-shot, exited 0; api replicas started after it)
|
||||
messenger-lab3_migrate_1 Exited (0) About a minute ago
|
||||
|
||||
$ podman logs messenger-lab3_migrate_1
|
||||
1/u init (10.119477ms)
|
||||
2/u message_idempotency (14.878187ms)
|
||||
$ podman logs migrate (schema already at version 2 in the kept volume; repeated run is safe)
|
||||
no change
|
||||
no change
|
||||
no change
|
||||
|
||||
$ podman logs messenger-lab3_api_1 | grep -i migrat (API replicas do not run migrations)
|
||||
0
|
||||
$ API replicas run no migrations (count of "migrat" in logs)
|
||||
api_1: 0
|
||||
api_2: 0
|
||||
|
||||
$ psql: SELECT version, dirty FROM schema_migrations
|
||||
$ SELECT version, dirty FROM schema_migrations
|
||||
2|f
|
||||
|
||||
$ \dt
|
||||
chat_members
|
||||
chats
|
||||
message_idempotency
|
||||
messages
|
||||
schema_migrations
|
||||
users
|
||||
|
||||
+13
-13
@@ -1,10 +1,10 @@
|
||||
$ containers before
|
||||
messenger-lab3_postgres_1 8c509b4f5143 Up About a minute (healthy)
|
||||
messenger-lab3_redis_1 b5db2a433051 Up About a minute (healthy)
|
||||
messenger-lab3_migrate_1 9fa71320cdba Exited (0) About a minute ago
|
||||
messenger-lab3_api_1 0405cf8a4a54 Up About a minute
|
||||
messenger-lab3_api_2 2751ae93e6d1 Up About a minute
|
||||
messenger-lab3_lb_1 d3a35fe2295d Up About a minute
|
||||
messenger-lab3_api_1 d63350b561c6 Up 42 seconds (healthy)
|
||||
messenger-lab3_api_2 56e8d4ca6c28 Up About a minute (healthy)
|
||||
messenger-lab3_lb_1 d0ec4e5311dd Up 4 minutes
|
||||
messenger-lab3_migrate_1 82c01bec10d9 Exited (0) 14 minutes ago
|
||||
messenger-lab3_postgres_1 0d8a6c71fcaf Up 13 minutes (healthy)
|
||||
messenger-lab3_redis_1 37f40fddaa42 Up 13 minutes (healthy)
|
||||
|
||||
$ make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
@@ -12,13 +12,13 @@ Executing explicit persistence hook; hook output suppressed to avoid leaking sec
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
|
||||
$ containers after (same IDs, fresh Up time = restarted, volumes kept)
|
||||
messenger-lab3_postgres_1 8c509b4f5143 Up 1 second (healthy)
|
||||
messenger-lab3_redis_1 b5db2a433051 Up 1 second (healthy)
|
||||
messenger-lab3_migrate_1 9fa71320cdba Exited (0) Less than a second ago
|
||||
messenger-lab3_api_1 0405cf8a4a54 Up Less than a second
|
||||
messenger-lab3_api_2 2751ae93e6d1 Up Less than a second
|
||||
messenger-lab3_lb_1 d3a35fe2295d Up Less than a second
|
||||
$ containers after (same IDs, restarted, volumes kept)
|
||||
messenger-lab3_api_1 d63350b561c6 Up 2 seconds (healthy)
|
||||
messenger-lab3_api_2 56e8d4ca6c28 Up 1 second (healthy)
|
||||
messenger-lab3_lb_1 d0ec4e5311dd Up 2 seconds
|
||||
messenger-lab3_migrate_1 82c01bec10d9 Exited (0) 14 minutes ago
|
||||
messenger-lab3_postgres_1 0d8a6c71fcaf Up 2 seconds (healthy)
|
||||
messenger-lab3_redis_1 37f40fddaa42 Up 1 second (healthy)
|
||||
|
||||
$ podman volume ls
|
||||
messenger-lab3_pgdata
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 60.0,
|
||||
"elapsed_seconds": 60.013,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 17311,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 288.453,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.202,
|
||||
"p95": 17.032,
|
||||
"p99": 20.075
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
$ api replicas before
|
||||
messenger-lab3_api_1 container=ab01920585fe image=3fcba3e82597 revision=1793930539 health=healthy
|
||||
messenger-lab3_api_2 container=5a9ffe83734a image=3fcba3e82597 revision=1793930539 health=healthy
|
||||
|
||||
$ python3 tests/load.py --duration 60 --concurrency 4 --output evidence/rolling-load.json & (background load)
|
||||
$ podman stats --no-stream (during load, before update)
|
||||
messenger-lab3_api_1 cpu=5.87% mem=106.7MB / 268.4MB
|
||||
messenger-lab3_api_2 cpu=2.47% mem=47.7MB / 268.4MB
|
||||
messenger-lab3_lb_1 cpu=4.56% mem=31.62MB / 67.11MB
|
||||
messenger-lab3_postgres_1 cpu=12.33% mem=156.8MB / 536.9MB
|
||||
messenger-lab3_redis_1 cpu=0.59% mem=7.361MB / 134.2MB
|
||||
|
||||
$ scripts/rolling-update.sh (build new image -> migrate once -> replace replicas one by one)
|
||||
exit=0 duration=21s
|
||||
migrate: no change
|
||||
|
||||
$ podman stats --no-stream (during load, after update)
|
||||
messenger-lab3_api_1 cpu=4.77% mem=12.05MB / 268.4MB
|
||||
messenger-lab3_api_2 cpu=7.42% mem=11.21MB / 268.4MB
|
||||
messenger-lab3_lb_1 cpu=4.78% mem=31.83MB / 67.11MB
|
||||
messenger-lab3_postgres_1 cpu=12.49% mem=159.6MB / 536.9MB
|
||||
messenger-lab3_redis_1 cpu=0.59% mem=7.119MB / 134.2MB
|
||||
|
||||
$ background tests/load.py result
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 60.0,
|
||||
"elapsed_seconds": 60.013,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 17311,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 288.453,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.202,
|
||||
"p95": 17.032,
|
||||
"p99": 20.075
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
|
||||
$ api replicas after (new containers, new image and revision)
|
||||
messenger-lab3_api_1 container=d63350b561c6 image=3ab3deda3163 revision=1793930771 health=healthy
|
||||
messenger-lab3_api_2 container=56e8d4ca6c28 image=3ab3deda3163 revision=1793930771 health=healthy
|
||||
+19
-18
@@ -2,41 +2,42 @@
|
||||
? messenger/internal/config [no test files]
|
||||
? messenger/internal/domain [no test files]
|
||||
=== RUN TestSessionExpiry
|
||||
--- PASS: TestSessionExpiry (3.09s)
|
||||
--- PASS: TestSessionExpiry (3.11s)
|
||||
=== RUN TestLogout
|
||||
--- PASS: TestLogout (0.08s)
|
||||
--- PASS: TestLogout (0.13s)
|
||||
=== RUN TestWrongPassword
|
||||
--- PASS: TestWrongPassword (0.10s)
|
||||
--- PASS: TestWrongPassword (0.11s)
|
||||
=== RUN TestRedisUnavailable
|
||||
redis: 2026/11/06 03:16:52 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 03:16:52 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 03:16:53 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 03:16:53 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 03:16:54 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 03:16:54 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
--- PASS: TestRedisUnavailable (2.89s)
|
||||
redis: 2026/11/06 05:07:57 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 05:07:57 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 05:07:58 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 05:07:58 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 05:07:59 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
redis: 2026/11/06 05:07:59 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
--- PASS: TestRedisUnavailable (2.94s)
|
||||
=== RUN TestDatabaseUnavailable
|
||||
--- PASS: TestDatabaseUnavailable (0.17s)
|
||||
--- PASS: TestDatabaseUnavailable (0.20s)
|
||||
=== RUN TestInvalidJSON
|
||||
--- PASS: TestInvalidJSON (0.01s)
|
||||
=== RUN TestForeignChat
|
||||
--- PASS: TestForeignChat (0.17s)
|
||||
redis: 2026/11/06 05:08:00 pool.go:770: redis: connection pool: failed to dial after 5 attempts: dial tcp 127.0.0.1:1: connect: connection refused
|
||||
--- PASS: TestForeignChat (0.23s)
|
||||
=== RUN TestEmptyMessage
|
||||
--- PASS: TestEmptyMessage (0.08s)
|
||||
=== RUN TestDuplicateUsername
|
||||
--- PASS: TestDuplicateUsername (0.09s)
|
||||
--- PASS: TestDuplicateUsername (0.10s)
|
||||
=== RUN TestPaginationMultiplePages
|
||||
--- PASS: TestPaginationMultiplePages (0.08s)
|
||||
--- PASS: TestPaginationMultiplePages (0.09s)
|
||||
=== RUN TestConcurrentMessages
|
||||
--- PASS: TestConcurrentMessages (0.52s)
|
||||
--- PASS: TestConcurrentMessages (0.57s)
|
||||
PASS
|
||||
ok messenger/internal/handler 7.273s
|
||||
ok messenger/internal/handler 7.578s
|
||||
=== RUN TestRegisterUsernameRace
|
||||
--- PASS: TestRegisterUsernameRace (0.02s)
|
||||
--- PASS: TestRegisterUsernameRace (0.07s)
|
||||
=== RUN TestCreateChatRollback
|
||||
--- PASS: TestCreateChatRollback (0.02s)
|
||||
PASS
|
||||
ok messenger/internal/repository/postgres 0.043s
|
||||
ok messenger/internal/repository/postgres 0.088s
|
||||
? messenger/internal/repository/redis [no test files]
|
||||
? messenger/internal/router [no test files]
|
||||
? messenger/internal/service [no test files]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
global
|
||||
log stdout format raw local0 info
|
||||
maxconn 2000
|
||||
|
||||
defaults
|
||||
mode http
|
||||
@@ -9,6 +10,7 @@ defaults
|
||||
timeout client 30s
|
||||
timeout server 20s
|
||||
timeout http-request 10s
|
||||
timeout check 1s
|
||||
retries 3
|
||||
option redispatch
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
|
||||
"messenger/internal/service"
|
||||
)
|
||||
@@ -17,6 +18,7 @@ type Handler struct {
|
||||
messages *service.MessageService
|
||||
deps []Pinger
|
||||
log *slog.Logger
|
||||
draining atomic.Bool
|
||||
}
|
||||
|
||||
func New(users *service.UserService, auth *service.AuthService, chats *service.ChatService,
|
||||
@@ -24,6 +26,10 @@ func New(users *service.UserService, auth *service.AuthService, chats *service.C
|
||||
return &Handler{users: users, auth: auth, chats: chats, messages: messages, deps: deps, log: log}
|
||||
}
|
||||
|
||||
func (h *Handler) StartDraining() {
|
||||
h.draining.Store(true)
|
||||
}
|
||||
|
||||
func (h *Handler) NotFound(w http.ResponseWriter, _ *http.Request) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "resource not found")
|
||||
}
|
||||
|
||||
@@ -16,6 +16,10 @@ func (h *Handler) Live(w http.ResponseWriter, _ *http.Request) {
|
||||
}
|
||||
|
||||
func (h *Handler) Ready(w http.ResponseWriter, r *http.Request) {
|
||||
if h.draining.Load() {
|
||||
h.fail(w, r, fmt.Errorf("%w: draining", domain.ErrUnavailable))
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), readyTimeout)
|
||||
defer cancel()
|
||||
for _, dep := range h.deps {
|
||||
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
COMPOSE=${COMPOSE:-docker compose}
|
||||
|
||||
{
|
||||
echo "# scripts/explain.sh; date: $(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
||||
$COMPOSE exec -T postgres psql -U messenger -d messenger -X -q < scripts/explain.sql
|
||||
} > evidence/explain.txt
|
||||
cat evidence/explain.txt
|
||||
@@ -1,66 +0,0 @@
|
||||
\set ON_ERROR_STOP on
|
||||
\set big_chat '''00000000-0000-4000-8000-00000000e001'''
|
||||
\set seed_user '''00000000-0000-4000-8000-00000000e000'''
|
||||
|
||||
INSERT INTO users (id, username, display_name, password_hash, created_at)
|
||||
VALUES (:seed_user, 'explain_seed', 'Explain seed', 'not-a-login', now())
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO chats (id, title, created_at)
|
||||
SELECT CASE WHEN g = 0 THEN :big_chat::uuid
|
||||
ELSE ('00000000-0000-4000-8000-' || lpad(to_hex(57345 + g), 12, '0'))::uuid END,
|
||||
'explain chat ' || g, now()
|
||||
FROM generate_series(0, 20) AS g
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO chat_members (chat_id, user_id, position)
|
||||
SELECT id, :seed_user, 0 FROM chats WHERE title LIKE 'explain chat %'
|
||||
ON CONFLICT DO NOTHING;
|
||||
|
||||
INSERT INTO messages (id, chat_id, sender_id, text, created_at)
|
||||
SELECT gen_random_uuid(), c.id, :seed_user, 'synthetic message ' || n,
|
||||
timestamptz '2026-01-01' + n * interval '1 second'
|
||||
FROM chats c
|
||||
CROSS JOIN LATERAL generate_series(1, CASE WHEN c.id = :big_chat THEN 20000 ELSE 2000 END) AS n
|
||||
WHERE c.title LIKE 'explain chat %'
|
||||
AND NOT EXISTS (SELECT 1 FROM messages m WHERE m.chat_id = c.id);
|
||||
|
||||
ANALYZE users;
|
||||
ANALYZE chats;
|
||||
ANALYZE chat_members;
|
||||
ANALYZE messages;
|
||||
|
||||
SELECT count(*) AS messages_total,
|
||||
count(*) FILTER (WHERE chat_id = :big_chat) AS messages_in_big_chat
|
||||
FROM messages;
|
||||
|
||||
SELECT created_at AS cur_ts, id AS cur_id
|
||||
FROM messages WHERE chat_id = :big_chat
|
||||
ORDER BY created_at, id OFFSET 9999 LIMIT 1 \gset
|
||||
|
||||
\echo '=== 1. First page (limit=50 -> LIMIT 51), index messages_chat_created_id_idx'
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = :big_chat
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
|
||||
\echo '=== 2. Page in the middle by keyset cursor (after 10 000 messages)'
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = :big_chat AND (created_at, id) > (:'cur_ts', :'cur_id')
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
|
||||
\echo '=== 3. Same middle page WITHOUT the index (dropped inside a rolled-back transaction)'
|
||||
BEGIN;
|
||||
DROP INDEX messages_chat_created_id_idx;
|
||||
EXPLAIN (ANALYZE, BUFFERS)
|
||||
SELECT id, chat_id, sender_id, text, created_at
|
||||
FROM messages
|
||||
WHERE chat_id = :big_chat AND (created_at, id) > (:'cur_ts', :'cur_id')
|
||||
ORDER BY created_at, id
|
||||
LIMIT 51;
|
||||
ROLLBACK;
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/bin/sh
|
||||
set -u
|
||||
cd "$(dirname "$0")/.."
|
||||
B=http://localhost:8080
|
||||
C=${COMPOSE:-docker compose}
|
||||
u="outage_$(head -c4 /dev/urandom | od -An -tx1 | tr -d ' \n')"; p="Outage-demo-password"
|
||||
code() { curl -s -o /dev/null -w '%{http_code}' "$@"; }
|
||||
wait_ready() { for i in $(seq 1 60); do [ "$(code $B/health/ready)" = 200 ] && return; sleep 1; done; }
|
||||
echo "# Dependency outage demo on the live stand; date: $(date -u +%Y-%m-%dT%H:%M:%SZ). Token is kept in a shell variable, never printed."
|
||||
curl -s -o /dev/null -X POST $B/api/v1/users -H 'Content-Type: application/json' -d "{\"username\":\"$u\",\"display_name\":\"Demo\",\"password\":\"$p\"}"
|
||||
T=$(curl -s -X POST $B/api/v1/auth/sessions -u "$u:$p" | python3 -c 'import sys,json;print(json.load(sys.stdin)["token"])')
|
||||
echo "baseline: GET /users/me (Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
|
||||
echo; echo "## $C stop redis"; $C stop redis >/dev/null 2>&1
|
||||
echo "GET /health/ready -> $(code $B/health/ready)"
|
||||
echo "GET /users/me (valid Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
|
||||
echo "GET /users/me (no auth) -> $(code $B/api/v1/users/me)"
|
||||
echo "POST /auth/sessions (Basic) -> $(code -X POST $B/api/v1/auth/sessions -u "$u:$p")"
|
||||
echo "body: $(curl -s $B/api/v1/users/me -H "Authorization: Bearer $T")"
|
||||
$C start redis >/dev/null 2>&1; wait_ready
|
||||
echo "## $C start redis -> ready; same token: GET /users/me -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
|
||||
echo; echo "## $C stop postgres"; $C stop postgres >/dev/null 2>&1
|
||||
echo "GET /health/ready -> $(code $B/health/ready)"
|
||||
echo "GET /users/me (valid Bearer) -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
|
||||
echo "GET /chats (valid Bearer) -> $(code $B/api/v1/chats -H "Authorization: Bearer $T")"
|
||||
echo "body: $(curl -s $B/api/v1/chats -H "Authorization: Bearer $T")"
|
||||
$C start postgres >/dev/null 2>&1; wait_ready
|
||||
echo "## $C start postgres -> ready; same token: GET /users/me -> $(code $B/api/v1/users/me -H "Authorization: Bearer $T")"
|
||||
+2
-1
@@ -1,7 +1,8 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
COMPOSE=${COMPOSE:-podman compose}
|
||||
ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)}
|
||||
COMPOSE=${COMPOSE:-$([ "$ENGINE" = podman ] && echo podman-compose || echo "docker compose")}
|
||||
|
||||
$COMPOSE stop lb api redis postgres
|
||||
$COMPOSE start postgres redis api lb
|
||||
|
||||
Executable
+37
@@ -0,0 +1,37 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)}
|
||||
COMPOSE=${COMPOSE:-$([ "$ENGINE" = podman ] && echo podman-compose || echo "docker compose")}
|
||||
PROJECT=${PROJECT:-messenger-lab3}
|
||||
|
||||
replicas() {
|
||||
$ENGINE ps -aq --no-trunc --filter "label=com.docker.compose.project=$PROJECT" --filter "label=com.docker.compose.service=api" | sort
|
||||
}
|
||||
|
||||
wait_healthy() {
|
||||
for _ in $(seq 1 60); do
|
||||
if [ "$($ENGINE inspect -f '{{.State.Health.Status}}' "$1" 2>/dev/null)" = "healthy" ]; then
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo "replica $1 did not become healthy" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
$ENGINE build --build-arg "BUILD_ID=${BUILD_ID:-$(date +%s)}" -t localhost/messenger-api:lab3 .
|
||||
$COMPOSE run --rm migrate
|
||||
|
||||
for old in $(replicas); do
|
||||
before=$(replicas)
|
||||
$ENGINE stop -t 10 "$old"
|
||||
$ENGINE rm "$old"
|
||||
$COMPOSE up -d --no-deps --no-recreate api
|
||||
for id in $(replicas); do
|
||||
if ! echo "$before" | grep -q "$id"; then
|
||||
wait_healthy "$id"
|
||||
fi
|
||||
done
|
||||
sleep 2
|
||||
done
|
||||
+2
-2
@@ -1,12 +1,12 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
ENGINE=${ENGINE:-podman}
|
||||
ENGINE=${ENGINE:-$(command -v podman >/dev/null 2>&1 && echo podman || echo docker)}
|
||||
PROJECT=${PROJECT:-messenger-lab3}
|
||||
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
|
||||
|
||||
for id in $($ENGINE ps -q --filter "label=com.docker.compose.project=$PROJECT" --filter "label=com.docker.compose.service=api"); do
|
||||
if [ "$($ENGINE inspect -f '{{.Config.Hostname}}' "$id")" = "$TARGET_INSTANCE_ID" ]; then
|
||||
$ENGINE stop -t 5 "$id" >/dev/null
|
||||
$ENGINE stop -t 10 "$id" >/dev/null
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
Reference in New Issue
Block a user