ЛР 3 — БВТ2301 — Курлова — на 5 #61
+20
-5
@@ -1,13 +1,28 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
# Credentials live in secrets/ (see REPORT.md), only file paths are set here.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
SESSION_STORE=valkey
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
GRACEFUL_TIMEOUT=8
|
||||
MAX_INFLIGHT_REQUESTS=64
|
||||
MAX_BODY_BYTES=65536
|
||||
MAX_RSS_MB=0
|
||||
DB_POOL_MIN=1
|
||||
DB_POOL_MAX=10
|
||||
DB_POOL_TIMEOUT=3
|
||||
DB_CONNECT_TIMEOUT=3
|
||||
DB_STATEMENT_TIMEOUT_MS=5000
|
||||
DB_OP_TIMEOUT=6
|
||||
REQUEST_TIMEOUT=8
|
||||
DB_READY_TIMEOUT=1.5
|
||||
READY_CHECK_SECONDS=1
|
||||
READY_FAILURES=2
|
||||
SESSION_CACHE_TTL_SECONDS=30
|
||||
SESSION_SWEEP_SECONDS=60
|
||||
CACHE_TIMEOUT=0.2
|
||||
SESSION_REVOKED_TTL_SECONDS=60
|
||||
|
||||
+31
@@ -0,0 +1,31 @@
|
||||
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_DEFAULT_TIMEOUT=120 \
|
||||
PIP_RETRIES=10
|
||||
|
||||
RUN groupadd --system --gid 10001 app \
|
||||
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-deps -r requirements.txt
|
||||
|
||||
COPY app/ app/
|
||||
COPY migrations/ migrations/
|
||||
COPY scripts/migrate.py scripts/migrate.py
|
||||
|
||||
USER 10001:10001
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"]
|
||||
|
||||
STOPSIGNAL SIGTERM
|
||||
|
||||
CMD ["sh", "-c", "exec uvicorn app.main:app --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""]
|
||||
@@ -1,37 +1,203 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Курлова Алёна, БВТ2301
|
||||
- Целевая оценка; выбранный трек 5, если нужен: 5, Трек 1 — Цена масштабирования
|
||||
- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab3/pulls/61, ветка `solution/bvt2301-kurlova`, основной commit fbb8916
|
||||
- Использованные источники, библиотеки и помощь ИИ: FastAPI, Uvicorn, psycopg 3, Valkey, Argon2id, Nginx. Помощь Claude (Anthropic) при написании кода и отчёта.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
**Версии:** Python 3.12.8, PostgreSQL 17.7, Valkey 8.1, Nginx 1.27.3, Docker Compose v2.
|
||||
|
||||
**Prerequisites:** Docker, Docker Compose, Python 3.9+.
|
||||
|
||||
```sh
|
||||
git clone <fork-url> && cd mtusi2026lab3
|
||||
|
||||
# Создать секреты (значения для локального стенда; не коммитить)
|
||||
mkdir -p secrets
|
||||
PG_PASSWORD=$(openssl rand -hex 16)
|
||||
printf '%s' "$PG_PASSWORD" > secrets/postgres_password
|
||||
printf 'postgresql://messenger:%s@postgres:5432/messenger' "$PG_PASSWORD" > secrets/database_url
|
||||
printf 'redis://valkey:6379/0' > secrets/cache_url
|
||||
|
||||
# Создать .env (скопировать из .env.example или использовать приложенный)
|
||||
cp .env.example .env # при необходимости отредактировать
|
||||
|
||||
# Сборка и запуск
|
||||
docker compose --env-file .env build
|
||||
docker compose --env-file .env up -d
|
||||
|
||||
# Дождаться готовности (обе реплики + nginx)
|
||||
until curl -sf http://localhost:8080/health/ready; do sleep 1; done
|
||||
|
||||
# Тесты
|
||||
make test
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
|
||||
# После failover восстановить остановленную реплику
|
||||
docker compose up -d api-1 api-2
|
||||
|
||||
# Остановка (без удаления данных)
|
||||
docker compose down
|
||||
|
||||
# Удаление данных
|
||||
docker compose down -v
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
Клиент ──► Nginx (LB, :8080)
|
||||
├──► api-1 (FastAPI/Uvicorn) ──┐
|
||||
└──► api-2 (FastAPI/Uvicorn) ──┤
|
||||
├──► PostgreSQL (источник истины)
|
||||
└──► Valkey (кэш сессий, без persistence)
|
||||
```
|
||||
|
||||
**Что перенесено из lab2 без изменений:** модули cursor, errors, passwords, db, sessions; миграция 0001_init.sql; Dockerfile; requirements.txt; скрипт миграций.
|
||||
|
||||
**Что изменено/добавлено в lab3:**
|
||||
|
||||
| Компонент | Изменение |
|
||||
|---|---|
|
||||
| `compose.yaml` | Два сервиса `api-1`, `api-2` за Nginx; CPU/RAM лимиты; API не публикуют порты; нет `container_name` |
|
||||
| `infra/nginx.conf` | Round-robin; список серверов upstream подключается из файла, который ведёт `infra/nginx-health.sh`; пассивная защита `max_fails=1 fail_timeout=10s`; повтор на другую реплику при 502/503/ошибке соединения: для GET везде, для POST только на маршруте `/api/v1/chats/{id}/messages` (`non_idempotent`), где обязателен Idempotency-Key; `resolver 127.0.0.11` + `resolve`, чтобы nginx видел новый IP реплики после её пересоздания; keepalive-соединения к API |
|
||||
| `infra/nginx-health.sh` | Активная проверка готовности внутри контейнера nginx: раз в 1 с `GET /health/ready` каждой реплики; 2 неудачи подряд → сервер помечается `down` в upstream и выполняется `nginx -s reload`; 2 успеха подряд → возвращается. Если неготовы все, никто не помечается `down` |
|
||||
| `app/config.py` | Добавлен `IDEMPOTENCY_TTL_SECONDS` |
|
||||
| `app/repo.py` | `create_message_idempotent()` — advisory lock, затем проверка членства в чате (и для нового запроса, и для повтора), затем INSERT message + INSERT idempotency_keys в одной транзакции |
|
||||
| `app/main.py` | Валидация `Idempotency-Key`, вызов idempotent-метода, очистка просроченных ключей в sweeper; фоновая проверка связи с PostgreSQL раз в 1 с, после 2 неудач подряд бизнес-запросы получают 503 до восстановления связи |
|
||||
| `migrations/0002_idempotency.sql` | Таблица `idempotency_keys` (PK: user_id, chat_id, key) |
|
||||
| `scripts/stop-one.sh` | Останавливает реплику по `TARGET_INSTANCE_ID` (= имя сервиса) |
|
||||
| `scripts/restart.sh` | Перезапуск всего стенда с сохранением volumes |
|
||||
| `infra/scaling/`, `student-tests/` | Override-файлы и скрипты эксперимента Трека 1, rolling update, тест параллельной идемпотентности |
|
||||
|
||||
**Источник истины:** PostgreSQL. Все данные (users, chats, messages, sessions, idempotency_keys) хранятся в PostgreSQL. Valkey — кэш сессий без persistence (cache-aside с NX + TTL).
|
||||
|
||||
**Транзакционные границы:** Отправка сообщения с Idempotency-Key обёрнута в одну PostgreSQL-транзакцию. `pg_advisory_xact_lock(hash)` сериализует конкурентные повторы одного ключа. Вставка сообщения и запись idempotency-результата атомарны.
|
||||
|
||||
**Идемпотентность:** Область ключа — (user_id, chat_id, key). Тело нормализуется: `sha256(json.dumps({"text": ...}, sort_keys=True))`. Повтор с тем же body → тот же 201 + Message. Другой body → 409. TTL по умолчанию 24 часа; sweeper удаляет просроченные записи.
|
||||
|
||||
**Миграции:** Запускаются отдельным одноразовым контейнером `migrate` до старта API. Используют advisory lock для защиты от гонки.
|
||||
|
||||
**Переменные .env → конфигурация:**
|
||||
|
||||
| Переменная | Назначение | Значение по умолчанию |
|
||||
|---|---|---|
|
||||
| `INSTANCE_ID` | X-Instance-Id; задаётся в compose per-service | hostname |
|
||||
| `IDEMPOTENCY_TTL_SECONDS` | Время жизни idempotency-ключа | 86400 (24 ч) |
|
||||
| `SESSION_TTL_SECONDS` | TTL сессии | 3600 |
|
||||
| `DB_POOL_MAX` | Макс. подключений к PostgreSQL | 10 |
|
||||
| `GRACEFUL_TIMEOUT` | Время graceful shutdown Uvicorn | 8 |
|
||||
| `READY_CHECK_SECONDS`, `READY_FAILURES` | Интервал фоновой проверки PostgreSQL и число неудач подряд до режима «не готов» | 1, 2 |
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
|---|---|---|---|
|
||||
| Публичный smoke | `make test` | PASS: lab3 public API scenario | evidence/smoke.txt |
|
||||
| ≥2 реплики, один entrypoint, общая БД и сессии | `curl -s http://localhost:8080/health/live` (несколько раз) | X-Instance-Id чередуется: api-1, api-2 | evidence/instance_ids.txt |
|
||||
| Разные запросы одного пользователя — разные реплики без повторного входа | smoke test + многократные запросы с одним Bearer | X-Instance-Id меняется | evidence/instance_ids.txt |
|
||||
| Остановка одной реплики — чтение/запись продолжаются ≤10 с | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS: failover preserved confirmed data | evidence/failover_both.txt |
|
||||
| Повтор POST с тем же Idempotency-Key — нет дубля | smoke test (idempotency replay section) | replay == first | evidence/smoke.txt |
|
||||
| Конкурентный повтор через разные реплики | `python3 student-tests/test_parallel_idempotency.py` | PASS: 8 parallel requests, 1 message | evidence/parallel_idempotency.txt |
|
||||
| Health/readiness, timeouts, graceful shutdown | SIGTERM → лог "draining", 503 на новые запросы; Nginx убирает backend | docker compose logs api-1 | evidence/graceful_shutdown.txt |
|
||||
| Nginx исключает упавший или неготовый backend | Активная проверка `/health/ready` (`infra/nginx-health.sh`) + `max_fails=1`; после stop — запросы идут на живую реплику | failover test; лог nginx `health: api-1 not ready, marked down` | evidence/failover_both.txt, evidence/review_scenarios.txt |
|
||||
| Максимальное время обнаружения отказа | Остановленная реплика: первая же ошибка соединения повторяется на другой реплике и выключает сервер (`max_fails=1`). Неготовая реплика: ≤ 2 проверки × (1 с интервал + 1 с таймаут) ≈ 4 с, всё это время GET и POST сообщений повторяются на живой реплике | first_recovery_seconds: 0.33 с | evidence/failover_both.txt |
|
||||
| Последовательная остановка каждой реплики | stop api-1 → test → restore → stop api-2 → test | Оба теста PASS | evidence/failover_both.txt |
|
||||
| RPS, ошибки, время восстановления | failover output JSON | successful_writes, failed_attempts, first_recovery_seconds | evidence/failover_both.txt |
|
||||
| CPU/RAM лимиты | `docker compose config \| grep -A2 limits` | api: 0.5 CPU / 256M; nginx: 0.25 CPU / 64M; pg: 1.0 CPU / 512M | evidence/limits.txt |
|
||||
| API не публикуют порты мимо LB | `docker compose ps` — только nginx на 8080 | Порты: "127.0.0.1:8080->8080/tcp" только у nginx | evidence/ports.txt |
|
||||
| Нет container_name | compose.yaml | Нет ни одного container_name | compose.yaml |
|
||||
| Rolling update под нагрузкой | `student-tests/rolling_update.sh`: нагрузка 60 с, `up -d --no-deps --force-recreate api-1` → ждём healthy → api-2 | Оба контейнера заменены (разные ID), 910 RPS, 0 ошибок | evidence/rolling_update.txt |
|
||||
| Замечания ревью PR #61 (повтор после потери членства, повтор POST /chats балансировщиком, неготовая реплика в upstream) | `python3 student-tests/test_review_scenarios.py` | До исправлений 5 FAIL (повтор → 201; 3 POST /chats ушли на две реплики; неготовая api-1 обслужила 20 из 40 GET), после — PASS | evidence/review_scenarios.txt |
|
||||
| Миграции однократны | Отдельный контейнер `migrate`; advisory lock; checksum | `docker compose logs migrate` → "skip 0001 / apply 0002" | evidence/migrate.txt |
|
||||
| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: persistence preserved confirmed data | evidence/persistence.txt |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
### Трек 1. Цена масштабирования
|
||||
|
||||
**Гипотеза:** при фиксированном суммарном CPU рост числа реплик сначала помогает, потому что один процесс uvicorn не использует больше ~0.5 CPU, но при очень маленьком лимите на реплику эффект пропадает. При росте ресурсов RPS упирается в общий компонент: Nginx или PostgreSQL.
|
||||
|
||||
**Методика:**
|
||||
- Hardware: Apple M4, 16 GB RAM, macOS, Docker Desktop. Клиент нагрузки работает на том же хосте.
|
||||
- Профиль: `tests/load.py --duration 30 --concurrency 16`, closed-loop POST сообщений с Idempotency-Key.
|
||||
- Прогрев 5 с, затем 3 прогона по 30 с на конфигурацию; в таблицах медиана, в скобках min..max.
|
||||
- Лимиты API задаются override-файлами `infra/scaling/limits.yaml` и `infra/scaling/replicas-4.yaml`, список реплик для nginx задаётся переменной `API_UPSTREAMS` (в замерах ниже использовались отдельные файлы `nginx-{1,2,4}.conf` с тем же upstream; после ревью они заменены активной проверкой, путь запросов через nginx не изменился). Фактические лимиты проверяются через `docker inspect` (`*_limits.txt`).
|
||||
- Лимиты остальных сервисов не меняются: nginx 0.25 CPU, PostgreSQL 1.0 CPU, Valkey 0.25 CPU.
|
||||
- В середине каждого прогона снимается `docker stats` (`*_stats.txt`), чтобы видеть, какой контейнер упёрся в лимит.
|
||||
- Скрипты: `student-tests/scaling_experiment.sh`, `student-tests/scaling_summary.py`. Сырые данные: `evidence/scaling/*.json`, сводка `evidence/scaling/summary.csv`, график `evidence/scaling/rps.svg`.
|
||||
|
||||
**Эксперимент A — фиксированный бюджет API: всего 1.0 CPU / 512M:**
|
||||
|
||||
| Реплики | CPU/реплику | RAM/реплику | RPS медиана [min..max] | p50 мс | p95 мс | p99 мс | Ошибки |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| 1 | 1.00 | 512M | 347 [140..473] | 13.5 | 205.8 | 519.1 | 0 |
|
||||
| 2 | 0.50 | 256M | 784 [282..811] | 11.7 | 66.2 | 87.8 | 0 |
|
||||
| 4 | 0.25 | 128M | 615 [594..620] | 9.5 | 92.9 | 167.7 | 0 |
|
||||
|
||||
**Эксперимент B — растущие ресурсы, каждая реплика 0.5 CPU / 256M:**
|
||||
|
||||
| Реплики | Суммарный CPU API | RPS медиана [min..max] | p50 мс | p95 мс | p99 мс | Ошибки |
|
||||
|---|---|---|---|---|---|---|
|
||||
| 1 | 0.5 | 414 [98..466] | 15.8 | 97.0 | 297.0 | 0 |
|
||||
| 2 | 1.0 | 784 [282..811] (= A2) | 11.7 | 66.2 | 87.8 | 0 |
|
||||
| 4 | 2.0 | 1014 [993..1018] | 10.8 | 47.1 | 74.1 | 0 |
|
||||
|
||||

|
||||
|
||||
**Что показали замеры:**
|
||||
- Одна реплика с 1.0 CPU не быстрее одной реплики с 0.5 CPU (347 против 414, в пределах разброса). Uvicorn работает одним процессом с одним event loop, и дополнительное CPU ему не помогает. Поэтому при том же бюджете 2 × 0.5 CPU дают в 2.3 раза больше RPS.
|
||||
- 4 × 0.25 CPU хуже, чем 2 × 0.5 (615 против 784, p99 168 против 88 мс): на таком маленьком лимите каждая реплика чаще упирается в CFS-квоту, а постоянные накладные расходы процесса делятся на меньший CPU.
|
||||
- При росте ресурсов переход 2 → 4 реплики удваивает CPU API, но даёт только +29% RPS.
|
||||
- Разброс прогонов у 1 реплики большой (98..466): одиночная реплика под квотой CPU нестабильна. Поэтому в выводах используется медиана.
|
||||
|
||||
**Saturation point.** По `docker stats` в B4 (`evidence/scaling/B4_run*_stats.txt`) API используют ~40–49% из лимита 50%, PostgreSQL ~45–48% из 100%, а nginx 20–26% при лимите 25%. Упирается балансировщик. В логах API видно, что nginx ходит к upstream по `HTTP/1.0`: на каждый запрос открывается новое TCP-соединение, это и съедает его CPU.
|
||||
|
||||
**Подтверждённое улучшение: keepalive от nginx к API.** В `infra/nginx.conf` добавлены `keepalive 32`, `keepalive_timeout 4s` (меньше 5 с keep-alive у uvicorn, чтобы nginx не взял соединение, которое API уже закрывает), `proxy_http_version 1.1` и пустой `Connection`. Ресурсы не менялись. Повторный замер по той же методике (`evidence/scaling/keepalive/`):
|
||||
|
||||
| Конфигурация | RPS до | RPS после | p95 до → после, мс | p99 до → после, мс | Ошибки после |
|
||||
|---|---|---|---|---|---|
|
||||
| 2 × 0.5 CPU (основной стенд) | 784 | 1021 (+30%) | 66.2 → 44.1 | 87.8 → 73.4 | 0 |
|
||||
| 4 × 0.5 CPU | 1014 | 1216 (+20%) | 47.1 → 34.1 | 74.1 → 54.3 | 0 |
|
||||
|
||||
После улучшения узкое место сместилось. На 2 репликах nginx потребляет ~16%, а обе API стоят на своём лимите 50%, то есть теперь ограничивают сами реплики. На 4 репликах nginx снова на пределе 25%, PostgreSQL поднялся до ~72–75%, а API недогружены (~45%). Дальше добавлять реплики бесполезно без увеличения CPU для nginx и PostgreSQL.
|
||||
|
||||
**Отклонение при замере.** В первом замере keepalive (до добавления `resolve` в upstream) один прогон B4 дал 319 ошибок из ~37 000 запросов. В трёх повторных прогонах и в итоговом замере ошибок 0, в логах nginx (`*_nginx_problems.txt`) и PostgreSQL ошибок нет. Причину установить не удалось: логи контейнеров того прогона потерялись при пересоздании. Подробности в `evidence/scaling/keepalive/B4_recheck.txt`. Скрипт теперь сохраняет ошибки nginx на каждую конфигурацию.
|
||||
|
||||
### Отказы
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
|---|---|---|---|
|
||||
| Остановка api-1 | Nginx направляет трафик на api-2; ≤10 с до восстановления; 0 потерь подтверждённых сообщений | 130 записей, 0 ошибок | 0.33 с / 0 потерь |
|
||||
| Остановка api-2 (после восстановления api-1) | Аналогично | 130 записей, 0 ошибок | 0.33 с / 0 потерь |
|
||||
| Перезапуск всего стенда | Данные и сессии сохраняются (volumes) | PASS: все данные и сессия сохранены | ~5 с перезапуск / 0 потерь |
|
||||
| Rolling update api-1, затем api-2 под нагрузкой | Кратковременный рост p99; 0 потерь | 910 RPS, 0 ошибок из 54 625, p99=17.9 мс | 0 потерь |
|
||||
| Пересоздание реплики с новым IP без перезапуска nginx | nginx находит новый адрес | До `resolve` nginx слал запросы на старый IP и failover-тест падал; после `resolve` обе реплики снова в балансировке | ≤5 с (valid резолвера) |
|
||||
| Живая api-1 без связи с PostgreSQL (`/health/ready`=503) | Реплика исключается из бизнес-маршрутизации | Через ~2–4 с `infra/nginx-health.sh` помечает api-1 `down`; 40 из 40 GET с прогретой сессией обслужила api-2; POST сообщений 20/20 успешны; POST /chats в первые секунды до исключения получили 503 без повтора и без создания чата; после восстановления api-1 вернулась в upstream | 0 потерь, лишних записей нет |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
**Выдерживаемые отказы:**
|
||||
- Остановка любой одной API-реплики при работающей другой
|
||||
- Перезапуск всего стенда с сохранением volumes
|
||||
|
||||
**Не выдерживаемые отказы:**
|
||||
- Потеря PostgreSQL (единственная БД, нет репликации) — все записи и чтения недоступны
|
||||
- Потеря Nginx (единственный LB) — единая точка входа
|
||||
- Потеря Docker-хоста — всё на одном узле
|
||||
|
||||
**Учебные упрощения:**
|
||||
- Один Docker-хост, нет multi-node failover
|
||||
- Nginx open-source не умеет активные health checks, поэтому они сделаны shell-скриптом в контейнере nginx (`nginx -s reload` при смене состояния). Если скрипт упадёт, upstream останется в последнем состоянии, и защитой будут только `max_fails` и проверка готовности внутри API
|
||||
- В первые ~2–4 с после потери связи с БД не-идемпотентные POST (кроме сообщений), попавшие на неготовую реплику, получают 503 без повтора. Это сделано намеренно: nginx не знает, успела ли реплика что-то записать, и прозрачный повтор мог бы создать дубль
|
||||
- Idempotency-ключи хранятся в PostgreSQL — при высокой нагрузке advisory lock на одном ключе может стать bottleneck
|
||||
- HTTP без TLS (допустимо для локального стенда с синтетическими данными, lab7 добавит HTTPS)
|
||||
|
||||
**Возможные улучшения:**
|
||||
- PostgreSQL streaming replication + automatic failover (Patroni)
|
||||
- HAProxy или Nginx Plus со встроенными active health checks вместо скрипта
|
||||
- Idempotency через distributed lock (Valkey) + PostgreSQL для снижения contention
|
||||
- Multi-node deployment (Docker Swarm / Kubernetes)
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import os
|
||||
import socket
|
||||
|
||||
|
||||
def _int(name, default):
|
||||
value = os.environ.get(name)
|
||||
return int(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _float(name, default):
|
||||
value = os.environ.get(name)
|
||||
return float(value) if value not in (None, "") else default
|
||||
|
||||
|
||||
def _secret(name, default=""):
|
||||
path = os.environ.get(name + "_FILE")
|
||||
if path:
|
||||
with open(path, encoding="utf-8") as handle:
|
||||
return handle.read().strip()
|
||||
return os.environ.get(name, default)
|
||||
|
||||
|
||||
DATABASE_URL = _secret("DATABASE_URL")
|
||||
CACHE_URL = _secret("CACHE_URL")
|
||||
SESSION_STORE = os.environ.get("SESSION_STORE", "postgres")
|
||||
SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600)
|
||||
SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30)
|
||||
SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60)
|
||||
DB_POOL_MIN = _int("DB_POOL_MIN", 1)
|
||||
DB_POOL_MAX = _int("DB_POOL_MAX", 10)
|
||||
DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0)
|
||||
DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3)
|
||||
DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000)
|
||||
DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0)
|
||||
REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0)
|
||||
DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5)
|
||||
READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0)
|
||||
READY_FAILURES = _int("READY_FAILURES", 2)
|
||||
CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2)
|
||||
MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64)
|
||||
MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536)
|
||||
MAX_RSS_MB = _int("MAX_RSS_MB", 0)
|
||||
PASSWORD_MIN = 12
|
||||
PASSWORD_MAX = 128
|
||||
SESSION_REVOKED_TTL_SECONDS = max(
|
||||
_int("SESSION_REVOKED_TTL_SECONDS", 60),
|
||||
SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5,
|
||||
)
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname()
|
||||
IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400)
|
||||
@@ -0,0 +1,43 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z")
|
||||
TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
|
||||
|
||||
class BadCursor(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def fmt(ts: datetime) -> str:
|
||||
return ts.astimezone(timezone.utc).strftime(TS_FORMAT)
|
||||
|
||||
|
||||
def encode_cursor(scope: str, created_at: str, item_id: str) -> str:
|
||||
raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip("=")
|
||||
|
||||
|
||||
def decode_cursor(scope: str, cursor: str):
|
||||
try:
|
||||
raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4))
|
||||
data = json.loads(raw)
|
||||
except Exception:
|
||||
raise BadCursor() from None
|
||||
if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope:
|
||||
raise BadCursor()
|
||||
ts, item_id = data["t"], data["i"]
|
||||
if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str):
|
||||
raise BadCursor()
|
||||
try:
|
||||
parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc)
|
||||
if str(uuid.UUID(item_id)) != item_id:
|
||||
raise BadCursor()
|
||||
except ValueError:
|
||||
raise BadCursor() from None
|
||||
if encode_cursor(scope, ts, item_id) != cursor:
|
||||
raise BadCursor()
|
||||
return parsed, item_id
|
||||
@@ -0,0 +1,99 @@
|
||||
import asyncio
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import psycopg
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import AsyncConnectionPool, PoolTimeout
|
||||
|
||||
from app import config
|
||||
from app.errors import unavailable
|
||||
|
||||
BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout)
|
||||
|
||||
|
||||
async def discard(conn) -> None:
|
||||
try:
|
||||
await conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
class Guarded:
|
||||
def __init__(self, conn, timeout: float):
|
||||
self.conn = conn
|
||||
self.timeout = timeout
|
||||
|
||||
async def execute(self, *args, **kwargs):
|
||||
try:
|
||||
return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout)
|
||||
except asyncio.TimeoutError:
|
||||
await discard(self.conn)
|
||||
raise unavailable("Хранилище не ответило за отведённое время") from None
|
||||
|
||||
def transaction(self):
|
||||
return self.conn.transaction()
|
||||
|
||||
|
||||
class Database:
|
||||
def __init__(self) -> None:
|
||||
options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % (
|
||||
config.DB_STATEMENT_TIMEOUT_MS,
|
||||
config.DB_STATEMENT_TIMEOUT_MS,
|
||||
config.DB_STATEMENT_TIMEOUT_MS * 2,
|
||||
)
|
||||
self.pool = AsyncConnectionPool(
|
||||
conninfo=config.DATABASE_URL,
|
||||
min_size=config.DB_POOL_MIN,
|
||||
max_size=config.DB_POOL_MAX,
|
||||
timeout=config.DB_POOL_TIMEOUT,
|
||||
max_lifetime=600.0,
|
||||
open=False,
|
||||
kwargs={
|
||||
"autocommit": True,
|
||||
"row_factory": dict_row,
|
||||
"connect_timeout": config.DB_CONNECT_TIMEOUT,
|
||||
"options": options,
|
||||
},
|
||||
)
|
||||
|
||||
async def open(self) -> None:
|
||||
await self.pool.open(wait=False)
|
||||
|
||||
async def close(self) -> None:
|
||||
await self.pool.close()
|
||||
|
||||
@asynccontextmanager
|
||||
async def connection(self):
|
||||
try:
|
||||
async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn:
|
||||
yield Guarded(conn, config.DB_OP_TIMEOUT)
|
||||
except BROKEN:
|
||||
raise unavailable("Хранилище недоступно") from None
|
||||
|
||||
@asynccontextmanager
|
||||
async def transaction(self):
|
||||
async with self.connection() as conn:
|
||||
async with conn.transaction():
|
||||
yield conn
|
||||
|
||||
async def in_transaction(self, action):
|
||||
async def run():
|
||||
async with self.transaction() as conn:
|
||||
return await action(conn)
|
||||
|
||||
try:
|
||||
return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
raise unavailable("Хранилище не ответило за отведённое время") from None
|
||||
|
||||
async def ready(self) -> bool:
|
||||
try:
|
||||
async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn:
|
||||
try:
|
||||
await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
await discard(conn)
|
||||
return False
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
@@ -0,0 +1,37 @@
|
||||
from typing import Optional
|
||||
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
|
||||
BEARER_CHALLENGE = 'Bearer realm="messenger"'
|
||||
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None):
|
||||
self.status, self.code, self.message, self.headers = status, code, message, headers or {}
|
||||
|
||||
|
||||
def bad(message: str) -> ApiError:
|
||||
return ApiError(400, "invalid_request", message)
|
||||
|
||||
|
||||
def unauthorized(scheme: str = "Bearer") -> ApiError:
|
||||
challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE
|
||||
message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия"
|
||||
return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge})
|
||||
|
||||
|
||||
def not_found() -> ApiError:
|
||||
return ApiError(404, "not_found", "Объект не найден")
|
||||
|
||||
|
||||
def unavailable(message: str) -> ApiError:
|
||||
return ApiError(503, "unavailable", message, {"Retry-After": "1"})
|
||||
|
||||
|
||||
def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse:
|
||||
return JSONResponse(
|
||||
status_code=status,
|
||||
content={"error": {"code": code, "message": message, "request_id": rid}},
|
||||
headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})},
|
||||
)
|
||||
+436
@@ -0,0 +1,436 @@
|
||||
import asyncio
|
||||
import base64
|
||||
import binascii
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import resource
|
||||
import signal
|
||||
import sys
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from app import config, passwords, repo
|
||||
from app.cursor import BadCursor, decode_cursor, encode_cursor
|
||||
from app.db import Database
|
||||
from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable
|
||||
from app.sessions import SessionStore
|
||||
|
||||
logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
log = logging.getLogger("messenger")
|
||||
|
||||
USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$")
|
||||
IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
||||
ZERO_WIDTH = str.maketrans({char: None for char in ""})
|
||||
EXEMPT_PATHS = ("/health/live", "/health/ready")
|
||||
PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096
|
||||
|
||||
db = Database()
|
||||
sessions = SessionStore(db)
|
||||
state = {"draining": False, "ready": True, "inflight": 0, "rejected": {}}
|
||||
|
||||
|
||||
def rss_mb() -> float:
|
||||
try:
|
||||
with open("/proc/self/statm") as handle:
|
||||
return int(handle.read().split()[1]) * PAGE_SIZE / 1048576
|
||||
except OSError:
|
||||
peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
|
||||
return peak / 1048576 if sys.platform == "darwin" else peak / 1024
|
||||
|
||||
|
||||
def count_rejection(code: str) -> None:
|
||||
state["rejected"][code] = state["rejected"].get(code, 0) + 1
|
||||
|
||||
|
||||
async def sweeper() -> None:
|
||||
while True:
|
||||
await asyncio.sleep(config.SESSION_SWEEP_SECONDS)
|
||||
try:
|
||||
removed = await sessions.sweep()
|
||||
if removed:
|
||||
log.info("sessions_expired removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("session sweep skipped: storage unavailable")
|
||||
try:
|
||||
async with db.connection() as conn:
|
||||
removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS)
|
||||
if removed:
|
||||
log.info("idempotency_cleanup removed=%d", removed)
|
||||
except Exception:
|
||||
log.warning("idempotency cleanup skipped")
|
||||
|
||||
|
||||
async def readiness_watch() -> None:
|
||||
failures = 0
|
||||
while True:
|
||||
if await db.ready():
|
||||
failures = 0
|
||||
if not state["ready"]:
|
||||
state["ready"] = True
|
||||
log.info("ready again: database reachable")
|
||||
else:
|
||||
failures += 1
|
||||
if failures >= config.READY_FAILURES and state["ready"]:
|
||||
state["ready"] = False
|
||||
log.warning("not ready: database unreachable, business requests get 503")
|
||||
await asyncio.sleep(config.READY_CHECK_SECONDS)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(_app: FastAPI):
|
||||
previous = {}
|
||||
|
||||
def on_signal(signum, frame):
|
||||
if not state["draining"]:
|
||||
state["draining"] = True
|
||||
log.info("signal %s: draining, new requests get 503", signal.Signals(signum).name)
|
||||
handler = previous.get(signum)
|
||||
if callable(handler):
|
||||
handler(signum, frame)
|
||||
|
||||
for sig in (signal.SIGTERM, signal.SIGINT):
|
||||
previous[sig] = signal.getsignal(sig)
|
||||
signal.signal(sig, on_signal)
|
||||
await db.open()
|
||||
await sessions.open()
|
||||
tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch())]
|
||||
log.info(
|
||||
"started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d",
|
||||
config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS,
|
||||
config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES,
|
||||
config.IDEMPOTENCY_TTL_SECONDS,
|
||||
)
|
||||
yield
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
await sessions.close()
|
||||
await db.close()
|
||||
log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"])
|
||||
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
|
||||
|
||||
@app.middleware("http")
|
||||
async def guard(request: Request, call_next):
|
||||
rid = uuid.uuid4().hex
|
||||
request.state.rid = rid
|
||||
exempt = request.url.path in EXEMPT_PATHS
|
||||
if state["draining"] and request.url.path != "/health/live":
|
||||
count_rejection("draining")
|
||||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"})
|
||||
if not exempt and not state["ready"]:
|
||||
count_rejection("not_ready")
|
||||
return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"})
|
||||
if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS:
|
||||
count_rejection("inflight")
|
||||
return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"})
|
||||
state["inflight"] += 1
|
||||
try:
|
||||
response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT)
|
||||
except asyncio.TimeoutError:
|
||||
count_rejection("request_timeout")
|
||||
log.warning("request timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path)
|
||||
response = error_body(
|
||||
rid, config.INSTANCE_ID, 503, "unavailable",
|
||||
"Запрос не удалось обработать за отведённое время", {"Retry-After": "1"},
|
||||
)
|
||||
except Exception:
|
||||
log.exception("unhandled error request_id=%s", rid)
|
||||
response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера")
|
||||
finally:
|
||||
state["inflight"] -= 1
|
||||
response.headers["X-Request-Id"] = rid
|
||||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||||
return response
|
||||
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
if exc.status in (413, 429, 503):
|
||||
count_rejection(exc.code)
|
||||
return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers)
|
||||
|
||||
|
||||
def parse_uuid(value: object) -> Optional[str]:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return str(uuid.UUID(value))
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def non_blank(value: object, max_len: int) -> bool:
|
||||
if not isinstance(value, str) or not 1 <= len(value) <= max_len:
|
||||
return False
|
||||
return value.translate(ZERO_WIDTH).strip() != ""
|
||||
|
||||
|
||||
async def read_json(request: Request, fields: Tuple[str, ...]) -> dict:
|
||||
if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json":
|
||||
raise ApiError(415, "unsupported_media_type", "Ожидается application/json")
|
||||
too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"Connection": "close"})
|
||||
declared = request.headers.get("content-length")
|
||||
if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES:
|
||||
raise too_large
|
||||
raw = bytearray()
|
||||
async for chunk in request.stream():
|
||||
raw += chunk
|
||||
if len(raw) > config.MAX_BODY_BYTES:
|
||||
raise too_large
|
||||
try:
|
||||
body = json.loads(bytes(raw).decode("utf-8"))
|
||||
except (UnicodeDecodeError, ValueError, RecursionError):
|
||||
raise bad("Некорректный JSON") from None
|
||||
if not isinstance(body, dict):
|
||||
raise bad("Ожидается JSON-объект")
|
||||
if set(body) - set(fields):
|
||||
raise bad("Лишние поля в запросе")
|
||||
if set(fields) - set(body):
|
||||
raise bad("Отсутствуют обязательные поля")
|
||||
return body
|
||||
|
||||
|
||||
def ensure_capacity() -> None:
|
||||
if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB:
|
||||
raise unavailable("Достигнут лимит памяти процесса, запись недоступна")
|
||||
|
||||
|
||||
def bearer_token(request: Request) -> str:
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
token = value.strip()
|
||||
if scheme.lower() != "bearer" or not token:
|
||||
raise unauthorized()
|
||||
return token
|
||||
|
||||
|
||||
async def actor(request: Request) -> dict:
|
||||
user = await sessions.resolve(bearer_token(request))
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
return user
|
||||
|
||||
|
||||
def basic_credentials(request: Request) -> Tuple[str, str]:
|
||||
scheme, _, value = request.headers.get("authorization", "").partition(" ")
|
||||
if scheme.lower() != "basic" or not value.strip():
|
||||
raise unauthorized("Basic")
|
||||
try:
|
||||
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
|
||||
except (binascii.Error, ValueError, UnicodeDecodeError):
|
||||
raise unauthorized("Basic") from None
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator:
|
||||
raise unauthorized("Basic")
|
||||
return username, password
|
||||
|
||||
|
||||
async def member_chat(user: dict, chat_id: str) -> dict:
|
||||
cid = parse_uuid(chat_id)
|
||||
if cid is None:
|
||||
raise bad("chat_id должен быть UUID")
|
||||
async with db.connection() as conn:
|
||||
chat = await repo.chat_for_member(conn, cid, user["id"])
|
||||
if chat is None:
|
||||
raise not_found()
|
||||
return chat
|
||||
|
||||
|
||||
def page_params(request: Request) -> Tuple[int, Optional[str]]:
|
||||
query = request.query_params
|
||||
limit = 50
|
||||
if "limit" in query:
|
||||
if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100:
|
||||
raise bad("limit должен быть целым 1..100")
|
||||
limit = int(query["limit"])
|
||||
cursor = query.get("cursor")
|
||||
if cursor is not None and cursor == "":
|
||||
raise bad("Некорректный cursor")
|
||||
return limit, cursor
|
||||
|
||||
|
||||
def keyset(scope: str, cursor: Optional[str]):
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
return decode_cursor(scope, cursor)
|
||||
except BadCursor:
|
||||
raise bad("Некорректный cursor") from None
|
||||
|
||||
|
||||
def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse:
|
||||
next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None
|
||||
return JSONResponse({"items": items, "next_cursor": next_cursor})
|
||||
|
||||
|
||||
@app.get("/health/live")
|
||||
async def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/health/ready")
|
||||
async def ready():
|
||||
if not await db.ready():
|
||||
raise unavailable("База данных недоступна")
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.post("/api/v1/users", status_code=201)
|
||||
async def create_user(request: Request):
|
||||
body = await read_json(request, ("username", "display_name", "password"))
|
||||
if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]):
|
||||
raise bad("username должен соответствовать [a-z0-9_]{3,32}")
|
||||
if not non_blank(body["display_name"], 100):
|
||||
raise bad("display_name: 1..100 символов, не только пробелы")
|
||||
password = body["password"]
|
||||
if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||||
raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX))
|
||||
ensure_capacity()
|
||||
password_hash = await run_in_threadpool(passwords.hash_password, password)
|
||||
async with db.connection() as conn:
|
||||
user = await repo.create_user(conn, body["username"], body["display_name"], password_hash)
|
||||
if user is None:
|
||||
raise ApiError(409, "conflict", "username уже занят")
|
||||
log.info("user_created id=%s", user["id"])
|
||||
return JSONResponse(user, status_code=201)
|
||||
|
||||
|
||||
@app.post("/api/v1/auth/sessions", status_code=201)
|
||||
async def login(request: Request):
|
||||
if await request.body():
|
||||
raise bad("POST /auth/sessions не принимает тело запроса")
|
||||
username, password = basic_credentials(request)
|
||||
row = None
|
||||
if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX:
|
||||
async with db.connection() as conn:
|
||||
row = await repo.credentials(conn, username)
|
||||
stored = row["password_hash"] if row else passwords.DUMMY_HASH
|
||||
matches = await run_in_threadpool(passwords.verify_password, stored, password)
|
||||
if row is None or not matches:
|
||||
log.info("login_failed username_known=%s", row is not None)
|
||||
raise unauthorized("Basic")
|
||||
user = repo.public_user(row)
|
||||
token, expires_at = await sessions.create(user)
|
||||
log.info("session_created user=%s", user["id"])
|
||||
return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201)
|
||||
|
||||
|
||||
@app.delete("/api/v1/auth/sessions/current", status_code=204)
|
||||
async def logout(request: Request):
|
||||
token = bearer_token(request)
|
||||
user = await sessions.resolve(token)
|
||||
if user is None:
|
||||
raise unauthorized()
|
||||
await sessions.revoke(token)
|
||||
log.info("session_revoked user=%s", user["id"])
|
||||
return Response(status_code=204)
|
||||
|
||||
|
||||
@app.get("/api/v1/users/me")
|
||||
async def me(request: Request):
|
||||
return await actor(request)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats", status_code=201)
|
||||
async def create_chat(request: Request):
|
||||
user = await actor(request)
|
||||
body = await read_json(request, ("title", "member_ids"))
|
||||
if not non_blank(body["title"], 100):
|
||||
raise bad("title: 1..100 символов, не только пробелы")
|
||||
raw_ids = body["member_ids"]
|
||||
if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100:
|
||||
raise bad("member_ids: массив из 1..100 UUID")
|
||||
ids: List[str] = []
|
||||
for value in raw_ids:
|
||||
parsed = parse_uuid(value)
|
||||
if parsed is None:
|
||||
raise bad("member_ids содержит некорректный UUID")
|
||||
ids.append(parsed)
|
||||
if len(set(ids)) != len(ids):
|
||||
raise bad("member_ids содержит дубликаты")
|
||||
if user["id"] not in ids:
|
||||
raise bad("Создатель должен входить в member_ids")
|
||||
ensure_capacity()
|
||||
|
||||
async def insert(conn):
|
||||
return await repo.create_chat(conn, body["title"], ids)
|
||||
|
||||
try:
|
||||
chat = await db.in_transaction(insert)
|
||||
except repo.UnknownMember:
|
||||
raise bad("member_ids содержит неизвестного пользователя") from None
|
||||
log.info("chat_created id=%s members=%d", chat["id"], len(ids))
|
||||
return JSONResponse(chat, status_code=201)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats")
|
||||
async def list_chats(request: Request):
|
||||
user = await actor(request)
|
||||
limit, cursor = page_params(request)
|
||||
scope = "chats:" + user["id"]
|
||||
after = keyset(scope, cursor)
|
||||
async with db.connection() as conn:
|
||||
items, has_more = await repo.list_chats(conn, user["id"], limit, after)
|
||||
return page(scope, items, has_more)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}")
|
||||
async def get_chat(chat_id: str, request: Request):
|
||||
return await member_chat(await actor(request), chat_id)
|
||||
|
||||
|
||||
@app.post("/api/v1/chats/{chat_id}/messages", status_code=201)
|
||||
async def create_message(chat_id: str, request: Request):
|
||||
user = await actor(request)
|
||||
cid = parse_uuid(chat_id)
|
||||
if cid is None:
|
||||
raise bad("chat_id должен быть UUID")
|
||||
|
||||
idem_key = request.headers.get("idempotency-key", "").strip()
|
||||
if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key):
|
||||
raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
|
||||
|
||||
body = await read_json(request, ("text",))
|
||||
if not non_blank(body["text"], 4000):
|
||||
raise bad("text: 1..4000 символов, не только пробелы")
|
||||
ensure_capacity()
|
||||
|
||||
body_hash = hashlib.sha256(
|
||||
json.dumps({"text": body["text"]}, sort_keys=True, ensure_ascii=False).encode()
|
||||
).hexdigest()
|
||||
|
||||
async def do(conn):
|
||||
return await repo.create_message_idempotent(conn, cid, user["id"], body["text"], idem_key, body_hash)
|
||||
|
||||
status, message = await db.in_transaction(do)
|
||||
|
||||
if status == "conflict":
|
||||
raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса")
|
||||
if status == "not_member":
|
||||
raise not_found()
|
||||
return JSONResponse(message, status_code=201)
|
||||
|
||||
|
||||
@app.get("/api/v1/chats/{chat_id}/messages")
|
||||
async def list_messages(chat_id: str, request: Request):
|
||||
chat = await member_chat(await actor(request), chat_id)
|
||||
limit, cursor = page_params(request)
|
||||
scope = "messages:" + chat["id"]
|
||||
after = keyset(scope, cursor)
|
||||
async with db.connection() as conn:
|
||||
items, has_more = await repo.list_messages(conn, chat["id"], limit, after)
|
||||
return page(scope, items, has_more)
|
||||
|
||||
|
||||
@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"])
|
||||
async def fallback(request: Request):
|
||||
raise not_found()
|
||||
@@ -0,0 +1,16 @@
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import Argon2Error, VerifyMismatchError
|
||||
|
||||
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = HASHER.hash("there-is-no-such-user-password")
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return HASHER.hash(password)
|
||||
|
||||
|
||||
def verify_password(stored: str, password: str) -> bool:
|
||||
try:
|
||||
return HASHER.verify(stored, password)
|
||||
except (VerifyMismatchError, Argon2Error, ValueError):
|
||||
return False
|
||||
+192
@@ -0,0 +1,192 @@
|
||||
import hashlib
|
||||
import json
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import psycopg
|
||||
|
||||
from app.cursor import fmt
|
||||
|
||||
MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at"
|
||||
CHAT_QUERY = (
|
||||
"SELECT c.id, c.title, c.created_at,"
|
||||
" ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids"
|
||||
)
|
||||
|
||||
|
||||
class UnknownMember(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def decoded(value) -> str:
|
||||
return bytes(value).decode("utf-8")
|
||||
|
||||
|
||||
def public_user(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"username": row["username"],
|
||||
"display_name": decoded(row["display_name"]),
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _chat(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"title": decoded(row["title"]),
|
||||
"member_ids": [str(value) for value in row["member_ids"]],
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
def _message(row) -> dict:
|
||||
return {
|
||||
"id": str(row["id"]),
|
||||
"chat_id": str(row["chat_id"]),
|
||||
"sender_id": str(row["sender_id"]),
|
||||
"text": decoded(row["text"]),
|
||||
"created_at": fmt(row["created_at"]),
|
||||
}
|
||||
|
||||
|
||||
async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]:
|
||||
try:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO users (id, username, display_name, password_hash, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())"
|
||||
" RETURNING id, username, display_name, created_at",
|
||||
(username, display_name.encode("utf-8"), password_hash),
|
||||
)
|
||||
except psycopg.errors.UniqueViolation:
|
||||
return None
|
||||
return public_user(await cur.fetchone())
|
||||
|
||||
|
||||
async def credentials(conn, username: str):
|
||||
cur = await conn.execute(
|
||||
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
|
||||
(username,),
|
||||
)
|
||||
return await cur.fetchone()
|
||||
|
||||
|
||||
async def create_chat(conn, title: str, member_ids: List[str]) -> dict:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chats (id, title, created_at)"
|
||||
" VALUES (gen_random_uuid(), %s, clock_timestamp())"
|
||||
" RETURNING id, title, created_at",
|
||||
(title.encode("utf-8"),),
|
||||
)
|
||||
chat = await cur.fetchone()
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])",
|
||||
(chat["id"], member_ids),
|
||||
)
|
||||
if cur.rowcount != len(member_ids):
|
||||
raise UnknownMember()
|
||||
chat["member_ids"] = list(member_ids)
|
||||
return _chat(chat)
|
||||
|
||||
|
||||
async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s",
|
||||
(user_id, chat_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _chat(row) if row else None
|
||||
|
||||
|
||||
async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
||||
ts, last_id = after if after else (None, None)
|
||||
cur = await conn.execute(
|
||||
CHAT_QUERY
|
||||
+ " FROM chats c JOIN chat_members m ON m.chat_id = c.id"
|
||||
" WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))"
|
||||
" ORDER BY c.created_at, c.id LIMIT %s",
|
||||
(user_id, ts, ts, last_id, limit + 1),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [_chat(row) for row in rows[:limit]], len(rows) > limit
|
||||
|
||||
|
||||
async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]:
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
||||
" RETURNING " + MESSAGE_COLUMNS,
|
||||
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
return _message(row) if row else None
|
||||
|
||||
|
||||
async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]:
|
||||
ts, last_id = after if after else (None, None)
|
||||
cur = await conn.execute(
|
||||
"SELECT " + MESSAGE_COLUMNS + " FROM messages"
|
||||
" WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))"
|
||||
" ORDER BY created_at, id LIMIT %s",
|
||||
(chat_id, ts, ts, last_id, limit + 1),
|
||||
)
|
||||
rows = await cur.fetchall()
|
||||
return [_message(row) for row in rows[:limit]], len(rows) > limit
|
||||
|
||||
|
||||
def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int:
|
||||
raw = "%s:%s:%s" % (user_id, chat_id, key)
|
||||
return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16)
|
||||
|
||||
|
||||
async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str, idem_key: str, body_hash: str):
|
||||
lock_id = _idem_lock_id(sender_id, chat_id, idem_key)
|
||||
await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,))
|
||||
|
||||
cur = await conn.execute(
|
||||
"SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
|
||||
(chat_id, sender_id),
|
||||
)
|
||||
if await cur.fetchone() is None:
|
||||
return "not_member", None
|
||||
|
||||
cur = await conn.execute(
|
||||
"SELECT request_body_hash, response_json FROM idempotency_keys"
|
||||
" WHERE user_id = %s AND chat_id = %s AND key = %s",
|
||||
(sender_id, chat_id, idem_key),
|
||||
)
|
||||
existing = await cur.fetchone()
|
||||
|
||||
if existing:
|
||||
if existing["request_body_hash"] != body_hash:
|
||||
return "conflict", None
|
||||
return "replay", json.loads(existing["response_json"])
|
||||
|
||||
cur = await conn.execute(
|
||||
"INSERT INTO messages (id, chat_id, sender_id, text, created_at)"
|
||||
" SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()"
|
||||
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)"
|
||||
" RETURNING " + MESSAGE_COLUMNS,
|
||||
(chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return "not_member", None
|
||||
|
||||
message = _message(row)
|
||||
|
||||
await conn.execute(
|
||||
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)"
|
||||
" VALUES (%s, %s, %s, %s, %s, clock_timestamp())",
|
||||
(sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)),
|
||||
)
|
||||
|
||||
return "created", message
|
||||
|
||||
|
||||
async def cleanup_idempotency(conn, ttl_seconds: int) -> int:
|
||||
cur = await conn.execute(
|
||||
"DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)",
|
||||
(ttl_seconds,),
|
||||
)
|
||||
return cur.rowcount
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional
|
||||
|
||||
import valkey.asyncio as valkey
|
||||
|
||||
from app import config, repo
|
||||
from app.cursor import fmt
|
||||
from app.errors import unavailable
|
||||
|
||||
log = logging.getLogger("messenger.sessions")
|
||||
|
||||
REVOKED = "revoked"
|
||||
|
||||
|
||||
class SessionStore:
|
||||
def __init__(self, db) -> None:
|
||||
self.db = db
|
||||
self.client = None
|
||||
self.stats = {"hits": 0, "misses": 0, "errors": 0}
|
||||
|
||||
async def open(self) -> None:
|
||||
if not config.CACHE_URL:
|
||||
log.info("session cache disabled: CACHE_URL is empty")
|
||||
return
|
||||
self.client = valkey.Valkey.from_url(
|
||||
config.CACHE_URL,
|
||||
socket_timeout=config.CACHE_TIMEOUT,
|
||||
socket_connect_timeout=config.CACHE_TIMEOUT,
|
||||
decode_responses=True,
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
if self.client is not None:
|
||||
await self.client.aclose()
|
||||
self.client = None
|
||||
|
||||
@staticmethod
|
||||
def digest(token: str) -> bytes:
|
||||
return hashlib.sha256(token.encode("utf-8")).digest()
|
||||
|
||||
@staticmethod
|
||||
def key(digest: bytes) -> str:
|
||||
return "session:" + digest.hex()
|
||||
|
||||
async def create(self, user: dict) -> tuple:
|
||||
token = secrets.token_urlsafe(32)
|
||||
digest = self.digest(token)
|
||||
expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
async with self.db.connection() as conn:
|
||||
await conn.execute(
|
||||
"INSERT INTO sessions (token_hash, user_id, created_at, expires_at)"
|
||||
" VALUES (%s, %s, clock_timestamp(), %s)",
|
||||
(digest, user["id"], expires_at),
|
||||
)
|
||||
await self._cache_put(digest, user, expires_at)
|
||||
return token, fmt(expires_at)
|
||||
|
||||
async def resolve(self, token: str) -> Optional[dict]:
|
||||
digest = self.digest(token)
|
||||
cached = await self._cache_get(digest)
|
||||
if cached == REVOKED:
|
||||
self.stats["hits"] += 1
|
||||
return None
|
||||
if cached is not None:
|
||||
self.stats["hits"] += 1
|
||||
return cached
|
||||
self.stats["misses"] += 1
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute(
|
||||
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
|
||||
" FROM sessions s JOIN users u ON u.id = s.user_id"
|
||||
" WHERE s.token_hash = %s AND s.expires_at > now()",
|
||||
(digest,),
|
||||
)
|
||||
row = await cur.fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
user = repo.public_user(row)
|
||||
await self._cache_put(digest, user, row["expires_at"])
|
||||
return user
|
||||
|
||||
async def revoke(self, token: str) -> None:
|
||||
digest = self.digest(token)
|
||||
await self._cache_revoke(digest)
|
||||
try:
|
||||
async with self.db.connection() as conn:
|
||||
await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,))
|
||||
except Exception:
|
||||
await self._cache_release(digest)
|
||||
raise
|
||||
|
||||
async def sweep(self) -> int:
|
||||
async with self.db.connection() as conn:
|
||||
cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()")
|
||||
return cur.rowcount
|
||||
|
||||
def _cache_ttl(self, expires_at: datetime) -> int:
|
||||
remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds())
|
||||
return min(config.SESSION_CACHE_TTL_SECONDS, remaining)
|
||||
|
||||
async def _cache_get(self, digest: bytes) -> Optional[dict]:
|
||||
if self.client is None:
|
||||
return None
|
||||
try:
|
||||
raw = await self.client.get(self.key(digest))
|
||||
except Exception:
|
||||
self._cache_failed("get")
|
||||
return None
|
||||
if not raw:
|
||||
return None
|
||||
if raw == REVOKED:
|
||||
return REVOKED
|
||||
try:
|
||||
return json.loads(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
ttl = self._cache_ttl(expires_at)
|
||||
if ttl <= 0:
|
||||
return
|
||||
payload = json.dumps(user)
|
||||
try:
|
||||
await self.client.set(self.key(digest), payload, ex=ttl, nx=True)
|
||||
except Exception:
|
||||
self._cache_failed("set")
|
||||
|
||||
async def _cache_release(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.delete(self.key(digest))
|
||||
except Exception:
|
||||
self._cache_failed("release")
|
||||
|
||||
async def _cache_revoke(self, digest: bytes) -> None:
|
||||
if self.client is None:
|
||||
return
|
||||
try:
|
||||
await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS)
|
||||
except Exception:
|
||||
self._cache_failed("revoke")
|
||||
raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None
|
||||
|
||||
def _cache_failed(self, operation: str) -> None:
|
||||
self.stats["errors"] += 1
|
||||
if self.stats["errors"] % 50 == 1:
|
||||
log.warning("session cache %s failed, falling back to postgres (errors=%d)", operation, self.stats["errors"])
|
||||
+167
@@ -0,0 +1,167 @@
|
||||
name: mtusi-lab3
|
||||
|
||||
x-api: &api-defaults
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab3
|
||||
env_file:
|
||||
- .env
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 256M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
services:
|
||||
nginx:
|
||||
image: nginx:1.27.3-alpine
|
||||
command: ["/bin/sh", "/opt/nginx-health.sh"]
|
||||
environment:
|
||||
API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2}
|
||||
HEALTH_INTERVAL: "1"
|
||||
HEALTH_FALL: "2"
|
||||
HEALTH_RISE: "2"
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- ./infra/nginx-health.sh:/opt/nginx-health.sh:ro
|
||||
depends_on:
|
||||
api-1:
|
||||
condition: service_healthy
|
||||
api-2:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 64M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
api-1:
|
||||
<<: *api-defaults
|
||||
environment:
|
||||
INSTANCE_ID: api-1
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
|
||||
api-2:
|
||||
<<: *api-defaults
|
||||
environment:
|
||||
INSTANCE_ID: api-2
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
|
||||
postgres:
|
||||
image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_INITDB_ARGS: --data-checksums
|
||||
command:
|
||||
- postgres
|
||||
- -c
|
||||
- synchronous_commit=on
|
||||
- -c
|
||||
- shared_buffers=128MB
|
||||
- -c
|
||||
- work_mem=8MB
|
||||
- -c
|
||||
- max_connections=50
|
||||
- -c
|
||||
- track_io_timing=on
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.00"
|
||||
memory: 512M
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 20
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
valkey:
|
||||
image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507
|
||||
command:
|
||||
- valkey-server
|
||||
- --save
|
||||
- ""
|
||||
- --appendonly
|
||||
- "no"
|
||||
- --maxmemory
|
||||
- 64mb
|
||||
- --maxmemory-policy
|
||||
- allkeys-lru
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 5s
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
image: mtusi-messenger:lab3
|
||||
command: ["python", "scripts/migrate.py"]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets:
|
||||
- database_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.25"
|
||||
memory: 128M
|
||||
restart: "no"
|
||||
|
||||
volumes:
|
||||
pgdata:
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
database_url:
|
||||
file: ./secrets/database_url
|
||||
cache_url:
|
||||
file: ./secrets/cache_url
|
||||
@@ -0,0 +1,11 @@
|
||||
=== Failover run 1 ===
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.333}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
|
||||
=== Failover run 2 (other replica) ===
|
||||
python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.325}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,37 @@
|
||||
=== Graceful shutdown test ===
|
||||
Sending SIGTERM to api-1...
|
||||
Container mtusi-lab3-api-1-1 Stopping
|
||||
Container mtusi-lab3-api-1-1 Stopped
|
||||
|
||||
=== api-1 logs (last 15 lines) ===
|
||||
api-1-1 | INFO: 172.19.0.6:58008 - "POST /api/v1/chats HTTP/1.0" 201 Created
|
||||
api-1-1 | INFO: 172.19.0.6:58018 - "POST /api/v1/chats/0a0b21a2-0a79-4182-ac0d-b9fbac88fadc/messages HTTP/1.0" 201 Created
|
||||
api-1-1 | INFO: 172.19.0.6:58022 - "POST /api/v1/chats/0a0b21a2-0a79-4182-ac0d-b9fbac88fadc/messages HTTP/1.0" 201 Created
|
||||
api-1-1 | INFO: 172.19.0.6:58036 - "POST /api/v1/chats/0a0b21a2-0a79-4182-ac0d-b9fbac88fadc/messages HTTP/1.0" 409 Conflict
|
||||
api-1-1 | INFO: 127.0.0.1:44656 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api-1-1 | INFO: 127.0.0.1:48164 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api-1-1 | INFO: 172.19.0.6:35016 - "GET /health/live HTTP/1.0" 200 OK
|
||||
api-1-1 | INFO: 127.0.0.1:48392 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api-1-1 | INFO: 127.0.0.1:48412 - "GET /health/ready HTTP/1.1" 200 OK
|
||||
api-1-1 | 2026-10-28 12:21:22,692 INFO messenger signal SIGTERM: draining, new requests get 503
|
||||
api-1-1 | INFO: Shutting down
|
||||
api-1-1 | INFO: Waiting for application shutdown.
|
||||
api-1-1 | 2026-10-28 12:21:22,836 INFO messenger stopped: cache={'hits': 4, 'misses': 2, 'errors': 0} rejected={}
|
||||
api-1-1 | INFO: Application shutdown complete.
|
||||
api-1-1 | INFO: Finished server process [1]
|
||||
|
||||
=== Restoring api-1 ===
|
||||
Container mtusi-lab3-postgres-1 Running
|
||||
Container mtusi-lab3-valkey-1 Running
|
||||
Container mtusi-lab3-postgres-1 Waiting
|
||||
Container mtusi-lab3-postgres-1 Healthy
|
||||
Container mtusi-lab3-migrate-1 Starting
|
||||
Container mtusi-lab3-migrate-1 Started
|
||||
Container mtusi-lab3-migrate-1 Waiting
|
||||
Container mtusi-lab3-postgres-1 Waiting
|
||||
Container mtusi-lab3-valkey-1 Waiting
|
||||
Container mtusi-lab3-valkey-1 Healthy
|
||||
Container mtusi-lab3-postgres-1 Healthy
|
||||
Container mtusi-lab3-migrate-1 Exited
|
||||
Container mtusi-lab3-api-1-1 Starting
|
||||
Container mtusi-lab3-api-1-1 Started
|
||||
@@ -0,0 +1,6 @@
|
||||
x-instance-id: api-2
|
||||
x-instance-id: api-1
|
||||
x-instance-id: api-2
|
||||
x-instance-id: api-1
|
||||
x-instance-id: api-2
|
||||
x-instance-id: api-1
|
||||
@@ -0,0 +1,34 @@
|
||||
limits:
|
||||
cpus: 0.5
|
||||
memory: "268435456"
|
||||
environment:
|
||||
--
|
||||
limits:
|
||||
cpus: 0.5
|
||||
memory: "268435456"
|
||||
environment:
|
||||
--
|
||||
limits:
|
||||
cpus: 0.25
|
||||
memory: "134217728"
|
||||
environment:
|
||||
--
|
||||
limits:
|
||||
cpus: 0.25
|
||||
memory: "67108864"
|
||||
healthcheck:
|
||||
--
|
||||
limits:
|
||||
cpus: 1
|
||||
memory: "536870912"
|
||||
environment:
|
||||
--
|
||||
limits:
|
||||
cpus: 0.25
|
||||
memory: "134217728"
|
||||
healthcheck:
|
||||
--
|
||||
limits:
|
||||
cpus: "0.50"
|
||||
memory: 256M
|
||||
env_file:
|
||||
@@ -0,0 +1,20 @@
|
||||
migrate-1 | skip 0002_idempotency
|
||||
migrate-1 | migrations complete: 2 file(s)
|
||||
migrate-1 | skip 0001_init
|
||||
migrate-1 | skip 0002_idempotency
|
||||
migrate-1 | migrations complete: 2 file(s)
|
||||
migrate-1 | skip 0001_init
|
||||
migrate-1 | skip 0002_idempotency
|
||||
migrate-1 | migrations complete: 2 file(s)
|
||||
migrate-1 | skip 0001_init
|
||||
migrate-1 | skip 0002_idempotency
|
||||
migrate-1 | migrations complete: 2 file(s)
|
||||
migrate-1 | skip 0001_init
|
||||
migrate-1 | skip 0002_idempotency
|
||||
migrate-1 | migrations complete: 2 file(s)
|
||||
migrate-1 | skip 0001_init
|
||||
migrate-1 | skip 0002_idempotency
|
||||
migrate-1 | migrations complete: 2 file(s)
|
||||
migrate-1 | skip 0001_init
|
||||
migrate-1 | skip 0002_idempotency
|
||||
migrate-1 | migrations complete: 2 file(s)
|
||||
@@ -0,0 +1,2 @@
|
||||
PASS: 8 parallel requests, 1 message created, 2 distinct replicas observed: ['api-1', 'api-2']
|
||||
PASS: same key + different body → 409
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080"
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,6 @@
|
||||
NAME PORTS STATUS
|
||||
mtusi-lab3-api-1-1 8080/tcp Up 12 seconds (healthy)
|
||||
mtusi-lab3-api-2-1 8080/tcp Up 12 seconds (healthy)
|
||||
mtusi-lab3-nginx-1 127.0.0.1:8080->8080/tcp Up 12 seconds (healthy)
|
||||
mtusi-lab3-postgres-1 5432/tcp Up 12 seconds (healthy)
|
||||
mtusi-lab3-valkey-1 6379/tcp Up 12 seconds (healthy)
|
||||
@@ -0,0 +1,39 @@
|
||||
### Before fixes (commit fbb8916)
|
||||
== 1. Idempotent replay after membership removal
|
||||
PASS: first POST with key -> 201 (got 201)
|
||||
PASS: history after removal -> 404 (got 404)
|
||||
FAIL: replay with same key and body after removal -> 404 (got 201)
|
||||
== 2. Live but not ready api-1 (no route to PostgreSQL)
|
||||
FAIL: nginx active check marks api-1 down in upstream
|
||||
PASS: Docker marks api-1 unhealthy
|
||||
FAIL: 40 GET /users/me with warm session: served by not-ready api-1 = 20
|
||||
PASS: 40 GET /users/me: 200 responses = 40
|
||||
PASS: 20 POST messages during outage -> all 201 (got [201])
|
||||
== 3. LB does not transparently retry non-idempotent POST /chats
|
||||
PASS: chats in DB (11) == 201 responses (11), POST /chats codes=[201, 201, 201, 201, 201, 201, 201, 201, 201, 201]
|
||||
FAIL: nginx access log: 12 POST /chats, sent to two upstreams: 3
|
||||
FAIL: non_idempotent only on messages route: {'/': True}
|
||||
PASS: api-1 healthy again after PostgreSQL route restored
|
||||
PASS: api-1 returns to LB rotation
|
||||
PASS: nginx upstream has no servers marked down
|
||||
RESULT: FAIL (5)
|
||||
|
||||
### After fixes
|
||||
== 1. Idempotent replay after membership removal
|
||||
PASS: first POST with key -> 201 (got 201)
|
||||
PASS: history after removal -> 404 (got 404)
|
||||
PASS: replay with same key and body after removal -> 404 (got 404)
|
||||
== 2. Live but not ready api-1 (no route to PostgreSQL)
|
||||
PASS: nginx active check marks api-1 down in upstream
|
||||
PASS: Docker marks api-1 unhealthy
|
||||
PASS: 40 GET /users/me with warm session: served by not-ready api-1 = 0
|
||||
PASS: 40 GET /users/me: 200 responses = 40
|
||||
PASS: 20 POST messages during outage -> all 201 (got [201])
|
||||
== 3. LB does not transparently retry non-idempotent POST /chats
|
||||
PASS: chats in DB (9) == 201 responses (9), POST /chats codes=[503, 201, 503, 201, 201, 201, 201, 201, 201, 201]
|
||||
PASS: nginx access log: 12 POST /chats, sent to two upstreams: 0
|
||||
PASS: non_idempotent only on messages route: {'~ ^/api/v1/chats/[^/]+/messages$': True, '/': False}
|
||||
PASS: api-1 healthy again after PostgreSQL route restored
|
||||
PASS: api-1 returns to LB rotation
|
||||
PASS: nginx upstream has no servers marked down
|
||||
RESULT: PASS
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 60.0,
|
||||
"elapsed_seconds": 60.012,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 54625,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 910.231,
|
||||
"successful_latency_ms": {
|
||||
"p50": 3.848,
|
||||
"p95": 5.7,
|
||||
"p99": 17.851
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
14:14:27 recreate api-1 (old container 572155b7facd)
|
||||
14:14:33 api-1 healthy (new container a81474023dca)
|
||||
14:14:43 recreate api-2 (old container 35c30c16ece5)
|
||||
14:14:49 api-2 healthy (new container 1ac67f968005)
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 60.0,
|
||||
"elapsed_seconds": 60.012,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 54625,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 910.231,
|
||||
"successful_latency_ms": {
|
||||
"p50": 3.848,
|
||||
"p95": 5.7,
|
||||
"p99": 17.851
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
config=A1 replicas=1 api_cpus=1.00 api_mem=512M concurrency=16 duration=30
|
||||
api-1 NanoCpus=1000000000 Memory=536870912
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.049,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 10411,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 346.466,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.495,
|
||||
"p95": 205.76,
|
||||
"p99": 519.105
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
mtusi-lab3-nginx-1 25.01% 10.68MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 11.14% 64.5MiB / 512MiB
|
||||
mtusi-lab3-postgres-1 3.80% 59.32MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 0.70% 12.3MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.047,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 4193,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 139.548,
|
||||
"successful_latency_ms": {
|
||||
"p50": 86.111,
|
||||
"p95": 491.678,
|
||||
"p99": 697.125
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
mtusi-lab3-nginx-1 24.54% 11.58MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 12.71% 64.38MiB / 512MiB
|
||||
mtusi-lab3-postgres-1 7.29% 64.33MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 0.65% 12.26MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.166,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 14260,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 472.712,
|
||||
"successful_latency_ms": {
|
||||
"p50": 13.454,
|
||||
"p95": 108.372,
|
||||
"p99": 498.509
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
mtusi-lab3-nginx-1 24.67% 11.53MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 31.22% 64.43MiB / 512MiB
|
||||
mtusi-lab3-postgres-1 7.25% 77.34MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 0.61% 12.08MiB / 128MiB
|
||||
@@ -0,0 +1,3 @@
|
||||
config=A2 replicas=2 api_cpus=0.50 api_mem=256M concurrency=16 duration=30
|
||||
api-1 NanoCpus=500000000 Memory=268435456
|
||||
api-2 NanoCpus=500000000 Memory=268435456
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.046,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 24369,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 811.047,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.499,
|
||||
"p95": 63.911,
|
||||
"p99": 86.902
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mtusi-lab3-nginx-1 18.84% 10.38MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 51.03% 63.71MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 43.37% 63.35MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 46.22% 115.9MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 2.79% 12.41MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.08,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 8481,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 281.951,
|
||||
"successful_latency_ms": {
|
||||
"p50": 32.217,
|
||||
"p95": 112.993,
|
||||
"p99": 297.73
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mtusi-lab3-nginx-1 25.09% 11.69MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 27.42% 75.51MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 31.68% 63.59MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 15.93% 129.2MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 1.08% 12.16MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.106,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 23597,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 783.804,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.682,
|
||||
"p95": 66.159,
|
||||
"p99": 87.785
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mtusi-lab3-nginx-1 18.01% 11.25MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 51.18% 64.11MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 43.66% 63.78MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 43.18% 147.1MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 2.88% 12.25MiB / 128MiB
|
||||
@@ -0,0 +1,5 @@
|
||||
config=A4 replicas=4 api_cpus=0.25 api_mem=128M concurrency=16 duration=30
|
||||
api-1 NanoCpus=250000000 Memory=134217728
|
||||
api-2 NanoCpus=250000000 Memory=134217728
|
||||
api-3 NanoCpus=250000000 Memory=134217728
|
||||
api-4 NanoCpus=250000000 Memory=134217728
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.048,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 18635,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 620.172,
|
||||
"successful_latency_ms": {
|
||||
"p50": 10.376,
|
||||
"p95": 89.425,
|
||||
"p99": 105.535
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 8.28% 9.543MiB / 64MiB
|
||||
mtusi-lab3-api-2-1 25.56% 63.45MiB / 128MiB
|
||||
mtusi-lab3-api-1-1 23.44% 63.25MiB / 128MiB
|
||||
mtusi-lab3-api-3-1 24.29% 63.04MiB / 128MiB
|
||||
mtusi-lab3-api-4-1 25.30% 63.8MiB / 128MiB
|
||||
mtusi-lab3-postgres-1 22.11% 216.8MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 2.30% 12.11MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.064,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 17869,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 594.358,
|
||||
"successful_latency_ms": {
|
||||
"p50": 9.511,
|
||||
"p95": 93.884,
|
||||
"p99": 174.611
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 16.58% 10.88MiB / 64MiB
|
||||
mtusi-lab3-api-2-1 24.09% 74.36MiB / 128MiB
|
||||
mtusi-lab3-api-1-1 11.84% 63.49MiB / 128MiB
|
||||
mtusi-lab3-api-3-1 24.96% 76.1MiB / 128MiB
|
||||
mtusi-lab3-api-4-1 12.84% 63.58MiB / 128MiB
|
||||
mtusi-lab3-postgres-1 21.19% 233.9MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 1.24% 12.81MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.131,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 18533,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 615.075,
|
||||
"successful_latency_ms": {
|
||||
"p50": 9.242,
|
||||
"p95": 92.912,
|
||||
"p99": 167.713
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 16.15% 11.08MiB / 64MiB
|
||||
mtusi-lab3-api-2-1 26.09% 63.18MiB / 128MiB
|
||||
mtusi-lab3-api-1-1 21.89% 63.21MiB / 128MiB
|
||||
mtusi-lab3-api-3-1 25.19% 72.58MiB / 128MiB
|
||||
mtusi-lab3-api-4-1 24.23% 63.88MiB / 128MiB
|
||||
mtusi-lab3-postgres-1 36.57% 252.6MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 2.19% 12.38MiB / 128MiB
|
||||
@@ -0,0 +1,2 @@
|
||||
config=B1 replicas=1 api_cpus=0.50 api_mem=256M concurrency=16 duration=30
|
||||
api-1 NanoCpus=500000000 Memory=268435456
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.168,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 12482,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 413.753,
|
||||
"successful_latency_ms": {
|
||||
"p50": 15.779,
|
||||
"p95": 97.028,
|
||||
"p99": 296.954
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
mtusi-lab3-nginx-1 9.25% 10.25MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 50.06% 64MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 13.37% 177.7MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 2.17% 12.04MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.068,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 2946,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 97.979,
|
||||
"successful_latency_ms": {
|
||||
"p50": 96.015,
|
||||
"p95": 601.496,
|
||||
"p99": 898.354
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
mtusi-lab3-nginx-1 24.01% 10.98MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 12.34% 64.64MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 7.14% 178.9MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 0.57% 12.72MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.485,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 14214,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 466.261,
|
||||
"successful_latency_ms": {
|
||||
"p50": 14.117,
|
||||
"p95": 94.171,
|
||||
"p99": 196.651
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
mtusi-lab3-nginx-1 6.65% 10.91MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 50.15% 64.57MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 15.69% 180MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 1.93% 12.37MiB / 128MiB
|
||||
@@ -0,0 +1,5 @@
|
||||
config=B4 replicas=4 api_cpus=0.50 api_mem=256M concurrency=16 duration=30
|
||||
api-1 NanoCpus=500000000 Memory=268435456
|
||||
api-2 NanoCpus=500000000 Memory=268435456
|
||||
api-3 NanoCpus=500000000 Memory=268435456
|
||||
api-4 NanoCpus=500000000 Memory=268435456
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.049,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 30483,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1014.431,
|
||||
"successful_latency_ms": {
|
||||
"p50": 10.888,
|
||||
"p95": 46.73,
|
||||
"p99": 74.057
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 20.00% 10.18MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 41.68% 72.96MiB / 256MiB
|
||||
mtusi-lab3-api-4-1 49.10% 63.3MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 40.04% 74.45MiB / 256MiB
|
||||
mtusi-lab3-api-3-1 45.58% 73.81MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 48.32% 246.8MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 4.01% 12MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.095,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 29869,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 992.482,
|
||||
"successful_latency_ms": {
|
||||
"p50": 10.804,
|
||||
"p95": 48.199,
|
||||
"p99": 76.479
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 25.93% 11.25MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 37.18% 62.95MiB / 256MiB
|
||||
mtusi-lab3-api-4-1 38.93% 63.41MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 37.11% 63.85MiB / 256MiB
|
||||
mtusi-lab3-api-3-1 37.95% 62.87MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 59.80% 252.1MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 4.81% 11.68MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.045,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 30576,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1017.679,
|
||||
"successful_latency_ms": {
|
||||
"p50": 10.762,
|
||||
"p95": 47.06,
|
||||
"p99": 71.198
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 25.66% 11.43MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 39.97% 62.99MiB / 256MiB
|
||||
mtusi-lab3-api-4-1 43.05% 62.9MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 38.31% 64.04MiB / 256MiB
|
||||
mtusi-lab3-api-3-1 42.32% 63.35MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 44.79% 253.3MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 3.87% 12.4MiB / 128MiB
|
||||
@@ -0,0 +1,26 @@
|
||||
=== A1: 1 x 1.00 CPU / 512M ===
|
||||
run 1: rps=346.466 err=0 p50=13.495 p95=205.76 p99=519.105
|
||||
run 2: rps=139.548 err=0 p50=86.111 p95=491.678 p99=697.125
|
||||
run 3: rps=472.712 err=0 p50=13.454 p95=108.372 p99=498.509
|
||||
=== A2: 2 x 0.50 CPU / 256M ===
|
||||
run 1: rps=811.047 err=0 p50=11.499 p95=63.911 p99=86.902
|
||||
run 2: rps=281.951 err=0 p50=32.217 p95=112.993 p99=297.73
|
||||
run 3: rps=783.804 err=0 p50=11.682 p95=66.159 p99=87.785
|
||||
=== A4: 4 x 0.25 CPU / 128M ===
|
||||
run 1: rps=620.172 err=0 p50=10.376 p95=89.425 p99=105.535
|
||||
run 2: rps=594.358 err=0 p50=9.511 p95=93.884 p99=174.611
|
||||
run 3: rps=615.075 err=0 p50=9.242 p95=92.912 p99=167.713
|
||||
=== B1: 1 x 0.50 CPU / 256M ===
|
||||
run 1: rps=413.753 err=0 p50=15.779 p95=97.028 p99=296.954
|
||||
run 2: rps=97.979 err=0 p50=96.015 p95=601.496 p99=898.354
|
||||
run 3: rps=466.261 err=0 p50=14.117 p95=94.171 p99=196.651
|
||||
=== B4: 4 x 0.50 CPU / 256M ===
|
||||
run 1: rps=1014.431 err=0 p50=10.888 p95=46.73 p99=74.057
|
||||
run 2: rps=992.482 err=0 p50=10.804 p95=48.199 p99=76.479
|
||||
run 3: rps=1017.679 err=0 p50=10.762 p95=47.06 p99=71.198
|
||||
A1 1x1.00 CPU rps=346.5 [139.5..472.7] p50=13.5 p95=205.8 p99=519.1 errors=0
|
||||
A2 2x0.50 CPU rps=783.8 [282.0..811.0] p50=11.7 p95=66.2 p99=87.8 errors=0
|
||||
A4 4x0.25 CPU rps=615.1 [594.4..620.2] p50=9.5 p95=92.9 p99=167.7 errors=0
|
||||
B1 1x0.50 CPU rps=413.8 [98.0..466.3] p50=15.8 p95=97.0 p99=297.0 errors=0
|
||||
B4 4x0.50 CPU rps=1014.4 [992.5..1017.7] p50=10.8 p95=47.1 p99=74.1 errors=0
|
||||
wrote evidence/scaling/summary.csv evidence/scaling/rps.svg
|
||||
@@ -0,0 +1,3 @@
|
||||
config=A2 replicas=2 api_cpus=0.50 api_mem=256M concurrency=16 duration=30
|
||||
api-1 NanoCpus=500000000 Memory=268435456
|
||||
api-2 NanoCpus=500000000 Memory=268435456
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.064,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 30683,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1020.576,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.476,
|
||||
"p95": 44.082,
|
||||
"p99": 73.405
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mtusi-lab3-nginx-1 15.65% 7.613MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 50.39% 63.44MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 48.75% 63.96MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 40.91% 116.4MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 1.95% 12.75MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.059,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 28447,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 946.385,
|
||||
"successful_latency_ms": {
|
||||
"p50": 12.05,
|
||||
"p95": 48.707,
|
||||
"p99": 78.884
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mtusi-lab3-nginx-1 18.79% 7.812MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 47.23% 63.85MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 50.58% 64.79MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 48.47% 144MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 2.36% 12.38MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.07,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 31667,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1053.114,
|
||||
"successful_latency_ms": {
|
||||
"p50": 11.35,
|
||||
"p95": 41.956,
|
||||
"p99": 71.286
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
mtusi-lab3-nginx-1 15.00% 8.219MiB / 64MiB
|
||||
mtusi-lab3-api-1-1 51.30% 76.93MiB / 256MiB
|
||||
mtusi-lab3-api-2-1 50.54% 76.83MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 38.70% 170.6MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 1.88% 13.06MiB / 128MiB
|
||||
@@ -0,0 +1,5 @@
|
||||
config=B4 replicas=4 api_cpus=0.50 api_mem=256M concurrency=16 duration=30
|
||||
api-1 NanoCpus=500000000 Memory=268435456
|
||||
api-2 NanoCpus=500000000 Memory=268435456
|
||||
api-3 NanoCpus=500000000 Memory=268435456
|
||||
api-4 NanoCpus=500000000 Memory=268435456
|
||||
@@ -0,0 +1,9 @@
|
||||
B4 (4 x 0.50 CPU / 256M, keepalive) recheck after run1 showed 319 failed requests.
|
||||
3 more runs, 30 s, concurrency 16, nginx/api logs captured live:
|
||||
run 1: successful_rps=1118.0 failed_requests=0
|
||||
run 2: successful_rps=1074.2 failed_requests=0
|
||||
run 3: successful_rps=1027.3 failed_requests=0
|
||||
nginx 4xx/5xx lines: 0; nginx [error]/[warn]: 0; api 4xx/5xx: 0
|
||||
postgres log (FATAL/ERROR/too many clients) for the whole session: 0
|
||||
Cause of the 319 errors in that earlier run (raw JSON superseded by the final measurement) not identified: nginx/api containers were recreated
|
||||
by the script afterwards and their logs were lost. Script now saves <label>_nginx_problems.txt.
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.045,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 36686,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1221.047,
|
||||
"successful_latency_ms": {
|
||||
"p50": 10.126,
|
||||
"p95": 34.09,
|
||||
"p99": 54.566
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 24.64% 7.605MiB / 64MiB
|
||||
mtusi-lab3-api-2-1 45.05% 63.07MiB / 256MiB
|
||||
mtusi-lab3-api-1-1 47.89% 63.44MiB / 256MiB
|
||||
mtusi-lab3-api-3-1 43.50% 63.16MiB / 256MiB
|
||||
mtusi-lab3-api-4-1 43.64% 64.22MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 75.01% 245.8MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 4.06% 12.41MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.049,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 36215,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1205.189,
|
||||
"successful_latency_ms": {
|
||||
"p50": 10.131,
|
||||
"p95": 34.418,
|
||||
"p99": 54.313
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 26.08% 8.008MiB / 64MiB
|
||||
mtusi-lab3-api-2-1 47.67% 63.12MiB / 256MiB
|
||||
mtusi-lab3-api-1-1 48.22% 63.36MiB / 256MiB
|
||||
mtusi-lab3-api-3-1 45.32% 62.89MiB / 256MiB
|
||||
mtusi-lab3-api-4-1 48.28% 63.55MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 76.58% 250.4MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 4.48% 11.78MiB / 128MiB
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.048,
|
||||
"concurrency": 16,
|
||||
"successful_requests": 36534,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 1215.869,
|
||||
"successful_latency_ms": {
|
||||
"p50": 10.404,
|
||||
"p95": 33.065,
|
||||
"p99": 51.443
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
mtusi-lab3-nginx-1 25.96% 7.855MiB / 64MiB
|
||||
mtusi-lab3-api-2-1 47.21% 63.2MiB / 256MiB
|
||||
mtusi-lab3-api-1-1 47.33% 63.57MiB / 256MiB
|
||||
mtusi-lab3-api-3-1 43.69% 63.66MiB / 256MiB
|
||||
mtusi-lab3-api-4-1 45.41% 63.65MiB / 256MiB
|
||||
mtusi-lab3-postgres-1 71.86% 249.2MiB / 512MiB
|
||||
mtusi-lab3-valkey-1 4.27% 12.85MiB / 128MiB
|
||||
@@ -0,0 +1,34 @@
|
||||
=== A2: 2 x 0.50 CPU / 256M ===
|
||||
run 1: rps=1020.576 err=0 p50=11.476 p95=44.082 p99=73.405
|
||||
run 2: rps=946.385 err=0 p50=12.05 p95=48.707 p99=78.884
|
||||
run 3: rps=1053.114 err=0 p50=11.35 p95=41.956 p99=71.286
|
||||
=== B4: 4 x 0.50 CPU / 256M ===
|
||||
run 1: rps=1221.047 err=0 p50=10.126 p95=34.09 p99=54.566
|
||||
run 2: rps=1205.189 err=0 p50=10.131 p95=34.418 p99=54.313
|
||||
run 3: rps=1215.869 err=0 p50=10.404 p95=33.065 p99=51.443
|
||||
A2 2x0.50 CPU rps=1020.6 [946.4..1053.1] p50=11.5 p95=44.1 p99=73.4 errors=0
|
||||
B4 4x0.50 CPU rps=1215.9 [1205.2..1221.0] p50=10.1 p95=34.1 p99=54.3 errors=0
|
||||
wrote evidence/scaling/keepalive/summary.csv evidence/scaling/keepalive/rps.svg
|
||||
|
||||
=== rolling update ===
|
||||
14:22:50 recreate api-1 (old container df4ba056eb90)
|
||||
14:22:56 api-1 healthy (new container 53a60bfc9073)
|
||||
14:23:06 recreate api-2 (old container 6b53175a3693)
|
||||
14:23:12 api-2 healthy (new container 047f8e956a97)
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 60.0,
|
||||
"elapsed_seconds": 60.012,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 53067,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 884.267,
|
||||
"successful_latency_ms": {
|
||||
"p50": 3.936,
|
||||
"p95": 6.112,
|
||||
"p99": 18.691
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
config,replicas,api_cpus,api_mem,runs,rps_median,rps_min,rps_max,p50_ms,p95_ms,p99_ms,errors
|
||||
A2,2,0.5,256M,3,1020.6,946.4,1053.1,11.5,44.1,73.4,0
|
||||
B4,4,0.5,256M,3,1215.9,1205.2,1221.0,10.1,34.1,54.3,0
|
||||
|
@@ -0,0 +1,38 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" width="640" height="360" font-family="sans-serif" font-size="12">
|
||||
<rect width="100%" height="100%" fill="white"/>
|
||||
<text x="60" y="22" font-size="14">RPS (медиана 3 прогонов) от числа API-реплик</text>
|
||||
<line x1="60" x2="490" y1="310.0" y2="310.0" stroke="#ddd"/>
|
||||
<text x="54" y="314.0" text-anchor="end">0</text>
|
||||
<line x1="60" x2="490" y1="256.0" y2="256.0" stroke="#ddd"/>
|
||||
<text x="54" y="260.0" text-anchor="end">234</text>
|
||||
<line x1="60" x2="490" y1="202.0" y2="202.0" stroke="#ddd"/>
|
||||
<text x="54" y="206.0" text-anchor="end">468</text>
|
||||
<line x1="60" x2="490" y1="148.0" y2="148.0" stroke="#ddd"/>
|
||||
<text x="54" y="152.0" text-anchor="end">702</text>
|
||||
<line x1="60" x2="490" y1="94.0" y2="94.0" stroke="#ddd"/>
|
||||
<text x="54" y="98.0" text-anchor="end">936</text>
|
||||
<line x1="60" x2="490" y1="40.0" y2="40.0" stroke="#ddd"/>
|
||||
<text x="54" y="44.0" text-anchor="end">1170</text>
|
||||
<text x="60.0" y="328" text-anchor="middle">1</text>
|
||||
<text x="275.0" y="328" text-anchor="middle">2</text>
|
||||
<text x="490.0" y="328" text-anchor="middle">4</text>
|
||||
<text x="275.0" y="348" text-anchor="middle">реплики</text>
|
||||
<polyline fill="none" stroke="#2a6fdb" stroke-width="2" points="60.0,230.1 275.0,129.2 490.0,168.1"/>
|
||||
<circle cx="60.0" cy="230.1" r="4" fill="#2a6fdb"/>
|
||||
<text x="60.0" y="222.1" fill="#2a6fdb" text-anchor="middle">346</text>
|
||||
<circle cx="275.0" cy="129.2" r="4" fill="#2a6fdb"/>
|
||||
<text x="275.0" y="121.2" fill="#2a6fdb" text-anchor="middle">783</text>
|
||||
<circle cx="490.0" cy="168.1" r="4" fill="#2a6fdb"/>
|
||||
<text x="490.0" y="160.1" fill="#2a6fdb" text-anchor="middle">615</text>
|
||||
<rect x="502" y="40" width="12" height="12" fill="#2a6fdb"/>
|
||||
<text x="518" y="50">A: всего 1.0 CPU</text>
|
||||
<polyline fill="none" stroke="#d9480f" stroke-width="2" points="60.0,214.5 275.0,129.2 490.0,76.0"/>
|
||||
<circle cx="60.0" cy="214.5" r="4" fill="#d9480f"/>
|
||||
<text x="60.0" y="196.5" fill="#d9480f" text-anchor="middle">413</text>
|
||||
<circle cx="275.0" cy="129.2" r="4" fill="#d9480f"/>
|
||||
<text x="275.0" y="111.2" fill="#d9480f" text-anchor="middle">783</text>
|
||||
<circle cx="490.0" cy="76.0" r="4" fill="#d9480f"/>
|
||||
<text x="490.0" y="58.0" fill="#d9480f" text-anchor="middle">1014</text>
|
||||
<rect x="502" y="60" width="12" height="12" fill="#d9480f"/>
|
||||
<text x="518" y="70">B: 0.5 CPU на реплику</text>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.3 KiB |
@@ -0,0 +1,6 @@
|
||||
config,replicas,api_cpus,api_mem,runs,rps_median,rps_min,rps_max,p50_ms,p95_ms,p99_ms,errors
|
||||
A1,1,1.0,512M,3,346.5,139.5,472.7,13.5,205.8,519.1,0
|
||||
A2,2,0.5,256M,3,783.8,282.0,811.0,11.7,66.2,87.8,0
|
||||
A4,4,0.25,128M,3,615.1,594.4,620.2,9.5,92.9,167.7,0
|
||||
B1,1,0.5,256M,3,413.8,98.0,466.3,15.8,97.0,297.0,0
|
||||
B4,4,0.5,256M,3,1014.4,992.5,1017.7,10.8,47.1,74.1,0
|
||||
|
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/bin/sh
|
||||
set -u
|
||||
|
||||
SERVERS="${API_UPSTREAMS:-api-1 api-2}"
|
||||
INTERVAL="${HEALTH_INTERVAL:-1}"
|
||||
FALL="${HEALTH_FALL:-2}"
|
||||
RISE="${HEALTH_RISE:-2}"
|
||||
OUT=/etc/nginx/upstream/servers.conf
|
||||
|
||||
mkdir -p "$(dirname "$OUT")"
|
||||
|
||||
var() { echo "$1" | tr -c 'a-zA-Z0-9\n' '_'; }
|
||||
|
||||
render() {
|
||||
any_up=0
|
||||
for s in $SERVERS; do
|
||||
eval "[ \"\$up_$(var "$s")\" = 1 ]" && any_up=1
|
||||
done
|
||||
for s in $SERVERS; do
|
||||
mark=""
|
||||
if [ "$any_up" = 1 ] && eval "[ \"\$up_$(var "$s")\" = 0 ]"; then
|
||||
mark=" down"
|
||||
fi
|
||||
echo "server $s:8080 resolve max_fails=1 fail_timeout=10s$mark;"
|
||||
done
|
||||
}
|
||||
|
||||
for s in $SERVERS; do
|
||||
v=$(var "$s")
|
||||
eval "up_$v=1 ok_$v=0 bad_$v=0"
|
||||
done
|
||||
render > "$OUT"
|
||||
|
||||
watch() {
|
||||
pending=0
|
||||
while sleep "$INTERVAL"; do
|
||||
changed=0
|
||||
for s in $SERVERS; do
|
||||
v=$(var "$s")
|
||||
if wget -q -T 1 -O /dev/null "http://$s:8080/health/ready" 2>/dev/null; then
|
||||
eval "ok_$v=\$((ok_$v + 1)) bad_$v=0"
|
||||
if eval "[ \"\$up_$v\" = 0 ] && [ \"\$ok_$v\" -ge $RISE ]"; then
|
||||
eval "up_$v=1"; changed=1; echo "health: $s ready, back in upstream"
|
||||
fi
|
||||
else
|
||||
eval "bad_$v=\$((bad_$v + 1)) ok_$v=0"
|
||||
if eval "[ \"\$up_$v\" = 1 ] && [ \"\$bad_$v\" -ge $FALL ]"; then
|
||||
eval "up_$v=0"; changed=1; echo "health: $s not ready, marked down"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
if [ "$changed" = 1 ]; then
|
||||
render > "$OUT"
|
||||
pending=1
|
||||
fi
|
||||
if [ "$pending" = 1 ] && nginx -s reload 2>/dev/null; then
|
||||
pending=0
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
watch &
|
||||
exec nginx -g 'daemon off;'
|
||||
@@ -0,0 +1,47 @@
|
||||
worker_processes auto;
|
||||
|
||||
events {
|
||||
worker_connections 256;
|
||||
}
|
||||
|
||||
http {
|
||||
resolver 127.0.0.11 valid=5s ipv6=off;
|
||||
|
||||
upstream api {
|
||||
zone api 64k;
|
||||
include /etc/nginx/upstream/servers.conf;
|
||||
keepalive 32;
|
||||
keepalive_timeout 4s;
|
||||
}
|
||||
|
||||
log_format main '$remote_addr - $upstream_addr [$time_local] '
|
||||
'"$request" $status $body_bytes_sent '
|
||||
'"$upstream_response_time" "$request_time"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 10s;
|
||||
proxy_send_timeout 5s;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 5s;
|
||||
|
||||
location ~ ^/api/v1/chats/[^/]+/messages$ {
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout http_502 http_503 non_idempotent;
|
||||
}
|
||||
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
proxy_next_upstream error timeout http_502 http_503;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
services:
|
||||
api-1:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPUS:-0.50}"
|
||||
memory: ${API_MEM:-256M}
|
||||
|
||||
api-2:
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPUS:-0.50}"
|
||||
memory: ${API_MEM:-256M}
|
||||
@@ -0,0 +1,48 @@
|
||||
x-api-extra: &api-extra
|
||||
image: mtusi-messenger:lab3
|
||||
env_file:
|
||||
- .env
|
||||
secrets:
|
||||
- database_url
|
||||
- cache_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "${API_CPUS:-0.50}"
|
||||
memory: ${API_MEM:-256M}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 3
|
||||
start_period: 10s
|
||||
restart: unless-stopped
|
||||
|
||||
services:
|
||||
nginx:
|
||||
depends_on:
|
||||
api-3:
|
||||
condition: service_healthy
|
||||
api-4:
|
||||
condition: service_healthy
|
||||
|
||||
api-3:
|
||||
<<: *api-extra
|
||||
environment:
|
||||
INSTANCE_ID: api-3
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
|
||||
api-4:
|
||||
<<: *api-extra
|
||||
environment:
|
||||
INSTANCE_ID: api-4
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
@@ -0,0 +1,46 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE
|
||||
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name bytea NOT NULL
|
||||
CONSTRAINT users_display_name_size CHECK (octet_length(display_name) BETWEEN 1 AND 400),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title bytea NOT NULL
|
||||
CONSTRAINT chats_title_size CHECK (octet_length(title) BETWEEN 1 AND 400),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX chats_created_idx ON chats (created_at, id);
|
||||
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
|
||||
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
|
||||
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
sender_id uuid NOT NULL REFERENCES users (id) ON DELETE RESTRICT,
|
||||
text bytea NOT NULL
|
||||
CONSTRAINT messages_text_size CHECK (octet_length(text) BETWEEN 1 AND 16000),
|
||||
created_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
|
||||
|
||||
CREATE TABLE sessions (
|
||||
token_hash bytea PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
created_at timestamptz NOT NULL,
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
|
||||
@@ -0,0 +1,12 @@
|
||||
CREATE TABLE idempotency_keys (
|
||||
user_id uuid NOT NULL REFERENCES users (id) ON DELETE CASCADE,
|
||||
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
|
||||
key text NOT NULL
|
||||
CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:\-]{1,128}$'),
|
||||
request_body_hash text NOT NULL,
|
||||
response_json text NOT NULL,
|
||||
created_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
|
||||
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
|
||||
@@ -0,0 +1,22 @@
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
cffi==2.1.1
|
||||
click==8.5.0
|
||||
fastapi==0.115.6
|
||||
h11==0.16.0
|
||||
httptools==0.9.0
|
||||
idna==3.20
|
||||
psycopg==3.3.6
|
||||
psycopg-binary==3.3.6
|
||||
psycopg-pool==3.3.3
|
||||
pycparser==3.11
|
||||
pydantic==2.14.1
|
||||
pydantic-core==2.50.1
|
||||
starlette==0.41.3
|
||||
typing-extensions==4.16.0
|
||||
typing-inspection==0.4.4
|
||||
uvicorn==0.34.0
|
||||
uvloop==0.23.0
|
||||
valkey==6.2.0
|
||||
@@ -0,0 +1,60 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Versioned SQL migrations: applied once, checksum-verified, safe to re-run."""
|
||||
import hashlib
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
import psycopg
|
||||
|
||||
ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
MIGRATIONS = ROOT / "migrations"
|
||||
LOCK_ID = 2026_0003
|
||||
|
||||
|
||||
def database_url() -> str:
|
||||
path = os.environ.get("DATABASE_URL_FILE")
|
||||
if path:
|
||||
return pathlib.Path(path).read_text(encoding="utf-8").strip()
|
||||
url = os.environ.get("DATABASE_URL", "").strip()
|
||||
if not url:
|
||||
sys.exit("DATABASE_URL or DATABASE_URL_FILE is required")
|
||||
return url
|
||||
|
||||
|
||||
def main() -> None:
|
||||
files = sorted(MIGRATIONS.glob("*.sql"))
|
||||
if not files:
|
||||
sys.exit("no migration files found in %s" % MIGRATIONS)
|
||||
with psycopg.connect(database_url(), autocommit=True, connect_timeout=10) as conn:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS schema_migrations ("
|
||||
" version text PRIMARY KEY,"
|
||||
" checksum text NOT NULL,"
|
||||
" applied_at timestamptz NOT NULL DEFAULT now())"
|
||||
)
|
||||
conn.execute("SELECT pg_advisory_lock(%s)", (LOCK_ID,))
|
||||
rows = conn.execute("SELECT version, checksum FROM schema_migrations").fetchall()
|
||||
applied = {row[0]: row[1] for row in rows}
|
||||
for path in files:
|
||||
version = path.stem
|
||||
body = path.read_text(encoding="utf-8")
|
||||
checksum = hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
if version in applied:
|
||||
if applied[version] != checksum:
|
||||
sys.exit("migration %s changed after it was applied" % version)
|
||||
print("skip %s" % version)
|
||||
continue
|
||||
with conn.transaction():
|
||||
conn.execute(body)
|
||||
conn.execute(
|
||||
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
|
||||
(version, checksum),
|
||||
)
|
||||
print("apply %s" % version)
|
||||
conn.execute("SELECT pg_advisory_unlock(%s)", (LOCK_ID,))
|
||||
print("migrations complete: %d file(s)" % len(files))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose restart -t 5 postgres valkey api-1 api-2 nginx
|
||||
Executable
+5
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
cd "$(dirname "$0")/.."
|
||||
docker compose stop -t 3 "$TARGET_INSTANCE_ID"
|
||||
Executable
+28
@@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
OUT="${OUT:-evidence}"
|
||||
DURATION="${DURATION:-60}"
|
||||
|
||||
wait_healthy() {
|
||||
local deadline=$((SECONDS + 60))
|
||||
until [ "$(docker inspect -f '{{.State.Health.Status}}' "$(docker compose ps -q "$1")")" = healthy ]; do
|
||||
[ $SECONDS -lt $deadline ] || { echo "ERROR: $1 not healthy" >&2; return 1; }
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
python3 tests/load.py --duration "$DURATION" --concurrency 4 --output "$OUT/rolling_load.json" > /dev/null &
|
||||
load_pid=$!
|
||||
sleep 10
|
||||
for api in api-1 api-2; do
|
||||
old=$(docker compose ps -q "$api")
|
||||
echo "$(date -u +%H:%M:%S) recreate $api (old container ${old:0:12})"
|
||||
docker compose up -d --no-deps --force-recreate "$api" > /dev/null 2>&1
|
||||
wait_healthy "$api"
|
||||
echo "$(date -u +%H:%M:%S) $api healthy (new container $(docker compose ps -q "$api" | cut -c1-12))"
|
||||
sleep 10
|
||||
done
|
||||
wait "$load_pid"
|
||||
cat "$OUT/rolling_load.json"
|
||||
Executable
+70
@@ -0,0 +1,70 @@
|
||||
#!/bin/bash
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8080}"
|
||||
DURATION="${DURATION:-30}"
|
||||
CONCURRENCY="${CONCURRENCY:-16}"
|
||||
RUNS="${RUNS:-3}"
|
||||
OUT="${OUT:-evidence/scaling}"
|
||||
CONFIGS="${CONFIGS:-A1:1:1.00:512M A2:2:0.50:256M A4:4:0.25:128M B1:1:0.50:256M B4:4:0.50:256M}"
|
||||
|
||||
mkdir -p "$OUT"
|
||||
|
||||
wait_instances() {
|
||||
local want=$1 deadline=$((SECONDS + 90)) seen
|
||||
while [ $SECONDS -lt $deadline ]; do
|
||||
seen=$(for _ in $(seq 1 $((want * 4))); do
|
||||
curl -s -D - -o /dev/null "$BASE_URL/health/ready" | tr -d '\r' | awk -F': ' 'tolower($1)=="x-instance-id"{print $2}'
|
||||
done | sort -u | grep -c . || true)
|
||||
[ "$seen" -eq "$want" ] && return 0
|
||||
sleep 2
|
||||
done
|
||||
echo "ERROR: expected $want instances, saw $seen" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
for cfg in $CONFIGS; do
|
||||
IFS=: read -r label replicas cpus mem <<< "$cfg"
|
||||
apis=$(seq -f "api-%g" 1 "$replicas" | tr '\n' ' ')
|
||||
export API_CPUS=$cpus API_MEM=$mem API_UPSTREAMS="${apis% }"
|
||||
files=(-f compose.yaml -f infra/scaling/limits.yaml)
|
||||
[ "$replicas" -eq 4 ] && files+=(-f infra/scaling/replicas-4.yaml)
|
||||
|
||||
echo "=== $label: $replicas x ${cpus} CPU / $mem ==="
|
||||
docker compose "${files[@]}" up -d --force-recreate $apis >/dev/null 2>&1
|
||||
for extra in api-2 api-3 api-4; do
|
||||
n=${extra#api-}
|
||||
[ "$n" -gt "$replicas" ] && docker compose -f compose.yaml -f infra/scaling/limits.yaml -f infra/scaling/replicas-4.yaml stop "$extra" >/dev/null 2>&1 || true
|
||||
done
|
||||
docker compose "${files[@]}" up -d --no-deps --force-recreate nginx >/dev/null 2>&1
|
||||
wait_instances "$replicas"
|
||||
|
||||
{
|
||||
echo "config=$label replicas=$replicas api_cpus=$cpus api_mem=$mem concurrency=$CONCURRENCY duration=$DURATION"
|
||||
for c in $apis; do
|
||||
docker inspect --format "$c NanoCpus={{.HostConfig.NanoCpus}} Memory={{.HostConfig.Memory}}" \
|
||||
"$(docker compose "${files[@]}" ps -q "$c")"
|
||||
done
|
||||
} > "$OUT/${label}_limits.txt"
|
||||
|
||||
python3 tests/load.py --base-url "$BASE_URL" --duration 5 --concurrency "$CONCURRENCY" >/dev/null
|
||||
since=$(date -u +%Y-%m-%dT%H:%M:%SZ)
|
||||
|
||||
for run in $(seq 1 "$RUNS"); do
|
||||
(sleep $((DURATION / 2)); docker stats --no-stream --format '{{.Name}} {{.CPUPerc}} {{.MemUsage}}' \
|
||||
> "$OUT/${label}_run${run}_stats.txt") &
|
||||
python3 tests/load.py --base-url "$BASE_URL" --duration "$DURATION" --concurrency "$CONCURRENCY" \
|
||||
--output "$OUT/${label}_run${run}.json" >/dev/null
|
||||
wait
|
||||
python3 -c "import json;d=json.load(open('$OUT/${label}_run${run}.json'));l=d['successful_latency_ms'];print(' run $run: rps=%s err=%s p50=%s p95=%s p99=%s'%(d['successful_rps'],d['failed_requests'],l['p50'],l['p95'],l['p99']))"
|
||||
done
|
||||
docker compose "${files[@]}" logs --since "$since" nginx 2>&1 \
|
||||
| grep -E '" [45][0-9][0-9] |\[(error|warn)\]' > "$OUT/${label}_nginx_problems.txt" || true
|
||||
done
|
||||
|
||||
unset API_CPUS API_MEM API_UPSTREAMS
|
||||
docker compose -f compose.yaml -f infra/scaling/replicas-4.yaml rm -sf api-3 api-4 >/dev/null 2>&1 || true
|
||||
docker compose up -d --force-recreate api-1 api-2 nginx >/dev/null 2>&1
|
||||
wait_instances 2
|
||||
python3 student-tests/scaling_summary.py "$OUT"
|
||||
Executable
+98
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import statistics
|
||||
import sys
|
||||
|
||||
out = sys.argv[1] if len(sys.argv) > 1 else "evidence/scaling"
|
||||
|
||||
rows = {}
|
||||
for path in sorted(glob.glob(os.path.join(out, "*_run*.json"))):
|
||||
label = re.match(r"(.+)_run\d+\.json", os.path.basename(path)).group(1)
|
||||
d = json.load(open(path))
|
||||
rows.setdefault(label, []).append(d)
|
||||
|
||||
meta = {}
|
||||
for label in rows:
|
||||
with open(os.path.join(out, label + "_limits.txt")) as f:
|
||||
meta[label] = dict(kv.split("=") for kv in f.readline().split())
|
||||
|
||||
|
||||
def med(runs, key):
|
||||
return statistics.median(key(r) for r in runs)
|
||||
|
||||
|
||||
table = []
|
||||
for label in sorted(rows):
|
||||
runs = rows[label]
|
||||
m = meta[label]
|
||||
table.append({
|
||||
"config": label,
|
||||
"replicas": int(m["replicas"]),
|
||||
"api_cpus": float(m["api_cpus"]),
|
||||
"api_mem": m["api_mem"],
|
||||
"runs": len(runs),
|
||||
"rps_median": round(med(runs, lambda r: r["successful_rps"]), 1),
|
||||
"rps_min": round(min(r["successful_rps"] for r in runs), 1),
|
||||
"rps_max": round(max(r["successful_rps"] for r in runs), 1),
|
||||
"p50_ms": round(med(runs, lambda r: r["successful_latency_ms"]["p50"]), 1),
|
||||
"p95_ms": round(med(runs, lambda r: r["successful_latency_ms"]["p95"]), 1),
|
||||
"p99_ms": round(med(runs, lambda r: r["successful_latency_ms"]["p99"]), 1),
|
||||
"errors": sum(r["failed_requests"] for r in runs),
|
||||
})
|
||||
|
||||
cols = list(table[0])
|
||||
with open(os.path.join(out, "summary.csv"), "w") as f:
|
||||
f.write(",".join(cols) + "\n")
|
||||
for t in table:
|
||||
f.write(",".join(str(t[c]) for c in cols) + "\n")
|
||||
|
||||
for t in table:
|
||||
print("%(config)s %(replicas)dx%(api_cpus).2f CPU rps=%(rps_median)s [%(rps_min)s..%(rps_max)s] "
|
||||
"p50=%(p50_ms)s p95=%(p95_ms)s p99=%(p99_ms)s errors=%(errors)s" % t)
|
||||
|
||||
|
||||
def series(prefix):
|
||||
return sorted((t["replicas"], t["rps_median"], t["p95_ms"]) for t in table if t["config"].startswith(prefix))
|
||||
|
||||
|
||||
W, H, L, B, T, R = 640, 360, 60, 50, 40, 150
|
||||
xs = {1: 0, 2: 1, 4: 2}
|
||||
ymax = max(t["rps_max"] for t in table) * 1.15
|
||||
|
||||
|
||||
def px(n):
|
||||
return L + xs[n] * (W - L - R) / 2
|
||||
|
||||
|
||||
def py(v):
|
||||
return H - B - v * (H - B - T) / ymax
|
||||
|
||||
|
||||
svg = ['<svg xmlns="http://www.w3.org/2000/svg" width="%d" height="%d" font-family="sans-serif" font-size="12">' % (W, H),
|
||||
'<rect width="100%" height="100%" fill="white"/>',
|
||||
'<text x="%d" y="22" font-size="14">RPS (медиана 3 прогонов) от числа API-реплик</text>' % L]
|
||||
for i in range(6):
|
||||
v = ymax * i / 5
|
||||
svg.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="#ddd"/>' % (L, W - R, py(v), py(v)))
|
||||
svg.append('<text x="%d" y="%.1f" text-anchor="end">%d</text>' % (L - 6, py(v) + 4, v))
|
||||
for n in xs:
|
||||
svg.append('<text x="%.1f" y="%d" text-anchor="middle">%d</text>' % (px(n), H - B + 18, n))
|
||||
svg.append('<text x="%.1f" y="%d" text-anchor="middle">реплики</text>' % ((L + W - R) / 2, H - 12))
|
||||
for (prefix, color, name), dy in zip((("A", "#2a6fdb", "A: всего 1.0 CPU"), ("B", "#d9480f", "B: 0.5 CPU на реплику")), (0, 20)):
|
||||
pts = series(prefix)
|
||||
if prefix == "B":
|
||||
pts = sorted(set(pts) | {(t["replicas"], t["rps_median"], t["p95_ms"]) for t in table if t["config"] == "A2"})
|
||||
svg.append('<polyline fill="none" stroke="%s" stroke-width="2" points="%s"/>' %
|
||||
(color, " ".join("%.1f,%.1f" % (px(n), py(v)) for n, v, _ in pts)))
|
||||
for n, v, _ in pts:
|
||||
svg.append('<circle cx="%.1f" cy="%.1f" r="4" fill="%s"/>' % (px(n), py(v), color))
|
||||
svg.append('<text x="%.1f" y="%.1f" fill="%s" text-anchor="middle">%d</text>' % (px(n), py(v) - 8 - dy / 2, color, v))
|
||||
svg.append('<rect x="%d" y="%d" width="12" height="12" fill="%s"/>' % (W - R + 12, T + dy, color))
|
||||
svg.append('<text x="%d" y="%d">%s</text>' % (W - R + 28, T + dy + 10, name))
|
||||
svg.append("</svg>")
|
||||
with open(os.path.join(out, "rps.svg"), "w") as f:
|
||||
f.write("\n".join(svg) + "\n")
|
||||
print("wrote", os.path.join(out, "summary.csv"), os.path.join(out, "rps.svg"))
|
||||
@@ -0,0 +1,99 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test shared idempotency: N parallel POST requests with the same Idempotency-Key
|
||||
must create exactly one message and return identical responses."""
|
||||
import argparse
|
||||
import base64
|
||||
import concurrent.futures
|
||||
import json
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default="http://localhost:8080")
|
||||
p.add_argument("--concurrency", type=int, default=8)
|
||||
args = p.parse_args()
|
||||
base = args.base_url.rstrip("/")
|
||||
|
||||
def api(method, path, body=None, headers=None):
|
||||
h = {"Accept": "application/json"}
|
||||
h.update(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(base + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
except urllib.error.HTTPError as e:
|
||||
resp = e
|
||||
return resp.code, json.loads(resp.read())
|
||||
|
||||
pw = "TestPassword-" + uuid.uuid4().hex[:8]
|
||||
u1 = "u_idem_" + uuid.uuid4().hex[:8]
|
||||
u2 = "u_idem_" + uuid.uuid4().hex[:8]
|
||||
|
||||
_, alice = api("POST", "/api/v1/users", {"username": u1, "display_name": "Alice", "password": pw})
|
||||
_, bob = api("POST", "/api/v1/users", {"username": u2, "display_name": "Bob", "password": pw})
|
||||
|
||||
cred = base64.b64encode((u1 + ":" + pw).encode()).decode()
|
||||
_, sess = api("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
token = sess["token"]
|
||||
|
||||
_, chat = api("POST", "/api/v1/chats", {"title": "Idempotency test", "member_ids": [alice["id"], bob["id"]]},
|
||||
headers={"Authorization": "Bearer " + token})
|
||||
|
||||
idem_key = uuid.uuid4().hex
|
||||
text = "Parallel idempotency test " + idem_key
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
|
||||
def send(_):
|
||||
h = {"Authorization": "Bearer " + token, "Idempotency-Key": idem_key,
|
||||
"Content-Type": "application/json", "Accept": "application/json"}
|
||||
data = json.dumps({"text": text}).encode()
|
||||
req = urllib.request.Request(base + path, data=data, headers=h, method="POST")
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
except urllib.error.HTTPError as e:
|
||||
resp = e
|
||||
body = json.loads(resp.read())
|
||||
return resp.code, body, resp.headers.get("X-Instance-Id", "")
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=args.concurrency) as pool:
|
||||
results = list(pool.map(send, range(args.concurrency)))
|
||||
|
||||
codes = [r[0] for r in results]
|
||||
bodies = [r[1] for r in results]
|
||||
instances = set(r[2] for r in results)
|
||||
|
||||
assert all(c == 201 for c in codes), "not all responses are 201: %s" % codes
|
||||
|
||||
message_ids = set(b["id"] for b in bodies)
|
||||
assert len(message_ids) == 1, "expected exactly 1 message, got %d different IDs" % len(message_ids)
|
||||
|
||||
first = bodies[0]
|
||||
for b in bodies[1:]:
|
||||
assert b == first, "response mismatch: %s vs %s" % (first, b)
|
||||
|
||||
print("PASS: %d parallel requests, 1 message created, %d distinct replicas observed: %s"
|
||||
% (args.concurrency, len(instances), sorted(instances)))
|
||||
|
||||
# Verify different body with same key gives 409
|
||||
h = {"Authorization": "Bearer " + token, "Idempotency-Key": idem_key,
|
||||
"Content-Type": "application/json", "Accept": "application/json"}
|
||||
data = json.dumps({"text": "different body"}).encode()
|
||||
req = urllib.request.Request(base + path, data=data, headers=h, method="POST")
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
code = resp.code
|
||||
except urllib.error.HTTPError as e:
|
||||
code = e.code
|
||||
assert code == 409, "expected 409 for different body with same key, got %d" % code
|
||||
print("PASS: same key + different body → 409")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
BASE = "http://localhost:8080"
|
||||
failures = []
|
||||
|
||||
|
||||
def check(cond, message):
|
||||
print(("PASS: " if cond else "FAIL: ") + message)
|
||||
if not cond:
|
||||
failures.append(message)
|
||||
|
||||
|
||||
def compose(*args, check_rc=True):
|
||||
return subprocess.run(["docker", "compose", *args], cwd=ROOT, check=check_rc,
|
||||
capture_output=True, text=True).stdout
|
||||
|
||||
|
||||
def psql(sql):
|
||||
return compose("exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger",
|
||||
"-tAc", sql).strip()
|
||||
|
||||
|
||||
def api(method, path, body=None, headers=None):
|
||||
h = {"Accept": "application/json"}
|
||||
h.update(headers or {})
|
||||
data = None
|
||||
if body is not None:
|
||||
data = json.dumps(body).encode()
|
||||
h["Content-Type"] = "application/json"
|
||||
req = urllib.request.Request(BASE + path, data=data, headers=h, method=method)
|
||||
try:
|
||||
resp = urllib.request.urlopen(req, timeout=10)
|
||||
except urllib.error.HTTPError as e:
|
||||
resp = e
|
||||
raw = resp.read()
|
||||
return resp.code, (json.loads(raw) if raw else None), resp.headers.get("X-Instance-Id", "")
|
||||
|
||||
|
||||
def setup():
|
||||
pw = "Review-" + uuid.uuid4().hex[:12]
|
||||
users = []
|
||||
for name in ("alice", "bob"):
|
||||
username = "rv_%s_%s" % (name, uuid.uuid4().hex[:8])
|
||||
_, user, _ = api("POST", "/api/v1/users", {"username": username, "display_name": name, "password": pw})
|
||||
users.append((username, user))
|
||||
cred = base64.b64encode(("%s:%s" % (users[0][0], pw)).encode()).decode()
|
||||
_, sess, _ = api("POST", "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred})
|
||||
auth = {"Authorization": "Bearer " + sess["token"]}
|
||||
return users[0][1], users[1][1], auth
|
||||
|
||||
|
||||
def new_chat(alice, bob, auth, title):
|
||||
return api("POST", "/api/v1/chats", {"title": title, "member_ids": [alice["id"], bob["id"]]}, auth)
|
||||
|
||||
|
||||
def scenario_replay_acl(alice, bob, auth):
|
||||
print("== 1. Idempotent replay after membership removal")
|
||||
_, chat, _ = new_chat(alice, bob, auth, "ACL replay")
|
||||
key = uuid.uuid4().hex
|
||||
path = "/api/v1/chats/%s/messages" % chat["id"]
|
||||
code, _, _ = api("POST", path, {"text": "before removal"}, dict(auth, **{"Idempotency-Key": key}))
|
||||
check(code == 201, "first POST with key -> 201 (got %s)" % code)
|
||||
psql("DELETE FROM chat_members WHERE chat_id = '%s' AND user_id = '%s'" % (chat["id"], alice["id"]))
|
||||
code, _, _ = api("GET", path, headers=auth)
|
||||
check(code == 404, "history after removal -> 404 (got %s)" % code)
|
||||
code, _, _ = api("POST", path, {"text": "before removal"}, dict(auth, **{"Idempotency-Key": key}))
|
||||
check(code == 404, "replay with same key and body after removal -> 404 (got %s)" % code)
|
||||
|
||||
|
||||
def api1_ip():
|
||||
cid = compose("ps", "-q", "api-1").strip()
|
||||
return subprocess.run(["docker", "inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", cid],
|
||||
check=True, capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def api1_health():
|
||||
cid = compose("ps", "-q", "api-1").strip()
|
||||
return subprocess.run(["docker", "inspect", "-f", "{{.State.Health.Status}}", cid],
|
||||
capture_output=True, text=True).stdout.strip()
|
||||
|
||||
|
||||
def break_api1_db():
|
||||
compose("exec", "-T", "-u", "root", "api-1", "sh", "-c",
|
||||
"cp /etc/hosts /tmp/hosts.orig && echo '10.255.255.1 postgres' >> /etc/hosts")
|
||||
psql("SELECT count(pg_terminate_backend(pid)) FROM pg_stat_activity WHERE client_addr = '%s'" % api1_ip())
|
||||
|
||||
|
||||
def restore_api1_db():
|
||||
compose("exec", "-T", "-u", "root", "api-1", "sh", "-c",
|
||||
"[ -f /tmp/hosts.orig ] && cat /tmp/hosts.orig > /etc/hosts", check_rc=False)
|
||||
|
||||
|
||||
def wait_for(pred, timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if pred():
|
||||
return True
|
||||
time.sleep(1)
|
||||
return False
|
||||
|
||||
|
||||
def nginx_upstreams():
|
||||
return compose("exec", "-T", "nginx", "cat", "/etc/nginx/upstream/servers.conf", check_rc=False)
|
||||
|
||||
|
||||
def scenario_unready_replica(alice, bob, auth):
|
||||
print("== 2. Live but not ready api-1 (no route to PostgreSQL)")
|
||||
for _ in range(4):
|
||||
api("GET", "/api/v1/users/me", headers=auth)
|
||||
since = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
break_api1_db()
|
||||
try:
|
||||
before = int(psql("SELECT count(*) FROM chat_members WHERE user_id = '%s'" % alice["id"]))
|
||||
codes = []
|
||||
for i in range(10):
|
||||
codes.append(new_chat(alice, bob, auth, "no-retry %d" % i)[0])
|
||||
time.sleep(0.5)
|
||||
|
||||
marked = wait_for(lambda: "api-1:8080 resolve max_fails=1 fail_timeout=10s down;" in nginx_upstreams(), 20)
|
||||
check(marked, "nginx active check marks api-1 down in upstream")
|
||||
became = wait_for(lambda: api1_health() == "unhealthy", 40)
|
||||
check(became, "Docker marks api-1 unhealthy")
|
||||
|
||||
served = []
|
||||
for _ in range(40):
|
||||
code, _, inst = api("GET", "/api/v1/users/me", headers=auth)
|
||||
served.append((code, inst))
|
||||
time.sleep(0.5)
|
||||
by_api1 = sum(1 for _, inst in served if inst == "api-1")
|
||||
ok = sum(1 for code, _ in served if code == 200)
|
||||
check(by_api1 == 0, "40 GET /users/me with warm session: served by not-ready api-1 = %d" % by_api1)
|
||||
check(ok == 40, "40 GET /users/me: 200 responses = %d" % ok)
|
||||
|
||||
code, chat, _ = new_chat(alice, bob, auth, "messages during outage")
|
||||
if code == 201:
|
||||
msg_codes = []
|
||||
for i in range(20):
|
||||
msg_codes.append(api("POST", "/api/v1/chats/%s/messages" % chat["id"], {"text": "m%d" % i},
|
||||
dict(auth, **{"Idempotency-Key": uuid.uuid4().hex}))[0])
|
||||
check(msg_codes.count(201) == 20, "20 POST messages during outage -> all 201 (got %s)" % sorted(set(msg_codes)))
|
||||
after = int(psql("SELECT count(*) FROM chat_members WHERE user_id = '%s'" % alice["id"]))
|
||||
|
||||
print("== 3. LB does not transparently retry non-idempotent POST /chats")
|
||||
created = codes.count(201) + (1 if code == 201 else 0)
|
||||
check(after - before == created, "chats in DB (%d) == 201 responses (%d), POST /chats codes=%s"
|
||||
% (after - before, created, codes))
|
||||
log = compose("logs", "--no-log-prefix", "--since", since, "nginx")
|
||||
chat_posts = [l for l in log.splitlines() if '"POST /api/v1/chats HTTP' in l]
|
||||
retried = [l for l in chat_posts if re.search(r":8080, [0-9]", l)]
|
||||
check(bool(chat_posts) and not retried,
|
||||
"nginx access log: %d POST /chats, sent to two upstreams: %d" % (len(chat_posts), len(retried)))
|
||||
conf = compose("exec", "-T", "nginx", "nginx", "-T")
|
||||
blocks = re.findall(r"location ([^{]+)\{([^}]*)\}", conf)
|
||||
policy = {loc.strip(): "non_idempotent" in body for loc, body in blocks}
|
||||
check(policy.get("/") is False and policy.get("~ ^/api/v1/chats/[^/]+/messages$") is True,
|
||||
"non_idempotent only on messages route: %s" % policy)
|
||||
finally:
|
||||
restore_api1_db()
|
||||
back = wait_for(lambda: api1_health() == "healthy", 40)
|
||||
check(back, "api-1 healthy again after PostgreSQL route restored")
|
||||
seen = wait_for(lambda: any(api("GET", "/health/live")[2] == "api-1" for _ in range(6)), 20)
|
||||
check(seen, "api-1 returns to LB rotation")
|
||||
check("down" not in nginx_upstreams(), "nginx upstream has no servers marked down")
|
||||
|
||||
|
||||
def main():
|
||||
global BASE
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--base-url", default=BASE)
|
||||
BASE = p.parse_args().base_url.rstrip("/")
|
||||
alice, bob, auth = setup()
|
||||
scenario_replay_acl(alice, bob, auth)
|
||||
scenario_unready_replica(alice, bob, auth)
|
||||
print("RESULT: %s" % ("PASS" if not failures else "FAIL (%d)" % len(failures)))
|
||||
sys.exit(1 if failures else 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user