diff --git a/.dockerignore b/.dockerignore index bc16e556..40bc41c9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,9 @@ __pycache__ node_modules evidence .DS_Store +research +student-tests +scripts +docs +requirements-research.txt +*.md diff --git a/.env.example b/.env.example index d32b9be0..86cc1dbb 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,24 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Секреты (пароль БД, DATABASE_URL) создаёт scripts/init-secrets.sh в каталоге secrets/ — не в .env. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +# Срок жизни сессии; для проверки expiry запускайте с SESSION_TTL_SECONDS=2 (см. student-tests/test_expiry.sh) SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: +# TTL записи в кэше Valkey (не больше остатка жизни сессии) +SESSION_CACHE_TTL_SECONDS=60 +# Источник сессий: PostgreSQL (истина) + Valkey (кэш). Адрес кэша задан в compose.yaml (VALKEY_URL). SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +DB_POOL_MAX=10 +# Файлы секретов внутри контейнера (создаются scripts/init-secrets.sh) +DATABASE_URL_FILE=/run/secrets/database_url +# --- lab3 --- +# Число API-реплик (compose deploy.replicas); также можно: docker compose up -d --scale api=4 API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 +API_CPUS=1.0 +API_MEM=256m +POSTGRES_CPUS=1.0 +LB_CPUS=1.0 +# Пауза между SIGTERM и закрытием listener: LB успевает исключить реплику (rolling update без ошибок) +DRAIN_SECONDS=4 +SEND_STRATEGY=function diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..ee31f7ab --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +# syntax=docker/dockerfile:1 +# Базовый образ закреплён тегом и digest (python 3.12.12, Debian bookworm). +ARG PYTHON_IMAGE=python:3.12.12-slim-bookworm@sha256:593bd06efe90efa80dc4eee3948be7c0fde4134606dd40d8dd8dbcade98e669c + +# --- сборка зависимостей в отдельном слое (в runtime не попадают pip и кэш) --- +FROM ${PYTHON_IMAGE} AS builder +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_NO_CACHE_DIR=1 +WORKDIR /build +COPY requirements.txt . +RUN python -m venv /opt/venv && /opt/venv/bin/pip install -r requirements.txt + +# --- runtime --- +FROM ${PYTHON_IMAGE} +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 PYTHONPATH=/srv/src \ + APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab LIMIT_CONCURRENCY=1000 MIGRATIONS_DIR=/srv/migrations +RUN groupadd --system --gid 10001 app && useradd --system --uid 10001 --gid app --no-create-home app +COPY --from=builder /opt/venv /opt/venv +WORKDIR /srv +COPY src/ /srv/src/ +COPY migrations/ /srv/migrations/ +# ревизия образа (меняется при rolling update; видна в журнале запуска) +ARG REV=dev +LABEL org.opencontainers.image.revision=$REV +ENV APP_REVISION=$REV +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=3 \ + CMD ["python", "-c", "import urllib.request,os,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT','8080'), timeout=2).status == 200 else 1)"] +# exec-форма: сервер = PID 1. Состояние хранится в PostgreSQL/Valkey, процесс stateless; +# масштабирование — репликами контейнера. SIGTERM: drain (readiness 503) -> graceful shutdown (см. app/serve.py). +CMD ["python", "-m", "app.serve"] diff --git a/Makefile b/Makefile index 5d86d171..a0bffdd9 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,21 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --- собственные цели студента --- +.PHONY: secrets up down student-test test-topology +secrets: + sh scripts/init-secrets.sh + +up: secrets + docker compose --env-file .env up -d --build + +down: + docker compose --env-file .env down + +student-test: + $(PYTHON) -m unittest discover -s student-tests -p "test_api.py" -v + $(PYTHON) student-tests/test_idempotency.py + +test-topology: + $(PYTHON) student-tests/test_topology.py diff --git a/REPORT.md b/REPORT.md index c2679005..f5fb4cff 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,142 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Кушнаренко Николай Вячеславович, БПИ2302 +- Целевая оценка; выбранный трек 5, если нужен: **5**, трек 1 «Цена масштабирования» +- Commit/PR: ветка `solution/bpi2302-kushnarenko`; исходный коммит реализации `7e6f66c`; [PR №132](https://gitea.privcloud.tech/mtusi2026/mtusi2026lab3/pulls/132) — из `kolya/mtusi2026lab3` в `mtusi2026/mtusi2026lab3`, ветка `main`. +- Использованные источники, библиотеки и помощь ИИ: Python 3.12, uvicorn, asyncpg, redis-py, argon2-cffi; HAProxy 3.0; PostgreSQL 17; Valkey 8; aiohttp/matplotlib (только для экспериментов). Код lab2 перенесён с объявленными изменениями (`Idempotency-Key`, `X-Instance-Id`). ИИ-ассистенты использовались для помощи с кодом, тестами и отчётом; Codex проверил подготовку к сдаче, исправил фиксацию суммарной RAM в исследовании и повторил проверки. + +Повторная проверка 11.11.2026: smoke, persistence, failover и 31 собственный тест прошли. Команды и полный вывод — `evidence/checks_2026-11-11.txt`. Исторические замеры ниже датированы октябрём; повтор серии A с фиксированными CPU и RAM приведён отдельно. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker Compose v2, bash (Git Bash на Windows), Python ≥ 3.9. На Windows перед `tests/*.py` установите `PYTHONUTF8=1`. +Версии: `python:3.12.12-slim-bookworm@sha256:593bd06e…`, `postgres:17.11-alpine@sha256:b0f9560a…`, `valkey/valkey:8.1.10-alpine@sha256:081c2f5c…`, `haproxy:3.0-alpine@sha256:56b887da…` (3.0.29), зависимости — `requirements.txt` (закреплены). + +```sh +sh scripts/init-secrets.sh # secrets/postgres_password, secrets/database_url (в Git не попадают) +cp .env.example .env # API_REPLICAS=2 и лимиты ресурсов +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build # postgres, valkey -> migrate (1 раз) -> api x N -> lb +curl http://127.0.0.1:8080/health/ready # единственная публичная точка входа (LB), только 127.0.0.1 +python3 tests/smoke.py --base-url http://127.0.0.1:8080 # make test +python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +sh scripts/restore-replicas.sh # вернуть остановленную реплику (удаляет exited-контейнеры api и поднимает N) +``` + +Windows: `scripts/restart.cmd`, `scripts/stop-one.cmd` — аналоги `.sh` (CreateProcess не запускает `.sh`). Масштабирование: `docker compose --env-file .env up -d --no-deps --scale api=4 api`. Rolling update: `sh scripts/rolling-update.sh [revision]`. Остановка с сохранением данных: `docker compose --env-file .env stop`. **Намеренное удаление учебных данных**: `docker compose --env-file .env down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +Клиент ─▶ HAProxy (127.0.0.1:8080) ─▶ API ×N (stateless, без публикации портов) ─▶ PostgreSQL (истина, идемпотентность, сессии) + └▶ Valkey (общий кэш сессий) +``` + +| Роль | Сервис | Версия / digest | Health | Volume | Сеть/порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| LB | `lb` | HAProxy 3.0.29 | `monitor-uri /healthz` | — | `127.0.0.1:8080` (единственный публичный) | — | +| API | `api` (≥2 реплики, без `container_name`) | собирается из Dockerfile | `/health/ready` | — | только сеть `backend` | secret `database_url` | +| БД | `postgres` | 17.11-alpine | `pg_isready` | `pgdata` | `backend` | secret `postgres_password` | +| Сессии/кэш | `valkey` | 8.1.10-alpine, AOF | `valkey-cli ping` | `valkeydata` | `backend` | не опубликован | +| Миграции | `migrate` (one-shot) | образ api | код выхода 0 | — | `backend` | secret `database_url` | + +- **Общее состояние**: пользователи, чаты, сообщения, сессии и ключи идемпотентности — в PostgreSQL; кэш сессий — в общем Valkey. В процессе API локального состояния нет, sticky-сессий в LB нет. Любой запрос любой реплики обслуживается одинаково; видно по `X-Instance-Id` (= имя хоста контейнера, 12 hex = короткий ID контейнера, поэтому его легко сопоставить с `docker ps` в `stop-one.sh`). +- **Миграции** выполняются отдельным one-shot сервисом `migrate` до старта реплик (`depends_on: service_completed_successfully`), а не гонкой реплик; при ручном запуске защищены advisory lock. Новые миграции делаются обратно совместимыми (expand/contract): при rolling update старая версия работает со схемой новой. +- **Идемпотентность `POST /chats/{id}/messages`** (`Idempotency-Key` обязателен, `[A-Za-z0-9._:-]{1,128}`): таблица `idempotency_keys (user_id, chat_id, idem_key)` с PK. Запись ключа и сообщение — **одна транзакция** (серверная функция `send_message`, миграция 0003). Одновременные повторы, в том числе через разные реплики, сериализуются уникальным индексом: второй `INSERT … ON CONFLICT DO NOTHING` ждёт COMMIT первого и возвращает сохранённое сообщение; другое тело при том же ключе → `409`; тело нормализуется (порядок ключей JSON, отсутствие `attachment_ids` == `[]`), хеш тела хранится. Повтор заново проверяет сессию и членство. Транзиентный сбой откатывает транзакцию целиком — ключ не фиксируется, повтор безопасен. TTL ключа 24 ч (просроченный ключ той же области считается свободным; фоновая очистка раз в час). +- **Балансировщик (HAProxy)**: активная проверка `GET /health/ready` каждые **1 с**, `fall 2` / `rise 2` — максимальное время обнаружения отказа ≈ **2–3 с**; `retries 3` + `redispatch` — запросы с ошибкой соединения повторяются на другой реплике; адреса реплик обновляются через Docker DNS (`resolvers docker`, `server-template api 1-8`), поэтому scale/пересоздание не требует перезапуска LB. Ошибки, созданные самим LB (нет живых реплик), отдаются в контрактном формате `503` JSON + `Retry-After` (errorfile). +- **Согласованное завершение**: SIGTERM → readiness сразу `503` → ещё 4 с (`DRAIN_SECONDS`) реплика обслуживает запросы, пока LB её исключает → uvicorn прекращает приём и дожидается текущих запросов (≤ 8 с). `stop_grace_period: 15s`. Измерено: реплика перестаёт получать трафик через 1.1–1.7 с, контейнер завершается за 4.6 с, ошибок клиента 0 (`evidence/drain_timing.json`). +- **Ресурсы и порты**: у всех сервисов `cpus`/`mem_limit` (API 1 CPU/256 MiB, PostgreSQL 1 CPU/512 MiB, LB 1 CPU/128 MiB); API-порты не публикуются (проверяет `student-tests/test_topology.py`); `container_name` не используется. +- Источник хранилища сессий — PostgreSQL + общий Valkey (как в lab2); readiness API зависит только от PostgreSQL. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Весь стенд в compose.yaml, ≥2 реплики, единая точка входа, общие БД и сессии | `docker compose up -d`; `tests/smoke.py` | PASS; ответы через LB с ≥2 разными `X-Instance-Id` без повторного входа | `evidence/checks.txt` | +| Разные запросы одного пользователя — разные реплики | `student-tests/test_idempotency.py::test_session_is_shared_between_replicas_without_relogin` | 30 запросов с одним токеном обслужены ≥2 репликами | `evidence/checks.txt` | +| Остановка любой одной реплики: чтение/запись ≤10 с, данные целы | `tests/resilience.py --mode failover` для каждой реплики | обе: PASS; восстановление записи 2.4 с и 1.5 с, ошибочных попыток 0 | `evidence/checks.txt` | +| Повтор POST с тем же ключом без дубля, в т. ч. конкурентный и через разные реплики | `test_idempotency.py` (9 тестов) | 24 параллельных запроса с одним ключом через ≥2 реплики → 1 сообщение, одинаковый ответ | `evidence/checks.txt` | +| make test, persistence, failover | см. выше | PASS; при persistence `StartedAt` всех сервисов изменился | `evidence/checks.txt` | +| Health/readiness/SIGTERM согласованы, период проверок и время обнаружения | `research/drain_timing.py` | SIGTERM: 1.1–1.7 с до исключения из ротации, 4.6 с до выхода, 0 ошибок | `evidence/drain_timing.json` | +| Последовательная остановка каждой реплики: RPS, ошибки, время восстановления | `research/failover_sequence.py` (70 с нагрузки, 16 клиентов, restore между отказами) | см. таблицу ниже | `evidence/failover_sequence.json\|png` | +| CPU/RAM-лимиты, порты API не опубликованы, нет `container_name` | `student-tests/test_topology.py` | 6 тестов OK | `evidence/checks.txt` | +| Rolling update под нагрузкой, миграции однократно | `research/rolling_update.py` | обе реплики заменены, 0 ошибок, 0 повторов | `evidence/rolling_update.json` | +| Трек 1 | `research/run_scaling_all.sh` | см. ниже | `evidence/scaling_*.json`, `scaling.png` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +**Последовательная остановка (под нагрузкой 16 клиентов, записи повторяются клиентом с тем же ключом):** -## Эксперименты и отказы +| Реплика | RPS до | RPS на одной реплике | RPS после восстановления | Ошибочных попыток за 10 с | Ошибки прекратились через | +| --- | --- | --- | --- | --- | --- | +| 1-я (t=10 с) | 1460 | 1411 | 1441 | 14 (`ServerDisconnectedError`) | 1.55 с | +| 2-я (t=40 с) | 1436 | 1350 | 1376 | 16 | 1.57 с | -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Итого 91 677 запросов, ошибочных 30 (0.03 %), после повторов с тем же ключом **финальных отказов 0**, подтверждённых 91 647 = записано 91 647, **потерь 0, дублей 0** (сверка по тексту уникальных сообщений). Ограничение: один клиентский процесс упирается в ≈1.4 тыс. RPS, поэтому RPS одной и двух реплик здесь схожи; для оценки ёмкости см. трек 1. + +**Rolling update** (60 с нагрузки, обновление на t=10..45 с, `scripts/rolling-update.sh`): запросов 39 597 за время обновления, ошибок **0**, повторов 0, RPS до/во время/после 1263/1123/1268, обе реплики заменены (4 разных `X-Instance-Id` видны во время обновления). В первом прогоне с `on-marked-down shutdown-sessions` в HAProxy наблюдалось 9 ошибок соединения (разрыв активных сессий при исключении сервера); опция удалена, повтор дал 0 ошибок — конфигурация в Git — финальная. + +## Эксперименты (трек 1): цена масштабирования + +**Гипотеза.** Масштабирование API репликами линейно увеличивает пропускную способность, пока не упрётся в общий компонент; при фиксированном суммарном CPU выгоднее меньше процессов лучшей загрузки, чем много «тонких». Узкое место — PostgreSQL (CPU), его можно снять и ресурсами, и сокращением числа обращений к БД на запрос. + +**Методика** (`research/scaling.py`, `research/run_scaling_all.sh`). Docker Desktop (WSL2), хост 24 логических CPU. Профиль: 64 одновременных клиента (8 процессов aiohttp, closed-loop, чтобы клиент не был узким местом), 90 % `POST` сообщения (с `Idempotency-Key`) + 10 % `GET` первой страницы истории, 40 пользователей/40 чатов, 5 с прогрева + 30 с замера, **2 повтора** на конфигурацию, ресурсы меняются на лету (`docker update`), загрузка CPU снимается `docker stats`. Лимиты по умолчанию: PostgreSQL 2.0 CPU, HAProxy 2.0 CPU, API по 256 MiB. + +**Историческая серия A (октябрь) — фиксированный суммарный CPU API = 2.0** (RPS = среднее двух повторов; p95 POST): + +| Конфигурация | До: RPS | p95 | После: RPS | p95 | CPU PostgreSQL (из 200 %) | +| --- | --- | --- | --- | --- | --- | +| 1 × 2.0 CPU | 1353 | 82 мс | 2289 | 43 мс | 97 % → 115 % | +| **2 × 1.0 CPU** | **2364** | 73 мс | **3280** | 37 мс | 163 % → 187 % | +| 4 × 0.5 CPU | 2107 | 119 мс | 2575 | 81 мс | 142 % → 171 % | + +Вывод A: один процесс Python использует ≈1 ядро — лимит 2.0 CPU у единственной реплики не используется (CPU API ≈ 98 %); 2 × 1.0 лучшая конфигурация; 4 × 0.5 хуже из-за троттлинга CPU (каждая реплика упирается в 50 %) и большего числа соединений к БД. + +**Серия B — рост ресурсов (реплики по 1.0 CPU), RPS:** + +| Реплик | БД 2 CPU, до | БД 4 CPU, до | БД 2 CPU, после | +| --- | --- | --- | --- | +| 1 | 1352 | 1418 | 2153 | +| 2 | 2453 | 2451 | 3313 | +| 4 | 2741 | 3834 | 2995 | +| 8 | 2593 | 4636 | 2381 | + +CPU PostgreSQL (до, лимит 200 %): 98 → 161 → **203 → 204 %** при 1/2/4/8 репликах; CPU одной реплики API падает 97 → 38 %, HAProxy 16 → 48 % (лимит 200 %). + +**Точка насыщения и почему рост реплик перестаёт помогать.** С 4 реплик PostgreSQL упирается в лимит 2.0 CPU (≈204 %), RPS выходит на плато ≈2.6–2.7 тыс.; дальнейшие реплики лишь ждут ответов БД (CPU API на реплику падает). Это подтверждено вторым прогоном с лимитом БД 4 CPU: RPS продолжает расти (3834 при 4 и 4636 при 8 репликах), т. е. причина плато именно CPU БД, а не LB (HAProxy 78 % из 200 %), пул или клиент. При 8 репликах БД с 4 CPU использует 395 %: ещё не насыщена. + +**Подтверждённое улучшение** — уменьшение числа обращений к БД на одну запись: вместо проверки членства + транзакции из 3 запросов (`DELETE` просроченного ключа, `INSERT` ключа, `INSERT` сообщения) — один вызов серверной функции `send_message` (миграция 0003; тот же код идемпотентности и атомарности, тесты `test_idempotency.py` проходят на обеих реализациях, переключатель `SEND_STRATEGY=multi|function`). При тех же лимитах (БД 2 CPU, реплики по 1.0 CPU): пик RPS **2741 → 3313 (+21 %)**, на 1 реплике 1352 → 2153 (**+59 %**), p95 при 2 репликах 74 → 40 мс (−45 %). Почему работает: на запись приходится в разы меньше сетевых round-trip и разборов запросов между приложением и БД — ограниченный CPU БД тратится на полезную работу. **Ограничение (честно):** при 4 и 8 репликах «после» ниже, чем при 2 (2995 и 2381): БД уже на пределе CPU, избыточные соединения дают конкуренцию и переключения контекста — оптимум смещается к 2 репликам; точку оптимума нужно определять под лимиты БД. + +Ограничения методики: один ноутбук, 2 повтора (разброс ≈±4 %), p95 включает задержку клиента; цифры RPS не переносимы между машинами, соотношения — переносимы. Данные: `evidence/scaling_A_base.json`, `scaling_A_improved.json`, `scaling_B_base.json`, `scaling_B_base_pg4.json`, `scaling_B_improved.json` (включая сырые прогоны и загрузку CPU компонентов); график `evidence/scaling.png`. + +### Повтор серии A с фиксированными CPU и RAM — 11.11.2026 + +В октябрьской серии A фиксировался только CPU, а общий лимит RAM API рос с числом реплик. Эти результаты сохранены как исторические и не служат доказательством фиксированного бюджета памяти. Ошибка методики исправлена: суммарно 2 CPU и 1024 MiB RAM на API; память на одну реплику — 1024/512/256 MiB для 1/2/4 реплик. PostgreSQL — 2 CPU/512 MiB, LB — 2 CPU/128 MiB, Valkey — 0.5 CPU/128 MiB. Скрипт проверяет фактические лимиты API через docker inspect и прекращает работу при ошибке Docker. + +Профиль: 64 клиента в 8 процессах, 90 % POST / 10 % GET, прогрев 5 с и замер 30 с, по 2 повтора. До оптимизации — SEND_STRATEGY=multi, после — function. Показатели относятся к успешным запросам; ошибки попыток и CPU компонентов сохранены в JSON. На машине параллельно работал другой Docker-проект, поэтому результаты не полностью изолированы и не сопоставляются напрямую с октябрьскими абсолютными значениями. + +| Реплик | RAM каждой, MiB | До, RPS | После, RPS | Оба повтора до | Оба повтора после | +| --- | --- | --- | --- | --- | --- | +| 1 | 1024 | 1436.3 | 1363.6 | [1462.1, 1410.5] | [1356.9, 1370.3] | +| 2 | 512 | 2139.7 | 2657.6 | [2341.4, 1938.0] | [2690.8, 2624.3] | +| 4 | 256 | 1900.2 | 2675.1 | [1906.7, 1893.6] | [2676.2, 2673.9] | + +Изменение пикового среднего RPS при тех же лимитах: +25.0 %. Это результат двух повторов, а не оценка статистической значимости. Число реплик и лимиты CPU/RAM должны рассматриваться вместе; рост реплик не гарантирует линейного роста пропускной способности. + +Данные: `evidence/scaling_A_base_fixed_resources.json`, `evidence/scaling_A_improved_fixed_resources.json`. Обновлённый `evidence/scaling.png` показывает новую серию A и историческую серию B. Воспроизведение — `research/run_scaling_all.sh`; этот скрипт также повторяет серию B. + +### Отказы | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка одной реплики (мгновенно, `docker stop --time 0`) | чтение/запись продолжаются ≤ 10 с | см. таблицу выше | ≈1.5 с; потерь и дублей нет | +| SIGTERM реплики | без ошибок клиента | 0 ошибок, исключение из LB за 1.1–1.7 с | — | +| Остановка PostgreSQL | реплики исключаются (readiness 503), клиент получает 503 JSON | `test_dependencies.py` OK | после старта БД — автоматически | +| Остановка Valkey | кэш обходится | OK | мгновенно | +| Рестарт всего стенда с volumes | токен и данные на месте | persistence PASS | < 90 с | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Не выдерживается: потеря PostgreSQL/LB/хоста Docker (единичные экземпляры, SPOF); отказ узла не исследовался (трек 2 не выбран). HAProxy — единственная точка входа. +- Окно инвалидации кэша сессий (≤ 60 с) при недоступном Valkey в момент logout — как в lab2. +- Идемпотентный ключ хранится 24 ч в PostgreSQL; очистка просроченных — фоновая задача каждой реплики. +- Подтверждено экспериментами: масштабирование, насыщение БД, улучшение +21 %/+59 %; гипотеза без доказательства: поведение при >8 репликах и при нескольких хостах. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e3ab7f01 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,122 @@ +# Стенд lab3: HAProxy -> API x N (stateless) -> PostgreSQL + Valkey. Запуск и проверки — см. REPORT.md. +# Секреты: scripts/init-secrets.sh создаёт secrets/* (в Git не попадают). +name: mtusi-lab3 + +x-common: &common + restart: unless-stopped + logging: + driver: json-file + options: {max-size: "10m", max-file: "3"} + +services: + postgres: + <<: *common + image: postgres:17.11-alpine@sha256:b0f9560a2de083e2cc7382e75f808c7381a32852a7ec49117deedb300e552b24 + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql/data + # fsync=on, synchronous_commit=on: подтверждённая запись переживает рестарт; max_connections хватает для 8 реплик по пулу 10. + command: ["postgres", "-c", "max_connections=200", "-c", "shared_buffers=128MB", "-c", "log_min_duration_statement=500"] + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 12 + mem_limit: 512m + cpus: ${POSTGRES_CPUS:-1.0} + networks: [backend] + + valkey: + <<: *common + image: valkey/valkey:8.1.10-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + # Хранилище сессий-кэша общее для всех реплик (источник истины — PostgreSQL). + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec", + "--maxmemory", "64mb", "--maxmemory-policy", "volatile-ttl", "--save", ""] + volumes: + - valkeydata:/data + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 12 + mem_limit: 128m + cpus: 0.5 + networks: [backend] + + # Миграции выполняются ОДИН раз до старта реплик (а не гонкой всех реплик). + migrate: + build: . + image: mtusi-messenger:lab3 + command: ["python", "-m", "app.migrate"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + postgres: {condition: service_healthy} + restart: "no" + networks: [backend] + + # Без container_name и без published ports: реплики масштабируются `--scale api=N`, доступны только через LB. + api: + <<: *common + build: . + image: mtusi-messenger:lab3 + environment: + APP_ENV: ${APP_ENV:-lab} + DATABASE_URL_FILE: /run/secrets/database_url + VALKEY_URL: redis://valkey:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DRAIN_SECONDS: ${DRAIN_SECONDS:-4} + MAX_INFLIGHT: ${MAX_INFLIGHT:-64} + ACCESS_LOG: ${ACCESS_LOG:-1} + SEND_STRATEGY: ${SEND_STRATEGY:-function} + secrets: [database_url] + depends_on: + migrate: {condition: service_completed_successfully} + valkey: {condition: service_healthy} + # Нужно > DRAIN_SECONDS + GRACEFUL_SHUTDOWN_SECONDS, иначе docker убьёт реплику посреди завершения. + stop_grace_period: 15s + mem_limit: ${API_MEM:-256m} + cpus: ${API_CPUS:-1.0} + deploy: + replicas: ${API_REPLICAS:-2} + networks: [backend] + + lb: + <<: *common + image: haproxy:3.0-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6 + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + - ./infra/haproxy/errors:/usr/local/etc/haproxy/errors:ro + depends_on: + - api + ports: + - "127.0.0.1:8080:8080" + healthcheck: + test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8081/healthz"] + interval: 5s + timeout: 3s + retries: 6 + mem_limit: 128m + cpus: ${LB_CPUS:-1.0} + stop_grace_period: 10s + networks: [backend] + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + +volumes: + pgdata: + valkeydata: + +networks: + backend: diff --git a/docs/pull_request_lab3.md b/docs/pull_request_lab3.md new file mode 100644 index 00000000..aa7d15e8 --- /dev/null +++ b/docs/pull_request_lab3.md @@ -0,0 +1,48 @@ +## Лабораторная 3 + +Автор / группа: Кушнаренко Николай Вячеславович, БПИ2302 + +Целевая оценка (3/4/5): **5** + +Трек 5: **Трек 1 — цена масштабирования**. + +Что реализовано и что перенесено: +Перенесён мессенджер из ЛР2: PostgreSQL, сессии Bearer, кэш Valkey, миграции и проверки доступа. Добавлены HAProxy и несколько stateless API-реплик с общими сессиями; наружу опубликован только балансировщик. Отправка сообщений требует Idempotency-Key: повторы через разные реплики возвращают прежнее сообщение, другое тело с тем же ключом даёт 409. Ключ и сообщение сохраняются атомарно в PostgreSQL. Добавлены X-Instance-Id, graceful shutdown, скрипты перезапуска, остановки реплики, восстановления и rolling update. + +Трек 1: исследованы 1/2/4 реплики при фиксированных суммарных CPU и RAM, а также рост ресурсов до 8 реплик. Измерена загрузка компонентов и выявлено насыщение PostgreSQL. Сравнены несколько SQL-запросов на отправку с одной серверной функцией. Историческая серия A имела постоянный CPU, но растущую RAM; методика исправлена, серия повторена 11.11.2026 при суммарных 2 CPU и 1024 MiB RAM. Новые данные сохранены отдельно от октябрьских. Пиковый средний RPS в повторной серии A вырос на 25.0 % при одинаковых ресурсах. + +Команды запуска из чистого clone: + +```sh +sh scripts/init-secrets.sh +cp .env.example .env +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build +# Дождаться /health/ready, затем: +python3 tests/smoke.py --base-url http://127.0.0.1:8080 +python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +sh scripts/restore-replicas.sh +``` + +На Windows: `PYTHONUTF8=1`, вместо `.sh` в тестах использовать `scripts/restart.cmd` и `scripts/stop-one.cmd`. Подробности — [REPORT.md](https://gitea.privcloud.tech/kolya/mtusi2026lab3/src/branch/solution/bpi2302-kushnarenko/REPORT.md). + +Результаты проверок: + +- 11.11.2026: smoke, persistence и failover — PASS; 31 собственный тест API, идемпотентности, топологии, expiry и отказов зависимостей — OK. +- Полная валидация OpenAPI — PASS. Команды и вывод — `evidence/checks_2026-11-11.txt`. +- Исторические эксперименты: поочерёдный отказ реплик без потерь и дублей подтверждённых сообщений; rolling update без клиентских ошибок. Данные — `evidence/failover_sequence.json`, `evidence/rolling_update.json`. +- Исследование масштабирования: исходные JSON, график и разбор лимитов, узкого места и улучшения — в `evidence/` и `REPORT.md`. Даты новых и исторических замеров указаны раздельно. + +Ограничения / известные ошибки: +Стенд на одном Docker-хосте, HAProxy и PostgreSQL остаются единичными точками отказа. Отказ хоста не покрывается. Возможна временно устаревшая запись кэша сессии, если Valkey недоступен при logout. Эксперименты выполнены на одном компьютере; в повторе 11.11.2026 присутствовала фоновая нагрузка другого Docker-проекта. Два повтора не дают оценки статистической значимости. + +Источники и использование ИИ: +Python, uvicorn, asyncpg, redis-py, argon2-cffi; PostgreSQL, Valkey, HAProxy; aiohttp и matplotlib для экспериментов. ИИ-ассистенты использовались для помощи с кодом, тестами и отчётом. Codex проверил подготовку к сдаче, исправил методику фиксации RAM и повторил проверки. + +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Проверки, эквивалентные make check и make test, выполнены; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Секреты и приватные ключи находятся вне Git; отчёт не содержит токенов. +- [x] README задания сохранён, REPORT.md содержит инструкцию запуска решения. diff --git a/docs/report_lab3.docx b/docs/report_lab3.docx new file mode 100644 index 00000000..2e656cab Binary files /dev/null and b/docs/report_lab3.docx differ diff --git a/evidence/checks.txt b/evidence/checks.txt new file mode 100644 index 00000000..6c5e2829 --- /dev/null +++ b/evidence/checks.txt @@ -0,0 +1,45 @@ +$ python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +$ python3 tests/smoke.py +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +$ [персистентность] before: +/mtusi-lab3-api-8 id=c07d43275734 StartedAt=2026-10-02T23:21:08.027285124Z +/mtusi-lab3-api-7 id=7a132f87de78 StartedAt=2026-10-02T23:19:20.697214715Z +$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh (Windows: restart.cmd) +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +after: +/mtusi-lab3-api-8 id=c07d43275734 StartedAt=2026-10-02T23:21:33.360551488Z +/mtusi-lab3-api-7 id=7a132f87de78 StartedAt=2026-10-02T23:21:33.2184745Z +$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh (1-я реплика) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 0, "first_recovery_seconds": 2.438} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab3-api-8 Up 24 seconds (healthy) +mtusi-lab3-api-7 Exited (137) 15 seconds ago +mtusi-lab2-api-1 Exited (128) 51 minutes ago +(реплика восстановлена: scripts/restore-replicas.sh) +mtusi-lab3-api-9 Up 12 seconds (healthy) +mtusi-lab3-api-8 Up 37 seconds (healthy) +$ ... (2-я реплика) +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 119, "failed_attempts": 0, "first_recovery_seconds": 1.453} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +mtusi-lab3-api-9 Exited (137) 15 seconds ago +mtusi-lab3-api-8 Up 53 seconds (healthy) +mtusi-lab2-api-1 Exited (128) 51 minutes ago +$ python -m unittest student-tests/test_api.py +Ran 12 tests in 2.888s +OK +$ python student-tests/test_idempotency.py +Ran 9 tests in 2.556s +OK +$ python student-tests/test_topology.py +Ran 6 tests in 1.318s +OK +$ python student-tests/test_expiry.py +Ran 2 tests in 22.272s +OK +$ python student-tests/test_dependencies.py (остановка Valkey / PostgreSQL; при остановке БД LB отдаёт 503 JSON из errorfile) +Ran 2 tests in 9.940s +OK diff --git a/evidence/checks_2026-11-11.txt b/evidence/checks_2026-11-11.txt new file mode 100644 index 00000000..35a0baa3 --- /dev/null +++ b/evidence/checks_2026-11-11.txt @@ -0,0 +1,104 @@ +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +exit=0 +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. + +exit=0 +$ python -m unittest discover -s student-tests -p test_api.py -v +test_bearer_rules (test_api.AuthTest.test_bearer_rules) ... ok +test_concurrent_registration_of_same_username (test_api.AuthTest.test_concurrent_registration_of_same_username) ... ok +test_logout_revokes_in_database_and_cache (test_api.AuthTest.test_logout_revokes_in_database_and_cache) ... ok +test_password_is_not_exposed_or_stored_plain (test_api.AuthTest.test_password_is_not_exposed_or_stored_plain) ... ok +test_register_validation (test_api.AuthTest.test_register_validation) ... ok +test_session_shape_entropy_and_logout (test_api.AuthTest.test_session_shape_entropy_and_logout) ... ok +test_token_only_stored_as_hash (test_api.AuthTest.test_token_only_stored_as_hash) ... ok +test_wrong_password_and_unknown_user_are_indistinguishable (test_api.AuthTest.test_wrong_password_and_unknown_user_are_indistinguishable) ... ok +test_chat_creation_is_atomic (test_api.DataTest.test_chat_creation_is_atomic) ... ok +test_foreign_chat_empty_message_bad_json (test_api.DataTest.test_foreign_chat_empty_message_bad_json) ... ok +test_multi_page_history_and_cursor_binding (test_api.DataTest.test_multi_page_history_and_cursor_binding) ... ok +test_parallel_writes_from_many_clients (test_api.DataTest.test_parallel_writes_from_many_clients) ... ok + +---------------------------------------------------------------------- +Ran 12 tests in 2.437s + +OK + +exit=0 +$ python -m unittest discover -s student-tests -p test_idempotency.py -v +test_concurrent_same_key_creates_exactly_one_message_across_replicas (test_idempotency.IdempotencyTest.test_concurrent_same_key_creates_exactly_one_message_across_replicas) ... ok +test_different_body_same_key_is_409 (test_idempotency.IdempotencyTest.test_different_body_same_key_is_409) ... ok +test_expired_key_is_reusable_after_ttl (test_idempotency.IdempotencyTest.test_expired_key_is_reusable_after_ttl) ... ok +test_key_validation (test_idempotency.IdempotencyTest.test_key_validation) ... ok +test_replay_rechecks_session_and_membership (test_idempotency.IdempotencyTest.test_replay_rechecks_session_and_membership) ... ok +test_replay_returns_same_message_and_creates_no_duplicate (test_idempotency.IdempotencyTest.test_replay_returns_same_message_and_creates_no_duplicate) ... ok +test_scope_is_user_and_chat (test_idempotency.IdempotencyTest.test_scope_is_user_and_chat) ... ok +test_semantically_equal_body_is_replayed (test_idempotency.IdempotencyTest.test_semantically_equal_body_is_replayed) ... ok +test_session_is_shared_between_replicas_without_relogin (test_idempotency.IdempotencyTest.test_session_is_shared_between_replicas_without_relogin) ... ok + +---------------------------------------------------------------------- +Ran 9 tests in 2.503s + +OK + +exit=0 +$ python -m unittest discover -s student-tests -p test_topology.py -v +test_api_runs_as_non_root (test_topology.TopologyTest.test_api_runs_as_non_root) ... ok +test_at_least_two_api_replicas_without_published_ports (test_topology.TopologyTest.test_at_least_two_api_replicas_without_published_ports) ... ok +test_no_fixed_container_names (test_topology.TopologyTest.test_no_fixed_container_names) ... ok +test_only_lb_is_published_and_only_on_localhost (test_topology.TopologyTest.test_only_lb_is_published_and_only_on_localhost) ... ok +test_resource_limits_are_set (test_topology.TopologyTest.test_resource_limits_are_set) ... ok +test_stop_grace_period_covers_drain (test_topology.TopologyTest.test_stop_grace_period_covers_drain) ... ok + +---------------------------------------------------------------------- +Ran 6 tests in 1.344s + +OK + +exit=0 +$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. + +exit=0 +$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 118, "failed_attempts": 0, "first_recovery_seconds": 2.438} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +exit=0 +$ docker compose --env-file .env start api + Container mtusi-lab3-postgres-1 Waiting + Container mtusi-lab3-postgres-1 Healthy + Container mtusi-lab3-migrate-1 Starting + Container mtusi-lab3-migrate-1 Started + Container mtusi-lab3-migrate-1 Waiting + Container mtusi-lab3-valkey-1 Waiting + Container mtusi-lab3-valkey-1 Healthy + Container mtusi-lab3-migrate-1 Exited + Container mtusi-lab3-api-9 Starting + Container mtusi-lab3-api-9 Started + +exit=0 +$ python student-tests/test_expiry.py +.. +---------------------------------------------------------------------- +Ran 2 tests in 23.234s + +OK + +exit=0 +$ python student-tests/test_dependencies.py +.. +---------------------------------------------------------------------- +Ran 2 tests in 8.236s + +OK + +exit=0 +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. + +exit=0 +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/drain_timing.json b/evidence/drain_timing.json new file mode 100644 index 00000000..cdb0498c --- /dev/null +++ b/evidence/drain_timing.json @@ -0,0 +1,50 @@ +{ + "date": "2026-10-03 02:21:16", + "lb_health_check": "inter 1s, fall 2, rise 2", + "drain_seconds": 4, + "graceful_shutdown_seconds": 8, + "trials": [ + { + "kind": "sigterm", + "seconds_until_replica_stops_serving": 1.58, + "seconds_until_container_exit": 4.59, + "client_errors": 0, + "seconds_until_errors_stop": 0.0 + }, + { + "kind": "sigterm", + "seconds_until_replica_stops_serving": 1.1, + "seconds_until_container_exit": 4.59, + "client_errors": 0, + "seconds_until_errors_stop": 0.0 + }, + { + "kind": "sigterm", + "seconds_until_replica_stops_serving": 1.66, + "seconds_until_container_exit": 4.61, + "client_errors": 0, + "seconds_until_errors_stop": 0.0 + }, + { + "kind": "sigkill", + "seconds_until_replica_stops_serving": 0.03, + "seconds_until_container_exit": 0.42, + "client_errors": 0, + "seconds_until_errors_stop": 0.0 + }, + { + "kind": "sigkill", + "seconds_until_replica_stops_serving": 0.05, + "seconds_until_container_exit": 0.45, + "client_errors": 0, + "seconds_until_errors_stop": 0.0 + }, + { + "kind": "sigkill", + "seconds_until_replica_stops_serving": 0.05, + "seconds_until_container_exit": 0.44, + "client_errors": 0, + "seconds_until_errors_stop": 0.0 + } + ] +} \ No newline at end of file diff --git a/evidence/failover_sequence.json b/evidence/failover_sequence.json new file mode 100644 index 00000000..3d969b01 --- /dev/null +++ b/evidence/failover_sequence.json @@ -0,0 +1,579 @@ +{ + "date": "2026-10-03 01:37:34", + "args": { + "out": "evidence", + "seconds": 70, + "concurrency": 16 + }, + "marks": [ + { + "t": 10.0, + "action": "stop", + "container": "e219bacd60d1" + }, + { + "t": 25.0, + "action": "restore" + }, + { + "t": 40.0, + "action": "stop", + "container": "9229a3349759" + }, + { + "t": 55.0, + "action": "restore" + } + ], + "windows": [ + { + "stopped_container": "e219bacd60d1", + "stop_at_s": 10.0, + "rps_before": 1460.0, + "rps_one_replica": 1411.3, + "rps_after_restore": 1440.7, + "failed_attempts_within_10s": 14, + "error_statuses": [ + "ServerDisconnectedError" + ], + "seconds_until_errors_stop": 1.55, + "p95_ms_one_replica": 13.15 + }, + { + "stopped_container": "9229a3349759", + "stop_at_s": 40.0, + "rps_before": 1436.1, + "rps_one_replica": 1350.4, + "rps_after_restore": 1376.3, + "failed_attempts_within_10s": 16, + "error_statuses": [ + "ServerDisconnectedError" + ], + "seconds_until_errors_stop": 1.57, + "p95_ms_one_replica": 14.74 + } + ], + "total": { + "requests": 91677, + "ok": 91647, + "errors": { + "ServerDisconnectedError": 30 + }, + "seconds": 69.92, + "ok_rps": 1310.7, + "error_ratio": 0.0003, + "p50_ms": 9.16, + "p95_ms": 32.46, + "p99_ms": 45.66 + }, + "verification": { + "stored_messages": 91647, + "confirmed_201": 91647, + "confirmed_missing_in_history": 0, + "duplicate_texts_in_history": 0, + "unconfirmed_requests": 0, + "unconfirmed_that_were_written": 0, + "retries": 30, + "final_failures": 0 + }, + "invariants_ok": true, + "timeline": [ + { + "t": 0, + "ok": 1216, + "err": 0, + "p95_ms": 40.31, + "instances": 2 + }, + { + "t": 1, + "ok": 1380, + "err": 0, + "p95_ms": 37.96, + "instances": 2 + }, + { + "t": 2, + "ok": 1419, + "err": 0, + "p95_ms": 35.69, + "instances": 2 + }, + { + "t": 3, + "ok": 1481, + "err": 0, + "p95_ms": 36.23, + "instances": 2 + }, + { + "t": 4, + "ok": 1443, + "err": 0, + "p95_ms": 21.97, + "instances": 2 + }, + { + "t": 5, + "ok": 1433, + "err": 0, + "p95_ms": 25.91, + "instances": 2 + }, + { + "t": 6, + "ok": 1422, + "err": 0, + "p95_ms": 38.71, + "instances": 2 + }, + { + "t": 7, + "ok": 1511, + "err": 0, + "p95_ms": 34.34, + "instances": 2 + }, + { + "t": 8, + "ok": 1480, + "err": 0, + "p95_ms": 37.92, + "instances": 2 + }, + { + "t": 9, + "ok": 1486, + "err": 0, + "p95_ms": 35.88, + "instances": 2 + }, + { + "t": 10, + "ok": 234, + "err": 0, + "p95_ms": 37.62, + "instances": 2 + }, + { + "t": 11, + "ok": 615, + "err": 14, + "p95_ms": 15.22, + "instances": 1 + }, + { + "t": 12, + "ok": 1420, + "err": 0, + "p95_ms": 14.2, + "instances": 1 + }, + { + "t": 13, + "ok": 1499, + "err": 0, + "p95_ms": 12.59, + "instances": 1 + }, + { + "t": 14, + "ok": 1484, + "err": 0, + "p95_ms": 12.66, + "instances": 1 + }, + { + "t": 15, + "ok": 1425, + "err": 0, + "p95_ms": 13.15, + "instances": 1 + }, + { + "t": 16, + "ok": 1508, + "err": 0, + "p95_ms": 12.15, + "instances": 1 + }, + { + "t": 17, + "ok": 1203, + "err": 0, + "p95_ms": 23.7, + "instances": 1 + }, + { + "t": 18, + "ok": 1433, + "err": 0, + "p95_ms": 13.18, + "instances": 1 + }, + { + "t": 19, + "ok": 1467, + "err": 0, + "p95_ms": 12.77, + "instances": 1 + }, + { + "t": 20, + "ok": 1328, + "err": 0, + "p95_ms": 14.37, + "instances": 1 + }, + { + "t": 21, + "ok": 1464, + "err": 0, + "p95_ms": 12.67, + "instances": 1 + }, + { + "t": 22, + "ok": 1424, + "err": 0, + "p95_ms": 13.96, + "instances": 1 + }, + { + "t": 23, + "ok": 1450, + "err": 0, + "p95_ms": 13.01, + "instances": 1 + }, + { + "t": 24, + "ok": 1491, + "err": 0, + "p95_ms": 12.56, + "instances": 1 + }, + { + "t": 25, + "ok": 1340, + "err": 0, + "p95_ms": 14.65, + "instances": 1 + }, + { + "t": 26, + "ok": 1182, + "err": 0, + "p95_ms": 19.25, + "instances": 2 + }, + { + "t": 27, + "ok": 1402, + "err": 0, + "p95_ms": 39.28, + "instances": 2 + }, + { + "t": 28, + "ok": 1428, + "err": 0, + "p95_ms": 39.82, + "instances": 2 + }, + { + "t": 29, + "ok": 1475, + "err": 0, + "p95_ms": 36.5, + "instances": 2 + }, + { + "t": 30, + "ok": 1408, + "err": 0, + "p95_ms": 37.79, + "instances": 2 + }, + { + "t": 31, + "ok": 1376, + "err": 0, + "p95_ms": 38.6, + "instances": 2 + }, + { + "t": 32, + "ok": 1378, + "err": 0, + "p95_ms": 37.01, + "instances": 2 + }, + { + "t": 33, + "ok": 1454, + "err": 0, + "p95_ms": 36.88, + "instances": 2 + }, + { + "t": 34, + "ok": 1490, + "err": 0, + "p95_ms": 32.11, + "instances": 2 + }, + { + "t": 35, + "ok": 1416, + "err": 0, + "p95_ms": 38.2, + "instances": 2 + }, + { + "t": 36, + "ok": 1459, + "err": 0, + "p95_ms": 36.62, + "instances": 2 + }, + { + "t": 37, + "ok": 1349, + "err": 0, + "p95_ms": 39.64, + "instances": 2 + }, + { + "t": 38, + "ok": 1476, + "err": 0, + "p95_ms": 37.98, + "instances": 2 + }, + { + "t": 39, + "ok": 1481, + "err": 0, + "p95_ms": 38.98, + "instances": 2 + }, + { + "t": 40, + "ok": 207, + "err": 0, + "p95_ms": 41.16, + "instances": 2 + }, + { + "t": 41, + "ok": 470, + "err": 16, + "p95_ms": 21.42, + "instances": 1 + }, + { + "t": 42, + "ok": 1191, + "err": 0, + "p95_ms": 16.5, + "instances": 1 + }, + { + "t": 43, + "ok": 1223, + "err": 0, + "p95_ms": 15.65, + "instances": 1 + }, + { + "t": 44, + "ok": 1216, + "err": 0, + "p95_ms": 16.13, + "instances": 1 + }, + { + "t": 45, + "ok": 1191, + "err": 0, + "p95_ms": 21.64, + "instances": 1 + }, + { + "t": 46, + "ok": 1209, + "err": 0, + "p95_ms": 16.61, + "instances": 1 + }, + { + "t": 47, + "ok": 1377, + "err": 0, + "p95_ms": 14.21, + "instances": 1 + }, + { + "t": 48, + "ok": 1180, + "err": 0, + "p95_ms": 32.49, + "instances": 1 + }, + { + "t": 49, + "ok": 1479, + "err": 0, + "p95_ms": 12.5, + "instances": 1 + }, + { + "t": 50, + "ok": 1417, + "err": 0, + "p95_ms": 13.78, + "instances": 1 + }, + { + "t": 51, + "ok": 1435, + "err": 0, + "p95_ms": 13.28, + "instances": 1 + }, + { + "t": 52, + "ok": 1355, + "err": 0, + "p95_ms": 14.66, + "instances": 1 + }, + { + "t": 53, + "ok": 1511, + "err": 0, + "p95_ms": 12.44, + "instances": 1 + }, + { + "t": 54, + "ok": 1454, + "err": 0, + "p95_ms": 13.14, + "instances": 1 + }, + { + "t": 55, + "ok": 1267, + "err": 0, + "p95_ms": 15.69, + "instances": 1 + }, + { + "t": 56, + "ok": 1205, + "err": 0, + "p95_ms": 15.98, + "instances": 2 + }, + { + "t": 57, + "ok": 1154, + "err": 0, + "p95_ms": 43.65, + "instances": 2 + }, + { + "t": 58, + "ok": 974, + "err": 0, + "p95_ms": 50.35, + "instances": 2 + }, + { + "t": 59, + "ok": 1054, + "err": 0, + "p95_ms": 48.29, + "instances": 2 + }, + { + "t": 60, + "ok": 1144, + "err": 0, + "p95_ms": 47.32, + "instances": 2 + }, + { + "t": 61, + "ok": 1169, + "err": 0, + "p95_ms": 46.31, + "instances": 2 + }, + { + "t": 62, + "ok": 1230, + "err": 0, + "p95_ms": 42.09, + "instances": 2 + }, + { + "t": 63, + "ok": 1322, + "err": 0, + "p95_ms": 43.44, + "instances": 2 + }, + { + "t": 64, + "ok": 1422, + "err": 0, + "p95_ms": 39.26, + "instances": 2 + }, + { + "t": 65, + "ok": 1411, + "err": 0, + "p95_ms": 38.88, + "instances": 2 + }, + { + "t": 66, + "ok": 1358, + "err": 0, + "p95_ms": 32.46, + "instances": 2 + }, + { + "t": 67, + "ok": 1402, + "err": 0, + "p95_ms": 32.67, + "instances": 2 + }, + { + "t": 68, + "ok": 1343, + "err": 0, + "p95_ms": 39.82, + "instances": 2 + }, + { + "t": 69, + "ok": 1431, + "err": 0, + "p95_ms": 29.97, + "instances": 2 + }, + { + "t": 70, + "ok": 16, + "err": 0, + "p95_ms": 43.02, + "instances": 2 + } + ] +} \ No newline at end of file diff --git a/evidence/failover_sequence.png b/evidence/failover_sequence.png new file mode 100644 index 00000000..dbba8f4c Binary files /dev/null and b/evidence/failover_sequence.png differ diff --git a/evidence/rolling_update.json b/evidence/rolling_update.json new file mode 100644 index 00000000..733a7148 --- /dev/null +++ b/evidence/rolling_update.json @@ -0,0 +1,84 @@ +{ + "date": "2026-10-03 01:41:11", + "update": { + "start": 10.0, + "end": 45.25, + "script_exit": 0, + "script_output": [ + "revision=rolling-1790980807 replicas=2", + "заменена реплика d25e9431db25 -> 0dc60295dff6", + "заменена реплика cfc5f1eb28cd -> 861c1ff7289a" + ] + }, + "revision": "rolling-1790980807", + "instances_before": [ + "cfc5f1eb28cd", + "d25e9431db25" + ], + "instances_during": [ + "0dc60295dff6", + "861c1ff7289a", + "cfc5f1eb28cd", + "d25e9431db25" + ], + "instances_after": [ + "0dc60295dff6", + "861c1ff7289a" + ], + "all_replicas_replaced": true, + "before": { + "requests": 10101, + "ok": 10101, + "errors": {}, + "seconds": 8.0, + "ok_rps": 1262.6, + "error_ratio": 0.0, + "p50_ms": 8.21, + "p95_ms": 43.04, + "p99_ms": 50.58 + }, + "during": { + "requests": 39597, + "ok": 39597, + "errors": {}, + "seconds": 35.25, + "ok_rps": 1123.3, + "error_ratio": 0.0, + "p50_ms": 7.88, + "p95_ms": 51.48, + "p99_ms": 57.88 + }, + "after": { + "requests": 16179, + "ok": 16179, + "errors": {}, + "seconds": 12.76, + "ok_rps": 1268.0, + "error_ratio": 0.0, + "p50_ms": 7.77, + "p95_ms": 44.37, + "p99_ms": 49.68 + }, + "total": { + "requests": 71069, + "ok": 71069, + "errors": {}, + "seconds": 60.0, + "ok_rps": 1184.6, + "error_ratio": 0.0, + "p50_ms": 7.91, + "p95_ms": 47.99, + "p99_ms": 56.25 + }, + "verification": { + "stored_messages": 71069, + "confirmed_201": 71069, + "confirmed_missing_in_history": 0, + "duplicate_texts_in_history": 0, + "unconfirmed_requests": 0, + "unconfirmed_that_were_written": 0, + "retries": 0, + "final_failures": 0 + }, + "invariants_ok": true +} \ No newline at end of file diff --git a/evidence/scaling.png b/evidence/scaling.png new file mode 100644 index 00000000..743b9da7 Binary files /dev/null and b/evidence/scaling.png differ diff --git a/evidence/scaling_A_base.json b/evidence/scaling_A_base.json new file mode 100644 index 00000000..a3b60b3e --- /dev/null +++ b/evidence/scaling_A_base.json @@ -0,0 +1,566 @@ +{ + "date": "2026-10-03 01:57:55", + "series": "A", + "tag": "base", + "args": { + "series": "A", + "out": "evidence", + "tag": "base", + "seconds": 30, + "concurrency": 64, + "procs": 8, + "pg_cpus": 2.0, + "lb_cpus": 2.0, + "repeat": 2 + }, + "results": [ + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 37440, + "ok": 37408, + "errors": { + "429": 32 + }, + "seconds": 30, + "ok_rps": 1246.9, + "error_ratio": 0.0009, + "p50_ms": 44.33, + "p95_ms": 81.79, + "p99_ms": 105.16 + }, + "get": { + "requests": 4041, + "ok": 4038, + "errors": { + "429": 3 + }, + "seconds": 30, + "ok_rps": 134.6, + "error_ratio": 0.0007, + "p50_ms": 39.64, + "p95_ms": 76.37, + "p99_ms": 98.48 + }, + "total_rps": 1352.9, + "replica_share": [ + 38056 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 98.4, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 15.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 96.7, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 4.1, + "samples": 15 + } + }, + "total_rps_runs": [ + 1381.5, + 1324.3 + ], + "post_p95_runs": [ + 81.79, + 86.79 + ], + "runs": [ + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 37440, + "ok": 37408, + "errors": { + "429": 32 + }, + "seconds": 30, + "ok_rps": 1246.9, + "error_ratio": 0.0009, + "p50_ms": 44.33, + "p95_ms": 81.79, + "p99_ms": 105.16 + }, + "get": { + "requests": 4041, + "ok": 4038, + "errors": { + "429": 3 + }, + "seconds": 30, + "ok_rps": 134.6, + "error_ratio": 0.0007, + "p50_ms": 39.64, + "p95_ms": 76.37, + "p99_ms": 98.48 + }, + "total_rps": 1381.5, + "replica_share": [ + 38056 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 98.4, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 15.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 96.7, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 4.1, + "samples": 15 + } + } + }, + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 35893, + "ok": 35861, + "errors": { + "429": 32 + }, + "seconds": 30, + "ok_rps": 1195.4, + "error_ratio": 0.0009, + "p50_ms": 45.86, + "p95_ms": 86.79, + "p99_ms": 115.37 + }, + "get": { + "requests": 3873, + "ok": 3869, + "errors": { + "429": 4 + }, + "seconds": 30, + "ok_rps": 129.0, + "error_ratio": 0.001, + "p50_ms": 41.21, + "p95_ms": 80.74, + "p99_ms": 109.19 + }, + "total_rps": 1324.3, + "replica_share": [ + 36573 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 96.4, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 15.5, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 97.8, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 4.3, + "samples": 15 + } + } + } + ] + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 66246, + "ok": 66246, + "errors": {}, + "seconds": 30, + "ok_rps": 2208.2, + "error_ratio": 0.0, + "p50_ms": 11.14, + "p95_ms": 73.19, + "p99_ms": 100.0 + }, + "get": { + "requests": 7219, + "ok": 7219, + "errors": {}, + "seconds": 30, + "ok_rps": 240.6, + "error_ratio": 0.0, + "p50_ms": 6.44, + "p95_ms": 68.57, + "p99_ms": 91.61 + }, + "total_rps": 2364.4, + "replica_share": [ + 33619, + 33643 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 86.1, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 30.5, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 163.0, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.1, + "samples": 15 + } + }, + "total_rps_runs": [ + 2448.8, + 2280.1 + ], + "post_p95_runs": [ + 73.19, + 77.63 + ], + "runs": [ + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 66246, + "ok": 66246, + "errors": {}, + "seconds": 30, + "ok_rps": 2208.2, + "error_ratio": 0.0, + "p50_ms": 11.14, + "p95_ms": 73.19, + "p99_ms": 100.0 + }, + "get": { + "requests": 7219, + "ok": 7219, + "errors": {}, + "seconds": 30, + "ok_rps": 240.6, + "error_ratio": 0.0, + "p50_ms": 6.44, + "p95_ms": 68.57, + "p99_ms": 91.61 + }, + "total_rps": 2448.8, + "replica_share": [ + 33619, + 33643 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 86.1, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 30.5, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 163.0, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.1, + "samples": 15 + } + } + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 61655, + "ok": 61653, + "errors": { + "429": 2 + }, + "seconds": 30, + "ok_rps": 2055.1, + "error_ratio": 0.0, + "p50_ms": 12.07, + "p95_ms": 77.63, + "p99_ms": 110.7 + }, + "get": { + "requests": 6749, + "ok": 6749, + "errors": {}, + "seconds": 30, + "ok_rps": 225.0, + "error_ratio": 0.0, + "p50_ms": 6.78, + "p95_ms": 73.12, + "p99_ms": 107.04 + }, + "total_rps": 2280.1, + "replica_share": [ + 31272, + 31315 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 83.4, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 29.4, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 154.8, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 7.6, + "samples": 15 + } + } + } + ] + }, + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 57027, + "ok": 57026, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 1900.9, + "error_ratio": 0.0, + "p50_ms": 7.08, + "p95_ms": 119.12, + "p99_ms": 192.52 + }, + "get": { + "requests": 6227, + "ok": 6227, + "errors": {}, + "seconds": 30, + "ok_rps": 207.6, + "error_ratio": 0.0, + "p50_ms": 3.78, + "p95_ms": 109.47, + "p99_ms": 193.7 + }, + "total_rps": 2107.2, + "replica_share": [ + 14413, + 14450, + 14474, + 14479 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 44.6, + "samples": 60 + }, + "lb": { + "avg_cpu_percent_per_container": 28.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 142.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.0, + "samples": 15 + } + }, + "total_rps_runs": [ + 2108.4, + 2105.9 + ], + "post_p95_runs": [ + 119.12, + 121.61 + ], + "runs": [ + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 57027, + "ok": 57026, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 1900.9, + "error_ratio": 0.0, + "p50_ms": 7.08, + "p95_ms": 119.12, + "p99_ms": 192.52 + }, + "get": { + "requests": 6227, + "ok": 6227, + "errors": {}, + "seconds": 30, + "ok_rps": 207.6, + "error_ratio": 0.0, + "p50_ms": 3.78, + "p95_ms": 109.47, + "p99_ms": 193.7 + }, + "total_rps": 2108.4, + "replica_share": [ + 14413, + 14450, + 14474, + 14479 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 44.6, + "samples": 60 + }, + "lb": { + "avg_cpu_percent_per_container": 28.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 142.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.0, + "samples": 15 + } + } + }, + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 56952, + "ok": 56950, + "errors": { + "429": 2 + }, + "seconds": 30, + "ok_rps": 1898.3, + "error_ratio": 0.0, + "p50_ms": 7.27, + "p95_ms": 121.61, + "p99_ms": 191.84 + }, + "get": { + "requests": 6226, + "ok": 6226, + "errors": {}, + "seconds": 30, + "ok_rps": 207.5, + "error_ratio": 0.0, + "p50_ms": 3.8, + "p95_ms": 115.23, + "p99_ms": 194.2 + }, + "total_rps": 2105.9, + "replica_share": [ + 14428, + 14448, + 14482, + 14526 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 45.1, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 30.0, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 142.8, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.1, + "samples": 14 + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/scaling_A_base_fixed_resources.json b/evidence/scaling_A_base_fixed_resources.json new file mode 100644 index 00000000..4b26057b --- /dev/null +++ b/evidence/scaling_A_base_fixed_resources.json @@ -0,0 +1,586 @@ +{ + "date": "2026-11-11 17:50:57", + "series": "A", + "tag": "base_fixed_resources", + "args": { + "series": "A", + "out": "evidence", + "tag": "base_fixed_resources", + "seconds": 30, + "concurrency": 64, + "procs": 8, + "pg_cpus": 2.0, + "lb_cpus": 2.0, + "repeat": 2 + }, + "results": [ + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 1024, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 39595, + "ok": 39561, + "errors": { + "429": 34 + }, + "seconds": 30, + "ok_rps": 1318.7, + "error_ratio": 0.0009, + "p50_ms": 41.51, + "p95_ms": 76.58, + "p99_ms": 106.5 + }, + "get": { + "requests": 4303, + "ok": 4302, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 143.4, + "error_ratio": 0.0002, + "p50_ms": 37.12, + "p95_ms": 72.26, + "p99_ms": 96.87 + }, + "total_rps": 1436.3, + "replica_share": [ + 40147 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 97.4, + "samples": 14 + }, + "lb": { + "avg_cpu_percent_per_container": 14.5, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 93.2, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 3.7, + "samples": 14 + } + }, + "total_rps_runs": [ + 1462.1, + 1410.5 + ], + "post_p95_runs": [ + 76.58, + 80.56 + ], + "runs": [ + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 1024, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 39595, + "ok": 39561, + "errors": { + "429": 34 + }, + "seconds": 30, + "ok_rps": 1318.7, + "error_ratio": 0.0009, + "p50_ms": 41.51, + "p95_ms": 76.58, + "p99_ms": 106.5 + }, + "get": { + "requests": 4303, + "ok": 4302, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 143.4, + "error_ratio": 0.0002, + "p50_ms": 37.12, + "p95_ms": 72.26, + "p99_ms": 96.87 + }, + "total_rps": 1462.1, + "replica_share": [ + 40147 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 97.4, + "samples": 14 + }, + "lb": { + "avg_cpu_percent_per_container": 14.5, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 93.2, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 3.7, + "samples": 14 + } + } + }, + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 1024, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 38213, + "ok": 38179, + "errors": { + "429": 34 + }, + "seconds": 30, + "ok_rps": 1272.6, + "error_ratio": 0.0009, + "p50_ms": 42.5, + "p95_ms": 80.56, + "p99_ms": 104.62 + }, + "get": { + "requests": 4139, + "ok": 4135, + "errors": { + "429": 4 + }, + "seconds": 30, + "ok_rps": 137.8, + "error_ratio": 0.001, + "p50_ms": 38.06, + "p95_ms": 75.18, + "p99_ms": 100.22 + }, + "total_rps": 1410.5, + "replica_share": [ + 38713 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 93.6, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 14.8, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 93.3, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 3.7, + "samples": 15 + } + } + } + ] + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 512, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 63318, + "ok": 63317, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 2110.6, + "error_ratio": 0.0, + "p50_ms": 21.51, + "p95_ms": 65.69, + "p99_ms": 104.19 + }, + "get": { + "requests": 6926, + "ok": 6926, + "errors": {}, + "seconds": 30, + "ok_rps": 230.9, + "error_ratio": 0.0, + "p50_ms": 15.75, + "p95_ms": 59.35, + "p99_ms": 93.11 + }, + "total_rps": 2139.7, + "replica_share": [ + 32033, + 32039 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 89.9, + "samples": 26 + }, + "lb": { + "avg_cpu_percent_per_container": 28.5, + "samples": 13 + }, + "postgres": { + "avg_cpu_percent_per_container": 177.7, + "samples": 13 + }, + "valkey": { + "avg_cpu_percent_per_container": 6.7, + "samples": 13 + } + }, + "total_rps_runs": [ + 2341.4, + 1938.0 + ], + "post_p95_runs": [ + 65.69, + 79.86 + ], + "runs": [ + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 512, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 63318, + "ok": 63317, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 2110.6, + "error_ratio": 0.0, + "p50_ms": 21.51, + "p95_ms": 65.69, + "p99_ms": 104.19 + }, + "get": { + "requests": 6926, + "ok": 6926, + "errors": {}, + "seconds": 30, + "ok_rps": 230.9, + "error_ratio": 0.0, + "p50_ms": 15.75, + "p95_ms": 59.35, + "p99_ms": 93.11 + }, + "total_rps": 2341.4, + "replica_share": [ + 32033, + 32039 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 89.9, + "samples": 26 + }, + "lb": { + "avg_cpu_percent_per_container": 28.5, + "samples": 13 + }, + "postgres": { + "avg_cpu_percent_per_container": 177.7, + "samples": 13 + }, + "valkey": { + "avg_cpu_percent_per_container": 6.7, + "samples": 13 + } + } + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 512, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 52397, + "ok": 52396, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 1746.5, + "error_ratio": 0.0, + "p50_ms": 27.25, + "p95_ms": 79.86, + "p99_ms": 124.25 + }, + "get": { + "requests": 5743, + "ok": 5743, + "errors": {}, + "seconds": 30, + "ok_rps": 191.4, + "error_ratio": 0.0, + "p50_ms": 19.31, + "p95_ms": 69.85, + "p99_ms": 112.83 + }, + "total_rps": 1938.0, + "replica_share": [ + 26564, + 26641 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 74.2, + "samples": 28 + }, + "lb": { + "avg_cpu_percent_per_container": 23.3, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 140.7, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 5.2, + "samples": 14 + } + } + } + ] + }, + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "api_memory_mib_each": 256, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 51560, + "ok": 51559, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 1718.6, + "error_ratio": 0.0, + "p50_ms": 17.19, + "p95_ms": 91.21, + "p99_ms": 197.61 + }, + "get": { + "requests": 5642, + "ok": 5642, + "errors": {}, + "seconds": 30, + "ok_rps": 188.1, + "error_ratio": 0.0, + "p50_ms": 7.24, + "p95_ms": 78.64, + "p99_ms": 180.88 + }, + "total_rps": 1900.2, + "replica_share": [ + 13055, + 13071, + 13087, + 13123 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 47.5, + "samples": 60 + }, + "lb": { + "avg_cpu_percent_per_container": 26.2, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 162.8, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 6.0, + "samples": 15 + } + }, + "total_rps_runs": [ + 1906.7, + 1893.6 + ], + "post_p95_runs": [ + 91.21, + 88.75 + ], + "runs": [ + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "api_memory_mib_each": 256, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 51560, + "ok": 51559, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 1718.6, + "error_ratio": 0.0, + "p50_ms": 17.19, + "p95_ms": 91.21, + "p99_ms": 197.61 + }, + "get": { + "requests": 5642, + "ok": 5642, + "errors": {}, + "seconds": 30, + "ok_rps": 188.1, + "error_ratio": 0.0, + "p50_ms": 7.24, + "p95_ms": 78.64, + "p99_ms": 180.88 + }, + "total_rps": 1906.7, + "replica_share": [ + 13055, + 13071, + 13087, + 13123 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 47.5, + "samples": 60 + }, + "lb": { + "avg_cpu_percent_per_container": 26.2, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 162.8, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 6.0, + "samples": 15 + } + } + }, + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "api_memory_mib_each": 256, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 51195, + "ok": 51195, + "errors": {}, + "seconds": 30, + "ok_rps": 1706.5, + "error_ratio": 0.0, + "p50_ms": 17.61, + "p95_ms": 88.75, + "p99_ms": 191.49 + }, + "get": { + "requests": 5614, + "ok": 5614, + "errors": {}, + "seconds": 30, + "ok_rps": 187.1, + "error_ratio": 0.0, + "p50_ms": 7.28, + "p95_ms": 74.12, + "p99_ms": 161.87 + }, + "total_rps": 1893.6, + "replica_share": [ + 12896, + 12938, + 12987, + 13020 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 47.7, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 26.1, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 161.4, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 6.2, + "samples": 14 + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/scaling_A_improved.json b/evidence/scaling_A_improved.json new file mode 100644 index 00000000..8d96e1b2 --- /dev/null +++ b/evidence/scaling_A_improved.json @@ -0,0 +1,566 @@ +{ + "date": "2026-10-03 02:13:17", + "series": "A", + "tag": "improved", + "args": { + "series": "A", + "out": "evidence", + "tag": "improved", + "seconds": 30, + "concurrency": 64, + "procs": 8, + "pg_cpus": 2.0, + "lb_cpus": 2.0, + "repeat": 2 + }, + "results": [ + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 62542, + "ok": 62512, + "errors": { + "429": 30 + }, + "seconds": 30, + "ok_rps": 2083.7, + "error_ratio": 0.0005, + "p50_ms": 25.23, + "p95_ms": 42.93, + "p99_ms": 53.69 + }, + "get": { + "requests": 6838, + "ok": 6833, + "errors": { + "429": 5 + }, + "seconds": 30, + "ok_rps": 227.8, + "error_ratio": 0.0007, + "p50_ms": 41.03, + "p95_ms": 61.33, + "p99_ms": 77.37 + }, + "total_rps": 2288.7, + "replica_share": [ + 63450 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 100.0, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 24.0, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 114.9, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 5.2, + "samples": 15 + } + }, + "total_rps_runs": [ + 2311.5, + 2265.9 + ], + "post_p95_runs": [ + 42.93, + 41.96 + ], + "runs": [ + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 62542, + "ok": 62512, + "errors": { + "429": 30 + }, + "seconds": 30, + "ok_rps": 2083.7, + "error_ratio": 0.0005, + "p50_ms": 25.23, + "p95_ms": 42.93, + "p99_ms": 53.69 + }, + "get": { + "requests": 6838, + "ok": 6833, + "errors": { + "429": 5 + }, + "seconds": 30, + "ok_rps": 227.8, + "error_ratio": 0.0007, + "p50_ms": 41.03, + "p95_ms": 61.33, + "p99_ms": 77.37 + }, + "total_rps": 2311.5, + "replica_share": [ + 63450 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 100.0, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 24.0, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 114.9, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 5.2, + "samples": 15 + } + } + }, + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 61318, + "ok": 61287, + "errors": { + "429": 31 + }, + "seconds": 30, + "ok_rps": 2042.9, + "error_ratio": 0.0005, + "p50_ms": 25.38, + "p95_ms": 41.96, + "p99_ms": 53.78 + }, + "get": { + "requests": 6691, + "ok": 6689, + "errors": { + "429": 2 + }, + "seconds": 30, + "ok_rps": 223.0, + "error_ratio": 0.0003, + "p50_ms": 41.04, + "p95_ms": 60.92, + "p99_ms": 76.23 + }, + "total_rps": 2265.9, + "replica_share": [ + 62120 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 96.8, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 23.1, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 106.8, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 4.9, + "samples": 15 + } + } + } + ] + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 91532, + "ok": 91531, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 3051.0, + "error_ratio": 0.0, + "p50_ms": 15.73, + "p95_ms": 36.88, + "p99_ms": 66.56 + }, + "get": { + "requests": 10001, + "ok": 10001, + "errors": {}, + "seconds": 30, + "ok_rps": 333.4, + "error_ratio": 0.0, + "p50_ms": 19.32, + "p95_ms": 55.61, + "p99_ms": 89.59 + }, + "total_rps": 3280.1, + "replica_share": [ + 46354, + 46460 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 88.0, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 41.8, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 186.9, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.8, + "samples": 15 + } + }, + "total_rps_runs": [ + 3384.4, + 3175.8 + ], + "post_p95_runs": [ + 36.88, + 45.16 + ], + "runs": [ + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 91532, + "ok": 91531, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 3051.0, + "error_ratio": 0.0, + "p50_ms": 15.73, + "p95_ms": 36.88, + "p99_ms": 66.56 + }, + "get": { + "requests": 10001, + "ok": 10001, + "errors": {}, + "seconds": 30, + "ok_rps": 333.4, + "error_ratio": 0.0, + "p50_ms": 19.32, + "p95_ms": 55.61, + "p99_ms": 89.59 + }, + "total_rps": 3384.4, + "replica_share": [ + 46354, + 46460 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 88.0, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 41.8, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 186.9, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.8, + "samples": 15 + } + } + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 85874, + "ok": 85872, + "errors": { + "429": 2 + }, + "seconds": 30, + "ok_rps": 2862.4, + "error_ratio": 0.0, + "p50_ms": 13.42, + "p95_ms": 45.16, + "p99_ms": 75.37 + }, + "get": { + "requests": 9403, + "ok": 9402, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 313.4, + "error_ratio": 0.0001, + "p50_ms": 18.73, + "p95_ms": 67.08, + "p99_ms": 98.23 + }, + "total_rps": 3175.8, + "replica_share": [ + 43530, + 43593 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 89.1, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 42.3, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 197.7, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.2, + "samples": 15 + } + } + } + ] + }, + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 68491, + "ok": 68491, + "errors": {}, + "seconds": 30, + "ok_rps": 2283.0, + "error_ratio": 0.0, + "p50_ms": 10.06, + "p95_ms": 80.64, + "p99_ms": 150.73 + }, + "get": { + "requests": 7469, + "ok": 7469, + "errors": {}, + "seconds": 30, + "ok_rps": 249.0, + "error_ratio": 0.0, + "p50_ms": 7.96, + "p95_ms": 93.92, + "p99_ms": 193.45 + }, + "total_rps": 2574.7, + "replica_share": [ + 17267, + 17301, + 17323, + 17372 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 47.5, + "samples": 60 + }, + "lb": { + "avg_cpu_percent_per_container": 36.4, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 170.9, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.9, + "samples": 15 + } + }, + "total_rps_runs": [ + 2532.0, + 2617.4 + ], + "post_p95_runs": [ + 80.64, + 79.28 + ], + "runs": [ + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 68491, + "ok": 68491, + "errors": {}, + "seconds": 30, + "ok_rps": 2283.0, + "error_ratio": 0.0, + "p50_ms": 10.06, + "p95_ms": 80.64, + "p99_ms": 150.73 + }, + "get": { + "requests": 7469, + "ok": 7469, + "errors": {}, + "seconds": 30, + "ok_rps": 249.0, + "error_ratio": 0.0, + "p50_ms": 7.96, + "p95_ms": 93.92, + "p99_ms": 193.45 + }, + "total_rps": 2532.0, + "replica_share": [ + 17267, + 17301, + 17323, + 17372 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 47.5, + "samples": 60 + }, + "lb": { + "avg_cpu_percent_per_container": 36.4, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 170.9, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.9, + "samples": 15 + } + } + }, + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 70791, + "ok": 70791, + "errors": {}, + "seconds": 30, + "ok_rps": 2359.7, + "error_ratio": 0.0, + "p50_ms": 9.31, + "p95_ms": 79.28, + "p99_ms": 107.94 + }, + "get": { + "requests": 7731, + "ok": 7731, + "errors": {}, + "seconds": 30, + "ok_rps": 257.7, + "error_ratio": 0.0, + "p50_ms": 7.77, + "p95_ms": 90.51, + "p99_ms": 183.11 + }, + "total_rps": 2617.4, + "replica_share": [ + 17886, + 17936, + 18004, + 18032 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 48.5, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 36.8, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 172.5, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.8, + "samples": 14 + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/scaling_A_improved_fixed_resources.json b/evidence/scaling_A_improved_fixed_resources.json new file mode 100644 index 00000000..c030745d --- /dev/null +++ b/evidence/scaling_A_improved_fixed_resources.json @@ -0,0 +1,576 @@ +{ + "date": "2026-11-11 17:55:14", + "series": "A", + "tag": "improved_fixed_resources", + "args": { + "series": "A", + "out": "evidence", + "tag": "improved_fixed_resources", + "seconds": 30, + "concurrency": 64, + "procs": 8, + "pg_cpus": 2.0, + "lb_cpus": 2.0, + "repeat": 2 + }, + "results": [ + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 1024, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 36780, + "ok": 36745, + "errors": { + "429": 35 + }, + "seconds": 30, + "ok_rps": 1224.8, + "error_ratio": 0.001, + "p50_ms": 41.98, + "p95_ms": 73.11, + "p99_ms": 106.4 + }, + "get": { + "requests": 3964, + "ok": 3963, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 132.1, + "error_ratio": 0.0003, + "p50_ms": 66.32, + "p95_ms": 114.9, + "p99_ms": 134.98 + }, + "total_rps": 1363.6, + "replica_share": [ + 37279 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 60.1, + "samples": 12 + }, + "lb": { + "avg_cpu_percent_per_container": 13.8, + "samples": 12 + }, + "postgres": { + "avg_cpu_percent_per_container": 63.7, + "samples": 12 + }, + "valkey": { + "avg_cpu_percent_per_container": 3.1, + "samples": 12 + } + }, + "total_rps_runs": [ + 1356.9, + 1370.3 + ], + "post_p95_runs": [ + 73.11, + 67.04 + ], + "runs": [ + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 1024, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 36780, + "ok": 36745, + "errors": { + "429": 35 + }, + "seconds": 30, + "ok_rps": 1224.8, + "error_ratio": 0.001, + "p50_ms": 41.98, + "p95_ms": 73.11, + "p99_ms": 106.4 + }, + "get": { + "requests": 3964, + "ok": 3963, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 132.1, + "error_ratio": 0.0003, + "p50_ms": 66.32, + "p95_ms": 114.9, + "p99_ms": 134.98 + }, + "total_rps": 1356.9, + "replica_share": [ + 37279 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 60.1, + "samples": 12 + }, + "lb": { + "avg_cpu_percent_per_container": 13.8, + "samples": 12 + }, + "postgres": { + "avg_cpu_percent_per_container": 63.7, + "samples": 12 + }, + "valkey": { + "avg_cpu_percent_per_container": 3.1, + "samples": 12 + } + } + }, + { + "config": "1x2.0", + "replicas": 1, + "api_cpus_each": 2.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 1024, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 37146, + "ok": 37113, + "errors": { + "429": 33 + }, + "seconds": 30, + "ok_rps": 1237.1, + "error_ratio": 0.0009, + "p50_ms": 41.04, + "p95_ms": 67.04, + "p99_ms": 108.44 + }, + "get": { + "requests": 3995, + "ok": 3995, + "errors": {}, + "seconds": 30, + "ok_rps": 133.2, + "error_ratio": 0.0, + "p50_ms": 65.15, + "p95_ms": 116.19, + "p99_ms": 137.05 + }, + "total_rps": 1370.3, + "replica_share": [ + 37696 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 60.5, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 13.6, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 63.3, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 3.2, + "samples": 15 + } + } + } + ] + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 512, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 72773, + "ok": 72773, + "errors": {}, + "seconds": 30, + "ok_rps": 2425.8, + "error_ratio": 0.0, + "p50_ms": 21.44, + "p95_ms": 35.99, + "p99_ms": 85.93 + }, + "get": { + "requests": 7950, + "ok": 7950, + "errors": {}, + "seconds": 30, + "ok_rps": 265.0, + "error_ratio": 0.0, + "p50_ms": 24.37, + "p95_ms": 53.55, + "p99_ms": 103.17 + }, + "total_rps": 2657.6, + "replica_share": [ + 36843, + 36962 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 66.2, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 30.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 138.1, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 5.9, + "samples": 15 + } + }, + "total_rps_runs": [ + 2690.8, + 2624.3 + ], + "post_p95_runs": [ + 35.99, + 41.08 + ], + "runs": [ + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 512, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 72773, + "ok": 72773, + "errors": {}, + "seconds": 30, + "ok_rps": 2425.8, + "error_ratio": 0.0, + "p50_ms": 21.44, + "p95_ms": 35.99, + "p99_ms": 85.93 + }, + "get": { + "requests": 7950, + "ok": 7950, + "errors": {}, + "seconds": 30, + "ok_rps": 265.0, + "error_ratio": 0.0, + "p50_ms": 24.37, + "p95_ms": 53.55, + "p99_ms": 103.17 + }, + "total_rps": 2690.8, + "replica_share": [ + 36843, + 36962 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 66.2, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 30.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 138.1, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 5.9, + "samples": 15 + } + } + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "api_memory_mib_each": 512, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 70978, + "ok": 70977, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 2365.9, + "error_ratio": 0.0, + "p50_ms": 22.0, + "p95_ms": 41.08, + "p99_ms": 86.02 + }, + "get": { + "requests": 7752, + "ok": 7752, + "errors": {}, + "seconds": 30, + "ok_rps": 258.4, + "error_ratio": 0.0, + "p50_ms": 24.87, + "p95_ms": 62.36, + "p99_ms": 102.83 + }, + "total_rps": 2624.3, + "replica_share": [ + 35999, + 36034 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 72.0, + "samples": 28 + }, + "lb": { + "avg_cpu_percent_per_container": 33.5, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 148.3, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 6.5, + "samples": 14 + } + } + } + ] + }, + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "api_memory_mib_each": 256, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 72370, + "ok": 72370, + "errors": {}, + "seconds": 30, + "ok_rps": 2412.3, + "error_ratio": 0.0, + "p50_ms": 12.69, + "p95_ms": 67.31, + "p99_ms": 142.31 + }, + "get": { + "requests": 7916, + "ok": 7916, + "errors": {}, + "seconds": 30, + "ok_rps": 263.9, + "error_ratio": 0.0, + "p50_ms": 7.99, + "p95_ms": 69.94, + "p99_ms": 200.78 + }, + "total_rps": 2675.1, + "replica_share": [ + 18332, + 18363, + 18384, + 18399 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 45.9, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 34.5, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 162.3, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.1, + "samples": 14 + } + }, + "total_rps_runs": [ + 2676.2, + 2673.9 + ], + "post_p95_runs": [ + 67.31, + 67.88 + ], + "runs": [ + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "api_memory_mib_each": 256, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 72370, + "ok": 72370, + "errors": {}, + "seconds": 30, + "ok_rps": 2412.3, + "error_ratio": 0.0, + "p50_ms": 12.69, + "p95_ms": 67.31, + "p99_ms": 142.31 + }, + "get": { + "requests": 7916, + "ok": 7916, + "errors": {}, + "seconds": 30, + "ok_rps": 263.9, + "error_ratio": 0.0, + "p50_ms": 7.99, + "p95_ms": 69.94, + "p99_ms": 200.78 + }, + "total_rps": 2676.2, + "replica_share": [ + 18332, + 18363, + 18384, + 18399 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 45.9, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 34.5, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 162.3, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.1, + "samples": 14 + } + } + }, + { + "config": "4x0.5", + "replicas": 4, + "api_cpus_each": 0.5, + "api_cpus_total": 2.0, + "api_memory_mib_each": 256, + "api_memory_mib_total": 1024, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 72314, + "ok": 72314, + "errors": {}, + "seconds": 30, + "ok_rps": 2410.5, + "error_ratio": 0.0, + "p50_ms": 13.28, + "p95_ms": 67.88, + "p99_ms": 116.45 + }, + "get": { + "requests": 7903, + "ok": 7903, + "errors": {}, + "seconds": 30, + "ok_rps": 263.4, + "error_ratio": 0.0, + "p50_ms": 8.06, + "p95_ms": 71.78, + "p99_ms": 190.09 + }, + "total_rps": 2673.9, + "replica_share": [ + 18245, + 18289, + 18294, + 18352 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 47.9, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 38.3, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 177.5, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 8.6, + "samples": 14 + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/scaling_B_base.json b/evidence/scaling_B_base.json new file mode 100644 index 00000000..3f14ed46 --- /dev/null +++ b/evidence/scaling_B_base.json @@ -0,0 +1,753 @@ +{ + "date": "2026-10-03 02:03:23", + "series": "B", + "tag": "base", + "args": { + "series": "B", + "out": "evidence", + "tag": "base", + "seconds": 30, + "concurrency": 64, + "procs": 8, + "pg_cpus": 2.0, + "lb_cpus": 2.0, + "repeat": 2 + }, + "results": [ + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 36157, + "ok": 36125, + "errors": { + "429": 32 + }, + "seconds": 30, + "ok_rps": 1204.2, + "error_ratio": 0.0009, + "p50_ms": 45.03, + "p95_ms": 88.08, + "p99_ms": 117.87 + }, + "get": { + "requests": 3897, + "ok": 3894, + "errors": { + "429": 3 + }, + "seconds": 30, + "ok_rps": 129.8, + "error_ratio": 0.0008, + "p50_ms": 40.61, + "p95_ms": 85.32, + "p99_ms": 110.44 + }, + "total_rps": 1352.3, + "replica_share": [ + 36792 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 97.0, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 15.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 98.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 4.2, + "samples": 15 + } + }, + "total_rps_runs": [ + 1334.0, + 1370.7 + ], + "post_p95_runs": [ + 88.08, + 83.27 + ], + "runs": [ + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 36157, + "ok": 36125, + "errors": { + "429": 32 + }, + "seconds": 30, + "ok_rps": 1204.2, + "error_ratio": 0.0009, + "p50_ms": 45.03, + "p95_ms": 88.08, + "p99_ms": 117.87 + }, + "get": { + "requests": 3897, + "ok": 3894, + "errors": { + "429": 3 + }, + "seconds": 30, + "ok_rps": 129.8, + "error_ratio": 0.0008, + "p50_ms": 40.61, + "p95_ms": 85.32, + "p99_ms": 110.44 + }, + "total_rps": 1334.0, + "replica_share": [ + 36792 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 97.0, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 15.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 98.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 4.2, + "samples": 15 + } + } + }, + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 37150, + "ok": 37117, + "errors": { + "429": 33 + }, + "seconds": 30, + "ok_rps": 1237.2, + "error_ratio": 0.0009, + "p50_ms": 44.63, + "p95_ms": 83.27, + "p99_ms": 108.08 + }, + "get": { + "requests": 4009, + "ok": 4004, + "errors": { + "429": 5 + }, + "seconds": 30, + "ok_rps": 133.5, + "error_ratio": 0.0012, + "p50_ms": 40.35, + "p95_ms": 78.2, + "p99_ms": 97.97 + }, + "total_rps": 1370.7, + "replica_share": [ + 37666 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 96.7, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 15.7, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 98.6, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 4.3, + "samples": 15 + } + } + } + ] + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 65066, + "ok": 65066, + "errors": {}, + "seconds": 30, + "ok_rps": 2168.9, + "error_ratio": 0.0, + "p50_ms": 11.37, + "p95_ms": 74.05, + "p99_ms": 100.65 + }, + "get": { + "requests": 7103, + "ok": 7103, + "errors": {}, + "seconds": 30, + "ok_rps": 236.8, + "error_ratio": 0.0, + "p50_ms": 6.54, + "p95_ms": 69.56, + "p99_ms": 99.34 + }, + "total_rps": 2452.9, + "replica_share": [ + 33043, + 33102 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 86.1, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 29.5, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 161.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 7.6, + "samples": 15 + } + }, + "total_rps_runs": [ + 2405.6, + 2500.2 + ], + "post_p95_runs": [ + 74.05, + 69.49 + ], + "runs": [ + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 65066, + "ok": 65066, + "errors": {}, + "seconds": 30, + "ok_rps": 2168.9, + "error_ratio": 0.0, + "p50_ms": 11.37, + "p95_ms": 74.05, + "p99_ms": 100.65 + }, + "get": { + "requests": 7103, + "ok": 7103, + "errors": {}, + "seconds": 30, + "ok_rps": 236.8, + "error_ratio": 0.0, + "p50_ms": 6.54, + "p95_ms": 69.56, + "p99_ms": 99.34 + }, + "total_rps": 2405.6, + "replica_share": [ + 33043, + 33102 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 86.1, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 29.5, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 161.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 7.6, + "samples": 15 + } + } + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 67641, + "ok": 67639, + "errors": { + "429": 2 + }, + "seconds": 30, + "ok_rps": 2254.6, + "error_ratio": 0.0, + "p50_ms": 11.23, + "p95_ms": 69.49, + "p99_ms": 96.09 + }, + "get": { + "requests": 7367, + "ok": 7367, + "errors": {}, + "seconds": 30, + "ok_rps": 245.6, + "error_ratio": 0.0, + "p50_ms": 6.26, + "p95_ms": 65.43, + "p99_ms": 93.53 + }, + "total_rps": 2500.2, + "replica_share": [ + 34297, + 34330 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 81.7, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 28.7, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 156.9, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 7.5, + "samples": 15 + } + } + } + ] + }, + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 76520, + "ok": 76520, + "errors": {}, + "seconds": 30, + "ok_rps": 2550.7, + "error_ratio": 0.0, + "p50_ms": 7.79, + "p95_ms": 83.29, + "p99_ms": 108.16 + }, + "get": { + "requests": 8333, + "ok": 8333, + "errors": {}, + "seconds": 30, + "ok_rps": 277.8, + "error_ratio": 0.0, + "p50_ms": 4.55, + "p95_ms": 75.03, + "p99_ms": 102.19 + }, + "total_rps": 2741.3, + "replica_share": [ + 19362, + 19387, + 19397, + 19398 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 61.0, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 42.7, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 203.2, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 10.8, + "samples": 14 + } + }, + "total_rps_runs": [ + 2828.4, + 2654.2 + ], + "post_p95_runs": [ + 83.29, + 87.17 + ], + "runs": [ + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 76520, + "ok": 76520, + "errors": {}, + "seconds": 30, + "ok_rps": 2550.7, + "error_ratio": 0.0, + "p50_ms": 7.79, + "p95_ms": 83.29, + "p99_ms": 108.16 + }, + "get": { + "requests": 8333, + "ok": 8333, + "errors": {}, + "seconds": 30, + "ok_rps": 277.8, + "error_ratio": 0.0, + "p50_ms": 4.55, + "p95_ms": 75.03, + "p99_ms": 102.19 + }, + "total_rps": 2828.4, + "replica_share": [ + 19362, + 19387, + 19397, + 19398 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 61.0, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 42.7, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 203.2, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 10.8, + "samples": 14 + } + } + }, + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 71780, + "ok": 71780, + "errors": {}, + "seconds": 30, + "ok_rps": 2392.7, + "error_ratio": 0.0, + "p50_ms": 8.3, + "p95_ms": 87.17, + "p99_ms": 123.36 + }, + "get": { + "requests": 7846, + "ok": 7846, + "errors": {}, + "seconds": 30, + "ok_rps": 261.5, + "error_ratio": 0.0, + "p50_ms": 4.84, + "p95_ms": 79.37, + "p99_ms": 111.11 + }, + "total_rps": 2654.2, + "replica_share": [ + 18263, + 18267, + 18326, + 18331 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 60.6, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 41.6, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 200.7, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 10.4, + "samples": 14 + } + } + } + ] + }, + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 69827, + "ok": 69827, + "errors": {}, + "seconds": 30, + "ok_rps": 2327.6, + "error_ratio": 0.0, + "p50_ms": 9.71, + "p95_ms": 76.16, + "p99_ms": 87.22 + }, + "get": { + "requests": 7623, + "ok": 7623, + "errors": {}, + "seconds": 30, + "ok_rps": 254.1, + "error_ratio": 0.0, + "p50_ms": 6.25, + "p95_ms": 66.55, + "p99_ms": 77.78 + }, + "total_rps": 2592.5, + "replica_share": [ + 8806, + 8816, + 8824, + 8847, + 8876, + 8886, + 8900, + 8908 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 38.2, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 48.3, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 204.3, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.9, + "samples": 14 + } + }, + "total_rps_runs": [ + 2581.7, + 2603.3 + ], + "post_p95_runs": [ + 76.16, + 77.12 + ], + "runs": [ + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 69827, + "ok": 69827, + "errors": {}, + "seconds": 30, + "ok_rps": 2327.6, + "error_ratio": 0.0, + "p50_ms": 9.71, + "p95_ms": 76.16, + "p99_ms": 87.22 + }, + "get": { + "requests": 7623, + "ok": 7623, + "errors": {}, + "seconds": 30, + "ok_rps": 254.1, + "error_ratio": 0.0, + "p50_ms": 6.25, + "p95_ms": 66.55, + "p99_ms": 77.78 + }, + "total_rps": 2581.7, + "replica_share": [ + 8806, + 8816, + 8824, + 8847, + 8876, + 8886, + 8900, + 8908 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 38.2, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 48.3, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 204.3, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.9, + "samples": 14 + } + } + }, + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 70405, + "ok": 70405, + "errors": {}, + "seconds": 30, + "ok_rps": 2346.8, + "error_ratio": 0.0, + "p50_ms": 9.52, + "p95_ms": 77.12, + "p99_ms": 87.97 + }, + "get": { + "requests": 7694, + "ok": 7694, + "errors": {}, + "seconds": 30, + "ok_rps": 256.5, + "error_ratio": 0.0, + "p50_ms": 6.15, + "p95_ms": 66.77, + "p99_ms": 79.42 + }, + "total_rps": 2603.3, + "replica_share": [ + 8893, + 8916, + 8926, + 8939, + 8949, + 8953, + 8955, + 8961 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 37.8, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 48.2, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 204.7, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.8, + "samples": 14 + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/scaling_B_base_pg4.json b/evidence/scaling_B_base_pg4.json new file mode 100644 index 00000000..aa36cdc2 --- /dev/null +++ b/evidence/scaling_B_base_pg4.json @@ -0,0 +1,751 @@ +{ + "date": "2026-10-03 02:08:51", + "series": "B", + "tag": "base_pg4", + "args": { + "series": "B", + "out": "evidence", + "tag": "base_pg4", + "seconds": 30, + "concurrency": 64, + "procs": 8, + "pg_cpus": 4.0, + "lb_cpus": 2.0, + "repeat": 2 + }, + "results": [ + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 38713, + "ok": 38680, + "errors": { + "429": 33 + }, + "seconds": 30, + "ok_rps": 1289.3, + "error_ratio": 0.0009, + "p50_ms": 42.69, + "p95_ms": 79.45, + "p99_ms": 103.58 + }, + "get": { + "requests": 4200, + "ok": 4200, + "errors": {}, + "seconds": 30, + "ok_rps": 140.0, + "error_ratio": 0.0, + "p50_ms": 38.5, + "p95_ms": 76.97, + "p99_ms": 101.05 + }, + "total_rps": 1417.7, + "replica_share": [ + 39136 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 95.3, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 14.5, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 90.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 3.8, + "samples": 15 + } + }, + "total_rps_runs": [ + 1429.3, + 1406.0 + ], + "post_p95_runs": [ + 79.45, + 82.74 + ], + "runs": [ + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 38713, + "ok": 38680, + "errors": { + "429": 33 + }, + "seconds": 30, + "ok_rps": 1289.3, + "error_ratio": 0.0009, + "p50_ms": 42.69, + "p95_ms": 79.45, + "p99_ms": 103.58 + }, + "get": { + "requests": 4200, + "ok": 4200, + "errors": {}, + "seconds": 30, + "ok_rps": 140.0, + "error_ratio": 0.0, + "p50_ms": 38.5, + "p95_ms": 76.97, + "p99_ms": 101.05 + }, + "total_rps": 1429.3, + "replica_share": [ + 39136 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 95.3, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 14.5, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 90.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 3.8, + "samples": 15 + } + } + }, + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 38101, + "ok": 38066, + "errors": { + "429": 35 + }, + "seconds": 30, + "ok_rps": 1268.9, + "error_ratio": 0.0009, + "p50_ms": 43.07, + "p95_ms": 82.74, + "p99_ms": 108.87 + }, + "get": { + "requests": 4120, + "ok": 4114, + "errors": { + "429": 6 + }, + "seconds": 30, + "ok_rps": 137.1, + "error_ratio": 0.0015, + "p50_ms": 38.99, + "p95_ms": 77.77, + "p99_ms": 103.32 + }, + "total_rps": 1406.0, + "replica_share": [ + 38626 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 96.6, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 15.6, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 97.5, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 4.1, + "samples": 15 + } + } + } + ] + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 67573, + "ok": 67572, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 2252.4, + "error_ratio": 0.0, + "p50_ms": 11.05, + "p95_ms": 71.44, + "p99_ms": 98.58 + }, + "get": { + "requests": 7366, + "ok": 7366, + "errors": {}, + "seconds": 30, + "ok_rps": 245.5, + "error_ratio": 0.0, + "p50_ms": 5.84, + "p95_ms": 66.76, + "p99_ms": 94.37 + }, + "total_rps": 2451.1, + "replica_share": [ + 34283, + 34312 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 83.8, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 29.4, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 157.1, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 7.6, + "samples": 15 + } + }, + "total_rps_runs": [ + 2497.9, + 2404.3 + ], + "post_p95_runs": [ + 71.44, + 74.63 + ], + "runs": [ + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 67573, + "ok": 67572, + "errors": { + "429": 1 + }, + "seconds": 30, + "ok_rps": 2252.4, + "error_ratio": 0.0, + "p50_ms": 11.05, + "p95_ms": 71.44, + "p99_ms": 98.58 + }, + "get": { + "requests": 7366, + "ok": 7366, + "errors": {}, + "seconds": 30, + "ok_rps": 245.5, + "error_ratio": 0.0, + "p50_ms": 5.84, + "p95_ms": 66.76, + "p99_ms": 94.37 + }, + "total_rps": 2497.9, + "replica_share": [ + 34283, + 34312 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 83.8, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 29.4, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 157.1, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 7.6, + "samples": 15 + } + } + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 65019, + "ok": 65019, + "errors": {}, + "seconds": 30, + "ok_rps": 2167.3, + "error_ratio": 0.0, + "p50_ms": 10.87, + "p95_ms": 74.63, + "p99_ms": 103.58 + }, + "get": { + "requests": 7109, + "ok": 7109, + "errors": {}, + "seconds": 30, + "ok_rps": 237.0, + "error_ratio": 0.0, + "p50_ms": 6.59, + "p95_ms": 70.52, + "p99_ms": 99.91 + }, + "total_rps": 2404.3, + "replica_share": [ + 33020, + 33044 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 85.1, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 29.9, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 164.4, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 7.9, + "samples": 15 + } + } + } + ] + }, + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 102257, + "ok": 102257, + "errors": {}, + "seconds": 30, + "ok_rps": 3408.6, + "error_ratio": 0.0, + "p50_ms": 7.69, + "p95_ms": 60.19, + "p99_ms": 91.22 + }, + "get": { + "requests": 11249, + "ok": 11249, + "errors": {}, + "seconds": 30, + "ok_rps": 375.0, + "error_ratio": 0.0, + "p50_ms": 4.41, + "p95_ms": 54.41, + "p99_ms": 89.36 + }, + "total_rps": 3833.9, + "replica_share": [ + 25948, + 25951, + 25967, + 26048 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 80.7, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 53.6, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 270.1, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 14.1, + "samples": 14 + } + }, + "total_rps_runs": [ + 3783.5, + 3884.4 + ], + "post_p95_runs": [ + 60.19, + 58.26 + ], + "runs": [ + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 102257, + "ok": 102257, + "errors": {}, + "seconds": 30, + "ok_rps": 3408.6, + "error_ratio": 0.0, + "p50_ms": 7.69, + "p95_ms": 60.19, + "p99_ms": 91.22 + }, + "get": { + "requests": 11249, + "ok": 11249, + "errors": {}, + "seconds": 30, + "ok_rps": 375.0, + "error_ratio": 0.0, + "p50_ms": 4.41, + "p95_ms": 54.41, + "p99_ms": 89.36 + }, + "total_rps": 3783.5, + "replica_share": [ + 25948, + 25951, + 25967, + 26048 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 80.7, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 53.6, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 270.1, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 14.1, + "samples": 14 + } + } + }, + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 104924, + "ok": 104924, + "errors": {}, + "seconds": 30, + "ok_rps": 3497.5, + "error_ratio": 0.0, + "p50_ms": 7.51, + "p95_ms": 58.26, + "p99_ms": 87.07 + }, + "get": { + "requests": 11609, + "ok": 11609, + "errors": {}, + "seconds": 30, + "ok_rps": 387.0, + "error_ratio": 0.0, + "p50_ms": 4.36, + "p95_ms": 53.0, + "p99_ms": 87.57 + }, + "total_rps": 3884.4, + "replica_share": [ + 26670, + 26705, + 26735, + 26743 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 78.9, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 53.2, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 271.1, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 13.6, + "samples": 14 + } + } + } + ] + }, + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 127664, + "ok": 127664, + "errors": {}, + "seconds": 30, + "ok_rps": 4255.5, + "error_ratio": 0.0, + "p50_ms": 9.69, + "p95_ms": 35.08, + "p99_ms": 59.56 + }, + "get": { + "requests": 14123, + "ok": 14123, + "errors": {}, + "seconds": 30, + "ok_rps": 470.8, + "error_ratio": 0.0, + "p50_ms": 6.03, + "p95_ms": 25.49, + "p99_ms": 57.83 + }, + "total_rps": 4636.1, + "replica_share": [ + 16143, + 16187, + 16195, + 16201, + 16204, + 16216, + 16231, + 16268 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 69.3, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 77.6, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 394.7, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 18.6, + "samples": 14 + } + }, + "total_rps_runs": [ + 4726.2, + 4546.0 + ], + "post_p95_runs": [ + 35.08, + 36.84 + ], + "runs": [ + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 127664, + "ok": 127664, + "errors": {}, + "seconds": 30, + "ok_rps": 4255.5, + "error_ratio": 0.0, + "p50_ms": 9.69, + "p95_ms": 35.08, + "p99_ms": 59.56 + }, + "get": { + "requests": 14123, + "ok": 14123, + "errors": {}, + "seconds": 30, + "ok_rps": 470.8, + "error_ratio": 0.0, + "p50_ms": 6.03, + "p95_ms": 25.49, + "p99_ms": 57.83 + }, + "total_rps": 4726.2, + "replica_share": [ + 16143, + 16187, + 16195, + 16201, + 16204, + 16216, + 16231, + 16268 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 69.3, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 77.6, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 394.7, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 18.6, + "samples": 14 + } + } + }, + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 4.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 122818, + "ok": 122818, + "errors": {}, + "seconds": 30, + "ok_rps": 4093.9, + "error_ratio": 0.0, + "p50_ms": 9.95, + "p95_ms": 36.84, + "p99_ms": 62.32 + }, + "get": { + "requests": 13562, + "ok": 13562, + "errors": {}, + "seconds": 30, + "ok_rps": 452.1, + "error_ratio": 0.0, + "p50_ms": 6.11, + "p95_ms": 27.26, + "p99_ms": 62.24 + }, + "total_rps": 4546.0, + "replica_share": [ + 15520, + 15532, + 15538, + 15539, + 15549, + 15569, + 15622, + 15638 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 68.1, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 78.5, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 386.0, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 18.1, + "samples": 14 + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/evidence/scaling_B_improved.json b/evidence/scaling_B_improved.json new file mode 100644 index 00000000..bd653081 --- /dev/null +++ b/evidence/scaling_B_improved.json @@ -0,0 +1,749 @@ +{ + "date": "2026-10-03 02:18:45", + "series": "B", + "tag": "improved", + "args": { + "series": "B", + "out": "evidence", + "tag": "improved", + "seconds": 30, + "concurrency": 64, + "procs": 8, + "pg_cpus": 2.0, + "lb_cpus": 2.0, + "repeat": 2 + }, + "results": [ + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 60581, + "ok": 60549, + "errors": { + "429": 32 + }, + "seconds": 30, + "ok_rps": 2018.3, + "error_ratio": 0.0005, + "p50_ms": 25.71, + "p95_ms": 44.31, + "p99_ms": 67.32 + }, + "get": { + "requests": 6611, + "ok": 6607, + "errors": { + "429": 4 + }, + "seconds": 30, + "ok_rps": 220.2, + "error_ratio": 0.0006, + "p50_ms": 41.8, + "p95_ms": 63.71, + "p99_ms": 91.71 + }, + "total_rps": 2153.1, + "replica_share": [ + 61428 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 95.9, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 23.7, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 111.3, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 5.2, + "samples": 15 + } + }, + "total_rps_runs": [ + 2238.5, + 2067.6 + ], + "post_p95_runs": [ + 44.31, + 46.43 + ], + "runs": [ + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 60581, + "ok": 60549, + "errors": { + "429": 32 + }, + "seconds": 30, + "ok_rps": 2018.3, + "error_ratio": 0.0005, + "p50_ms": 25.71, + "p95_ms": 44.31, + "p99_ms": 67.32 + }, + "get": { + "requests": 6611, + "ok": 6607, + "errors": { + "429": 4 + }, + "seconds": 30, + "ok_rps": 220.2, + "error_ratio": 0.0006, + "p50_ms": 41.8, + "p95_ms": 63.71, + "p99_ms": 91.71 + }, + "total_rps": 2238.5, + "replica_share": [ + 61428 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 95.9, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 23.7, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 111.3, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 5.2, + "samples": 15 + } + } + }, + { + "config": "1x1.0", + "replicas": 1, + "api_cpus_each": 1.0, + "api_cpus_total": 1.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 55934, + "ok": 55906, + "errors": { + "429": 28 + }, + "seconds": 30, + "ok_rps": 1863.5, + "error_ratio": 0.0005, + "p50_ms": 27.26, + "p95_ms": 46.43, + "p99_ms": 69.24 + }, + "get": { + "requests": 6123, + "ok": 6123, + "errors": {}, + "seconds": 30, + "ok_rps": 204.1, + "error_ratio": 0.0, + "p50_ms": 43.73, + "p95_ms": 70.09, + "p99_ms": 94.81 + }, + "total_rps": 2067.6, + "replica_share": [ + 56761 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 93.0, + "samples": 15 + }, + "lb": { + "avg_cpu_percent_per_container": 22.3, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 109.2, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 5.1, + "samples": 15 + } + } + } + ] + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 90895, + "ok": 90895, + "errors": {}, + "seconds": 30, + "ok_rps": 3029.8, + "error_ratio": 0.0, + "p50_ms": 13.56, + "p95_ms": 40.44, + "p99_ms": 68.62 + }, + "get": { + "requests": 9942, + "ok": 9942, + "errors": {}, + "seconds": 30, + "ok_rps": 331.4, + "error_ratio": 0.0, + "p50_ms": 17.19, + "p95_ms": 59.38, + "p99_ms": 90.83 + }, + "total_rps": 3312.7, + "replica_share": [ + 46049, + 46269 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 91.8, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 40.7, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 188.8, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.1, + "samples": 15 + } + }, + "total_rps_runs": [ + 3361.2, + 3264.2 + ], + "post_p95_runs": [ + 40.44, + 41.53 + ], + "runs": [ + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 90895, + "ok": 90895, + "errors": {}, + "seconds": 30, + "ok_rps": 3029.8, + "error_ratio": 0.0, + "p50_ms": 13.56, + "p95_ms": 40.44, + "p99_ms": 68.62 + }, + "get": { + "requests": 9942, + "ok": 9942, + "errors": {}, + "seconds": 30, + "ok_rps": 331.4, + "error_ratio": 0.0, + "p50_ms": 17.19, + "p95_ms": 59.38, + "p99_ms": 90.83 + }, + "total_rps": 3361.2, + "replica_share": [ + 46049, + 46269 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 91.8, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 40.7, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 188.8, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.1, + "samples": 15 + } + } + }, + { + "config": "2x1.0", + "replicas": 2, + "api_cpus_each": 1.0, + "api_cpus_total": 2.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 88264, + "ok": 88264, + "errors": {}, + "seconds": 30, + "ok_rps": 2942.1, + "error_ratio": 0.0, + "p50_ms": 13.7, + "p95_ms": 41.53, + "p99_ms": 74.72 + }, + "get": { + "requests": 9661, + "ok": 9661, + "errors": {}, + "seconds": 30, + "ok_rps": 322.0, + "error_ratio": 0.0, + "p50_ms": 19.16, + "p95_ms": 63.92, + "p99_ms": 96.5 + }, + "total_rps": 3264.2, + "replica_share": [ + 44929, + 45102 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 91.7, + "samples": 30 + }, + "lb": { + "avg_cpu_percent_per_container": 42.2, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 189.3, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.3, + "samples": 15 + } + } + } + ] + }, + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 81714, + "ok": 81714, + "errors": {}, + "seconds": 30, + "ok_rps": 2723.8, + "error_ratio": 0.0, + "p50_ms": 10.9, + "p95_ms": 61.52, + "p99_ms": 69.63 + }, + "get": { + "requests": 8955, + "ok": 8955, + "errors": {}, + "seconds": 30, + "ok_rps": 298.5, + "error_ratio": 0.0, + "p50_ms": 10.84, + "p95_ms": 65.71, + "p99_ms": 80.41 + }, + "total_rps": 2994.9, + "replica_share": [ + 20664, + 20677, + 20745, + 20784 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 56.5, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 44.3, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 203.7, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.9, + "samples": 14 + } + }, + "total_rps_runs": [ + 3022.3, + 2967.6 + ], + "post_p95_runs": [ + 61.52, + 61.62 + ], + "runs": [ + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 81714, + "ok": 81714, + "errors": {}, + "seconds": 30, + "ok_rps": 2723.8, + "error_ratio": 0.0, + "p50_ms": 10.9, + "p95_ms": 61.52, + "p99_ms": 69.63 + }, + "get": { + "requests": 8955, + "ok": 8955, + "errors": {}, + "seconds": 30, + "ok_rps": 298.5, + "error_ratio": 0.0, + "p50_ms": 10.84, + "p95_ms": 65.71, + "p99_ms": 80.41 + }, + "total_rps": 3022.3, + "replica_share": [ + 20664, + 20677, + 20745, + 20784 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 56.5, + "samples": 56 + }, + "lb": { + "avg_cpu_percent_per_container": 44.3, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 203.7, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.9, + "samples": 14 + } + } + }, + { + "config": "4x1.0", + "replicas": 4, + "api_cpus_each": 1.0, + "api_cpus_total": 4.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 80242, + "ok": 80242, + "errors": {}, + "seconds": 30, + "ok_rps": 2674.7, + "error_ratio": 0.0, + "p50_ms": 11.45, + "p95_ms": 61.62, + "p99_ms": 66.63 + }, + "get": { + "requests": 8785, + "ok": 8785, + "errors": {}, + "seconds": 30, + "ok_rps": 292.8, + "error_ratio": 0.0, + "p50_ms": 11.52, + "p95_ms": 63.05, + "p99_ms": 71.74 + }, + "total_rps": 2967.6, + "replica_share": [ + 20272, + 20322, + 20361, + 20409 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 55.0, + "samples": 60 + }, + "lb": { + "avg_cpu_percent_per_container": 45.8, + "samples": 15 + }, + "postgres": { + "avg_cpu_percent_per_container": 203.4, + "samples": 15 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.7, + "samples": 15 + } + } + } + ] + }, + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 63544, + "ok": 63544, + "errors": {}, + "seconds": 30, + "ok_rps": 2118.1, + "error_ratio": 0.0, + "p50_ms": 11.31, + "p95_ms": 75.99, + "p99_ms": 79.73 + }, + "get": { + "requests": 6951, + "ok": 6951, + "errors": {}, + "seconds": 30, + "ok_rps": 231.7, + "error_ratio": 0.0, + "p50_ms": 6.76, + "p95_ms": 69.51, + "p99_ms": 73.4 + }, + "total_rps": 2380.5, + "replica_share": [ + 7965, + 8023, + 8041, + 8042, + 8047, + 8060, + 8065, + 8134 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 34.6, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 44.7, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 206.5, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.0, + "samples": 14 + } + }, + "total_rps_runs": [ + 2349.8, + 2411.2 + ], + "post_p95_runs": [ + 75.99, + 74.87 + ], + "runs": [ + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 63544, + "ok": 63544, + "errors": {}, + "seconds": 30, + "ok_rps": 2118.1, + "error_ratio": 0.0, + "p50_ms": 11.31, + "p95_ms": 75.99, + "p99_ms": 79.73 + }, + "get": { + "requests": 6951, + "ok": 6951, + "errors": {}, + "seconds": 30, + "ok_rps": 231.7, + "error_ratio": 0.0, + "p50_ms": 6.76, + "p95_ms": 69.51, + "p99_ms": 73.4 + }, + "total_rps": 2349.8, + "replica_share": [ + 7965, + 8023, + 8041, + 8042, + 8047, + 8060, + 8065, + 8134 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 34.6, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 44.7, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 206.5, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.0, + "samples": 14 + } + } + }, + { + "config": "8x1.0", + "replicas": 8, + "api_cpus_each": 1.0, + "api_cpus_total": 8.0, + "postgres_cpus": 2.0, + "lb_cpus": 2.0, + "concurrency": 64, + "client_processes": 8, + "post": { + "requests": 65208, + "ok": 65208, + "errors": {}, + "seconds": 30, + "ok_rps": 2173.6, + "error_ratio": 0.0, + "p50_ms": 11.53, + "p95_ms": 74.87, + "p99_ms": 78.79 + }, + "get": { + "requests": 7128, + "ok": 7128, + "errors": {}, + "seconds": 30, + "ok_rps": 237.6, + "error_ratio": 0.0, + "p50_ms": 6.75, + "p95_ms": 68.3, + "p99_ms": 71.85 + }, + "total_rps": 2411.2, + "replica_share": [ + 8193, + 8202, + 8207, + 8233, + 8264, + 8279, + 8288, + 8290 + ], + "docker_stats": { + "api": { + "avg_cpu_percent_per_container": 34.8, + "samples": 112 + }, + "lb": { + "avg_cpu_percent_per_container": 47.2, + "samples": 14 + }, + "postgres": { + "avg_cpu_percent_per_container": 205.8, + "samples": 14 + }, + "valkey": { + "avg_cpu_percent_per_container": 9.5, + "samples": 14 + } + } + } + ] + } + ] +} \ No newline at end of file diff --git a/infra/haproxy/errors/503.http b/infra/haproxy/errors/503.http new file mode 100644 index 00000000..92c1df2a --- /dev/null +++ b/infra/haproxy/errors/503.http @@ -0,0 +1,9 @@ +HTTP/1.0 503 Service Unavailable +Cache-Control: no-cache +Connection: close +Content-Type: application/json +Retry-After: 1 +X-Request-Id: lb-unavailable +Content-Length: 117 + +{"error":{"code":"unavailable","message":"Нет доступных реплик API","request_id":"lb-unavailable"}} \ No newline at end of file diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..fe145bc3 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,54 @@ +# Балансировщик lab3: HAProxy 3.0. Единственная публичная точка входа; API-реплики наружу не опубликованы. +global + log stdout format raw local0 notice + maxconn 4096 + +defaults + mode http + log global + option dontlognull + option forwardfor + # connect короткий: недоступная реплика быстро заменяется другой (redispatch) + timeout connect 1s + timeout client 30s + timeout server 30s + timeout http-request 10s + timeout http-keep-alive 5s + timeout queue 5s + retries 3 + option redispatch 1 + retry-on conn-failure empty-response response-timeout 0rtt-rejected + # Ошибки, созданные самим LB, отдаются в контрактном формате (JSON + Retry-After + X-Request-Id). + errorfile 502 /usr/local/etc/haproxy/errors/503.http + errorfile 503 /usr/local/etc/haproxy/errors/503.http + errorfile 504 /usr/local/etc/haproxy/errors/503.http + +# Docker DNS (127.0.0.11): адреса реплик API обновляются без перезапуска LB (scale up/down, пересоздание контейнеров). +resolvers docker + nameserver dns 127.0.0.11:53 + accepted_payload_size 8192 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold nx 2s + hold other 2s + hold obsolete 2s + +frontend public + bind :8080 + default_backend api + +# Внутренний healthcheck самого LB (контейнерный HEALTHCHECK); наружу не публикуется. +frontend lb_health + bind :8081 + monitor-uri /healthz + +backend api + balance roundrobin + # Проверка готовности каждые 1 с; 2 неудачи подряд -> сервер исключён (обнаружение отказа <= ~3 с), 2 успеха -> возвращён. + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + default-server check inter 1s fall 2 rise 2 resolvers docker init-addr none + server-template api 1-8 api:8080 diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..2effe2e4 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,52 @@ +-- 0001: пользователи, чаты, участники, сообщения, сессии. +-- Источник истины для всех данных — PostgreSQL. Каждая миграция выполняется в транзакции. +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100) +); +-- Уникальность username: гонка двух регистраций решается ограничением БД, а не проверкой в приложении. +CREATE UNIQUE INDEX users_username_key ON users (username); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- chat_created_at денормализован ради keyset-пагинации списка чатов пользователя без join+sort. +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + chat_created_at timestamptz NOT NULL, + ord smallint NOT NULL, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_chats ON chat_members (user_id, chat_created_at, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users (id), + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_len CHECK (char_length(text) <= 4000) +); +-- История чата: WHERE chat_id = ? AND (created_at, id) > (?, ?) ORDER BY created_at, id LIMIT n +-- обслуживается этим индексом без сортировки и без сканирования всей истории. +CREATE INDEX messages_chat_history ON messages (chat_id, created_at, id); + +-- Серверные сессии. В БД хранится только SHA-256 токена: утечка БД не раскрывает рабочие токены. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT now(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz +); +CREATE INDEX sessions_expires_at ON sessions (expires_at); diff --git a/migrations/0002_idempotency.sql b/migrations/0002_idempotency.sql new file mode 100644 index 00000000..a18c45f7 --- /dev/null +++ b/migrations/0002_idempotency.sql @@ -0,0 +1,17 @@ +-- 0002: общая идемпотентность отправки сообщений (lab3). +-- Область уникальности: пользователь + чат + ключ (метод и путь фиксированы: POST /chats/{id}/messages). +-- Запись ключа и сообщения — одна транзакция: побочный эффект и результат фиксируются атомарно. +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + idem_key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, chat_id, idem_key), + CONSTRAINT idem_key_format CHECK (idem_key ~ '^[A-Za-z0-9._:-]{1,128}$'), + -- сообщение вставляется после ключа в той же транзакции -> проверка внешнего ключа откладывается до COMMIT + CONSTRAINT idem_message_fk FOREIGN KEY (message_id) REFERENCES messages (id) ON DELETE CASCADE + DEFERRABLE INITIALLY DEFERRED +); +CREATE INDEX idempotency_keys_created_at ON idempotency_keys (created_at); diff --git a/migrations/0003_send_message_function.sql b/migrations/0003_send_message_function.sql new file mode 100644 index 00000000..f9e8dd85 --- /dev/null +++ b/migrations/0003_send_message_function.sql @@ -0,0 +1,35 @@ +-- 0003: отправка сообщения одной серверной функцией (оптимизация по итогам трека 1: меньше обращений к БД на запрос). +-- Семантика идентична версии «несколько запросов в транзакции» (migrations 0002 + код приложения): +-- проверка членства -> просроченный ключ удаляется -> INSERT ключа (ON CONFLICT DO NOTHING) -> INSERT сообщения. +-- Функция выполняется атомарно (один вызов = одна транзакция); гонку одинаковых ключей по-прежнему решает уникальный индекс. +CREATE FUNCTION send_message(p_chat uuid, p_user uuid, p_key text, p_hash bytea, p_text text, p_ttl double precision) +RETURNS TABLE (outcome text, msg_id uuid, msg_chat uuid, msg_sender uuid, msg_text text, msg_created timestamptz, + request_hash bytea) +LANGUAGE plpgsql AS $$ +DECLARE + v_new uuid := gen_random_uuid(); + v_ins uuid; + v_created timestamptz; +BEGIN + IF NOT EXISTS (SELECT 1 FROM chat_members m WHERE m.chat_id = p_chat AND m.user_id = p_user) THEN + RETURN QUERY SELECT 'not_member'::text, NULL::uuid, NULL::uuid, NULL::uuid, NULL::text, NULL::timestamptz, NULL::bytea; + RETURN; + END IF; + DELETE FROM idempotency_keys k + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key + AND k.created_at < now() - make_interval(secs => p_ttl); + INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id) + VALUES (p_user, p_chat, p_key, p_hash, v_new) + ON CONFLICT DO NOTHING + RETURNING message_id INTO v_ins; + IF v_ins IS NOT NULL THEN + INSERT INTO messages (id, chat_id, sender_id, text) VALUES (v_new, p_chat, p_user, p_text) + RETURNING created_at INTO v_created; + RETURN QUERY SELECT 'created'::text, v_new, p_chat, p_user, p_text, v_created, p_hash; + ELSE + RETURN QUERY SELECT 'replay'::text, m.id, m.chat_id, m.sender_id, m.text, m.created_at, k.request_hash + FROM idempotency_keys k JOIN messages m ON m.id = k.message_id + WHERE k.user_id = p_user AND k.chat_id = p_chat AND k.idem_key = p_key; + END IF; +END; +$$; diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..427d0eed --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +argon2-cffi==23.1.0 +argon2-cffi-bindings==26.1.0 +asyncpg==0.30.0 +cffi==2.1.1 +click==8.5.0 +h11==0.16.0 +httptools==0.8.0 +pycparser==3.0 +redis==5.2.1 +typing_extensions==4.16.0 +uvicorn==0.34.0 +uvloop==0.21.0; sys_platform != "win32" diff --git a/research/drain_timing.py b/research/drain_timing.py new file mode 100644 index 00000000..1c457820 --- /dev/null +++ b/research/drain_timing.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Уровень 4: согласованность health/readiness/SIGTERM и время обнаружения отказа балансировщиком. + +Опыт 1 (штатная остановка, SIGTERM): под нагрузкой ~100 запросов/с на одну реплику посылаем SIGTERM; фиксируем + t_ready_fail — когда реплика перестала быть в ротации LB (последний ответ с её X-Instance-Id), + t_exit — когда контейнер завершился, число ошибок клиента за это время (ожидаем 0). +Опыт 2 (внезапный отказ, SIGKILL): время до прекращения ошибок клиента = время обнаружения отказа LB. +Повторяется N раз. Результат: evidence/drain_timing.json. +""" +import json +import os +import subprocess +import sys +import threading +import time +import urllib.request + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +BASE = "http://127.0.0.1:8080" +PROJECT = "mtusi-lab3" + + +def sh(*args): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip() + + +def api_ids(): + return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, "--filter", "label=com.docker.compose.service=api").split() + + +class Prober(threading.Thread): + def __init__(self): + super().__init__(daemon=True) + self.events, self.stop = [], threading.Event() + + def run(self): + while not self.stop.is_set(): + t = time.time() + try: + with urllib.request.urlopen(BASE + "/health/live", timeout=2) as r: + self.events.append((t, r.status, r.headers.get("X-Instance-Id"))) + except Exception as exc: + self.events.append((t, type(exc).__name__, None)) + time.sleep(0.01) + + +def trial(kind): + ids = api_ids() + target = ids[0] + prober = Prober() + prober.start() + time.sleep(2) + t0 = time.time() + if kind == "sigterm": + sh("docker", "kill", "--signal=SIGTERM", target) + else: + sh("docker", "kill", target) + deadline = time.time() + 30 + t_exit = None + while time.time() < deadline: + if sh("docker", "inspect", "-f", "{{.State.Running}}", target) == "false": + t_exit = time.time() - t0 + break + time.sleep(0.05) + time.sleep(3) + prober.stop.set() + prober.join() + served = [e for e in prober.events if e[0] >= t0 and e[2] == target] + last_served = max((e[0] for e in served), default=t0) - t0 + errors = [e for e in prober.events if e[0] >= t0 and e[1] != 200] + last_error = max((e[0] for e in errors), default=t0) - t0 + sh("docker", "rm", "-f", target) + sh("sh", "scripts/restore-replicas.sh") + time.sleep(8) + return {"kind": kind, "seconds_until_replica_stops_serving": round(last_served, 2), + "seconds_until_container_exit": round(t_exit, 2) if t_exit else None, "client_errors": len(errors), + "seconds_until_errors_stop": round(last_error, 2) if errors else 0.0} + + +def main(): + n = int(sys.argv[1]) if len(sys.argv) > 1 else 3 + results = [trial(k) for k in ("sigterm", "sigkill") for _ in range(n)] + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "lb_health_check": "inter 1s, fall 2, rise 2", + "drain_seconds": 4, "graceful_shutdown_seconds": 8, "trials": results} + with open(os.path.join(ROOT, "evidence", "drain_timing.json"), "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/research/failover_sequence.py b/research/failover_sequence.py new file mode 100644 index 00000000..1b06bde3 --- /dev/null +++ b/research/failover_sequence.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Уровень 4: последовательная остановка каждой API-реплики под нагрузкой (с восстановлением предыдущей). + +Сценарий (стенд `docker compose up -d` с 2 репликами; нагрузка идёт через LB непрерывно ~70 с): + t=10 остановить реплику A (docker stop --time 0 = мгновенный отказ процесса) + t=25 восстановить (scripts/restore-replicas.sh) + t=40 остановить реплику B + t=55 восстановить +Для каждого отказа считаем: RPS до/во время/после, число ошибочных попыток, время до прекращения ошибок +(= обнаружение отказа балансировщиком), сохранность подтверждённых сообщений и отсутствие дублей. + + python research/failover_sequence.py --out evidence +""" +import argparse +import asyncio +import json +import os +import subprocess +import sys +import threading +import time + +sys.path.insert(0, os.path.dirname(__file__)) +import loadgen # noqa: E402 + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +BASE = "http://127.0.0.1:8080" + + +def sh(*args): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True).stdout.strip() + + +def api_ids(): + return sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=mtusi-lab3", + "--filter", "label=com.docker.compose.service=api").split() + + +def scenario(rec, plan): + """Выполняется в отдельном потоке: действия по расписанию относительно rec.t0.""" + for at, action in plan: + time.sleep(max(0, rec.t0 + at - time.time())) + t = time.time() - rec.t0 + if action == "stop": + target = api_ids()[0] + subprocess.run(["docker", "stop", "--time", "0", target], capture_output=True) + rec.marks.append({"t": round(t, 2), "action": "stop", "container": target}) + else: + subprocess.run(["sh", "scripts/restore-replicas.sh"], cwd=ROOT, capture_output=True) + rec.marks.append({"t": round(t, 2), "action": "restore"}) + + +async def main(args): + stand = loadgen.Stand(BASE) + await stand.setup() + rec = loadgen.Recorder() + rec.marks = [] + plan = [(10, "stop"), (25, "restore"), (40, "stop"), (55, "restore")] + threading.Thread(target=scenario, args=(rec, plan), daemon=True).start() + await loadgen.run_load(stand, rec, args.seconds, args.concurrency) + await asyncio.sleep(2) + check = await loadgen.verify(stand, rec) + stops = [m for m in rec.marks if m["action"] == "stop"] + restores = [m for m in rec.marks if m["action"] == "restore"] + windows = [] + for i, stop in enumerate(stops): + t0 = stop["t"] + errs = [e for e in rec.events if e[1] == "POST" and e[2] != 201 and t0 - 0.2 <= e[0] <= t0 + 10] + last_err = max((e[0] for e in errs), default=t0) + restore_t = restores[i]["t"] if i < len(restores) else t0 + 15 + windows.append({"stopped_container": stop["container"], "stop_at_s": t0, + "rps_before": loadgen.summarize(rec, t0 - 8, t0 - 0.5).get("ok_rps"), + "rps_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("ok_rps"), + "rps_after_restore": loadgen.summarize(rec, restore_t + 8, restore_t + 14).get("ok_rps"), + "failed_attempts_within_10s": len(errs), + "error_statuses": sorted({str(e[2]) for e in errs}), + "seconds_until_errors_stop": round(last_err - t0, 2) if errs else 0.0, + "p95_ms_one_replica": loadgen.summarize(rec, t0 + 5, restore_t - 1).get("p95_ms")}) + series = loadgen.per_second(rec) + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "args": vars(args), "marks": rec.marks, "windows": windows, + "total": loadgen.summarize(rec), "verification": check, + "invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0, + "timeline": [{"t": t, "ok": b["ok"], "err": b["err"], "p95_ms": loadgen.pct(b["lat"], .95), "instances": len(b["instances"])} + for t, b in sorted(series.items())]} + with open(os.path.join(args.out, "failover_sequence.json"), "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + plot(report, os.path.join(args.out, "failover_sequence.png")) + print(json.dumps({k: v for k, v in report.items() if k != "timeline"}, ensure_ascii=False, indent=2)) + + +def plot(report, path): + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + tl = report["timeline"] + t = [x["t"] for x in tl] + fig, ax = plt.subplots(3, 1, figsize=(11, 8.5), sharex=True) + ax[0].plot(t, [x["ok"] for x in tl], color="#2e7d32"); ax[0].set_ylabel("успешных POST/с"); ax[0].set_title("Пропускная способность") + ax[1].bar(t, [x["err"] for x in tl], color="#c62828"); ax[1].set_ylabel("ошибочных попыток/с"); ax[1].set_title("Ошибки (до повтора с тем же ключом)") + ax[2].plot(t, [x["p95_ms"] or 0 for x in tl], color="#1565c0"); ax[2].set_ylabel("p95, мс"); ax[2].set_xlabel("время, с"); ax[2].set_title("Латентность успешных POST") + for a in ax: + for m in report["marks"]: + a.axvline(m["t"], color="red" if m["action"] == "stop" else "green", ls="--", lw=1) + ax[0].text(0.01, 0.05, "красная линия — остановка реплики, зелёная — восстановление", transform=ax[0].transAxes, fontsize=9) + fig.tight_layout(); fig.savefig(path, dpi=110) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="evidence") + ap.add_argument("--seconds", type=int, default=70) + ap.add_argument("--concurrency", type=int, default=16) + asyncio.run(main(ap.parse_args())) diff --git a/research/loadgen.py b/research/loadgen.py new file mode 100644 index 00000000..df38c7be --- /dev/null +++ b/research/loadgen.py @@ -0,0 +1,172 @@ +"""Генератор нагрузки для экспериментов lab3+ (asyncio + aiohttp, closed-loop). + +Каждый воркер выполняет цикл: POST сообщения с новым Idempotency-Key (90 %) или GET первой страницы истории (10 %). +При ошибке соединения/5xx POST повторяется с ТЕМ ЖЕ ключом (до RETRIES раз) — именно так клиент безопасно +повторяет неподтверждённую запись. Результат: по секундам успехи/ошибки/латентности, список подтверждённых +сообщений (для сверки с историей), число повторов. +""" +import asyncio +import base64 +import random +import time +import uuid + +import aiohttp + +RETRIES = 3 + + +class Stand: + """Набор пользователей/чатов, созданных через API (через LB), для одного эксперимента.""" + + def __init__(self, base, users=20, chats=20): + self.base, self.n_users, self.n_chats = base, users, chats + self.users, self.chats = [], [] + + async def setup(self): + timeout = aiohttp.ClientTimeout(total=30) + async with aiohttp.ClientSession(timeout=timeout) as s: + for i in range(self.n_users): + name = "ld_%s" % uuid.uuid4().hex[:14] + pw = "load-test-password-" + uuid.uuid4().hex[:8] + async with s.post(self.base + "/api/v1/users", json={"username": name, "display_name": "load %d" % i, "password": pw}) as r: + user = await r.json() + assert r.status == 201, user + cred = base64.b64encode(("%s:%s" % (name, pw)).encode()).decode() + async with s.post(self.base + "/api/v1/auth/sessions", headers={"Authorization": "Basic " + cred}) as r: + sess = await r.json() + assert r.status == 201, sess + self.users.append({"id": user["id"], "token": sess["token"]}) + for i in range(self.n_chats): + members = random.sample(self.users, 3) + owner = members[0] + async with s.post(self.base + "/api/v1/chats", json={"title": "load %d" % i, "member_ids": [m["id"] for m in members]}, + headers={"Authorization": "Bearer " + owner["token"]}) as r: + chat = await r.json() + assert r.status == 201, chat + self.chats.append({"id": chat["id"], "members": members}) + + +class Recorder: + def __init__(self): + self.t0 = time.time() + self.events = [] # (t, kind, status, latency_ms, instance) + self.confirmed = [] # (chat_id, message_id, text) + self.retries = 0 + self.final_failures = 0 + self.unconfirmed = [] # (chat_id, key, text) — финально не подтверждены (могли записаться!) + + +async def worker(stand, rec, deadline_fn, read_ratio=0.1, text_size=0, think=0.0): + connector = aiohttp.TCPConnector(limit=2) + timeout = aiohttp.ClientTimeout(total=5) + async with aiohttp.ClientSession(connector=connector, timeout=timeout) as s: + n = 0 + while not deadline_fn(): + chat = random.choice(stand.chats) + user = random.choice(chat["members"]) + hdr = {"Authorization": "Bearer " + user["token"]} + if random.random() < read_ratio: + t = time.perf_counter() + try: + async with s.get("%s/api/v1/chats/%s/messages?limit=20" % (stand.base, chat["id"]), headers=hdr) as r: + await r.read() + rec.events.append((time.time() - rec.t0, "GET", r.status, (time.perf_counter() - t) * 1000, r.headers.get("X-Instance-Id"))) + except Exception as exc: + rec.events.append((time.time() - rec.t0, "GET", type(exc).__name__, (time.perf_counter() - t) * 1000, None)) + continue + key = uuid.uuid4().hex + text = "m-%s-%d" % (key[:8], n) + "я" * text_size + n += 1 + for attempt in range(RETRIES + 1): + t = time.perf_counter() + try: + async with s.post("%s/api/v1/chats/%s/messages" % (stand.base, chat["id"]), json={"text": text}, + headers=dict(hdr, **{"Idempotency-Key": key})) as r: + body = await r.json(content_type=None) + status, inst = r.status, r.headers.get("X-Instance-Id") + except Exception as exc: + status, inst, body = type(exc).__name__, None, None + rec.events.append((time.time() - rec.t0, "POST", status, (time.perf_counter() - t) * 1000, inst)) + if status == 201: + rec.confirmed.append((chat["id"], body["id"], text)) + break + if attempt < RETRIES and (not isinstance(status, int) or status >= 500 or status == 429): + rec.retries += 1 + await asyncio.sleep(0.05 * (attempt + 1)) + continue + rec.final_failures += 1 + rec.unconfirmed.append((chat["id"], key, text)) + break + if think: + await asyncio.sleep(think) + + +async def run_load(stand, rec, seconds, concurrency, **kw): + end = time.time() + seconds + await asyncio.gather(*(worker(stand, rec, lambda: time.time() > end, **kw) for _ in range(concurrency))) + + +async def verify(stand, rec): + """Сверка: каждое подтверждённое сообщение читается, дублей по тексту нет, неподтверждённые встречаются не более 1 раза.""" + timeout = aiohttp.ClientTimeout(total=30) + stored = {} + async with aiohttp.ClientSession(timeout=timeout) as s: + for chat in stand.chats: + hdr = {"Authorization": "Bearer " + chat["members"][0]["token"]} + cursor, items = None, [] + while True: + url = "%s/api/v1/chats/%s/messages?limit=100%s" % (stand.base, chat["id"], "&cursor=" + cursor if cursor else "") + async with s.get(url, headers=hdr) as r: + page = await r.json() + assert r.status == 200, page + items += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + stored[chat["id"]] = items + ids = {m["id"] for items in stored.values() for m in items} + texts = [m["text"] for items in stored.values() for m in items] + confirmed_missing = [c for c in rec.confirmed if c[1] not in ids] + dup_texts = len(texts) - len(set(texts)) + unconfirmed_present = sum(1 for c in rec.unconfirmed if c[2] in set(texts)) + return {"stored_messages": len(ids), "confirmed_201": len(rec.confirmed), "confirmed_missing_in_history": len(confirmed_missing), + "duplicate_texts_in_history": dup_texts, "unconfirmed_requests": len(rec.unconfirmed), + "unconfirmed_that_were_written": unconfirmed_present, "retries": rec.retries, "final_failures": rec.final_failures} + + +def per_second(rec, kind="POST"): + buckets = {} + for t, k, status, lat, inst in rec.events: + if k != kind: + continue + b = buckets.setdefault(int(t), {"ok": 0, "err": 0, "lat": [], "instances": set()}) + if status == 201 or (kind == "GET" and status == 200): + b["ok"] += 1 + b["lat"].append(lat) + if inst: + b["instances"].add(inst) + else: + b["err"] += 1 + return buckets + + +def pct(values, p): + if not values: + return None + values = sorted(values) + return round(values[max(0, int(len(values) * p + 0.9999) - 1)], 2) + + +def summarize(rec, t_from=None, t_to=None, kind="POST"): + rows = [e for e in rec.events if e[1] == kind and (t_from is None or e[0] >= t_from) and (t_to is None or e[0] < t_to)] + if not rows: + return {} + dur = (t_to if t_to is not None else max(e[0] for e in rows)) - (t_from if t_from is not None else min(e[0] for e in rows)) + ok = [e[3] for e in rows if e[2] == 201 or (kind == "GET" and e[2] == 200)] + errs = {} + for e in rows: + if not (e[2] == 201 or (kind == "GET" and e[2] == 200)): + errs[str(e[2])] = errs.get(str(e[2]), 0) + 1 + return {"requests": len(rows), "ok": len(ok), "errors": errs, "seconds": round(dur, 2), "ok_rps": round(len(ok) / dur, 1) if dur > 0 else None, + "error_ratio": round((len(rows) - len(ok)) / len(rows), 4), "p50_ms": pct(ok, .5), "p95_ms": pct(ok, .95), "p99_ms": pct(ok, .99)} diff --git a/research/plot_scaling.py b/research/plot_scaling.py new file mode 100644 index 00000000..16b683cd --- /dev/null +++ b/research/plot_scaling.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Графики трека 1 из evidence/scaling_*.json: python research/plot_scaling.py --out evidence""" +import argparse +import json +import os + +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt # noqa: E402 + + +def load(out, series, tag): + path = os.path.join(out, "scaling_%s_%s.json" % (series, tag)) + return json.load(open(path, encoding="utf-8")) if os.path.exists(path) else None + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="evidence") + out = ap.parse_args().out + fig, ax = plt.subplots(1, 3, figsize=(17, 5)) + # A: фиксированный суммарный CPU + for tag, color, shift in (("base", "#1565c0", -0.2), ("improved", "#2e7d32", 0.2)): + d = load(out, "A", tag + "_fixed_resources") + if not d: + continue + names = [r["config"] for r in d["results"]] + ax[0].bar([i + shift for i in range(len(names))], [r["total_rps"] for r in d["results"]], width=0.4, color=color, + label={"base": "до (несколько запросов)", "improved": "после (функция БД)"}[tag]) + ax[0].set_xticks(range(len(names))) + ax[0].set_xticklabels(["%s" % n.replace("x", " × ") + " CPU" for n in names]) + ax[0].set_title("A: суммарно 2 CPU / 1024 MiB RAM на API"); ax[0].set_ylabel("успешных запросов/с"); ax[0].legend() + # B: рост ресурсов + for tag, color, label in (("base", "#1565c0", "БД 2 CPU, до"), ("base_pg4", "#ef6c00", "БД 4 CPU, до"), ("improved", "#2e7d32", "БД 2 CPU, после")): + d = load(out, "B", tag) + if not d: + continue + ax[1].plot([r["replicas"] for r in d["results"]], [r["total_rps"] for r in d["results"]], marker="o", color=color, label=label) + ax[1].set_xscale("log", base=2); ax[1].set_xticks([1, 2, 4, 8]); ax[1].set_xticklabels(["1", "2", "4", "8"]) + ax[1].set_title("B: реплики по 1.0 CPU — насыщение"); ax[1].set_xlabel("число API-реплик"); ax[1].set_ylabel("успешных запросов/с"); ax[1].legend() + # CPU компонентов в серии B (до) + d = load(out, "B", "base") + if d: + reps = [r["replicas"] for r in d["results"]] + for svc, color in (("postgres", "#c62828"), ("api", "#1565c0"), ("lb", "#6a1b9a")): + ax[2].plot(reps, [r["docker_stats"].get(svc, {}).get("avg_cpu_percent_per_container") for r in d["results"]], marker="s", color=color, + label={"postgres": "PostgreSQL (лимит 200 %)", "api": "API, на реплику (лимит 100 %)", "lb": "HAProxy (лимит 200 %)"}[svc]) + ax[2].set_xscale("log", base=2); ax[2].set_xticks([1, 2, 4, 8]); ax[2].set_xticklabels(["1", "2", "4", "8"]) + ax[2].set_title("B (до): загрузка CPU компонентов, % одного ядра"); ax[2].set_xlabel("число API-реплик"); ax[2].legend() + fig.tight_layout() + fig.savefig(os.path.join(out, "scaling.png"), dpi=110) + print("saved") + + +if __name__ == "__main__": + main() diff --git a/research/rolling_update.py b/research/rolling_update.py new file mode 100644 index 00000000..dc81c0e4 --- /dev/null +++ b/research/rolling_update.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +"""Уровень 4: rolling update под нагрузкой. Нагрузка идёт через LB 60 с; на t=10 запускается scripts/rolling-update.sh. +Проверяем: ошибки клиента (после повторов с тем же ключом), потерянные/задублированные сообщения, смену instance id.""" +import argparse +import asyncio +import json +import os +import subprocess +import sys +import threading +import time + +sys.path.insert(0, os.path.dirname(__file__)) +import loadgen # noqa: E402 + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") + + +async def main(args): + stand = loadgen.Stand("http://127.0.0.1:8080") + await stand.setup() + rec = loadgen.Recorder() + info = {} + + def update(): + time.sleep(max(0, rec.t0 + 10 - time.time())) + info["start"] = round(time.time() - rec.t0, 2) + out = subprocess.run(["sh", "scripts/rolling-update.sh", args.revision], cwd=ROOT, capture_output=True, text=True) + info["end"] = round(time.time() - rec.t0, 2) + info["script_exit"] = out.returncode + info["script_output"] = out.stdout.strip().splitlines()[-6:] + + threading.Thread(target=update, daemon=True).start() + await loadgen.run_load(stand, rec, args.seconds, args.concurrency) + await asyncio.sleep(1) + check = await loadgen.verify(stand, rec) + instances_before = {e[4] for e in rec.events if e[1] == "POST" and e[0] < info["start"] and e[4]} + instances_during = {e[4] for e in rec.events if e[1] == "POST" and info["start"] <= e[0] <= info["end"] and e[4]} + instances_after = {e[4] for e in rec.events if e[1] == "POST" and e[0] > info["end"] + 2 and e[4]} + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "update": info, "revision": args.revision, + "instances_before": sorted(instances_before), "instances_during": sorted(instances_during), + "instances_after": sorted(instances_after), "all_replicas_replaced": not (instances_before & instances_after), + "before": loadgen.summarize(rec, 2, info["start"]), "during": loadgen.summarize(rec, info["start"], info["end"]), + "after": loadgen.summarize(rec, info["end"] + 2, None), "total": loadgen.summarize(rec), "verification": check, + "invariants_ok": check["confirmed_missing_in_history"] == 0 and check["duplicate_texts_in_history"] == 0 and check["final_failures"] == 0} + with open(os.path.join(args.out, "rolling_update.json"), "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--out", default="evidence") + ap.add_argument("--seconds", type=int, default=60) + ap.add_argument("--concurrency", type=int, default=16) + ap.add_argument("--revision", default="rolling-%d" % int(time.time())) + asyncio.run(main(ap.parse_args())) diff --git a/research/run_scaling_all.sh b/research/run_scaling_all.sh new file mode 100755 index 00000000..0e4763f1 --- /dev/null +++ b/research/run_scaling_all.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Полная серия трека 1: base (SEND_STRATEGY=multi) и improved (function); см. REPORT.md. Занимает ~35 минут. +set -eu +cd "$(dirname "$0")/.." +PY="${PY:-python}" +export PYTHONUTF8=1 +set_strategy() { + SEND_STRATEGY="$1" docker compose --env-file .env up -d --no-deps --force-recreate api >/dev/null 2>&1 + sleep 12 +} +set_strategy multi +$PY research/scaling.py --series A --concurrency 64 --tag base_fixed_resources --out evidence +$PY research/scaling.py --series B --concurrency 64 --tag base --out evidence +$PY research/scaling.py --series B --concurrency 64 --pg-cpus 4 --tag base_pg4 --out evidence +set_strategy function +$PY research/scaling.py --series A --concurrency 64 --tag improved_fixed_resources --out evidence +$PY research/scaling.py --series B --concurrency 64 --tag improved --out evidence +echo ALL-DONE diff --git a/research/scaling.py b/research/scaling.py new file mode 100644 index 00000000..d248cc75 --- /dev/null +++ b/research/scaling.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Трек 1 (цена масштабирования), lab3. + +Серия A (фиксированный суммарный CPU/RAM API): 1×2.0 CPU, 2×1.0, 4×0.5 — один и тот же профиль нагрузки. +Серия B (рост ресурсов): 1, 2, 4, 8 реплик по 1.0 CPU; показывает точку насыщения (БД / LB / пул / клиент). +Для каждой конфигурации: прогрев 5 с, затем N с нагрузки несколькими процессами-клиентами (closed-loop, 90 % POST / 10 % GET), +параллельно семплируются `docker stats` (CPU каждого сервиса). Ресурсы меняются на лету `docker update`, реплики — `compose --scale`. + + python research/scaling.py --series A --out evidence --tag base + python research/scaling.py --series B --out evidence --tag base +Переменные окружения стенда (SEND_STRATEGY=...) задаются при запуске compose и описываются в --tag. +""" +import argparse +import asyncio +import json +import multiprocessing as mp +import os +import statistics +import subprocess +import sys +import threading +import time + +sys.path.insert(0, os.path.dirname(__file__)) +import loadgen # noqa: E402 + +ROOT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..") +BASE = "http://127.0.0.1:8080" +FILTER = ["--filter", "label=com.docker.compose.project=mtusi-lab3"] + + +def sh(*args, check=True): + return subprocess.run(args, cwd=ROOT, capture_output=True, text=True, check=check, env=dict(os.environ, **{"PYTHONUTF8": "1"})).stdout.strip() + + +def containers(service): + return sh("docker", "ps", "-q", *FILTER, "--filter", "label=com.docker.compose.service=" + service).split() + + +def configure(replicas, api_cpus, pg_cpus, lb_cpus, api_memory_mib=256): + sh("docker", "compose", "--env-file", ".env", "up", "-d", "--no-deps", "--no-recreate", "--scale", "api=%d" % replicas, "api") + # лишние реплики (если было больше) удаляем + ids = containers("api") + for extra in ids[replicas:]: + sh("docker", "stop", "--time", "5", extra) + sh("docker", "rm", extra) + for cid in containers("api"): + memory = "%dm" % api_memory_mib + sh("docker", "update", "--cpus", str(api_cpus), "--memory", memory, "--memory-swap", memory, cid) + limits = json.loads(sh("docker", "inspect", "-f", "{{json .HostConfig}}", cid)) + assert limits["Memory"] == api_memory_mib * 1024 * 1024 + assert limits["NanoCpus"] == int(api_cpus * 1_000_000_000) + for cid in containers("postgres"): + sh("docker", "update", "--cpus", str(pg_cpus), "--memory", "512m", "--memory-swap", "512m", cid) + for cid in containers("lb"): + sh("docker", "update", "--cpus", str(lb_cpus), "--memory", "128m", "--memory-swap", "128m", cid) + deadline = time.time() + 60 + while time.time() < deadline: + states = [sh("docker", "inspect", "-f", "{{.State.Health.Status}}", c) for c in containers("api")] + if len(states) == replicas and all(s == "healthy" for s in states): + break + time.sleep(1) + wait_lb(replicas) + + +def wait_lb(replicas, timeout=90): + """Ждём, пока LB реально раздаёт запросы всем репликам (каждая видна по X-Instance-Id).""" + import urllib.request + deadline = time.time() + timeout + while time.time() < deadline: + seen = set() + try: + for _ in range(replicas * 12): + with urllib.request.urlopen(BASE + "/health/ready", timeout=2) as r: + seen.add(r.headers.get("X-Instance-Id")) + except Exception: + time.sleep(1) + continue + if len(seen) >= replicas: + return + time.sleep(1) + raise RuntimeError("LB не раздаёт запросы всем %d репликам" % replicas) + + +def child(args): + stand_data, seconds, concurrency, seed, t0 = args + import random + random.seed(seed) + stand = loadgen.Stand(BASE) + stand.users, stand.chats = stand_data["users"], stand_data["chats"] + rec = loadgen.Recorder() + rec.t0 = t0 + + async def go(): + await loadgen.run_load(stand, rec, seconds, concurrency) + asyncio.run(go()) + return rec.events, len(rec.confirmed), rec.final_failures + + +class Sampler(threading.Thread): + def __init__(self): + super().__init__(daemon=True) + self.rows, self.stop = [], threading.Event() + + def run(self): + while not self.stop.is_set(): + out = subprocess.run(["docker", "stats", "--no-stream", "--format", "{{.Name}} {{.CPUPerc}} {{.MemUsage}}"], + capture_output=True, text=True).stdout.splitlines() + self.rows.append((time.time(), [l.split() for l in out if "mtusi-lab3" in l])) + + def summary(self, t_from, t_to): + agg = {} + for ts, rows in self.rows: + if not t_from <= ts <= t_to: + continue + for r in rows: + service = r[0].replace("mtusi-lab3-", "").rsplit("-", 1)[0] + agg.setdefault(service, []).append(float(r[1].rstrip("%"))) + return {k: {"avg_cpu_percent_per_container": round(statistics.mean(v), 1), "samples": len(v)} for k, v in agg.items()} + + +def run_config(name, replicas, api_cpus, pg_cpus, lb_cpus, args, stand_data): + api_memory_mib = 1024 // replicas if args.series == "A" else 256 + configure(replicas, api_cpus, pg_cpus, lb_cpus, api_memory_mib) + # прогрев + t0 = time.time() + with mp.get_context("spawn").Pool(args.procs) as pool: + pool.map(child, [(stand_data, 5, max(1, args.concurrency // args.procs), i, t0) for i in range(args.procs)]) + sampler = Sampler() + sampler.start() + t_start = time.time() + with mp.get_context("spawn").Pool(args.procs) as pool: + results = pool.map(child, [(stand_data, args.seconds, args.concurrency // args.procs, 100 + i, t_start) for i in range(args.procs)]) + t_end = time.time() + sampler.stop.set() + rec = loadgen.Recorder() + rec.t0 = t_start + for events, confirmed, failures in results: + rec.events.extend(events) + post = loadgen.summarize(rec, 0, args.seconds, "POST") + get = loadgen.summarize(rec, 0, args.seconds, "GET") + instances = {} + for e in rec.events: + if e[1] == "POST" and e[4]: + instances[e[4]] = instances.get(e[4], 0) + 1 + out = {"config": name, "replicas": replicas, "api_cpus_each": api_cpus, "api_cpus_total": round(replicas * api_cpus, 2), + "api_memory_mib_each": api_memory_mib, "api_memory_mib_total": api_memory_mib * replicas, + "postgres_cpus": pg_cpus, "lb_cpus": lb_cpus, "concurrency": args.concurrency, "client_processes": args.procs, + "post": post, "get": get, "total_rps": round((post.get("ok", 0) + get.get("ok", 0)) / args.seconds, 1), + "replica_share": sorted(instances.values()), "docker_stats": sampler.summary(t_start + 2, t_end)} + print(json.dumps({k: out[k] for k in ("config", "total_rps")} | {"p95": post.get("p95_ms"), "err": post.get("error_ratio"), + "stats": out["docker_stats"]}, ensure_ascii=False)) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--series", choices=["A", "B", "C"], required=True) + ap.add_argument("--out", default="evidence") + ap.add_argument("--tag", default="base") + ap.add_argument("--seconds", type=int, default=30) + ap.add_argument("--concurrency", type=int, default=256) + ap.add_argument("--procs", type=int, default=8) + ap.add_argument("--pg-cpus", type=float, default=2.0) + ap.add_argument("--lb-cpus", type=float, default=2.0) + ap.add_argument("--repeat", type=int, default=2) + args = ap.parse_args() + configure(2, 1.0, args.pg_cpus, args.lb_cpus) + stand = loadgen.Stand(BASE, users=40, chats=40) + asyncio.run(stand.setup()) + stand_data = {"users": stand.users, "chats": stand.chats} + if args.series == "A": + configs = [("1x2.0", 1, 2.0), ("2x1.0", 2, 1.0), ("4x0.5", 4, 0.5)] + elif args.series == "B": + configs = [("1x1.0", 1, 1.0), ("2x1.0", 2, 1.0), ("4x1.0", 4, 1.0), ("8x1.0", 8, 1.0)] + else: # C: только 2 и 4 реплики при увеличенном CPU БД — проверка гипотезы «узкое место — БД» + configs = [("2x1.0", 2, 1.0), ("4x1.0", 4, 1.0)] + results = [] + for name, n, cpus in configs: + runs = [run_config(name, n, cpus, args.pg_cpus, args.lb_cpus, args, stand_data) for _ in range(args.repeat)] + best = dict(runs[0]) + best["total_rps_runs"] = [r["total_rps"] for r in runs] + best["total_rps"] = round(statistics.mean(r["total_rps"] for r in runs), 1) + best["post_p95_runs"] = [r["post"].get("p95_ms") for r in runs] + best["runs"] = runs + results.append(best) + report = {"date": time.strftime("%Y-%m-%d %H:%M:%S"), "series": args.series, "tag": args.tag, "args": vars(args), "results": results} + path = os.path.join(args.out, "scaling_%s_%s.json" % (args.series, args.tag)) + with open(path, "w", encoding="utf-8") as fh: + json.dump(report, fh, ensure_ascii=False, indent=2) + configure(2, 1.0, 1.0, 1.0) + print("saved", path) + + +if __name__ == "__main__": + mp.freeze_support() + main() diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100755 index 00000000..05b791a0 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Создаёт локальные секреты стенда (secrets/ в .gitignore). Идемпотентно: существующие файлы не перезаписываются. +set -eu +cd "$(dirname "$0")/.." +mkdir -p secrets +if [ ! -s secrets/postgres_password ]; then + head -c 24 /dev/urandom | base64 | tr -d '=+/\n' > secrets/postgres_password + echo "создан secrets/postgres_password" +fi +if [ ! -s secrets/database_url ]; then + printf 'postgresql://messenger:%s@postgres:5432/messenger' "$(cat secrets/postgres_password)" > secrets/database_url + echo "создан secrets/database_url" +fi +chmod 644 secrets/* 2>/dev/null || true # читает непривилегированный пользователь контейнера (uid 10001) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..055bc928 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,5 @@ +@echo off +rem Windows analogue of restart.sh: restart the whole stand, volumes are kept. +cd /d "%~dp0.." +docker compose --env-file .env restart postgres valkey api lb >NUL 2>&1 +exit /b %errorlevel% diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..e609ecdf --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# Перезапуск ВСЕГО стенда с сохранением volumes: LB, API-реплики, PostgreSQL, Valkey. Данные не удаляются. +set -eu +cd "$(dirname "$0")/.." +docker compose --env-file .env restart postgres valkey api lb >/dev/null 2>&1 diff --git a/scripts/restore-replicas.sh b/scripts/restore-replicas.sh new file mode 100755 index 00000000..ae9b08b9 --- /dev/null +++ b/scripts/restore-replicas.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Возвращает стенд к заданному числу API-реплик (API_REPLICAS в .env, по умолчанию 2): +# остановленные контейнеры удаляются и создаются заново; данные (volumes) не затрагиваются. +set -eu +cd "$(dirname "$0")/.." +for cid in $(docker ps -aq --filter "label=com.docker.compose.project=mtusi-lab3" --filter "label=com.docker.compose.service=api" --filter "status=exited"); do + docker rm "$cid" >/dev/null +done +N="${API_REPLICAS:-$(grep -E '^API_REPLICAS=' .env | cut -d= -f2 | tr -d '\r')}" +docker compose --env-file .env up -d --no-deps --scale api="$N" api >/dev/null 2>&1 || { echo "не удалось вернуть $N реплик" >&2; exit 1; } diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100755 index 00000000..3dd749bf --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# Rolling update API без простоя: реплики заменяются по одной. +# 1) миграции выполняются ОДИН раз до замены (идемпотентно; должны быть обратно совместимы со старой версией); +# 2) для каждой старой реплики: поднять новую (из нового образа) рядом -> дождаться healthy и попадания в LB -> +# остановить старую штатно (SIGTERM -> drain 4 с -> graceful shutdown) -> удалить. +# Использование: scripts/rolling-update.sh [REVISION] +set -eu +cd "$(dirname "$0")/.." +REV="${1:-$(date +%s)}" +COMPOSE="docker compose --env-file .env" +API_FILTER="--filter label=com.docker.compose.project=mtusi-lab3 --filter label=com.docker.compose.service=api" + +$COMPOSE build --build-arg REV="$REV" api >/dev/null +$COMPOSE run --rm migrate >/dev/null +OLD=$(docker ps -q $API_FILTER) +N=$(echo "$OLD" | wc -w | tr -d ' ') +echo "revision=$REV replicas=$N" +for old in $OLD; do + BEFORE_FILE=$(mktemp) + docker ps -q $API_FILTER | sort > "$BEFORE_FILE" + $COMPOSE up -d --no-deps --no-recreate --scale api=$((N + 1)) api >/dev/null 2>&1 + NEW=$(docker ps -q $API_FILTER | sort | comm -13 "$BEFORE_FILE" -) + rm -f "$BEFORE_FILE" + for i in $(seq 1 60); do + [ "$(docker inspect -f '{{.State.Health.Status}}' "$NEW" 2>/dev/null)" = healthy ] && break + sleep 1 + done + sleep 3 # LB: rise=2 проверки по 1 с -> реплика в ротации + docker stop --time 15 "$old" >/dev/null + docker rm "$old" >/dev/null + echo "заменена реплика $old -> $NEW" +done diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..8c5f15c1 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,8 @@ +@echo off +rem Windows analogue of stop-one.sh: stop exactly one API replica and leave it stopped. +set "cid=" +echo %TARGET_INSTANCE_ID%| findstr /r "^[a-f0-9][a-f0-9]*$" >NUL || exit /b 2 +for /f %%i in ('docker ps -q --filter id^=%TARGET_INSTANCE_ID% --filter label^=com.docker.compose.project^=mtusi-lab3 --filter label^=com.docker.compose.service^=api') do set "cid=%%i" +if "%cid%"=="" exit /b 3 +docker stop --time 0 %cid% >NUL 2>&1 +exit /b %errorlevel% diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..d52eac94 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Останавливает РОВНО одну API-реплику (TARGET_INSTANCE_ID == X-Instance-Id == короткий ID контейнера) и +# оставляет её остановленной. Не трогает БД, Valkey, LB и volumes. Возвращается быстро (< 2 с). +# `docker stop --time 0`: SIGTERM и немедленный SIGKILL — имитация отказа процесса; политика restart +# после ручной остановки не поднимает контейнер. Восстановление: scripts/restore-replicas.sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +case "$TARGET_INSTANCE_ID" in + ''|*[!a-f0-9]*) echo "некорректный TARGET_INSTANCE_ID" >&2; exit 2 ;; +esac +cid=$(docker ps -q --filter "id=$TARGET_INSTANCE_ID" \ + --filter "label=com.docker.compose.project=mtusi-lab3" --filter "label=com.docker.compose.service=api") +[ -n "$cid" ] || { echo "реплика не найдена среди работающих api-контейнеров" >&2; exit 3; } +docker stop --time 0 "$cid" >/dev/null diff --git a/src/app/config.py b/src/app/config.py new file mode 100644 index 00000000..64e0ab92 --- /dev/null +++ b/src/app/config.py @@ -0,0 +1,58 @@ +"""Конфигурация из переменных окружения (см. .env.example).""" +import os + + +def _int(name, default): + value = os.environ.get(name) + return int(value) if value not in (None, "") else default + + +def _float(name, default): + value = os.environ.get(name) + return float(value) if value not in (None, "") else default + + +def _secret(name, default=""): + """Значение из переменной NAME либо из файла NAME_FILE (Docker secrets); файл читается один раз.""" + if os.environ.get(name): + return os.environ[name] + path = os.environ.get(name + "_FILE") + if path and os.path.exists(path): + with open(path, encoding="utf-8") as fh: + return fh.read().strip() + return default + + +APP_HOST = os.environ.get("APP_HOST", "0.0.0.0") +APP_PORT = _int("APP_PORT", 8080) +APP_ENV = os.environ.get("APP_ENV", "lab") + +DATABASE_URL = _secret("DATABASE_URL") +DB_POOL_MIN = _int("DB_POOL_MIN", 2) +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_ACQUIRE_TIMEOUT = _float("DB_ACQUIRE_TIMEOUT", 2.0) # ожидание свободного соединения пула +DB_COMMAND_TIMEOUT = _float("DB_COMMAND_TIMEOUT", 5.0) # таймаут одного запроса +DB_CONNECT_TIMEOUT = _float("DB_CONNECT_TIMEOUT", 2.0) + +VALKEY_URL = os.environ.get("VALKEY_URL", "") # пусто = кэш сессий выключен +VALKEY_TIMEOUT = _float("VALKEY_TIMEOUT", 0.2) +VALKEY_BREAKER_SECONDS = _float("VALKEY_BREAKER_SECONDS", 5.0) + +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 60) + +ARGON2_TIME_COST = _int("ARGON2_TIME_COST", 2) +ARGON2_MEMORY_KIB = _int("ARGON2_MEMORY_KIB", 19456) # 19 MiB, рекомендация OWASP +ARGON2_PARALLELISM = _int("ARGON2_PARALLELISM", 1) +HASH_CONCURRENCY = _int("HASH_CONCURRENCY", 4) + +MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536) +MAX_INFLIGHT = _int("MAX_INFLIGHT", 64) +LIMIT_CONCURRENCY = _int("LIMIT_CONCURRENCY", 1000) +RETRY_AFTER_SECONDS = _int("RETRY_AFTER_SECONDS", 1) +MIGRATIONS_DIR = os.environ.get("MIGRATIONS_DIR", os.path.join(os.path.dirname(__file__), "..", "..", "migrations")) +DRAIN_SECONDS = _float("DRAIN_SECONDS", 4.0) # после SIGTERM: readiness=503, сервер ещё принимает запросы +GRACEFUL_SHUTDOWN_SECONDS = _int("GRACEFUL_SHUTDOWN_SECONDS", 8) +KEEPALIVE_SECONDS = _int("KEEPALIVE_SECONDS", 5) +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 24 * 3600) +SEND_STRATEGY = os.environ.get("SEND_STRATEGY", "function") # function (миграция 0003) | multi (несколько запросов в транзакции) diff --git a/src/app/db.py b/src/app/db.py new file mode 100644 index 00000000..7e81fb93 --- /dev/null +++ b/src/app/db.py @@ -0,0 +1,71 @@ +"""PostgreSQL: ограниченный пул asyncpg, таймауты, перевод сбоев зависимости в 503.""" +import asyncio +from contextlib import asynccontextmanager + +import asyncpg + +from . import config +from .web import log, unavailable + +DEPENDENCY_ERRORS = (asyncpg.PostgresConnectionError, asyncpg.InterfaceError, asyncpg.exceptions.OperatorInterventionError, + asyncpg.exceptions.InsufficientResourcesError, asyncio.TimeoutError, OSError) +_RETRYABLE = (asyncpg.ConnectionDoesNotExistError, asyncpg.InterfaceError, ConnectionResetError) + + +class Database: + def __init__(self): + self.pool = None + + async def start(self): + if not config.DATABASE_URL: + log("error", "database_url_missing") + return + # min_size=0: приложение стартует и без БД (readiness покажет 503), пул подключается лениво. + self.pool = await asyncpg.create_pool( + dsn=config.DATABASE_URL, min_size=0, max_size=config.DB_POOL_MAX, + command_timeout=config.DB_COMMAND_TIMEOUT, timeout=config.DB_CONNECT_TIMEOUT, + max_inactive_connection_lifetime=60, + server_settings={"application_name": "messenger-api", "timezone": "UTC"}) + + async def stop(self): + if self.pool: + try: + await asyncio.wait_for(self.pool.close(), 3) + except Exception: + self.pool.terminate() + + @asynccontextmanager + async def conn(self): + if self.pool is None: + raise unavailable("База данных не настроена") + try: + async with self.pool.acquire(timeout=config.DB_ACQUIRE_TIMEOUT) as connection: + yield connection + except DEPENDENCY_ERRORS as exc: + log("warning", "db_unavailable", error_type=type(exc).__name__) + raise unavailable() from exc + + async def _run(self, method, sql, args, retry): + for attempt in (0, 1): + try: + async with self.conn() as c: + return await getattr(c, method)(sql, *args) + except Exception as exc: + if attempt == 0 and retry and isinstance(exc.__cause__, _RETRYABLE): + continue # разорванное соединение пула (рестарт БД): один повтор только для чтения + raise + + async def fetch(self, sql, *args): + return await self._run("fetch", sql, args, True) + + async def fetchrow(self, sql, *args, retry=True): + return await self._run("fetchrow", sql, args, retry) + + async def fetchval(self, sql, *args): + return await self._run("fetchval", sql, args, True) + + async def execute(self, sql, *args): + return await self._run("execute", sql, args, False) + + async def ping(self): + return await asyncio.wait_for(self.fetchval("SELECT 1"), 1.0) diff --git a/src/app/main.py b/src/app/main.py new file mode 100644 index 00000000..3803beb7 --- /dev/null +++ b/src/app/main.py @@ -0,0 +1,415 @@ +"""ASGI-приложение мессенджера (lab2): REST API, PostgreSQL, сессии Bearer, кэш Valkey.""" +import asyncio +import base64 +import binascii +import hashlib +import json +import os +import re +import time +import uuid + +import asyncpg + +from . import config, passwords +from .db import DEPENDENCY_ERRORS, Database +from .sessions import SessionCache, Sessions +from .web import (ApiError, Request, bad, dt_to_us, encode_cursor, format_dt, log, new_request_id, page_params, + parse_json_object, parse_uuid, respond, unavailable, us_to_dt, valid_text, USERNAME_RE) + +ROUTES = [ + ("health_live", re.compile(r"/health/live"), {"GET"}), + ("health_ready", re.compile(r"/health/ready"), {"GET"}), + ("users", re.compile(r"/api/v1/users"), {"POST"}), + ("users_me", re.compile(r"/api/v1/users/me"), {"GET"}), + ("create_session", re.compile(r"/api/v1/auth/sessions"), {"POST"}), + ("session_current", re.compile(r"/api/v1/auth/sessions/current"), {"DELETE"}), + ("chats", re.compile(r"/api/v1/chats"), {"GET", "POST"}), + ("chat", re.compile(r"/api/v1/chats/(?P[^/]+)"), {"GET"}), + ("messages", re.compile(r"/api/v1/chats/(?P[^/]+)/messages"), {"GET", "POST"}), +] +ROUTE_LABELS = { + "health_live": "/health/live", "health_ready": "/health/ready", "users": "/api/v1/users", + "users_me": "/api/v1/users/me", "create_session": "/api/v1/auth/sessions", + "session_current": "/api/v1/auth/sessions/current", "chats": "/api/v1/chats", + "chat": "/api/v1/chats/{chat_id}", "messages": "/api/v1/chats/{chat_id}/messages", +} +TOKEN_RE = re.compile(r"[A-Za-z0-9_-]{32,128}") +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") +BEARER = 'Bearer realm="messenger"' +BASIC = 'Basic realm="messenger", charset="UTF-8"' +# Идентификатор реплики: имя хоста контейнера (12 hex, == короткий ID контейнера) -> легко сопоставить с docker ps. +INSTANCE_ID = os.environ.get("INSTANCE_ID") or (os.environ.get("HOSTNAME") or uuid.uuid4().hex)[:12] + +CHAT_SELECT = ("SELECT c.id, c.title, c.created_at, " + "(SELECT array_agg(m2.user_id ORDER BY m2.ord) FROM chat_members m2 WHERE m2.chat_id = c.id) AS member_ids ") + + +def user_json(row): + return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], + "created_at": format_dt(row["created_at"])} + + +def chat_json(row): + return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in row["member_ids"]], + "created_at": format_dt(row["created_at"])} + + +def message_json(row): + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), + "text": row["text"], "created_at": format_dt(row["created_at"])} + + +def unauthorized(challenge=BEARER, message="Требуется действительная сессия"): + return ApiError(401, "unauthorized", message, {"www-authenticate": challenge}) + + +class Messenger: + def __init__(self): + self.db = Database() + self.cache = SessionCache() + self.sessions = Sessions(self.db, self.cache) + self.inflight = 0 + self.draining = False + + # --- аутентификация ------------------------------------------------------ + @staticmethod + def bearer_token(req): + header = req.headers.get("authorization", "") + scheme, _, token = header.partition(" ") + if scheme.lower() != "bearer" or not TOKEN_RE.fullmatch(token.strip()): + raise unauthorized() + return token.strip() + + async def authenticate(self, req): + token = self.bearer_token(req) + user_id = await self.sessions.resolve(token) + if user_id is None: + raise unauthorized() + req.user_id, req.token = user_id, token + return user_id + + # --- обработчики ----------------------------------------------------------- + async def health_live(self, req, params): + return 200, {"status": "ok"} + + async def health_ready(self, req, params): + # Критичная зависимость — PostgreSQL. Valkey — необязательный кэш: его отказ не останавливает API. + if self.draining: + raise unavailable("Сервис останавливается") + try: + await self.db.ping() + except ApiError: + raise + except Exception: + raise unavailable() from None + return 200, {"status": "ok"} + + async def users(self, req, params): + req.require_json_media_type() + data = parse_json_object(await req.body(), {"username", "display_name", "password"}, + {"username", "display_name", "password"}) + username = data["username"] + if not isinstance(username, str) or not USERNAME_RE.fullmatch(username): + raise bad("username должен соответствовать [a-z0-9_]{3,32}") + display_name = valid_text(data["display_name"], "display_name", 100) + password = valid_text(data["password"], "password", 128, allow_empty=True) + if len(password) < 12: + raise bad("password должен содержать от 12 до 128 символов") + pw_hash = await passwords.hash_password(password) + row = await self.db.fetchrow( + "INSERT INTO users (id, username, display_name, password_hash) VALUES ($1, $2, $3, $4) " + "ON CONFLICT (username) DO NOTHING RETURNING id, username, display_name, created_at", + uuid.uuid4(), username, display_name, pw_hash) + if row is None: + raise ApiError(409, "conflict", "username уже занят") + return 201, user_json(row) + + async def users_me(self, req, params): + user_id = await self.authenticate(req) + row = await self.db.fetchrow("SELECT id, username, display_name, created_at FROM users WHERE id = $1", user_id) + if row is None: + raise unauthorized() + return 200, user_json(row) + + async def create_session(self, req, params): + header = req.headers.get("authorization", "") + scheme, _, value = header.partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise unauthorized(BASIC, "Неверные учётные данные") + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + username, sep, password = decoded.partition(":") + if not sep: + raise ValueError + except (binascii.Error, ValueError, UnicodeDecodeError): + raise unauthorized(BASIC, "Неверные учётные данные") from None + row = None + if USERNAME_RE.fullmatch(username): + row = await self.db.fetchrow("SELECT id, username, display_name, created_at, password_hash FROM users " + "WHERE username = $1", username) + ok = await passwords.verify_password(row["password_hash"] if row else None, password) + if not ok: + raise unauthorized(BASIC, "Неверные учётные данные") + token, expires_at = await self.sessions.create(row["id"]) + return 201, {"token": token, "token_type": "Bearer", "expires_at": format_dt(expires_at), "user": user_json(row)} + + async def session_current(self, req, params): + await self.authenticate(req) + await self.sessions.revoke(req.token) + return 204, None + + async def chats(self, req, params): + user_id = await self.authenticate(req) + if req.method == "GET": + limit, after = page_params(req.query, "c", user_id) + if after is None: + rows = await self.db.fetch( + CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 " + "ORDER BY m.chat_created_at, m.chat_id LIMIT $2", user_id, limit + 1) + else: + rows = await self.db.fetch( + CHAT_SELECT + "FROM chat_members m JOIN chats c ON c.id = m.chat_id WHERE m.user_id = $1 " + "AND (m.chat_created_at, m.chat_id) > ($2, $3) ORDER BY m.chat_created_at, m.chat_id LIMIT $4", + user_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1) + more = len(rows) > limit + rows = rows[:limit] + cursor = encode_cursor("c", user_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None + return 200, {"items": [chat_json(r) for r in rows], "next_cursor": cursor} + req.require_json_media_type() + data = parse_json_object(await req.body(), {"title", "member_ids"}, {"title", "member_ids"}) + title = valid_text(data["title"], "title", 100) + members = data["member_ids"] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + raise bad("member_ids должен содержать от 1 до 100 элементов") + ids = [parse_uuid(m, "member_id") for m in members] + if len(set(ids)) != len(ids): + raise bad("member_ids содержит дубликаты") + if user_id not in ids: + raise bad("Создатель должен быть в member_ids") + chat_id = uuid.uuid4() + uuids = [uuid.UUID(i) for i in ids] + try: + async with self.db.conn() as c: + async with c.transaction(): # чат и участники создаются атомарно + # Существование участников проверяет внешний ключ chat_members.user_id: при неизвестном + # участнике вставка падает и откатывается вся транзакция, включая уже созданный чат. + created = await c.fetchval("INSERT INTO chats (id, title) VALUES ($1, $2) RETURNING created_at", + chat_id, title) + await c.execute( + "INSERT INTO chat_members (chat_id, user_id, chat_created_at, ord) " + "SELECT $1, u, $2, o FROM unnest($3::uuid[]) WITH ORDINALITY AS t(u, o)", + chat_id, created, uuids) + except asyncpg.ForeignKeyViolationError: + raise bad("Неизвестный участник") from None + return 201, {"id": str(chat_id), "title": title, "member_ids": ids, "created_at": format_dt(created)} + + async def _require_member(self, user_id, params): + chat_id = parse_uuid(params["chat_id"], "chat_id") + ok = await self.db.fetchval("SELECT 1 FROM chat_members WHERE chat_id = $1 AND user_id = $2", chat_id, user_id) + if not ok: + raise ApiError(404, "not_found", "Чат не найден") + return chat_id + + async def chat(self, req, params): + user_id = await self.authenticate(req) + chat_id = parse_uuid(params["chat_id"], "chat_id") + row = await self.db.fetchrow( + CHAT_SELECT + "FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = $2 WHERE c.id = $1", + chat_id, user_id) + if row is None: + raise ApiError(404, "not_found", "Чат не найден") + return 200, chat_json(row) + + async def messages(self, req, params): + user_id = await self.authenticate(req) + if req.method == "POST": + row = await self._post_message(req, parse_uuid(params["chat_id"], "chat_id"), user_id) + return 201, message_json(row) + chat_id = await self._require_member(user_id, params) + if req.method == "GET": + limit, after = page_params(req.query, "m", chat_id) + if after is None: + rows = await self.db.fetch( + "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 " + "ORDER BY created_at, id LIMIT $2", chat_id, limit + 1) + else: + rows = await self.db.fetch( + "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = $1 " + "AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT $4", + chat_id, us_to_dt(after[0]), uuid.UUID(after[1]), limit + 1) + more = len(rows) > limit + rows = rows[:limit] + cursor = encode_cursor("m", chat_id, (dt_to_us(rows[-1]["created_at"]), str(rows[-1]["id"]))) if more else None + return 200, {"items": [message_json(r) for r in rows], "next_cursor": cursor} + + async def _post_message(self, req, chat_id, user_id): + # Сначала разбор запроса; ошибка валидации (400) возвращается только участнику чата — постороннему всегда 404. + try: + key = req.headers.get("idempotency-key", "") + if not IDEMPOTENCY_KEY_RE.fullmatch(key): + raise bad("Требуется заголовок Idempotency-Key ([A-Za-z0-9._:-]{1,128})") + req.require_json_media_type() + data = parse_json_object(await req.body(), {"text"}, {"text"}) + text = valid_text(data["text"], "text", 4000) + except ApiError: + await self._require_member(user_id, {"chat_id": chat_id}) + raise + # Нормализованное представление запроса: порядок ключей JSON не важен, отсутствие attachment_ids == []. + request_hash = hashlib.sha256(json.dumps({"text": text, "attachment_ids": []}, sort_keys=True, + ensure_ascii=False).encode("utf-8")).digest() + if config.SEND_STRATEGY == "function": + return await self._send_message_function(chat_id, user_id, key, request_hash, text) + await self._require_member(user_id, {"chat_id": chat_id}) + return await self._send_message(chat_id, user_id, key, request_hash, text) + + async def _send_message_function(self, chat_id, user_id, key, request_hash, text): + """Один вызов серверной функции send_message (миграция 0003): членство + идемпотентность + вставка.""" + row = await self.db.fetchrow("SELECT * FROM send_message($1, $2, $3, $4, $5, $6)", uuid.UUID(chat_id), uuid.UUID(user_id), + key, request_hash, text, float(config.IDEMPOTENCY_TTL_SECONDS), retry=False) + if row["outcome"] == "not_member": + raise ApiError(404, "not_found", "Чат не найден") + if bytes(row["request_hash"]) != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса") + return {"id": row["msg_id"], "chat_id": row["msg_chat"], "sender_id": row["msg_sender"], "text": row["msg_text"], + "created_at": row["msg_created"]} + + async def _send_message(self, chat_id, user_id, key, request_hash, text): + """Идемпотентная отправка: ключ и сообщение пишутся одной транзакцией. + + Одновременные повторы (в том числе через разные реплики) сериализуются уникальным индексом ключа: + второй INSERT ждёт COMMIT первого, затем видит чужую запись и возвращает сохранённое сообщение. + Транзиентный сбой откатывает транзакцию целиком, ключ не фиксируется, повтор безопасен. + """ + chat_uuid, user_uuid, message_id = uuid.UUID(chat_id), uuid.UUID(user_id), uuid.uuid4() + async with self.db.conn() as c: + async with c.transaction(): + # просроченный (старше TTL) ключ этой же области считается свободным + await c.execute("DELETE FROM idempotency_keys WHERE user_id = $1 AND chat_id = $2 AND idem_key = $3 " + "AND created_at < now() - make_interval(secs => $4)", + user_uuid, chat_uuid, key, float(config.IDEMPOTENCY_TTL_SECONDS)) + inserted = await c.fetchval( + "INSERT INTO idempotency_keys (user_id, chat_id, idem_key, request_hash, message_id) " + "VALUES ($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING RETURNING message_id", + user_uuid, chat_uuid, key, request_hash, message_id) + if inserted is not None: + return await c.fetchrow( + "INSERT INTO messages (id, chat_id, sender_id, text) VALUES ($1, $2, $3, $4) " + "RETURNING id, chat_id, sender_id, text, created_at", message_id, chat_uuid, user_uuid, text) + existing = await c.fetchrow( + "SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at FROM idempotency_keys k " + "JOIN messages m ON m.id = k.message_id WHERE k.user_id = $1 AND k.chat_id = $2 AND k.idem_key = $3", + user_uuid, chat_uuid, key) + if existing is None: # ключ исчез между попытками (очистка) — клиент может повторить запрос + raise unavailable("Повторите запрос") + if bytes(existing["request_hash"]) != request_hash: + raise ApiError(409, "conflict", "Idempotency-Key уже использован с другим телом запроса") + return existing + + # --- ASGI ---------------------------------------------------------------------- + async def __call__(self, scope, receive, send): + if scope["type"] == "lifespan": + return await self.lifespan(receive, send) + if scope["type"] != "http": + return + started = time.perf_counter() + request_id = new_request_id() + route_name, status, headers = "unknown", 500, {} + try: + req = Request(scope, receive, request_id) + handler, params, route_name = self.route(req.method, req.path) + if not route_name.startswith("health") and self.inflight >= config.MAX_INFLIGHT: + raise ApiError(429, "rate_limited", "Слишком много одновременных запросов", + {"retry-after": str(config.RETRY_AFTER_SECONDS)}) + self.inflight += 1 + try: + await asyncio.sleep(0) + status, body = await handler(req, params) + finally: + self.inflight -= 1 + except ApiError as exc: + status, headers = exc.status, exc.headers + body = {"error": {"code": exc.code, "message": exc.message, "request_id": request_id}} + except DEPENDENCY_ERRORS as exc: + log("warning", "dependency_error", request_id=request_id, error_type=type(exc).__name__) + status, headers = 503, {"retry-after": str(config.RETRY_AFTER_SECONDS)} + body = {"error": {"code": "unavailable", "message": "Зависимость временно недоступна", "request_id": request_id}} + except Exception as exc: + log("error", "unhandled_exception", request_id=request_id, error_type=type(exc).__name__) + status = 500 + body = {"error": {"code": "internal_error", "message": "Внутренняя ошибка", "request_id": request_id}} + await respond(send, status, body, request_id, headers, INSTANCE_ID) + if os.environ.get("ACCESS_LOG", "1") != "0": + log("info", "request", request_id=request_id, method=scope.get("method"), + route=ROUTE_LABELS.get(route_name, "unmatched"), status=status, + duration_ms=round((time.perf_counter() - started) * 1000, 3)) + + async def janitor(self): + """Удаляет давно просроченные ключи идемпотентности и сессии (безопасно запускать на каждой реплике).""" + import random + await asyncio.sleep(random.uniform(5, 30)) + while True: + try: + await self.db.execute("DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys " + "WHERE created_at < now() - make_interval(secs => $1) LIMIT 5000)", + float(config.IDEMPOTENCY_TTL_SECONDS) + 3600) + await self.db.execute("DELETE FROM sessions WHERE ctid IN (SELECT ctid FROM sessions " + "WHERE expires_at < now() - interval '1 day' LIMIT 5000)") + except Exception as exc: + log("warning", "janitor_failed", error_type=type(exc).__name__) + await asyncio.sleep(3600) + + def route(self, method, path): + for name, pattern, methods in ROUTES: + m = pattern.fullmatch(path) + if m: + if method not in methods: + raise ApiError(405, "invalid_request", "Метод не поддерживается", {"allow": ", ".join(sorted(methods))}) + return getattr(self, name), m.groupdict(), name + raise ApiError(404, "not_found", "Ресурс не найден") + + async def lifespan(self, receive, send): + while True: + event = await receive() + if event["type"] == "lifespan.startup": + await self.db.start() + await self.cache.start() + self._janitor = asyncio.create_task(self.janitor()) + log("info", "startup", env=config.APP_ENV, instance=INSTANCE_ID, revision=os.environ.get("APP_REVISION", "dev"), cache=bool(config.VALKEY_URL), + session_ttl=config.SESSION_TTL_SECONDS, pool_max=config.DB_POOL_MAX) + await send({"type": "lifespan.startup.complete"}) + elif event["type"] == "lifespan.shutdown": + self.draining = True + self._janitor.cancel() + await self.cache.stop() + await self.db.stop() + log("info", "shutdown") + await send({"type": "lifespan.shutdown.complete"}) + return + + +async def overloaded_app(scope, receive, send): + """Ответ uvicorn при превышении --limit-concurrency в контрактном формате.""" + if scope["type"] != "http": + return + request_id = new_request_id() + body = {"error": {"code": "unavailable", "message": "Сервер перегружен: слишком много соединений", + "request_id": request_id}} + await respond(send, 503, body, request_id, {"retry-after": str(config.RETRY_AFTER_SECONDS), "connection": "close"}, INSTANCE_ID) + + +def _install_overload_response(): + for name in ("uvicorn.protocols.http.httptools_impl", "uvicorn.protocols.http.h11_impl"): + try: + module = __import__(name, fromlist=["service_unavailable"]) + except ImportError: + continue + module.service_unavailable = overloaded_app + + +def create_app(): + _install_overload_response() + return Messenger() + + +app = create_app() diff --git a/src/app/migrate.py b/src/app/migrate.py new file mode 100644 index 00000000..e979d89c --- /dev/null +++ b/src/app/migrate.py @@ -0,0 +1,64 @@ +"""Применение версионируемых SQL-миграций: python -m app.migrate + +Свойства: + * повторный запуск ничего не делает (таблица schema_migrations); + * одновременный запуск нескольких экземпляров безопасен (advisory lock); + * каждая миграция — одна транзакция вместе с записью о применении; + * ждёт готовности PostgreSQL до MIGRATE_WAIT_SECONDS. +""" +import asyncio +import hashlib +import os +import sys +import time + +import asyncpg + +from . import config +from .web import log + +LOCK_ID = 7352001 + + +async def connect(deadline): + while True: + try: + return await asyncpg.connect(config.DATABASE_URL, timeout=3) + except (OSError, asyncpg.PostgresError, asyncio.TimeoutError) as exc: + if time.monotonic() > deadline: + raise + log("info", "waiting_for_database", error_type=type(exc).__name__) + await asyncio.sleep(1) + + +async def main(): + wait = float(os.environ.get("MIGRATE_WAIT_SECONDS", "60")) + conn = await connect(time.monotonic() + wait) + try: + await conn.execute("SELECT pg_advisory_lock($1)", LOCK_ID) + await conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (" + "version text PRIMARY KEY, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())") + applied = {r["version"]: r["checksum"] for r in await conn.fetch("SELECT version, checksum FROM schema_migrations")} + directory = os.path.abspath(config.MIGRATIONS_DIR) + files = sorted(f for f in os.listdir(directory) if f.endswith(".sql")) + count = 0 + for name in files: + sql = open(os.path.join(directory, name), encoding="utf-8").read() + checksum = hashlib.sha256(sql.encode("utf-8")).hexdigest() + if name in applied: + if applied[name] != checksum: + log("error", "migration_modified", version=name) + sys.exit(2) + continue + async with conn.transaction(): + await conn.execute(sql) + await conn.execute("INSERT INTO schema_migrations (version, checksum) VALUES ($1, $2)", name, checksum) + count += 1 + log("info", "migration_applied", version=name) + log("info", "migrations_done", applied=count, total=len(files)) + finally: + await conn.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/src/app/passwords.py b/src/app/passwords.py new file mode 100644 index 00000000..650e7b39 --- /dev/null +++ b/src/app/passwords.py @@ -0,0 +1,31 @@ +"""Хеширование паролей: Argon2id (библиотека argon2-cffi), вычисления вне event loop.""" +import asyncio +from concurrent.futures import ThreadPoolExecutor + +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError + +from . import config + +_hasher = PasswordHasher(time_cost=config.ARGON2_TIME_COST, memory_cost=config.ARGON2_MEMORY_KIB, + parallelism=config.ARGON2_PARALLELISM) +_pool = ThreadPoolExecutor(max_workers=config.HASH_CONCURRENCY, thread_name_prefix="argon2") +# Для неизвестного username проверяем фиктивный хеш: время ответа не выдаёт существование пользователя. +_DUMMY = _hasher.hash("dummy-password-for-timing-equalisation") + + +async def hash_password(password): + return await asyncio.get_running_loop().run_in_executor(_pool, _hasher.hash, password) + + +def _verify(stored, password): + try: + return _hasher.verify(stored, password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + + +async def verify_password(stored, password): + """stored=None -> проверка фиктивного хеша, результат всегда False.""" + ok = await asyncio.get_running_loop().run_in_executor(_pool, _verify, stored or _DUMMY, password) + return ok and stored is not None diff --git a/src/app/serve.py b/src/app/serve.py new file mode 100644 index 00000000..9980988c --- /dev/null +++ b/src/app/serve.py @@ -0,0 +1,40 @@ +"""Точка входа: uvicorn с управляемым завершением (drain). + +SIGTERM -> readiness сразу 503 -> DRAIN_SECONDS продолжаем обслуживать запросы, пока балансировщик +исключает реплику -> uvicorn прекращает приём и дожидается текущих запросов (timeout_graceful_shutdown). +Так плановая остановка/rolling update не даёт ошибок клиентам. +""" +import signal +import threading + +import uvicorn + +from . import config +from .main import app +from .web import log + + +class DrainingServer(uvicorn.Server): + _draining = False + + def handle_exit(self, sig, frame): + if sig == signal.SIGTERM and not self._draining and config.DRAIN_SECONDS > 0: + self._draining = True + app.draining = True + log("info", "draining", seconds=config.DRAIN_SECONDS) + timer = threading.Timer(config.DRAIN_SECONDS, super().handle_exit, args=(sig, frame)) + timer.daemon = True + timer.start() + return + super().handle_exit(sig, frame) + + +def main(): + cfg = uvicorn.Config(app, host=config.APP_HOST, port=config.APP_PORT, workers=1, access_log=False, + log_level="warning", timeout_graceful_shutdown=config.GRACEFUL_SHUTDOWN_SECONDS, + timeout_keep_alive=config.KEEPALIVE_SECONDS, limit_concurrency=config.LIMIT_CONCURRENCY) + DrainingServer(cfg).run() + + +if __name__ == "__main__": + main() diff --git a/src/app/sessions.py b/src/app/sessions.py new file mode 100644 index 00000000..b07117c0 --- /dev/null +++ b/src/app/sessions.py @@ -0,0 +1,131 @@ +"""Сессии: PostgreSQL — источник истины; Valkey — кэш с TTL (read-through). + +Политики: + * токен — 256 бит из secrets; в БД и Valkey хранится только SHA-256 токена; + * expiry: expires_at в БД (по часам БД). В кэше TTL = min(остаток жизни, SESSION_CACHE_TTL_SECONDS); + * инвалидация: logout помечает revoked_at в БД и удаляет ключ в Valkey; + * Valkey недоступен -> кэш обходится (чтение из БД), анонимный запрос никогда не считается доверенным; + circuit breaker не даёт каждому запросу ждать таймаут; + * окно устаревания: если Valkey был недоступен во время logout, ключ может прожить до SESSION_CACHE_TTL_SECONDS. +""" +import hashlib +import secrets +import time + +from . import config +from .web import log + +try: + import redis.asyncio as aioredis +except ImportError: # кэш опционален + aioredis = None + + +def token_hash(token): + return hashlib.sha256(token.encode("ascii")).digest() + + +class SessionCache: + def __init__(self): + self.client = None + self.down_until = 0.0 + self.stats = {"hit": 0, "miss": 0, "error": 0} + + async def start(self): + if config.VALKEY_URL and aioredis: + self.client = aioredis.Redis.from_url(config.VALKEY_URL, socket_timeout=config.VALKEY_TIMEOUT, + socket_connect_timeout=config.VALKEY_TIMEOUT) + + async def stop(self): + if self.client: + await self.client.aclose() + + def _usable(self): + return self.client is not None and time.monotonic() >= self.down_until + + def _fail(self, exc): + self.stats["error"] += 1 + if time.monotonic() >= self.down_until: + log("warning", "session_cache_unavailable", error_type=type(exc).__name__) + self.down_until = time.monotonic() + config.VALKEY_BREAKER_SECONDS + + async def get(self, key): + if not self._usable(): + return None + try: + value = await self.client.get(b"s:" + key) + except Exception as exc: + self._fail(exc) + return None + self.stats["hit" if value is not None else "miss"] += 1 + return value + + async def put(self, key, value, ttl): + if not self._usable() or ttl <= 0: + return + try: + await self.client.set(b"s:" + key, value, ex=max(1, int(ttl))) + except Exception as exc: + self._fail(exc) + + async def delete(self, key): + if not self._usable(): + return + try: + await self.client.delete(b"s:" + key) + except Exception as exc: + self._fail(exc) + + async def ping(self): + if self.client is None: + return None + try: + return bool(await self.client.ping()) + except Exception: + return False + + +class Sessions: + def __init__(self, db, cache): + self.db, self.cache = db, cache + + @staticmethod + def _cache_value(user_id, expires_at): + return ("%s|%d" % (user_id, int(expires_at.timestamp() * 1000))).encode() + + async def create(self, user_id): + token = secrets.token_urlsafe(32) # 256 бит энтропии + digest = token_hash(token) + row = await self.db.fetchrow( + "INSERT INTO sessions (token_hash, user_id, expires_at) " + "VALUES ($1, $2, now() + make_interval(secs => $3)) RETURNING expires_at", + digest, user_id, float(config.SESSION_TTL_SECONDS)) + remaining = row["expires_at"].timestamp() - time.time() + await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]), + min(remaining, config.SESSION_CACHE_TTL_SECONDS)) + return token, row["expires_at"] + + async def resolve(self, token): + """token -> user_id (str) либо None, если сессии нет/истекла/отозвана.""" + digest = token_hash(token) + cached = await self.cache.get(digest) + if cached is not None: + user_id, _, expires_ms = cached.decode().partition("|") + if int(expires_ms) > time.time() * 1000: + return user_id + await self.cache.delete(digest) + row = await self.db.fetchrow( + "SELECT user_id, expires_at FROM sessions " + "WHERE token_hash = $1 AND revoked_at IS NULL AND expires_at > now()", digest) + if row is None: + return None + user_id = str(row["user_id"]) + remaining = row["expires_at"].timestamp() - time.time() + await self.cache.put(digest, self._cache_value(user_id, row["expires_at"]), + min(remaining, config.SESSION_CACHE_TTL_SECONDS)) + return user_id + + async def revoke(self, token): + digest = token_hash(token) + await self.db.execute("UPDATE sessions SET revoked_at = now() WHERE token_hash = $1 AND revoked_at IS NULL", digest) + await self.cache.delete(digest) diff --git a/src/app/web.py b/src/app/web.py new file mode 100644 index 00000000..f463b5d8 --- /dev/null +++ b/src/app/web.py @@ -0,0 +1,187 @@ +"""Общий HTTP-слой: ошибки, запрос, валидация, курсоры, ответы, журнал.""" +import base64 +import json +import re +import sys +import time +import uuid +from datetime import datetime, timedelta, timezone +from urllib.parse import parse_qsl + +from . import config + +UUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") +LIMIT_RE = re.compile(r"[0-9]{1,9}") +EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc) + + +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + super().__init__(message) + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +def bad(message): + return ApiError(400, "invalid_request", message) + + +def unavailable(message="Зависимость временно недоступна"): + return ApiError(503, "unavailable", message, {"retry-after": str(config.RETRY_AFTER_SECONDS)}) + + +def log(level, msg, **fields): + record = {"ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "level": level, "msg": msg} + record.update(fields) + sys.stdout.write(json.dumps(record, ensure_ascii=False) + "\n") + sys.stdout.flush() + + +# --- время ------------------------------------------------------------------- +def format_dt(value): + value = value.astimezone(timezone.utc) + return value.strftime("%Y-%m-%dT%H:%M:%S.") + "%06dZ" % value.microsecond + + +def dt_to_us(value): + return (value - EPOCH) // timedelta(microseconds=1) + + +def us_to_dt(us): + return EPOCH + timedelta(microseconds=us) + + +# --- валидация --------------------------------------------------------------- +def parse_uuid(value, what="id"): + if not isinstance(value, str) or not UUID_RE.fullmatch(value): + raise bad("Некорректный %s" % what) + return value.lower() + + +def valid_text(value, field, maximum, allow_empty=False): + if not isinstance(value, str): + raise bad("Поле %s должно быть строкой" % field) + try: + value.encode("utf-8") + except UnicodeEncodeError: + raise bad("Поле %s содержит некорректные символы" % field) from None + if "\x00" in value: + raise bad("Поле %s содержит NUL" % field) + if len(value) > maximum: + raise bad("Поле %s слишком длинное" % field) + if not allow_empty and not value.strip(): + raise bad("Поле %s не может быть пустым" % field) + return value + + +def _reject_constant(name): + raise ValueError(name) + + +def parse_json_object(raw, required, allowed): + try: + data = json.loads(raw.decode("utf-8"), parse_constant=_reject_constant) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad("Тело запроса — некорректный JSON") from None + if not isinstance(data, dict): + raise bad("Ожидается JSON-объект") + if set(data) - allowed: + raise bad("Лишние поля в запросе") + if any(key not in data for key in required): + raise bad("Не хватает обязательных полей") + return data + + +def encode_cursor(kind, resource, key): + payload = json.dumps({"k": kind, "r": resource, "t": key[0], "i": key[1]}, separators=(",", ":")) + return base64.urlsafe_b64encode(payload.encode()).decode().rstrip("=") + + +def decode_cursor(text, kind, resource): + try: + data = json.loads(base64.urlsafe_b64decode(text + "=" * (-len(text) % 4)).decode()) + if (not isinstance(data, dict) or data.get("k") != kind or data.get("r") != resource + or not isinstance(data.get("t"), int) or isinstance(data.get("t"), bool) + or not isinstance(data.get("i"), str) or not UUID_RE.fullmatch(data["i"]) + or not -62135596800000000 < data["t"] < 253402300799000000): + raise ValueError + return data["t"], data["i"] + except (ValueError, UnicodeError): + raise bad("Некорректный cursor") from None + + +def page_params(query, kind, resource): + limit, cursor = 50, None + if "limit" in query: + if len(query["limit"]) != 1 or not LIMIT_RE.fullmatch(query["limit"][0]): + raise bad("limit должен быть целым числом 1–100") + limit = int(query["limit"][0]) + if not 1 <= limit <= 100: + raise bad("limit должен быть целым числом 1–100") + if "cursor" in query: + if len(query["cursor"]) != 1 or not query["cursor"][0]: + raise bad("Некорректный cursor") + cursor = decode_cursor(query["cursor"][0], kind, resource) + return limit, cursor + + +def parse_query(raw): + result = {} + if raw: + for key, value in parse_qsl(raw.decode("latin-1"), keep_blank_values=True): + result.setdefault(key, []).append(value) + return result + + +class Request: + def __init__(self, scope, receive, request_id): + self.method = scope["method"] + self.path = scope["path"] + self.query = parse_query(scope.get("query_string", b"")) + self.headers = {k.decode("latin-1").lower(): v.decode("latin-1") for k, v in scope["headers"]} + self.receive = receive + self.request_id = request_id + self.user_id = None + + async def body(self): + declared = self.headers.get("content-length") + if declared is not None: + if not declared.isdigit(): + raise bad("Некорректный Content-Length") + if int(declared) > config.MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"}) + chunks, size = [], 0 + while True: + event = await self.receive() + if event["type"] == "http.disconnect": + raise bad("Соединение закрыто клиентом") + chunk = event.get("body", b"") + size += len(chunk) + if size > config.MAX_BODY_BYTES: + raise ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"connection": "close"}) + chunks.append(chunk) + if not event.get("more_body"): + return b"".join(chunks) + + def require_json_media_type(self): + media = self.headers.get("content-type", "").split(";")[0].strip().lower() + if media != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается Content-Type: application/json") + + +async def respond(send, status, body, request_id, extra=None, instance_id=None): + payload = b"" if status == 204 else json.dumps(body, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + headers = [(b"x-request-id", request_id.encode())] + if status != 204: + headers += [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())] + else: + headers.append((b"content-length", b"0")) + if instance_id: + headers.append((b"x-instance-id", instance_id.encode())) + headers += [(k.encode(), v.encode()) for k, v in (extra or {}).items()] + await send({"type": "http.response.start", "status": status, "headers": headers}) + await send({"type": "http.response.body", "body": payload}) + + +def new_request_id(): + return uuid.uuid4().hex diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..46e5e765 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,208 @@ +"""Собственные тесты lab2 против запущенного стенда (docker compose up -d). + + python -m unittest discover -s student-tests -v # BASE_URL по умолчанию http://127.0.0.1:8080 +Тесты используют только синтетические данные и ничего не удаляют. Проверки expiry и недоступности +зависимостей лежат отдельно (test_expiry.py, test_dependencies.py): они управляют контейнерами. +""" +import base64 +import json +import os +import subprocess +import unittest +import urllib.error +import urllib.request +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote + +BASE = os.environ.get("BASE_URL", "http://127.0.0.1:8080").rstrip("/") +ROOT = os.path.join(os.path.dirname(__file__), "..") + + +def call(method, path, body=None, token=None, headers=None, raw=None): + h = dict(headers or {}) + if method == "POST" and path.endswith("/messages") and headers is None: + h["Idempotency-Key"] = uuid.uuid4().hex # lab3: ключ обязателен; тесты без ключа — в test_idempotency.py + data = raw + if body is not None: + data = json.dumps(body, ensure_ascii=False).encode("utf-8") + h.setdefault("Content-Type", "application/json") + if token: + h["Authorization"] = "Bearer " + token + req = urllib.request.Request(BASE + path, data=data, headers=h, method=method) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + status, hdrs, payload = resp.status, resp.headers, resp.read() + except urllib.error.HTTPError as err: + status, hdrs, payload = err.code, err.headers, err.read() + return status, hdrs, (json.loads(payload) if payload else None) + + +def basic(username, password): + return {"Authorization": "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode()} + + +def register(prefix="u"): + name = "%s_%s" % (prefix, uuid.uuid4().hex[:12]) + password = "Pa55:word-" + uuid.uuid4().hex # двоеточие в пароле допустимо + status, _, user = call("POST", "/api/v1/users", {"username": name, "display_name": "Имя " + name, "password": password}) + assert status == 201, user + status, _, session = call("POST", "/api/v1/auth/sessions", headers=basic(name, password)) + assert status == 201, session + return {"user": user, "password": password, "token": session["token"], "session": session} + + +def psql(sql): + out = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", "psql", "-U", "messenger", + "-d", "messenger", "-Atc", sql], cwd=ROOT, capture_output=True, text=True, check=True) + return out.stdout.strip() + + +class AuthTest(unittest.TestCase): + def assertError(self, result, status, code): + self.assertEqual(result[0], status, result[2]) + self.assertEqual(result[2]["error"]["code"], code) + self.assertEqual(result[2]["error"]["request_id"], result[1]["X-Request-Id"]) + + def test_register_validation(self): + base = {"username": "val_" + uuid.uuid4().hex[:8], "display_name": "x", "password": "p" * 12} + for patch in ({"password": "short"}, {"password": "p" * 129}, {"password": 12345678901234}, {"extra": 1}, + {"username": "BAD"}, {"display_name": " "}): + body = dict(base, **patch) + self.assertError(call("POST", "/api/v1/users", body), 400, "invalid_request") + missing = {k: v for k, v in base.items() if k != "password"} + self.assertError(call("POST", "/api/v1/users", missing), 400, "invalid_request") + self.assertEqual(call("POST", "/api/v1/users", dict(base, password="я" * 12))[0], 201) # 12 code points + + def test_password_is_not_exposed_or_stored_plain(self): + actor = register() + self.assertEqual(set(actor["user"]), {"id", "username", "display_name", "created_at"}) + stored = psql("SELECT password_hash FROM users WHERE username = '%s'" % actor["user"]["username"]) + self.assertTrue(stored.startswith("$argon2id$"), stored[:12]) + self.assertNotIn(actor["password"], stored) + + def test_wrong_password_and_unknown_user_are_indistinguishable(self): + actor = register() + wrong = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], "wrong-password-123")) + unknown = call("POST", "/api/v1/auth/sessions", headers=basic("nobody_" + uuid.uuid4().hex[:8], "wrong-password-123")) + for r in (wrong, unknown): + self.assertError(r, 401, "unauthorized") + self.assertEqual(r[1]["WWW-Authenticate"], 'Basic realm="messenger", charset="UTF-8"') + self.assertEqual(wrong[2]["error"]["message"], unknown[2]["error"]["message"]) + for bad in ({}, {"Authorization": "Basic !!!"}, {"Authorization": "Basic " + base64.b64encode(b"nocolon").decode()}, + {"Authorization": "Bearer abc"}): + self.assertEqual(call("POST", "/api/v1/auth/sessions", headers=bad)[0], 401) + + def test_bearer_rules(self): + actor = register() + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[2], actor["user"]) + for headers in ({}, basic(actor["user"]["username"], actor["password"]), {"X-User-Id": actor["user"]["id"]}, + {"Authorization": "Bearer " + "A" * 43}, {"Authorization": "Bearer short"}): + r = call("GET", "/api/v1/users/me", headers=headers) + self.assertError(r, 401, "unauthorized") + self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"') + other = register() + me = call("GET", "/api/v1/users/me", token=actor["token"], headers={"X-User-Id": other["user"]["id"]})[2] + self.assertEqual(me["id"], actor["user"]["id"]) # X-User-Id игнорируется + + def test_session_shape_entropy_and_logout(self): + actor = register() + s = actor["session"] + self.assertEqual(s["token_type"], "Bearer") + self.assertGreaterEqual(len(s["token"]), 43) # 32 байта в base64url = 256 бит + self.assertNotIn(actor["user"]["id"], s["token"]) + self.assertNotEqual(register()["token"], s["token"]) + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"])[0], 204) + self.assertError(call("GET", "/api/v1/users/me", token=s["token"]), 401, "unauthorized") + self.assertError(call("DELETE", "/api/v1/auth/sessions/current", token=s["token"]), 401, "unauthorized") + + def test_logout_revokes_in_database_and_cache(self): + actor = register() + call("GET", "/api/v1/users/me", token=actor["token"]) # прогреваем кэш + call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"]) + revoked = psql("SELECT count(*) FROM sessions s JOIN users u ON u.id = s.user_id " + "WHERE u.username = '%s' AND s.revoked_at IS NOT NULL" % actor["user"]["username"]) + self.assertEqual(revoked, "1") + + def test_token_only_stored_as_hash(self): + actor = register() + count = psql("SELECT count(*) FROM sessions WHERE token_hash = '\\x00'::bytea OR encode(token_hash,'escape') = '%s'" % actor["token"]) + self.assertEqual(count, "0") + + def test_concurrent_registration_of_same_username(self): + name = "race_" + uuid.uuid4().hex[:10] + body = {"username": name, "display_name": "x", "password": "racing-password-1"} + with ThreadPoolExecutor(12) as pool: + codes = sorted(pool.map(lambda _: call("POST", "/api/v1/users", body)[0], range(12))) + self.assertEqual(codes, [201] + [409] * 11) + self.assertEqual(psql("SELECT count(*) FROM users WHERE username = '%s'" % name), "1") + + +class DataTest(unittest.TestCase): + def test_chat_creation_is_atomic(self): + a = register() + before = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)") + ghost = str(uuid.uuid4()) + r = call("POST", "/api/v1/chats", {"title": "откат", "member_ids": [a["user"]["id"], ghost]}, token=a["token"]) + self.assertEqual(r[0], 400) + after = psql("SELECT (SELECT count(*) FROM chats) || '/' || (SELECT count(*) FROM chat_members)") + self.assertEqual(before, after, "после неудачного создания не должно остаться чата или участников") + ok = call("POST", "/api/v1/chats", {"title": "ок", "member_ids": [a["user"]["id"]]}, token=a["token"]) + self.assertEqual(ok[0], 201) + + def test_foreign_chat_empty_message_bad_json(self): + a, b, eve = register(), register(), register() + chat = call("POST", "/api/v1/chats", {"title": "ЛС", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2] + path = "/api/v1/chats/%s/messages" % chat["id"] + self.assertEqual(call("POST", path, {"text": "чужое"}, eve["token"])[0], 404) + self.assertEqual(call("GET", path, token=eve["token"])[0], 404) + self.assertEqual(call("POST", path, {"text": " "}, a["token"])[0], 400) + self.assertEqual(call("POST", path, raw=b"{oops", token=a["token"], headers={"Content-Type": "application/json"})[0], 400) + self.assertEqual(call("POST", path, {"text": "x", "sender_id": b["user"]["id"]}, a["token"])[0], 400) + sent = call("POST", path, {"text": "Привет 👋"}, a["token"])[2] + self.assertEqual(sent["sender_id"], a["user"]["id"]) + + def test_multi_page_history_and_cursor_binding(self): + a, b = register(), register() + chat = call("POST", "/api/v1/chats", {"title": "стр", "member_ids": [a["user"]["id"], b["user"]["id"]]}, a["token"])[2] + other = call("POST", "/api/v1/chats", {"title": "др", "member_ids": [a["user"]["id"]]}, a["token"])[2] + path = "/api/v1/chats/%s/messages" % chat["id"] + sent = [call("POST", path, {"text": "m%d" % i}, (a if i % 2 else b)["token"])[2] for i in range(23)] + walked, cursor, pages = [], None, 0 + while True: + q = "?limit=%d" % (5 if pages % 2 == 0 else 8) + ("&cursor=" + quote(cursor, safe="") if cursor else "") + page = call("GET", path + q, token=b["token"])[2] + walked += page["items"] + pages += 1 + cursor = page["next_cursor"] + if not cursor: + break + self.assertEqual(walked, sent) + self.assertGreaterEqual(pages, 3) + first = call("GET", path + "?limit=1", token=a["token"])[2]["next_cursor"] + r = call("GET", "/api/v1/chats/%s/messages?cursor=%s" % (other["id"], quote(first, safe="")), token=a["token"]) + self.assertEqual(r[0], 400) + + def test_parallel_writes_from_many_clients(self): + users = [register() for _ in range(5)] + chat = call("POST", "/api/v1/chats", {"title": "гонки", "member_ids": [u["user"]["id"] for u in users]}, users[0]["token"])[2] + path = "/api/v1/chats/%s/messages" % chat["id"] + + def client(u): + return [call("POST", path, {"text": "%s-%d" % (u["user"]["username"], i)}, u["token"])[0] for i in range(20)] + + with ThreadPoolExecutor(5) as pool: + codes = [c for part in pool.map(client, users) for c in part] + self.assertEqual(codes, [201] * 100) + total, cursor = [], None + while True: + page = call("GET", path + "?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else ""), token=users[0]["token"])[2] + total += page["items"] + cursor = page["next_cursor"] + if not cursor: + break + self.assertEqual(len({m["id"] for m in total}), 100) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_dependencies.py b/student-tests/test_dependencies.py new file mode 100644 index 00000000..7d8519ef --- /dev/null +++ b/student-tests/test_dependencies.py @@ -0,0 +1,84 @@ +"""Поведение при недоступности зависимостей: Valkey (кэш) и PostgreSQL (источник истины). + + python student-tests/test_dependencies.py +Контейнеры останавливаются и запускаются обратно (volumes сохраняются). Ожидания: + * Valkey остановлен -> кэш обходится, API работает (чтение сессии из PostgreSQL), logout работает; + * PostgreSQL остановлен -> 503 в контрактном формате с Retry-After (не 500, не «анонимный» доступ), + /health/ready = 503, запрос без токена по-прежнему 401; после запуска БД API восстанавливается сам. +""" +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, basic, call, register # noqa: E402 + + +def compose(*args): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, capture_output=True, text=True) + + +def wait(predicate, seconds=60): + deadline = time.time() + seconds + while time.time() < deadline: + try: + if predicate(): + return + except Exception: + pass + time.sleep(0.5) + raise RuntimeError("условие не выполнено за %d с" % seconds) + + +class DependencyTest(unittest.TestCase): + def tearDown(self): + compose("start", "postgres", "valkey") + wait(lambda: call("GET", "/health/ready")[0] == 200) + + def test_valkey_down_is_bypassed(self): + actor = register("dep") + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) + compose("stop", "valkey") + started = time.time() + for _ in range(5): + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) + self.assertLess(time.time() - started, 3, "circuit breaker не должен давать каждому запросу ждать таймаут") + # вход и выход работают без кэша + s = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"])) + self.assertEqual(s[0], 201) + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=s[2]["token"])[0], 204) + self.assertEqual(call("GET", "/api/v1/users/me", token=s[2]["token"])[0], 401) # отзыв виден без кэша + self.assertEqual(call("GET", "/health/ready")[0], 200) # кэш не критичен для readiness + + def test_postgres_down_gives_bounded_503_never_anonymous_access(self): + actor = register("dep") + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # токен в кэше + compose("stop", "postgres") + time.sleep(1) + for method, path, body, token in (("GET", "/api/v1/users/me", None, actor["token"]), + ("GET", "/api/v1/chats", None, actor["token"]), + ("POST", "/api/v1/chats", {"title": "x", "member_ids": [actor["user"]["id"]]}, actor["token"]), + ("GET", "/api/v1/users/me", None, "Z" * 43)): # неизвестный токен: кэш пуст -> БД -> 503 + r = call(method, path, body, token) + self.assertEqual(r[0], 503, (path, r)) + self.assertEqual(r[2]["error"]["code"], "unavailable") # ответ API либо LB (errorfile) — в контрактном формате + self.assertGreaterEqual(int(r[1]["Retry-After"]), 1) + self.assertEqual(r[2]["error"]["request_id"], r[1]["X-Request-Id"]) + r = call("POST", "/api/v1/auth/sessions", headers=basic(actor["user"]["username"], actor["password"])) + self.assertEqual(r[0], 503) + self.assertEqual(call("GET", "/health/ready")[0], 503) + # через LB реплики исключены (readiness=503), но процессы живы: liveness каждой реплики проверяем изнутри + probe = ("import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=3).status)") + for cid in compose("ps", "-q", "api").stdout.split(): + out = subprocess.run(["docker", "exec", cid, "python", "-c", probe], capture_output=True, text=True).stdout.strip() + self.assertEqual(out, "200") + self.assertIn(call("GET", "/api/v1/users/me")[0], (401, 503)) # без токена — никогда не 200 (401 от API либо 503 от LB) + compose("start", "postgres") + wait(lambda: call("GET", "/health/ready")[0] == 200) + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # сессия пережила простой БД + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 00000000..293fd059 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,69 @@ +"""Проверка срока жизни сессии при коротком SESSION_TTL_SECONDS (по умолчанию 2). + + python student-tests/test_expiry.py +Скрипт пересоздаёт только контейнер api с SESSION_TTL_SECONDS=2, проверяет expiry и возвращает настройки +по умолчанию. Данные и volumes не затрагиваются. +""" +import os +import subprocess +import sys +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import ROOT, call, register # noqa: E402 + +TTL = 2 + + +def compose(*args, env=None): + return subprocess.run(["docker", "compose", "--env-file", ".env", *args], cwd=ROOT, check=True, + capture_output=True, text=True, env=dict(os.environ, **(env or {}))) + + +def wait_ready(seconds=60): + deadline = time.time() + seconds + while time.time() < deadline: + try: + if call("GET", "/health/ready")[0] == 200: + return + except Exception: + pass + time.sleep(0.5) + raise RuntimeError("api не стал ready") + + +class ExpiryTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + compose("up", "-d", "--no-deps", "--force-recreate", "api", env={"SESSION_TTL_SECONDS": str(TTL)}) + wait_ready() + + @classmethod + def tearDownClass(cls): + compose("up", "-d", "--no-deps", "--force-recreate", "api") + wait_ready() + + def test_session_expires_after_ttl(self): + actor = register("exp") + session = actor["session"] + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 200) # валидна, кэш прогрет + time.sleep(TTL + 1.5) + r = call("GET", "/api/v1/users/me", token=actor["token"]) + self.assertEqual(r[0], 401, "после TTL токен должен быть отвергнут (и кэшем, и БД)") + self.assertEqual(r[1]["WWW-Authenticate"], 'Bearer realm="messenger"') + self.assertEqual(r[2]["error"]["code"], "unauthorized") + self.assertEqual(call("DELETE", "/api/v1/auth/sessions/current", token=actor["token"])[0], 401) + self.assertIn("expires_at", session) + + def test_expiry_is_not_extended_by_use(self): + actor = register("exp") + for _ in range(3): + call("GET", "/api/v1/users/me", token=actor["token"]) + time.sleep(0.5) + time.sleep(1.5) + self.assertEqual(call("GET", "/api/v1/users/me", token=actor["token"])[0], 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..e421a8e9 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,121 @@ +"""Идемпотентность отправки сообщений (lab3) и распределение по репликам. Запуск: + python student-tests/test_idempotency.py (BASE_URL по умолчанию http://127.0.0.1:8080, через LB) +""" +import os +import sys +import unittest +import uuid +from concurrent.futures import ThreadPoolExecutor +from urllib.parse import quote + +sys.path.insert(0, os.path.dirname(__file__)) +from test_api import call, psql, register # noqa: E402 + + +def chat_with(owner, *others): + ids = [owner["user"]["id"]] + [o["user"]["id"] for o in others] + return call("POST", "/api/v1/chats", {"title": "идемпотентность", "member_ids": ids}, owner["token"])[2] + + +def history(token, chat_id): + items, cursor = [], None + while True: + page = call("GET", "/api/v1/chats/%s/messages?limit=100%s" % (chat_id, "&cursor=" + quote(cursor, safe="") if cursor else ""), token=token)[2] + items += page["items"] + cursor = page["next_cursor"] + if not cursor: + return items + + +class IdempotencyTest(unittest.TestCase): + def setUp(self): + self.a, self.b, self.eve = register(), register(), register() + self.chat = chat_with(self.a, self.b) + self.path = "/api/v1/chats/%s/messages" % self.chat["id"] + + def post(self, actor, body, key, path=None): + return call("POST", path or self.path, body, actor["token"], headers={"Idempotency-Key": key}) + + def test_key_validation(self): + for headers in ({}, {"Idempotency-Key": ""}, {"Idempotency-Key": "a b"}, + {"Idempotency-Key": "x" * 129}, {"Idempotency-Key": "a/b"}): + h = dict(headers) + h.setdefault("Content-Type", "application/json") + r = call("POST", self.path, raw=b'{"text":"x"}', token=self.a["token"], headers=h) + self.assertEqual(r[0], 400, headers) + self.assertEqual(r[2]["error"]["code"], "invalid_request") + self.assertEqual(self.post(self.a, {"text": "ok"}, "A-z_0.9:-" + "k" * 100)[0], 201) + + def test_replay_returns_same_message_and_creates_no_duplicate(self): + key = uuid.uuid4().hex + first = self.post(self.a, {"text": "один раз"}, key) + second = self.post(self.a, {"text": "один раз"}, key) + self.assertEqual((first[0], second[0]), (201, 201)) + self.assertEqual(first[2], second[2]) + self.assertEqual(len([m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "один раз"]), 1) + + def test_different_body_same_key_is_409(self): + key = uuid.uuid4().hex + self.post(self.a, {"text": "первое"}, key) + r = self.post(self.a, {"text": "другое"}, key) + self.assertEqual(r[0], 409) + self.assertEqual(r[2]["error"]["code"], "conflict") + + def test_scope_is_user_and_chat(self): + key = uuid.uuid4().hex + m1 = self.post(self.a, {"text": "общий ключ"}, key)[2] + m2 = self.post(self.b, {"text": "общий ключ"}, key)[2] # другой пользователь + other = chat_with(self.a) + m3 = self.post(self.a, {"text": "общий ключ"}, key, "/api/v1/chats/%s/messages" % other["id"])[2] # другой чат + self.assertEqual(len({m1["id"], m2["id"], m3["id"]}), 3) + + def test_replay_rechecks_session_and_membership(self): + key = uuid.uuid4().hex + self.post(self.a, {"text": "защищено"}, key) + self.assertEqual(self.post(self.eve, {"text": "защищено"}, key)[0], 404) # посторонний + r = call("POST", self.path, {"text": "защищено"}, "Z" * 43, headers={"Idempotency-Key": key}) + self.assertEqual(r[0], 401) # недействительный токен + logout = register() + chat = chat_with(logout) + path = "/api/v1/chats/%s/messages" % chat["id"] + self.post(logout, {"text": "x"}, key, path) + call("DELETE", "/api/v1/auth/sessions/current", token=logout["token"]) + self.assertEqual(call("POST", path, {"text": "x"}, logout["token"], headers={"Idempotency-Key": key})[0], 401) + + def test_semantically_equal_body_is_replayed(self): + key = uuid.uuid4().hex + first = self.post(self.a, {"text": "эквивалент"}, key) + raw = call("POST", self.path, raw=b'{ "text" : "\u044d\u043a\u0432\u0438\u0432\u0430\u043b\u0435\u043d\u0442" }', + token=self.a["token"], headers={"Idempotency-Key": key, "Content-Type": "application/json"}) + self.assertEqual((raw[0], raw[2]), (201, first[2])) + + def test_concurrent_same_key_creates_exactly_one_message_across_replicas(self): + key = uuid.uuid4().hex + with ThreadPoolExecutor(24) as pool: + results = list(pool.map(lambda _: self.post(self.a, {"text": "гонка ключа"}, key), range(24))) + self.assertEqual({r[0] for r in results}, {201}) + self.assertEqual(len({r[2]["id"] for r in results}), 1) + instances = {r[1]["X-Instance-Id"] for r in results} + self.assertGreaterEqual(len(instances), 2, "повторы должны обслуживаться разными репликами: %s" % instances) + found = [m for m in history(self.b["token"], self.chat["id"]) if m["text"] == "гонка ключа"] + self.assertEqual(len(found), 1) + + def test_expired_key_is_reusable_after_ttl(self): + key = uuid.uuid4().hex + first = self.post(self.a, {"text": "ttl"}, key)[2] + psql("UPDATE idempotency_keys SET created_at = now() - interval '25 hours' WHERE idem_key = '%s'" % key) + again = self.post(self.a, {"text": "ttl"}, key) + self.assertEqual(again[0], 201) + self.assertNotEqual(again[2]["id"], first["id"], "после 24 ч ключ считается новым") + + def test_session_is_shared_between_replicas_without_relogin(self): + seen = set() + for _ in range(30): + r = call("GET", "/api/v1/users/me", token=self.a["token"]) + self.assertEqual(r[0], 200) + seen.add(r[1]["X-Instance-Id"]) + self.assertGreaterEqual(len(seen), 2, seen) + + +if __name__ == "__main__": + unittest.main() diff --git a/student-tests/test_topology.py b/student-tests/test_topology.py new file mode 100644 index 00000000..6bc6c640 --- /dev/null +++ b/student-tests/test_topology.py @@ -0,0 +1,58 @@ +"""Проверка топологии стенда (уровень 4): число реплик, отсутствие публикации портов API, лимиты ресурсов, healthcheck, +отсутствие container_name, единая публичная точка входа. Запуск: python student-tests/test_topology.py""" +import json +import subprocess +import unittest + +PROJECT = "mtusi-lab3" + + +def sh(*args): + return subprocess.run(args, capture_output=True, text=True, encoding="utf-8").stdout.strip() + + +def containers(service): + ids = sh("docker", "ps", "-q", "--filter", "label=com.docker.compose.project=" + PROJECT, + "--filter", "label=com.docker.compose.service=" + service).split() + return [json.loads(sh("docker", "inspect", i))[0] for i in ids] + + +class TopologyTest(unittest.TestCase): + def test_at_least_two_api_replicas_without_published_ports(self): + api = containers("api") + self.assertGreaterEqual(len(api), 2) + for c in api: + self.assertFalse([p for p in c["HostConfig"]["PortBindings"] or {}], "реплика API публикует порт в обход LB") + self.assertEqual(c["State"]["Health"]["Status"], "healthy") + + def test_only_lb_is_published_and_only_on_localhost(self): + published = {} + for service in ("api", "postgres", "valkey", "lb"): + for c in containers(service): + for port, binds in (c["HostConfig"]["PortBindings"] or {}).items(): + published.setdefault(service, []).extend((port, b["HostIp"], b["HostPort"]) for b in binds) + self.assertEqual(list(published), ["lb"], published) + self.assertEqual(published["lb"], [("8080/tcp", "127.0.0.1", "8080")]) + + def test_resource_limits_are_set(self): + for service in ("api", "postgres", "valkey", "lb"): + for c in containers(service): + self.assertGreater(c["HostConfig"]["Memory"], 0, service) + self.assertGreater(c["HostConfig"]["NanoCpus"], 0, service) + + def test_no_fixed_container_names(self): + for c in containers("api"): + labels = c["Config"]["Labels"] + self.assertEqual(c["Name"].lstrip("/"), "%s-api-%s" % (PROJECT, labels["com.docker.compose.container-number"])) + + def test_api_runs_as_non_root(self): + for c in containers("api"): + self.assertEqual(c["Config"]["User"], "10001:10001") + + def test_stop_grace_period_covers_drain(self): + for c in containers("api"): + self.assertGreaterEqual(c["Config"].get("StopTimeout", 10), 15) + + +if __name__ == "__main__": + unittest.main()