diff --git a/.env.example b/.env.example index d32b9be0..5dddd7df 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,11 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab +POSTGRES_USER=messenger +POSTGRES_PASSWORD=change-me-before-running +POSTGRES_DB=messenger SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 +DB_POOL_MAX=10 +DB_POOL_MIN=1 +API_CPUS=1.0 +API_MEMORY=512m +CURSOR_SECRET=replace-with-a-random-secret-of-at-least-32-bytes +LB_CPUS=0.25 diff --git a/.gitignore b/.gitignore index b5a71896..ed8abe81 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +.pytest_cache/ diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1d4fbb04 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_DEFAULT_TIMEOUT=60 PIP_RETRIES=3 +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app +COPY --chown=app:app server.py ./server.py +COPY --chown=app:app migrations ./migrations +USER 10001:10001 +EXPOSE 8080 +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" +CMD ["python", "server.py"] diff --git a/Makefile b/Makefile index 5d86d171..8d3ad200 100644 --- a/Makefile +++ b/Makefile @@ -1,30 +1,33 @@ PYTHON ?= python3 -BASE_URL ?= http://localhost:8080 +BASE_URL ?= http://127.0.0.1:8080 CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover - +.PHONY: help check validate test load test-persistence test-failover test-idempotency track1 track1-offered track1-offered-scale rolling-update help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" - + @echo "check, validate, test, load, test-persistence, test-failover, test-idempotency, track1, track1-offered, track1-offered-scale, rolling-update" check: $(PYTHON) scripts/check_contract.py - validate: $(PYTHON) scripts/validate_openapi.py - test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) - load: $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) - test-persistence: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/restart.sh"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) - test-failover: - @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) + @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/stop-one.sh"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) +test-idempotency: + $(PYTHON) student-tests/idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS) +track1: + $(PYTHON) scripts/track1_experiment.py +track1-offered: + $(PYTHON) scripts/track1_offered_load.py +track1-offered-scale: + $(PYTHON) scripts/track1_offered_scale.py +rolling-update: + ./scripts/rolling-update.sh diff --git a/REPORT.md b/REPORT.md index c2679005..92c5dd4d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,101 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Койчакаев Багавдин, БВТ2301. +- Целевая оценка: 5; законченный трек: 1 — цена масштабирования. +- Состав реализации: API ЛР2, HAProxy, минимум две API-реплики, общие PostgreSQL и Valkey, конкурентная идемпотентность в PostgreSQL. +- Внешние источники: контракт и материалы курса. Использованы Python 3.12.8, psycopg 3.2.3, argon2-cffi 23.1.0, redis-py 5.2.1; при подготовке использовался Codex. -## Запуск из чистого clone +## Запуск -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +```sh +cp .env.example .env +# задайте POSTGRES_PASSWORD и CURSOR_SECRET (случайный ключ минимум 32 байта) +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180 +make check +make validate +make test BASE_URL=http://127.0.0.1:8080 +make test-idempotency BASE_URL=http://127.0.0.1:8080 +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 +``` + +Приложение слушает только внутреннюю сеть Compose. Единственный опубликованный порт — loopback порт HAProxy 8080. Миграции запускает одноразовый сервис `migrate` перед API, под advisory lock PostgreSQL. Первая схема сохраняется совместимой с базой ЛР2; миграция `002_idempotency` добавляет таблицу ключей и индекс срока хранения. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +HTTP client → HAProxy :8080 → API × 2+ → PostgreSQL (messages, sessions, idempotency) + ↘ общий Valkey (TTL lookup hint; PostgreSQL остаётся источником истины) +``` -## Доказательства критериев +HAProxy round-robin балансирует запросы между адресами DNS `api`, повторно разрешая адрес каждые 2 секунды. Проверка `GET /health/ready` выполняется раз в секунду; две ошибки исключают backend, два успеха возвращают его. Connect timeout — 1 секунда, timeout ожидания/ответа — 10 секунд. API слушает `0.0.0.0:8080` внутри сети и на SIGTERM прекращает `serve_forever`; Compose ждёт до 10 секунд для штатного завершения. -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Сессия проверяется PostgreSQL на каждом запросе; значение Valkey лишь ускоряет lookup и не авторизует пользователя без живой строки в PostgreSQL. Общий `CURSOR_SECRET` нужен, чтобы cursor истории, созданный одной репликой, принимался другой. Секрет и пароль БД находятся только в локальном `.env`, игнорируемом Git и Docker build context. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Для каждого POST сообщения обязателен ключ `[A-Za-z0-9._:-]{1,128}`. Уникальная область — пользователь, метод, путь чата и ключ; body хешируется после нормализации JSON, а пропущенный `attachment_ids` равен `[]`. `pg_advisory_xact_lock` сериализует одновременные запросы одной области во всех API-процессах; PostgreSQL хранит отпечаток body и точный JSON результата вместе с сообщением в одной транзакции. Совпавший запрос возвращает тот же `201` и Message; отличающийся body — `409`; запись можно использовать заново спустя 24 часа. Ошибка/откат транзакции не оставляет частично зарегистрированный ключ. Каждый API-ответ включая ошибку содержит непустой `X-Request-Id` и идентификатор процесса в `X-Instance-Id`. -## Эксперименты и отказы +### Роли и версии -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +| Роль | Service | Версия | Health/volume/порты | +|---|---|---|---| +| API | `api` (Compose scale) | Python 3.12.8 slim Bookworm, pinned digest в Dockerfile | `/health/ready`, без host-порта, без локального состояния | +| Балансировщик | `haproxy` | 3.0.8 Alpine | HAProxy backend checks, единственный порт `127.0.0.1:8080` | +| База | `postgres` | 16.4 Bookworm | `pg_isready`, общий volume `postgres_data`, внутренний порт 5432 | +| Кэш сессий | `valkey` | 8.0.1 Alpine | `PING`, volume `valkey_data`, AOF `everysec`, внутренний порт 6379 | +| Миграции | `migrate` | тот же образ API | одноразовый контейнер; выполняется до запуска реплик | -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +CPU/RAM limits заданы в compose для API, HAProxy, PostgreSQL и Valkey. `api` не имеет `container_name`, поэтому штатное горизонтальное масштабирование поддерживается. -## Ограничения и следующие шаги +## Проверки уровня 3 и 4 -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Публичный smoke проверяет обязательный API через HAProxy. Дополнительный параллельный тест запускает 32 запроса с одним ключом и одним body одновременно; все ответы вернули один ID сообщения, наблюдались две реплики. Persistence-сценарий перезапустил API, HAProxy, PostgreSQL и Valkey без удаления volumes; тот же Bearer-токен прочитал те же данные. Три failover-прогона остановили API-реплику по очереди и повторили сценарий после восстановления. Каждый наблюдал две реплики до отказа; новые записи восстановились за 2,902 с, 1,543 с и 1,447 с; ошибок было 0 во всех трёх прогонах (113, 125 и 114 успешных записей); сообщение до отказа и все подтверждённые записи остались доступны, дубли отсутствовали. В конце теста остановленная реплика оставалась остановленной; Compose восстановил её перед следующим прогоном и после финальной проверки. + +Rolling replacement под нагрузкой выполняется скриптом `scripts/rolling-update.sh`: пул сначала увеличивается до трёх, затем контейнеры заменяются по одному с сохранением активного backend. Под нагрузкой 60 секунд, concurrency 16: 50 291 успешная запись (836,949 RPS), 24 ошибки из 50 315 запросов (0,048%), p50/p95/p99 = 6,118/55,938/85,729 мс. После завершения пул восстановлен до двух здоровых реплик. Метрики сохранены в `evidence/rolling-update-load.json`; успешность запросов при замене не абсолютная, что отражено в ограничениях. + +## Трек 1 — цена масштабирования + +**Гипотеза.** При равном суммарном CPU/RAM перенос ресурса с одного API на несколько реплик улучшит RPS только до насыщения общей PostgreSQL или пула соединений. При фиксированной мощности на реплику добавление API в какой-то момент перестанет повышать throughput; увеличение пула сверх полезного уровня добавит конкуренцию за БД, а не пропускную способность. + +**Методика.** `scripts/track1_experiment.py` пересоздаёт API при каждом профиле, ждёт readiness и 5 секунд стабилизации, затем трижды измеряет закрытый цикл POST-сообщений по 30 секунд с concurrency 16. Перед каждым sample очищаются только учебные таблицы `messages` и `idempotency_keys`, затем выполняется `ANALYZE`; порядок вариантов перемешивается детерминированным seed. Сессии/пользователи остаются в PostgreSQL. Фиксированный бюджет: 2 vCPU и 1 GiB RAM суммарно для API, разделённые между 1/2/4 репликами. Серия роста: 0,5 vCPU и 256 MiB на реплику для 1/2/4/8 реплик. Отдельное сравнение при четырёх репликах меняет `DB_POOL_MAX` 5/10/20. PostgreSQL ограничен 2 CPU и 1 GiB; HAProxy — 0,25 CPU и 128 MiB. Записываются успешные RPS, ошибки, p50/p95/p99, CPU/RAM компонентов и число сообщений. Сводки и исходные данные — `evidence/track1-results.json`, график — `evidence/track1-rps.svg`. + +Чтобы определить точку насыщения, `scripts/track1_offered_load.py` подаёт фиксированный поток 250/500/750/1000 POST RPS на восемь API-реплик (0,5 CPU / 256 MiB каждая) по три раза в перемешанном порядке; HTTP-клиент переиспользуется в потоках, число in-flight ограничено, Docker stats во время нагрузочного окна не опрашивается. Throughput рассчитан по фактическому времени вместе с drain. Затем `scripts/track1_offered_scale.py` делает три парных прогона 8/12 реплик при предложенных 1000 RPS. Между повторами масштабирование возвращается к восьми репликам, чтобы освободить DB-соединения до очистки синтетических таблиц. Все исходные измерения лежат в `evidence/offered-load-results.json` и `evidence/offered-scale-results.json`; график rate sweep — `evidence/offered-load.svg`. + +| Профиль | Реплики | Пул/реплика | медиана RPS (min–max) | медиана ошибок | p50/p95/p99, ms | API CPU mean/peak, % | API RAM peak, MiB | +|---|---:|---:|---:|---:|---:|---:|---:| +| fixed-1 | 1 | 10 | 418,5 (389,7–456,3) | 0,017% | 32,3/53,0/82,4 | 174,3/190,2 | 61,9 | +| fixed-2 | 2 | 10 | 698,6 (653,6–749,2) | 0% | 8,0/62,4/92,9 | 74,0/109,5 | 60,9 | +| fixed-4 | 4 | 10 | 837,7 (590,7–900,3) | 0,007% | 5,2/57,5/101,9 | 38,2/53,3 | 62,4 | +| scaled-1 | 1 | 10 | 103,7 (102,1–108,8) | 0,064% | 111,1/313,3/484,9 | 47,8/53,3 | 83,5 | +| scaled-2 | 2 | 10 | 274,6 (244,8–276,3) | 0,048% | 22,9/183,7/290,4 | 37,6/52,9 | 140,0 | +| scaled-4 | 4 | 10 | 680,7 (635,3–875,7) | 0% | 6,0/71,0/123,1 | 40,8/53,2 | 79,2 | +| scaled-8 | 8 | 10 | 764,2 (655,4–787,6) | 0% | 9,5/70,6/111,2 | 29,0/50,1 | 82,0 | +| pool-5 | 4 | 5 | 921,2 (837,9–1061,3) | 0,003% | 5,3/49,2/89,8 | 40,4/53,3 | 62,6 | +| pool-10 | 4 | 10 | 808,9 (611,5–969,2) | 0,020% | 5,5/66,7/108,3 | 38,9/51,9 | 99,5 | +| pool-20 | 4 | 20 | 951,3 (573,5–1044,2) | 0,003% | 5,4/45,3/89,2 | 41,7/52,5 | 63,1 | + +Среда: macOS 26.5.1 arm64, 8 host CPU; Docker Engine 29.8.2; Docker VM 8 CPU / 4 106 605 444 байта RAM (~3 918 MiB); Python клиента 3.14.6; Python API 3.12.8; PostgreSQL 16.4. Закрытая серия длилась 30 секунд, rate-controlled серия — 15 секунд на sample. Во всех замерах JSON фиксирует размер синтетических таблиц; перед sample `messages` и `idempotency_keys` очищались, остальные пользовательские данные сохранялись. Closed-loop percentiles подвержены coordinated omission; rate-controlled серия имеет короткие окна и оценивает только эту среду. Все результаты ограничены одним Docker-host; потеря узла одновременно затрагивает API, HAProxy, PostgreSQL и локальные volumes. + +| Предложено, RPS | Медиана успешных RPS (min–max) | Медиана ошибок | Медиана p95, ms | +|---:|---:|---:|---:| +| 250 | 249,6 (249,6–250,0) | 0% | 9,9 | +| 500 | 499,6 (419,6–499,9) | 0% | 41,6 | +| 750 | 745,9 (693,2–748,7) | 0,018% | 100,9 | +| 1000 | 810,1 (553,2–879,3) | 2,753% | 1 611,3 | + +Парное сравнение при 1000 предложенных RPS: + +| API-реплик | Успешные RPS, медиана (min–max) | p95, медиана, ms | Ошибки POST за 3 повтора | Диагностика PostgreSQL | +|---:|---:|---:|---:|---| +| 8 | 727,0 (584,4–851,5) | 1 504,7 | 874 / 45 000 | `SELECT` прошёл | +| 12 | 555,8 (546,0–774,9) | 1 986,4 | 12 / 45 000 | Все 3 запроса отклонены: `too many clients already` | + +**Поиск насыщения и улучшение.** В rate-controlled серии 250 и 500 предложенных RPS полностью принимались без ошибок (медианный p95 9,9 и 41,6 мс). При 750 RPS медиана успешной пропускной способности составила 745,9 RPS, p95 — 100,9 мс. При 1000 RPS throughput вырос лишь до 810,1 RPS, медианная доля ошибок достигла 2,75%, а p95 — 1 611 мс. Это показывает knee общей цепочки между 750 и 1000 RPS; повторяемость ограничена тремя короткими прогонами, поэтому это не универсальная производственная ёмкость. + +Компонентное ограничение установлено отдельной проверкой DB-пула. При `DB_POOL_MAX=10` восемь реплик могут открыть до 80 соединений, а двенадцать — до 120. Фактический `max_connections` PostgreSQL равен 100. В трёх прогонах с 12 репликами и нагрузкой 1000 RPS, помимо роста задержки, PostgreSQL трижды отказал даже в диагностическом `SELECT` с `FATAL: sorry, too many clients already`; после возврата к восьми репликам запросы к БД снова проходили. Наблюдение сохранено в `evidence/db-saturation-observation.txt`, а метрики сравнения — `evidence/offered-scale-results.json`. Таким образом, насыщение пула/БД возникает между восемью и двенадцатью репликами при текущем лимите пула; дальнейшее увеличение API без снижения размера пула или повышения DB connection budget ухудшает доступность общих соединений. 8→12 RPS-диапазоны перекрываются, поэтому причинный вывод основан на подтверждённом исчерпании connection slots, а не только на падении медианного throughput. + +Подтверждённое улучшение при фиксированном суммарном бюджете 2 vCPU / 1 GiB: переход fixed-1→fixed-2 поднял медиану с 418,5 до 698,6 RPS (+66,9%); диапазоны трёх повторов не пересекаются (389,7–456,3 и 653,6–749,2 RPS). p50 снизился с 32,3 до 8,0 мс, p95 изменился с 53,0 до 62,4 мс. Серии DB_POOL_MAX=5/10/20 при низкой конкуренции и HAProxy 0,25/0,5 CPU имеют перекрывающиеся диапазоны; их эффект не заявляется как отдельное улучшение. Все измерения ограничены одной Docker VM и конкретным синтетическим профилем. + +## Результаты и ограничения + +Офлайн структурная проверка, API smoke, гонка идемпотентности, перезапуск и failover пройдены; подробный stdout сохранён в `evidence/lab3-checks.txt`. Полная OpenAPI/JSON Schema проверка требует `openapi-spec-validator==0.7.2` из `requirements-dev.txt`; установить зависимость в этой среде не удалось из-за недоступного DNS PyPI. HTTP используется только на loopback для синтетических данных. Эта схема выдерживает отказ одного API-процесса при работающих HAProxy, host, PostgreSQL и Valkey; она не выдерживает потерю Docker-host, HAProxy или PostgreSQL. При отказе Valkey действующие сессии проверяются по PostgreSQL. Отдельной HA базы данных и второго узла нет. Под нагрузкой при rolling replacement замерены 24 ошибки на 50 315 запросов (0,048%); для повторения именно операции создания сообщения клиент должен повторить запрос с тем же `Idempotency-Key`. Другие POST-операции не обещают прозрачную идемпотентность. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..bbcbc210 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,79 @@ +services: + migrate: + build: . + command: ["python", "server.py"] + environment: + RUN_MIGRATIONS_ONLY: "1" + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + CACHE_URL: redis://valkey:6379/0 + depends_on: + postgres: { condition: service_healthy } + valkey: { condition: service_healthy } + restart: "no" + api: + build: . + restart: unless-stopped + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CURSOR_SECRET: ${CURSOR_SECRET:?Set CURSOR_SECRET in .env} + DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger} + CACHE_URL: redis://valkey:6379/0 + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_MIN: ${DB_POOL_MIN:-1} + depends_on: + migrate: { condition: service_completed_successfully } + valkey: { condition: service_healthy } + cpus: ${API_CPUS:-1.0} + mem_limit: ${API_MEMORY:-512m} + stop_grace_period: 10s + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 2s + timeout: 3s + retries: 2 + start_period: 10s + postgres: + image: postgres:16.4-bookworm + restart: unless-stopped + environment: + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + POSTGRES_DB: ${POSTGRES_DB:-messenger} + volumes: ["postgres_data:/var/lib/postgresql/data"] + cpus: 2.0 + mem_limit: 1g + healthcheck: + test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""] + interval: 3s + timeout: 3s + retries: 20 + valkey: + image: valkey/valkey:8.0.1-alpine + restart: unless-stopped + command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: ["valkey_data:/data"] + cpus: 0.25 + mem_limit: 128m + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 3s + timeout: 3s + retries: 10 + haproxy: + image: haproxy:3.0.8-alpine + restart: unless-stopped + ports: ["127.0.0.1:${APP_PORT:-8080}:8080"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + depends_on: [api] + cpus: ${LB_CPUS:-0.25} + mem_limit: 128m + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 10s + timeout: 3s + retries: 3 +volumes: + postgres_data: + valkey_data: diff --git a/evidence/db-saturation-observation.txt b/evidence/db-saturation-observation.txt new file mode 100644 index 00000000..c18cf994 --- /dev/null +++ b/evidence/db-saturation-observation.txt @@ -0,0 +1,110 @@ +PostgreSQL connection saturation observation at 12 API replicas; no credentials or message content recorded. +$ docker compose --env-file .env ps api +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-10 sha256:4833b59a6d4c8406b779432843914b72e61ee92aecd4325e4d034e6839f2ff57 "python server.py" api 45 minutes ago Up 45 minutes (healthy) 8080/tcp +mtusi2026lab3-api-11 mtusi2026lab3-api "python server.py" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab3-api-12 mtusi2026lab3-api "python server.py" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab3-api-13 mtusi2026lab3-api "python server.py" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab3-api-14 mtusi2026lab3-api "python server.py" api About a minute ago Up About a minute (healthy) 8080/tcp +mtusi2026lab3-api-3 sha256:5f93cf319bad7c0e5f360f38e3e3f07708f2357101f6c5804f0194b1a02377cd "python server.py" api 50 minutes ago Up 50 minutes (healthy) 8080/tcp +mtusi2026lab3-api-4 sha256:5f93cf319bad7c0e5f360f38e3e3f07708f2357101f6c5804f0194b1a02377cd "python server.py" api 50 minutes ago Up 50 minutes (healthy) 8080/tcp +mtusi2026lab3-api-5 sha256:5f93cf319bad7c0e5f360f38e3e3f07708f2357101f6c5804f0194b1a02377cd "python server.py" api 50 minutes ago Up 50 minutes (healthy) 8080/tcp +mtusi2026lab3-api-6 sha256:5f93cf319bad7c0e5f360f38e3e3f07708f2357101f6c5804f0194b1a02377cd "python server.py" api 50 minutes ago Up 50 minutes (healthy) 8080/tcp +mtusi2026lab3-api-7 sha256:4833b59a6d4c8406b779432843914b72e61ee92aecd4325e4d034e6839f2ff57 "python server.py" api 45 minutes ago Up 45 minutes (healthy) 8080/tcp +mtusi2026lab3-api-8 sha256:4833b59a6d4c8406b779432843914b72e61ee92aecd4325e4d034e6839f2ff57 "python server.py" api 45 minutes ago Up 45 minutes (healthy) 8080/tcp +mtusi2026lab3-api-9 sha256:4833b59a6d4c8406b779432843914b72e61ee92aecd4325e4d034e6839f2ff57 "python server.py" api 45 minutes ago Up 45 minutes (healthy) 8080/tcp +exit_code=0 +$ docker compose --env-file .env exec -T postgres psql -U messenger -d messenger -Atc SHOW max_connections; SELECT count(*) FROM pg_stat_activity; +psql: error: connection to server on socket "/var/run/postgresql/.s.PGSQL.5432" failed: FATAL: sorry, too many clients already +exit_code=2 +$ compose scale to 8 API replicas +Container mtusi2026lab3-postgres-1 Running + Container mtusi2026lab3-valkey-1 Running + Container mtusi2026lab3-haproxy-1 Running + Container mtusi2026lab3-api-6 Recreate + Container mtusi2026lab3-api-12 Stopping + Container mtusi2026lab3-api-3 Recreate + Container mtusi2026lab3-api-11 Stopping + Container mtusi2026lab3-api-7 Recreate + Container mtusi2026lab3-api-13 Stopping + Container mtusi2026lab3-api-5 Recreate + Container mtusi2026lab3-api-14 Stopping + Container mtusi2026lab3-api-10 Recreate + Container mtusi2026lab3-api-4 Recreate + Container mtusi2026lab3-api-9 Recreate + Container mtusi2026lab3-api-8 Recreate + Container mtusi2026lab3-api-11 Stopped + Container mtusi2026lab3-api-11 Removing + Container mtusi2026lab3-api-11 Removed + Container mtusi2026lab3-api-13 Stopped + Container mtusi2026lab3-api-13 Removing + Container mtusi2026lab3-api-13 Removed + Container mtusi2026lab3-api-14 Stopped + Container mtusi2026lab3-api-14 Removing + Container mtusi2026lab3-api-14 Removed + Container mtusi2026lab3-api-12 Stopped + Container mtusi2026lab3-api-12 Removing + Container mtusi2026lab3-api-12 Removed + Container mtusi2026lab3-api-3 Recreated + Container mtusi2026lab3-api-7 Recreated + Container mtusi2026lab3-api-9 Recreated + Container mtusi2026lab3-api-4 Recreated + Container mtusi2026lab3-api-6 Recreated + Container mtusi2026lab3-api-8 Recreated + Container mtusi2026lab3-api-10 Recreated + Container mtusi2026lab3-api-5 Recreated + Container mtusi2026lab3-postgres-1 Waiting + Container mtusi2026lab3-valkey-1 Waiting + Container mtusi2026lab3-valkey-1 Healthy + Container mtusi2026lab3-postgres-1 Healthy + Container mtusi2026lab3-migrate-1 Starting + Container mtusi2026lab3-migrate-1 Started + Container mtusi2026lab3-valkey-1 Waiting + Container mtusi2026lab3-migrate-1 Waiting + Container mtusi2026lab3-valkey-1 Healthy + Container mtusi2026lab3-migrate-1 Exited + Container mtusi2026lab3-api-5 Starting + Container mtusi2026lab3-api-5 Started + Container mtusi2026lab3-api-3 Starting + Container mtusi2026lab3-api-3 Started + Container mtusi2026lab3-api-6 Starting + Container mtusi2026lab3-api-6 Started + Container mtusi2026lab3-api-7 Starting + Container mtusi2026lab3-api-7 Started + Container mtusi2026lab3-api-4 Starting + Container mtusi2026lab3-api-4 Started + Container mtusi2026lab3-api-8 Starting + Container mtusi2026lab3-api-8 Started + Container mtusi2026lab3-api-9 Starting + Container mtusi2026lab3-api-9 Started + Container mtusi2026lab3-api-10 Starting + Container mtusi2026lab3-api-10 Started + Container mtusi2026lab3-valkey-1 Waiting + Container mtusi2026lab3-haproxy-1 Waiting + Container mtusi2026lab3-migrate-1 Waiting + Container mtusi2026lab3-api-5 Waiting + Container mtusi2026lab3-api-3 Waiting + Container mtusi2026lab3-api-6 Waiting + Container mtusi2026lab3-api-7 Waiting + Container mtusi2026lab3-api-4 Waiting + Container mtusi2026lab3-api-8 Waiting + Container mtusi2026lab3-api-9 Waiting + Container mtusi2026lab3-api-10 Waiting + Container mtusi2026lab3-postgres-1 Waiting + Container mtusi2026lab3-haproxy-1 Healthy + Container mtusi2026lab3-valkey-1 Healthy + Container mtusi2026lab3-postgres-1 Healthy + Container mtusi2026lab3-migrate-1 Exited + Container mtusi2026lab3-api-5 Healthy + Container mtusi2026lab3-api-3 Healthy + Container mtusi2026lab3-api-6 Healthy + Container mtusi2026lab3-api-7 Healthy + Container mtusi2026lab3-api-4 Healthy + Container mtusi2026lab3-api-8 Healthy + Container mtusi2026lab3-api-9 Healthy + Container mtusi2026lab3-api-10 Healthy +exit_code=0 +$ query PostgreSQL connection limit and count after scale down +100 +15 +exit_code=0 diff --git a/evidence/final-checks.txt b/evidence/final-checks.txt new file mode 100644 index 00000000..54bda4a3 --- /dev/null +++ b/evidence/final-checks.txt @@ -0,0 +1,153 @@ +$ docker compose --env-file .env config --quiet +exit_code=0 +$ docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180 +#1 [internal] load local bake definitions +#1 reading from stdin 1.06kB done +#1 DONE 0.0s + +#2 [api internal] load build definition from Dockerfile +#2 transferring dockerfile: 785B done +#2 DONE 0.0s + +#3 [api internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#3 DONE 1.9s + +#4 [api internal] load .dockerignore +#4 transferring context: 196B done +#4 DONE 0.0s + +#5 [migrate internal] load build context +#5 transferring context: 194B done +#5 DONE 0.0s + +#6 [migrate 1/6] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 +#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done +#6 DONE 0.0s + +#7 [migrate 2/6] WORKDIR /app +#7 CACHED + +#8 [migrate 4/6] RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app +#8 CACHED + +#9 [migrate 3/6] COPY requirements.txt . +#9 CACHED + +#10 [migrate 5/6] COPY --chown=app:app server.py ./server.py +#10 CACHED + +#11 [migrate 6/6] COPY --chown=app:app migrations ./migrations +#11 CACHED + +#12 [api] exporting to image +#12 exporting layers done +#12 exporting manifest sha256:6d456f82ad1a9f5b8ce4b3a64cab8b843c36f037fcf79696a82db3ee58a97133 done +#12 exporting config sha256:019ccb0a5711c492cb5dbcc7b6c8d1cd8996a1f4ea28579e93493499ea6176f1 done +#12 exporting attestation manifest sha256:7a889781683085054b961e9bca3897ccc36a5174d3c5e020c834341f2c857ca7 done +#12 exporting manifest list sha256:b95f98ffcfcb7cd1b3a5c187f0e0d5d39d078a2f74e151032735e5fa16c230dd done +#12 naming to docker.io/library/mtusi2026lab3-api:latest done +#12 unpacking to docker.io/library/mtusi2026lab3-api:latest done +#12 DONE 0.1s + +#13 [migrate] exporting to image +#13 exporting layers done +#13 exporting manifest sha256:fd5e9154b8af2d17e52adf4b864ac3474ba1d0907ea6381808e7e10cf8e2e461 done +#13 exporting config sha256:682c09802432b2cc1f0e66f0658ac7eef8f82eeeca28fb344054cc2eaf9fb73b done +#13 exporting attestation manifest sha256:a4ab08e7eb901351bc35d3da999b1f46c58a723be0a20bec35af0c904af7f180 done +#13 exporting manifest list sha256:d80d6a8bd4dd17f178cc13b321dd7467c69425f5fcb35aeda2ebeed1ce056e68 done +#13 naming to docker.io/library/mtusi2026lab3-migrate:latest done +#13 unpacking to docker.io/library/mtusi2026lab3-migrate:latest done +#13 DONE 0.1s + +#14 [api] resolving provenance for metadata file +#14 DONE 0.0s + +#15 [migrate] resolving provenance for metadata file +#15 DONE 0.0s +Image mtusi2026lab3-migrate Building + Image mtusi2026lab3-api Building + Image mtusi2026lab3-migrate Built + Image mtusi2026lab3-api Built + Container mtusi2026lab3-valkey-1 Running + Container mtusi2026lab3-haproxy-1 Running + Container mtusi2026lab3-postgres-1 Running + Container mtusi2026lab3-api-6 Stopping + Container mtusi2026lab3-api-7 Stopping + Container mtusi2026lab3-api-3 Recreate + Container mtusi2026lab3-api-8 Stopping + Container mtusi2026lab3-api-5 Stopping + Container mtusi2026lab3-api-4 Recreate + Container mtusi2026lab3-api-10 Stopping + Container mtusi2026lab3-api-9 Stopping + Container mtusi2026lab3-api-3 Recreated + Container mtusi2026lab3-api-4 Recreated + Container mtusi2026lab3-api-10 Stopped + Container mtusi2026lab3-api-10 Removing + Container mtusi2026lab3-api-5 Stopped + Container mtusi2026lab3-api-5 Removing + Container mtusi2026lab3-api-10 Removed + Container mtusi2026lab3-api-7 Stopped + Container mtusi2026lab3-api-7 Removing + Container mtusi2026lab3-api-5 Removed + Container mtusi2026lab3-api-8 Stopped + Container mtusi2026lab3-api-8 Removing + Container mtusi2026lab3-api-7 Removed + Container mtusi2026lab3-api-9 Stopped + Container mtusi2026lab3-api-9 Removing + Container mtusi2026lab3-api-8 Removed + Container mtusi2026lab3-api-6 Stopped + Container mtusi2026lab3-api-6 Removing + Container mtusi2026lab3-api-9 Removed + Container mtusi2026lab3-api-6 Removed + Container mtusi2026lab3-postgres-1 Waiting + Container mtusi2026lab3-valkey-1 Waiting + Container mtusi2026lab3-postgres-1 Healthy + Container mtusi2026lab3-valkey-1 Healthy + Container mtusi2026lab3-migrate-1 Starting + Container mtusi2026lab3-migrate-1 Started + Container mtusi2026lab3-valkey-1 Waiting + Container mtusi2026lab3-migrate-1 Waiting + Container mtusi2026lab3-migrate-1 Exited + Container mtusi2026lab3-valkey-1 Healthy + Container mtusi2026lab3-api-4 Starting + Container mtusi2026lab3-api-4 Started + Container mtusi2026lab3-api-3 Starting + Container mtusi2026lab3-api-3 Started + Container mtusi2026lab3-haproxy-1 Waiting + Container mtusi2026lab3-migrate-1 Waiting + Container mtusi2026lab3-api-4 Waiting + Container mtusi2026lab3-api-3 Waiting + Container mtusi2026lab3-postgres-1 Waiting + Container mtusi2026lab3-valkey-1 Waiting + Container mtusi2026lab3-valkey-1 Healthy + Container mtusi2026lab3-haproxy-1 Healthy + Container mtusi2026lab3-migrate-1 Exited + Container mtusi2026lab3-postgres-1 Healthy + Container mtusi2026lab3-api-4 Healthy + Container mtusi2026lab3-api-3 Healthy +exit_code=0 +$ .venv/bin/python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +exit_code=0 +$ .venv/bin/python scripts/validate_openapi.py +Traceback (most recent call last): + File "/Users/koichakaev/Desktop/учеба/ВНП/mtusi2026lab3/scripts/validate_openapi.py", line 5, in + from openapi_spec_validator import validate +ModuleNotFoundError: No module named 'openapi_spec_validator' +exit_code=1 + +$ .venv/bin/python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +exit_code=0 +$ .venv/bin/python tests/idempotency_race.py --base-url http://127.0.0.1:8080 +PASS: 32 concurrent retries, one message, 2 API instances observed +exit_code=0 +$ docker compose --env-file .env ps +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi2026lab3-api-3 mtusi2026lab3-api "python server.py" api 30 seconds ago Up 27 seconds (healthy) 8080/tcp +mtusi2026lab3-api-4 mtusi2026lab3-api "python server.py" api 30 seconds ago Up 27 seconds (healthy) 8080/tcp +mtusi2026lab3-haproxy-1 haproxy:3.0.8-alpine "docker-entrypoint.s…" haproxy 42 minutes ago Up 42 minutes (healthy) 127.0.0.1:8080->8080/tcp +mtusi2026lab3-postgres-1 postgres:16.4-bookworm "docker-entrypoint.s…" postgres 21 hours ago Up About an hour (healthy) 5432/tcp +mtusi2026lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 21 hours ago Up About an hour (healthy) 6379/tcp +exit_code=0 +NOTE: Full OpenAPI validator unavailable: openapi-spec-validator is absent and PyPI DNS is unavailable; offline contract check passed. diff --git a/evidence/lab3-checks.txt b/evidence/lab3-checks.txt new file mode 100644 index 00000000..3f104c5b --- /dev/null +++ b/evidence/lab3-checks.txt @@ -0,0 +1,14 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +.venv/bin/python tests/idempotency_race.py --base-url "http://127.0.0.1:8080" +PASS: 32 concurrent retries, one message, 2 API instances observed +.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 0, "first_recovery_seconds": 1.447} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +NOTE: scripts/validate_openapi.py not run: openapi-spec-validator==0.7.2 is declared in requirements-dev.txt but pip install was blocked by unavailable DNS for pypi.org. Offline structural check passed. diff --git a/evidence/offered-load-results.json b/evidence/offered-load-results.json new file mode 100644 index 00000000..4e75d814 --- /dev/null +++ b/evidence/offered-load-results.json @@ -0,0 +1,371 @@ +{ + "method": "Rate-controlled POST sweep; 15s per sample; offered rate 250/500/750/1000 RPS; three randomized repeats; 8 API replicas at 0.5 CPU/256 MiB, DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; max in-flight 8192; per-thread HTTP client reuse; synthetic message/idempotency tables reset per sample; no Docker stats polling during load.", + "summary": [ + { + "offered_rate_rps": 250, + "completed_success_rps_total_elapsed_median": 249.622, + "completed_success_rps_total_elapsed_min": 249.559, + "completed_success_rps_total_elapsed_max": 249.959, + "success_ratio_median": 1.0, + "p95_ms_median": 9.931, + "effective_success_ratio_offered_median": 1.0, + "skipped_overload_median": 0 + }, + { + "offered_rate_rps": 500, + "completed_success_rps_total_elapsed_median": 499.579, + "completed_success_rps_total_elapsed_min": 419.571, + "completed_success_rps_total_elapsed_max": 499.857, + "success_ratio_median": 1.0, + "p95_ms_median": 41.601, + "effective_success_ratio_offered_median": 1.0, + "skipped_overload_median": 0 + }, + { + "offered_rate_rps": 750, + "completed_success_rps_total_elapsed_median": 745.875, + "completed_success_rps_total_elapsed_min": 693.24, + "completed_success_rps_total_elapsed_max": 748.66, + "success_ratio_median": 0.999822, + "p95_ms_median": 100.889, + "effective_success_ratio_offered_median": 0.999822, + "skipped_overload_median": 0 + }, + { + "offered_rate_rps": 1000, + "completed_success_rps_total_elapsed_median": 810.128, + "completed_success_rps_total_elapsed_min": 553.163, + "completed_success_rps_total_elapsed_max": 879.333, + "success_ratio_median": 0.972467, + "p95_ms_median": 1611.346, + "effective_success_ratio_offered_median": 0.972467, + "skipped_overload_median": 0 + } + ], + "samples": [ + { + "variant": "offered-500", + "sample": 1, + "offered_rate_rps": 500, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 7501, + "scheduled_requests": 7501, + "successful_requests": 7501, + "failed_requests": 0, + "failure_types": {}, + "skipped_overload": 0, + "offered_rps_actual": 500.067, + "submitted_rps_actual": 500.067, + "completed_success_rps_total_elapsed": 499.579, + "success_ratio": 1.0, + "effective_success_ratio_offered": 1.0, + "latency_successful_ms": { + "p50": 3.277, + "p95": 41.601, + "p99": 71.014 + }, + "drain_seconds_after_offer_window": 0.015, + "message_rows_after": 7501 + }, + { + "variant": "offered-750", + "sample": 1, + "offered_rate_rps": 750, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 11251, + "scheduled_requests": 11251, + "successful_requests": 11249, + "failed_requests": 2, + "failure_types": { + "Failure": 2 + }, + "skipped_overload": 0, + "offered_rps_actual": 750.067, + "submitted_rps_actual": 750.067, + "completed_success_rps_total_elapsed": 748.66, + "success_ratio": 0.999822, + "effective_success_ratio_offered": 0.999822, + "latency_successful_ms": { + "p50": 3.819, + "p95": 100.889, + "p99": 1021.492 + }, + "drain_seconds_after_offer_window": 0.026, + "message_rows_after": 11249 + }, + { + "variant": "offered-1000", + "sample": 1, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14976, + "failed_requests": 24, + "failure_types": { + "Failure": 24 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 810.128, + "success_ratio": 0.9984, + "effective_success_ratio_offered": 0.9984, + "latency_successful_ms": { + "p50": 109.324, + "p95": 1611.346, + "p99": 2175.384 + }, + "drain_seconds_after_offer_window": 3.486, + "message_rows_after": 14976 + }, + { + "variant": "offered-250", + "sample": 1, + "offered_rate_rps": 250, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 3751, + "scheduled_requests": 3751, + "successful_requests": 3751, + "failed_requests": 0, + "failure_types": {}, + "skipped_overload": 0, + "offered_rps_actual": 250.067, + "submitted_rps_actual": 250.067, + "completed_success_rps_total_elapsed": 249.959, + "success_ratio": 1.0, + "effective_success_ratio_offered": 1.0, + "latency_successful_ms": { + "p50": 2.744, + "p95": 9.931, + "p99": 48.516 + }, + "drain_seconds_after_offer_window": 0.006, + "message_rows_after": 3751 + }, + { + "variant": "offered-750", + "sample": 2, + "offered_rate_rps": 750, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 11251, + "scheduled_requests": 11251, + "successful_requests": 11251, + "failed_requests": 0, + "failure_types": {}, + "skipped_overload": 0, + "offered_rps_actual": 750.067, + "submitted_rps_actual": 750.067, + "completed_success_rps_total_elapsed": 745.875, + "success_ratio": 1.0, + "effective_success_ratio_offered": 1.0, + "latency_successful_ms": { + "p50": 5.657, + "p95": 79.469, + "p99": 164.156 + }, + "drain_seconds_after_offer_window": 0.084, + "message_rows_after": 11251 + }, + { + "variant": "offered-1000", + "sample": 2, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14283, + "failed_requests": 717, + "failure_types": { + "Failure": 717 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 879.333, + "success_ratio": 0.9522, + "effective_success_ratio_offered": 0.9522, + "latency_successful_ms": { + "p50": 150.455, + "p95": 1296.214, + "p99": 2320.975 + }, + "drain_seconds_after_offer_window": 1.243, + "message_rows_after": 14283 + }, + { + "variant": "offered-250", + "sample": 2, + "offered_rate_rps": 250, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 3751, + "scheduled_requests": 3751, + "successful_requests": 3751, + "failed_requests": 0, + "failure_types": {}, + "skipped_overload": 0, + "offered_rps_actual": 250.067, + "submitted_rps_actual": 250.067, + "completed_success_rps_total_elapsed": 249.559, + "success_ratio": 1.0, + "effective_success_ratio_offered": 1.0, + "latency_successful_ms": { + "p50": 2.689, + "p95": 8.215, + "p99": 44.123 + }, + "drain_seconds_after_offer_window": 0.031, + "message_rows_after": 3751 + }, + { + "variant": "offered-500", + "sample": 2, + "offered_rate_rps": 500, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 7501, + "scheduled_requests": 7501, + "successful_requests": 7501, + "failed_requests": 0, + "failure_types": {}, + "skipped_overload": 0, + "offered_rps_actual": 500.067, + "submitted_rps_actual": 500.067, + "completed_success_rps_total_elapsed": 499.857, + "success_ratio": 1.0, + "effective_success_ratio_offered": 1.0, + "latency_successful_ms": { + "p50": 3.567, + "p95": 34.969, + "p99": 65.8 + }, + "drain_seconds_after_offer_window": 0.006, + "message_rows_after": 7501 + }, + { + "variant": "offered-750", + "sample": 3, + "offered_rate_rps": 750, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 11251, + "scheduled_requests": 11251, + "successful_requests": 11236, + "failed_requests": 15, + "failure_types": { + "Failure": 15 + }, + "skipped_overload": 0, + "offered_rps_actual": 750.067, + "submitted_rps_actual": 750.067, + "completed_success_rps_total_elapsed": 693.24, + "success_ratio": 0.998667, + "effective_success_ratio_offered": 0.998667, + "latency_successful_ms": { + "p50": 8.339, + "p95": 858.144, + "p99": 1593.896 + }, + "drain_seconds_after_offer_window": 1.208, + "message_rows_after": 11236 + }, + { + "variant": "offered-250", + "sample": 3, + "offered_rate_rps": 250, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 3751, + "scheduled_requests": 3751, + "successful_requests": 3751, + "failed_requests": 0, + "failure_types": {}, + "skipped_overload": 0, + "offered_rps_actual": 250.067, + "submitted_rps_actual": 250.067, + "completed_success_rps_total_elapsed": 249.622, + "success_ratio": 1.0, + "effective_success_ratio_offered": 1.0, + "latency_successful_ms": { + "p50": 2.685, + "p95": 17.715, + "p99": 52.77 + }, + "drain_seconds_after_offer_window": 0.027, + "message_rows_after": 3751 + }, + { + "variant": "offered-500", + "sample": 3, + "offered_rate_rps": 500, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 7501, + "scheduled_requests": 7501, + "successful_requests": 7493, + "failed_requests": 8, + "failure_types": { + "Failure": 8 + }, + "skipped_overload": 0, + "offered_rps_actual": 500.067, + "submitted_rps_actual": 500.067, + "completed_success_rps_total_elapsed": 419.571, + "success_ratio": 0.998933, + "effective_success_ratio_offered": 0.998933, + "latency_successful_ms": { + "p50": 9.042, + "p95": 1812.817, + "p99": 2541.843 + }, + "drain_seconds_after_offer_window": 2.859, + "message_rows_after": 7493 + }, + { + "variant": "offered-1000", + "sample": 3, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14587, + "failed_requests": 413, + "failure_types": { + "Failure": 413 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 553.163, + "success_ratio": 0.972467, + "effective_success_ratio_offered": 0.972467, + "latency_successful_ms": { + "p50": 535.721, + "p95": 2011.397, + "p99": 4616.569 + }, + "drain_seconds_after_offer_window": 11.37, + "message_rows_after": 14587 + } + ] +} diff --git a/evidence/offered-load.svg b/evidence/offered-load.svg new file mode 100644 index 00000000..52da8b7c --- /dev/null +++ b/evidence/offered-load.svg @@ -0,0 +1,33 @@ + + +Насыщение при росте предложенной нагрузки + +0 + +250 + +500 + +750 + +1000 +250 +500 +750 +1000 + + +250 +500 +746 +810 +10ms +42ms +101ms +1611ms +Предложенная нагрузка, RPS +Успешные RPS за offer+drain +p95, ms +Успешные RPS +p95 успешных запросов + diff --git a/evidence/offered-scale-results.json b/evidence/offered-scale-results.json new file mode 100644 index 00000000..59ba78a6 --- /dev/null +++ b/evidence/offered-scale-results.json @@ -0,0 +1,209 @@ +{ + "method": "Randomized paired blocks at offered 1000 RPS, 15s per sample; three repeats; compare 8 versus 12 replicas at fixed 0.5 CPU/256 MiB per API replica; DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; HAProxy server-template supports 16 backends; scale to 8 before each DB reset to release idle connections, then scale to target; synthetic messages and idempotency keys truncated per sample; HTTP client reused per worker.", + "summary": [ + { + "replicas": 8, + "offered_rate_rps": 1000, + "completed_success_rps_total_elapsed_median": 726.979, + "completed_success_rps_min": 584.377, + "completed_success_rps_max": 851.508, + "success_ratio_median": 0.981267, + "p95_ms_median": 1504.722, + "error_count_total": 874, + "message_rows_after_median": 14719 + }, + { + "replicas": 12, + "offered_rate_rps": 1000, + "completed_success_rps_total_elapsed_median": 555.8, + "completed_success_rps_min": 545.991, + "completed_success_rps_max": 774.91, + "success_ratio_median": 0.999733, + "p95_ms_median": 1986.412, + "error_count_total": 12, + "message_rows_after_median": null + } + ], + "samples": [ + { + "variant": "offered-1000", + "sample": 1, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14719, + "failed_requests": 281, + "failure_types": { + "Failure": 281 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 851.508, + "success_ratio": 0.981267, + "effective_success_ratio_offered": 0.981267, + "latency_successful_ms": { + "p50": 206.696, + "p95": 1307.155, + "p99": 1984.206 + }, + "drain_seconds_after_offer_window": 2.286, + "message_rows_after": 14719, + "message_rows_after_error": null, + "replicas": 8 + }, + { + "variant": "offered-1000", + "sample": 1, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14996, + "failed_requests": 4, + "failure_types": { + "Failure": 4 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 774.91, + "success_ratio": 0.999733, + "effective_success_ratio_offered": 0.999733, + "latency_successful_ms": { + "p50": 400.696, + "p95": 1664.779, + "p99": 2378.651 + }, + "drain_seconds_after_offer_window": 4.352, + "message_rows_after": null, + "message_rows_after_error": "psql: error: connection to server on socket \"/var/run/postgresql/.s.PGSQL.5432\" failed: FATAL: sorry, too many clients already", + "replicas": 12 + }, + { + "variant": "offered-1000", + "sample": 2, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14804, + "failed_requests": 196, + "failure_types": { + "Failure": 196 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 584.377, + "success_ratio": 0.986933, + "effective_success_ratio_offered": 0.986933, + "latency_successful_ms": { + "p50": 496.284, + "p95": 2007.63, + "p99": 2684.382 + }, + "drain_seconds_after_offer_window": 10.333, + "message_rows_after": 14804, + "message_rows_after_error": null, + "replicas": 8 + }, + { + "variant": "offered-1000", + "sample": 2, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14999, + "failed_requests": 1, + "failure_types": { + "Failure": 1 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 545.991, + "success_ratio": 0.999933, + "effective_success_ratio_offered": 0.999933, + "latency_successful_ms": { + "p50": 605.99, + "p95": 2589.505, + "p99": 3420.92 + }, + "drain_seconds_after_offer_window": 12.471, + "message_rows_after": null, + "message_rows_after_error": "psql: error: connection to server on socket \"/var/run/postgresql/.s.PGSQL.5432\" failed: FATAL: sorry, too many clients already", + "replicas": 12 + }, + { + "variant": "offered-1000", + "sample": 3, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14993, + "failed_requests": 7, + "failure_types": { + "Failure": 7 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 555.8, + "success_ratio": 0.999533, + "effective_success_ratio_offered": 0.999533, + "latency_successful_ms": { + "p50": 717.667, + "p95": 1986.412, + "p99": 2959.931 + }, + "drain_seconds_after_offer_window": 11.976, + "message_rows_after": null, + "message_rows_after_error": "psql: error: connection to server on socket \"/var/run/postgresql/.s.PGSQL.5432\" failed: FATAL: sorry, too many clients already", + "replicas": 12 + }, + { + "variant": "offered-1000", + "sample": 3, + "offered_rate_rps": 1000, + "duration_seconds": 15, + "concurrency_workers": 512, + "max_inflight_requests": 8192, + "offered_slots": 15000, + "scheduled_requests": 15000, + "successful_requests": 14603, + "failed_requests": 397, + "failure_types": { + "Failure": 397 + }, + "skipped_overload": 0, + "offered_rps_actual": 1000.0, + "submitted_rps_actual": 1000.0, + "completed_success_rps_total_elapsed": 726.979, + "success_ratio": 0.973533, + "effective_success_ratio_offered": 0.973533, + "latency_successful_ms": { + "p50": 291.159, + "p95": 1504.722, + "p99": 2242.259 + }, + "drain_seconds_after_offer_window": 5.087, + "message_rows_after": 14603, + "message_rows_after_error": null, + "replicas": 8 + } + ] +} diff --git a/evidence/rolling-update-load.json b/evidence/rolling-update-load.json new file mode 100644 index 00000000..37a77193 --- /dev/null +++ b/evidence/rolling-update-load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 60.0, + "elapsed_seconds": 60.088, + "concurrency": 16, + "successful_requests": 50291, + "failed_requests": 24, + "error_ratio": 0.000477, + "successful_rps": 836.949, + "successful_latency_ms": { + "p50": 6.118, + "p95": 55.938, + "p99": 85.729 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/track1-results.json b/evidence/track1-results.json new file mode 100644 index 00000000..72652b38 --- /dev/null +++ b/evidence/track1-results.json @@ -0,0 +1,1654 @@ +{ + "method": "closed-loop POST, 5s stabilization after each topology change; each sample starts after TRUNCATE of synthetic messages/idempotency_keys and ANALYZE; deterministic randomized blocks; median summary plus every raw sample", + "duration_seconds": 30.0, + "concurrency": 16, + "repeat_count": 3, + "random_seed": 20261105, + "host_platform": "macOS-26.5.1-arm64-arm-64bit-Mach-O", + "host_cpu_count": 8, + "docker_server_version": "29.8.2", + "results": [ + { + "variant": "fixed-1", + "replicas": 1, + "aggregate_api_cpu_limit": 2, + "api_memory_limit_per_replica": "1g", + "db_pool_max_per_replica": 10, + "successful_rps": 418.547, + "successful_latency_ms": { + "p50": 32.318, + "p95": 52.96, + "p99": 82.381 + }, + "error_ratio": 0.00017, + "successful_requests": 12586, + "failed_requests": 2, + "message_rows_before": 0, + "message_rows_after": 12586, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 174.28, + "cpu_percent_peak": 190.25, + "memory_mib_peak": 61.86 + }, + "postgres": { + "cpu_percent_mean": 25.31, + "cpu_percent_peak": 28.33, + "memory_mib_peak": 182.0 + }, + "haproxy": { + "cpu_percent_mean": 11.02, + "cpu_percent_peak": 13.75, + "memory_mib_peak": 17.22 + } + } + }, + { + "variant": "fixed-2", + "replicas": 2, + "aggregate_api_cpu_limit": 2, + "api_memory_limit_per_replica": "512m", + "db_pool_max_per_replica": 10, + "successful_rps": 698.64, + "successful_latency_ms": { + "p50": 8.009, + "p95": 62.409, + "p99": 92.949 + }, + "error_ratio": 0.0, + "successful_requests": 21187, + "failed_requests": 0, + "message_rows_before": 0, + "message_rows_after": 21187, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 73.96, + "cpu_percent_peak": 109.51, + "memory_mib_peak": 60.91 + }, + "postgres": { + "cpu_percent_mean": 36.75, + "cpu_percent_peak": 52.21, + "memory_mib_peak": 205.5 + }, + "haproxy": { + "cpu_percent_mean": 14.15, + "cpu_percent_peak": 19.06, + "memory_mib_peak": 17.11 + } + } + }, + { + "variant": "fixed-4", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "successful_rps": 837.666, + "successful_latency_ms": { + "p50": 5.216, + "p95": 57.52, + "p99": 101.878 + }, + "error_ratio": 7.2e-05, + "successful_requests": 25909, + "failed_requests": 2, + "message_rows_before": 0, + "message_rows_after": 25909, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 38.22, + "cpu_percent_peak": 53.33, + "memory_mib_peak": 62.36 + }, + "postgres": { + "cpu_percent_mean": 55.73, + "cpu_percent_peak": 73.15, + "memory_mib_peak": 252.6 + }, + "haproxy": { + "cpu_percent_mean": 17.39, + "cpu_percent_peak": 26.04, + "memory_mib_peak": 19.65 + } + } + }, + { + "variant": "scaled-1", + "replicas": 1, + "aggregate_api_cpu_limit": 0.5, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "successful_rps": 103.693, + "successful_latency_ms": { + "p50": 111.091, + "p95": 313.264, + "p99": 484.917 + }, + "error_ratio": 0.000641, + "successful_requests": 3120, + "failed_requests": 2, + "message_rows_before": 0, + "message_rows_after": 3120, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 47.8, + "cpu_percent_peak": 53.34, + "memory_mib_peak": 83.55 + }, + "postgres": { + "cpu_percent_mean": 8.85, + "cpu_percent_peak": 16.73, + "memory_mib_peak": 181.8 + }, + "haproxy": { + "cpu_percent_mean": 3.7, + "cpu_percent_peak": 5.9, + "memory_mib_peak": 17.23 + } + } + }, + { + "variant": "scaled-2", + "replicas": 2, + "aggregate_api_cpu_limit": 1.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "successful_rps": 274.582, + "successful_latency_ms": { + "p50": 22.905, + "p95": 183.678, + "p99": 290.384 + }, + "error_ratio": 0.000484, + "successful_requests": 8255, + "failed_requests": 4, + "message_rows_before": 0, + "message_rows_after": 8255, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 37.63, + "cpu_percent_peak": 52.89, + "memory_mib_peak": 140.0 + }, + "postgres": { + "cpu_percent_mean": 34.63, + "cpu_percent_peak": 52.05, + "memory_mib_peak": 205.9 + }, + "haproxy": { + "cpu_percent_mean": 6.53, + "cpu_percent_peak": 9.27, + "memory_mib_peak": 17.08 + } + } + }, + { + "variant": "scaled-4", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "successful_rps": 680.697, + "successful_latency_ms": { + "p50": 5.995, + "p95": 70.964, + "p99": 123.083 + }, + "error_ratio": 0.0, + "successful_requests": 20500, + "failed_requests": 0, + "message_rows_before": 0, + "message_rows_after": 20500, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 40.83, + "cpu_percent_peak": 53.17, + "memory_mib_peak": 79.21 + }, + "postgres": { + "cpu_percent_mean": 60.14, + "cpu_percent_peak": 76.35, + "memory_mib_peak": 251.4 + }, + "haproxy": { + "cpu_percent_mean": 18.6, + "cpu_percent_peak": 26.04, + "memory_mib_peak": 17.38 + } + } + }, + { + "variant": "scaled-8", + "replicas": 8, + "aggregate_api_cpu_limit": 4.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "successful_rps": 764.228, + "successful_latency_ms": { + "p50": 9.513, + "p95": 70.586, + "p99": 111.195 + }, + "error_ratio": 0.0, + "successful_requests": 23095, + "failed_requests": 0, + "message_rows_before": 0, + "message_rows_after": 23095, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 29.04, + "cpu_percent_peak": 50.12, + "memory_mib_peak": 81.97 + }, + "postgres": { + "cpu_percent_mean": 61.36, + "cpu_percent_peak": 71.26, + "memory_mib_peak": 331.0 + }, + "haproxy": { + "cpu_percent_mean": 21.82, + "cpu_percent_peak": 26.08, + "memory_mib_peak": 17.2 + } + } + }, + { + "variant": "pool-5", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 5, + "successful_rps": 921.219, + "successful_latency_ms": { + "p50": 5.339, + "p95": 49.154, + "p99": 89.762 + }, + "error_ratio": 3.1e-05, + "successful_requests": 28361, + "failed_requests": 1, + "message_rows_before": 0, + "message_rows_after": 28361, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 40.4, + "cpu_percent_peak": 53.26, + "memory_mib_peak": 62.61 + }, + "postgres": { + "cpu_percent_mean": 50.52, + "cpu_percent_peak": 71.7, + "memory_mib_peak": 206.3 + }, + "haproxy": { + "cpu_percent_mean": 20.85, + "cpu_percent_peak": 26.66, + "memory_mib_peak": 17.16 + } + } + }, + { + "variant": "pool-10", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "successful_rps": 808.886, + "successful_latency_ms": { + "p50": 5.533, + "p95": 66.748, + "p99": 108.295 + }, + "error_ratio": 0.000203, + "successful_requests": 25052, + "failed_requests": 6, + "message_rows_before": 0, + "message_rows_after": 25052, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 38.91, + "cpu_percent_peak": 51.94, + "memory_mib_peak": 99.54 + }, + "postgres": { + "cpu_percent_mean": 59.95, + "cpu_percent_peak": 85.23, + "memory_mib_peak": 256.2 + }, + "haproxy": { + "cpu_percent_mean": 19.96, + "cpu_percent_peak": 26.02, + "memory_mib_peak": 17.22 + } + } + }, + { + "variant": "pool-20", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 20, + "successful_rps": 951.344, + "successful_latency_ms": { + "p50": 5.429, + "p95": 45.311, + "p99": 89.207 + }, + "error_ratio": 3.1e-05, + "successful_requests": 29376, + "failed_requests": 1, + "message_rows_before": 0, + "message_rows_after": 29376, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 41.67, + "cpu_percent_peak": 52.53, + "memory_mib_peak": 63.12 + }, + "postgres": { + "cpu_percent_mean": 51.46, + "cpu_percent_peak": 69.62, + "memory_mib_peak": 265.4 + }, + "haproxy": { + "cpu_percent_mean": 18.45, + "cpu_percent_peak": 26.49, + "memory_mib_peak": 17.28 + } + } + } + ], + "samples": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.27, + "concurrency": 16, + "successful_requests": 18511, + "failed_requests": 397, + "error_ratio": 0.020996, + "successful_rps": 611.521, + "successful_latency_ms": { + "p50": 5.8, + "p95": 66.748, + "p99": 195.939 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-10", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 134, + "message_rows_after": 21563, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 37.8, + "cpu_percent_peak": 53.82, + "memory_mib_peak": 81.91 + }, + "postgres": { + "cpu_percent_mean": 59.95, + "cpu_percent_peak": 85.23, + "memory_mib_peak": 263.2 + }, + "haproxy": { + "cpu_percent_mean": 18.38, + "cpu_percent_peak": 26.32, + "memory_mib_peak": 17.63 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.161, + "concurrency": 16, + "successful_requests": 11755, + "failed_requests": 2, + "error_ratio": 0.00017, + "successful_rps": 389.738, + "successful_latency_ms": { + "p50": 32.318, + "p95": 70.998, + "p99": 147.03 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-1", + "replicas": 1, + "aggregate_api_cpu_limit": 2, + "api_memory_limit_per_replica": "1g", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 11755, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 174.28, + "cpu_percent_peak": 189.63, + "memory_mib_peak": 51.67 + }, + "postgres": { + "cpu_percent_mean": 34.28, + "cpu_percent_peak": 53.98, + "memory_mib_peak": 181.1 + }, + "haproxy": { + "cpu_percent_mean": 11.64, + "cpu_percent_peak": 13.75, + "memory_mib_peak": 17.27 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.116, + "concurrency": 16, + "successful_requests": 20500, + "failed_requests": 1, + "error_ratio": 4.9e-05, + "successful_rps": 680.697, + "successful_latency_ms": { + "p50": 5.995, + "p95": 75.771, + "p99": 252.027 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-4", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 20500, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 40.83, + "cpu_percent_peak": 53.17, + "memory_mib_peak": 63.09 + }, + "postgres": { + "cpu_percent_mean": 42.49, + "cpu_percent_peak": 71.25, + "memory_mib_peak": 251.4 + }, + "haproxy": { + "cpu_percent_mean": 18.6, + "cpu_percent_peak": 26.04, + "memory_mib_peak": 17.38 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.174, + "concurrency": 16, + "successful_requests": 19775, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 655.366, + "successful_latency_ms": { + "p50": 11.757, + "p95": 78.12, + "p99": 136.589 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-8", + "replicas": 8, + "aggregate_api_cpu_limit": 4.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 19775, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 29.18, + "cpu_percent_peak": 51.37, + "memory_mib_peak": 81.97 + }, + "postgres": { + "cpu_percent_mean": 94.6, + "cpu_percent_peak": 156.61, + "memory_mib_peak": 334.1 + }, + "haproxy": { + "cpu_percent_mean": 21.51, + "cpu_percent_peak": 26.36, + "memory_mib_peak": 17.16 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.123, + "concurrency": 16, + "successful_requests": 25239, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 837.854, + "successful_latency_ms": { + "p50": 5.851, + "p95": 63.703, + "p99": 100.367 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-5", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 5, + "message_rows_before": 0, + "message_rows_after": 25239, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 40.4, + "cpu_percent_peak": 53.26, + "memory_mib_peak": 62.98 + }, + "postgres": { + "cpu_percent_mean": 44.85, + "cpu_percent_peak": 74.63, + "memory_mib_peak": 206.3 + }, + "haproxy": { + "cpu_percent_mean": 20.85, + "cpu_percent_peak": 26.66, + "memory_mib_peak": 17.41 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.99, + "concurrency": 16, + "successful_requests": 8562, + "failed_requests": 4, + "error_ratio": 0.000467, + "successful_rps": 276.28, + "successful_latency_ms": { + "p50": 13.069, + "p95": 153.656, + "p99": 1005.18 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-2", + "replicas": 2, + "aggregate_api_cpu_limit": 1.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 8562, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 35.16, + "cpu_percent_peak": 52.32, + "memory_mib_peak": 63.89 + }, + "postgres": { + "cpu_percent_mean": 16.19, + "cpu_percent_peak": 24.38, + "memory_mib_peak": 204.6 + }, + "haproxy": { + "cpu_percent_mean": 7.13, + "cpu_percent_peak": 11.17, + "memory_mib_peak": 17.25 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.089, + "concurrency": 16, + "successful_requests": 3120, + "failed_requests": 2, + "error_ratio": 0.000641, + "successful_rps": 103.693, + "successful_latency_ms": { + "p50": 109.4, + "p95": 322.451, + "p99": 491.534 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-1", + "replicas": 1, + "aggregate_api_cpu_limit": 0.5, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 3120, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 47.8, + "cpu_percent_peak": 54.41, + "memory_mib_peak": 83.55 + }, + "postgres": { + "cpu_percent_mean": 8.85, + "cpu_percent_peak": 16.73, + "memory_mib_peak": 181.4 + }, + "haproxy": { + "cpu_percent_mean": 3.7, + "cpu_percent_peak": 6.42, + "memory_mib_peak": 17.28 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.326, + "concurrency": 16, + "successful_requests": 21187, + "failed_requests": 2, + "error_ratio": 9.4e-05, + "successful_rps": 698.64, + "successful_latency_ms": { + "p50": 8.009, + "p95": 62.409, + "p99": 92.949 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-2", + "replicas": 2, + "aggregate_api_cpu_limit": 2, + "api_memory_limit_per_replica": "512m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 21187, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 77.33, + "cpu_percent_peak": 109.74, + "memory_mib_peak": 60.91 + }, + "postgres": { + "cpu_percent_mean": 39.7, + "cpu_percent_peak": 55.67, + "memory_mib_peak": 204.7 + }, + "haproxy": { + "cpu_percent_mean": 14.15, + "cpu_percent_peak": 19.67, + "memory_mib_peak": 17.11 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.878, + "concurrency": 16, + "successful_requests": 29376, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 951.344, + "successful_latency_ms": { + "p50": 5.429, + "p95": 45.311, + "p99": 83.344 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-20", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 20, + "message_rows_before": 0, + "message_rows_after": 29376, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 42.28, + "cpu_percent_peak": 52.69, + "memory_mib_peak": 63.12 + }, + "postgres": { + "cpu_percent_mean": 54.85, + "cpu_percent_peak": 69.62, + "memory_mib_peak": 257.9 + }, + "haproxy": { + "cpu_percent_mean": 21.82, + "cpu_percent_peak": 26.77, + "memory_mib_peak": 17.59 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.788, + "concurrency": 16, + "successful_requests": 27718, + "failed_requests": 2, + "error_ratio": 7.2e-05, + "successful_rps": 900.275, + "successful_latency_ms": { + "p50": 5.216, + "p95": 49.287, + "p99": 86.6 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-4", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 27718, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 42.62, + "cpu_percent_peak": 53.33, + "memory_mib_peak": 62.25 + }, + "postgres": { + "cpu_percent_mean": 55.73, + "cpu_percent_peak": 70.54, + "memory_mib_peak": 253.1 + }, + "haproxy": { + "cpu_percent_mean": 20.22, + "cpu_percent_peak": 26.5, + "memory_mib_peak": 17.38 + } + }, + "sample": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.08, + "concurrency": 16, + "successful_requests": 3072, + "failed_requests": 1, + "error_ratio": 0.000325, + "successful_rps": 102.129, + "successful_latency_ms": { + "p50": 112.332, + "p95": 313.264, + "p99": 484.917 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-1", + "replicas": 1, + "aggregate_api_cpu_limit": 0.5, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 3072, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 51.54, + "cpu_percent_peak": 53.34, + "memory_mib_peak": 70.85 + }, + "postgres": { + "cpu_percent_mean": 14.54, + "cpu_percent_peak": 19.63, + "memory_mib_peak": 181.8 + }, + "haproxy": { + "cpu_percent_mean": 3.77, + "cpu_percent_peak": 5.9, + "memory_mib_peak": 17.22 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.971, + "concurrency": 16, + "successful_requests": 25052, + "failed_requests": 4, + "error_ratio": 0.00016, + "successful_rps": 808.886, + "successful_latency_ms": { + "p50": 5.327, + "p95": 70.205, + "p99": 108.295 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-10", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 25052, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 38.91, + "cpu_percent_peak": 51.16, + "memory_mib_peak": 99.54 + }, + "postgres": { + "cpu_percent_mean": 47.39, + "cpu_percent_peak": 72.09, + "memory_mib_peak": 256.2 + }, + "haproxy": { + "cpu_percent_mean": 19.96, + "cpu_percent_peak": 25.73, + "memory_mib_peak": 17.22 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.22, + "concurrency": 16, + "successful_requests": 23095, + "failed_requests": 1, + "error_ratio": 4.3e-05, + "successful_rps": 764.228, + "successful_latency_ms": { + "p50": 9.513, + "p95": 70.586, + "p99": 94.662 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-8", + "replicas": 8, + "aggregate_api_cpu_limit": 4.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 23095, + "docker_stats_samples": 9, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 29.04, + "cpu_percent_peak": 46.11, + "memory_mib_peak": 119.3 + }, + "postgres": { + "cpu_percent_mean": 61.36, + "cpu_percent_peak": 68.76, + "memory_mib_peak": 331.0 + }, + "haproxy": { + "cpu_percent_mean": 23.02, + "cpu_percent_peak": 25.92, + "memory_mib_peak": 17.2 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.816, + "concurrency": 16, + "successful_requests": 7545, + "failed_requests": 5, + "error_ratio": 0.000662, + "successful_rps": 244.837, + "successful_latency_ms": { + "p50": 23.377, + "p95": 183.678, + "p99": 290.384 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-2", + "replicas": 2, + "aggregate_api_cpu_limit": 1.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 7545, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 39.38, + "cpu_percent_peak": 52.89, + "memory_mib_peak": 140.0 + }, + "postgres": { + "cpu_percent_mean": 35.97, + "cpu_percent_peak": 54.57, + "memory_mib_peak": 205.9 + }, + "haproxy": { + "cpu_percent_mean": 6.41, + "cpu_percent_peak": 7.51, + "memory_mib_peak": 17.08 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.969, + "concurrency": 16, + "successful_requests": 17760, + "failed_requests": 1, + "error_ratio": 5.6e-05, + "successful_rps": 573.474, + "successful_latency_ms": { + "p50": 6.704, + "p95": 85.95, + "p99": 248.032 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-20", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 20, + "message_rows_before": 0, + "message_rows_after": 17760, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 36.97, + "cpu_percent_peak": 51.65, + "memory_mib_peak": 62.94 + }, + "postgres": { + "cpu_percent_mean": 48.51, + "cpu_percent_peak": 74.98, + "memory_mib_peak": 274.7 + }, + "haproxy": { + "cpu_percent_mean": 16.79, + "cpu_percent_peak": 25.93, + "memory_mib_peak": 17.28 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.071, + "concurrency": 16, + "successful_requests": 12586, + "failed_requests": 2, + "error_ratio": 0.000159, + "successful_rps": 418.547, + "successful_latency_ms": { + "p50": 33.72, + "p95": 52.96, + "p99": 82.381 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-1", + "replicas": 1, + "aggregate_api_cpu_limit": 2, + "api_memory_limit_per_replica": "1g", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 12586, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 180.31, + "cpu_percent_peak": 194.64, + "memory_mib_peak": 64.15 + }, + "postgres": { + "cpu_percent_mean": 23.21, + "cpu_percent_peak": 26.19, + "memory_mib_peak": 189.3 + }, + "haproxy": { + "cpu_percent_mean": 11.02, + "cpu_percent_peak": 13.91, + "memory_mib_peak": 16.97 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.989, + "concurrency": 16, + "successful_requests": 20254, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 653.596, + "successful_latency_ms": { + "p50": 13.061, + "p95": 65.981, + "p99": 93.685 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-2", + "replicas": 2, + "aggregate_api_cpu_limit": 2, + "api_memory_limit_per_replica": "512m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 20254, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 73.96, + "cpu_percent_peak": 108.38, + "memory_mib_peak": 60.84 + }, + "postgres": { + "cpu_percent_mean": 31.2, + "cpu_percent_peak": 35.91, + "memory_mib_peak": 205.5 + }, + "haproxy": { + "cpu_percent_mean": 12.58, + "cpu_percent_peak": 16.94, + "memory_mib_peak": 17.02 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.93, + "concurrency": 16, + "successful_requests": 25909, + "failed_requests": 2, + "error_ratio": 7.7e-05, + "successful_rps": 837.666, + "successful_latency_ms": { + "p50": 4.656, + "p95": 57.52, + "p99": 101.878 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-4", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 25909, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 36.98, + "cpu_percent_peak": 51.94, + "memory_mib_peak": 62.36 + }, + "postgres": { + "cpu_percent_mean": 47.53, + "cpu_percent_peak": 73.15, + "memory_mib_peak": 250.8 + }, + "haproxy": { + "cpu_percent_mean": 17.39, + "cpu_percent_peak": 24.29, + "memory_mib_peak": 19.71 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.04, + "concurrency": 16, + "successful_requests": 19085, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 635.329, + "successful_latency_ms": { + "p50": 11.405, + "p95": 70.964, + "p99": 123.083 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-4", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 19085, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 35.41, + "cpu_percent_peak": 51.83, + "memory_mib_peak": 79.21 + }, + "postgres": { + "cpu_percent_mean": 145.26, + "cpu_percent_peak": 210.25, + "memory_mib_peak": 251.0 + }, + "haproxy": { + "cpu_percent_mean": 16.45, + "cpu_percent_peak": 23.81, + "memory_mib_peak": 17.38 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.786, + "concurrency": 16, + "successful_requests": 28361, + "failed_requests": 2, + "error_ratio": 7.1e-05, + "successful_rps": 921.219, + "successful_latency_ms": { + "p50": 5.339, + "p95": 49.154, + "p99": 89.762 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-5", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 5, + "message_rows_before": 0, + "message_rows_after": 28361, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 41.91, + "cpu_percent_peak": 52.03, + "memory_mib_peak": 62.31 + }, + "postgres": { + "cpu_percent_mean": 59.12, + "cpu_percent_peak": 71.7, + "memory_mib_peak": 206.1 + }, + "haproxy": { + "cpu_percent_mean": 20.74, + "cpu_percent_peak": 26.32, + "memory_mib_peak": 16.98 + } + }, + "sample": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.047, + "concurrency": 16, + "successful_requests": 13710, + "failed_requests": 3, + "error_ratio": 0.000219, + "successful_rps": 456.292, + "successful_latency_ms": { + "p50": 31.712, + "p95": 50.435, + "p99": 67.034 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-1", + "replicas": 1, + "aggregate_api_cpu_limit": 2, + "api_memory_limit_per_replica": "1g", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 13710, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 163.16, + "cpu_percent_peak": 190.25, + "memory_mib_peak": 61.86 + }, + "postgres": { + "cpu_percent_mean": 25.31, + "cpu_percent_peak": 28.33, + "memory_mib_peak": 182.0 + }, + "haproxy": { + "cpu_percent_mean": 10.31, + "cpu_percent_peak": 13.63, + "memory_mib_peak": 17.22 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.074, + "concurrency": 16, + "successful_requests": 23687, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 787.62, + "successful_latency_ms": { + "p50": 7.384, + "p95": 66.06, + "p99": 111.195 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-8", + "replicas": 8, + "aggregate_api_cpu_limit": 4.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 23687, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 28.67, + "cpu_percent_peak": 50.12, + "memory_mib_peak": 62.7 + }, + "postgres": { + "cpu_percent_mean": 57.67, + "cpu_percent_peak": 71.26, + "memory_mib_peak": 303.0 + }, + "haproxy": { + "cpu_percent_mean": 21.82, + "cpu_percent_peak": 26.08, + "memory_mib_peak": 21.52 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 31.048, + "concurrency": 16, + "successful_requests": 18340, + "failed_requests": 1, + "error_ratio": 5.5e-05, + "successful_rps": 590.693, + "successful_latency_ms": { + "p50": 12.065, + "p95": 80.633, + "p99": 180.003 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-4", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 18340, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 38.22, + "cpu_percent_peak": 53.98, + "memory_mib_peak": 82.16 + }, + "postgres": { + "cpu_percent_mean": 135.78, + "cpu_percent_peak": 209.98, + "memory_mib_peak": 252.6 + }, + "haproxy": { + "cpu_percent_mean": 16.3, + "cpu_percent_peak": 26.04, + "memory_mib_peak": 19.65 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.815, + "concurrency": 16, + "successful_requests": 26986, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 875.746, + "successful_latency_ms": { + "p50": 5.43, + "p95": 54.151, + "p99": 100.281 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-4", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 26986, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 45.52, + "cpu_percent_peak": 54.18, + "memory_mib_peak": 101.1 + }, + "postgres": { + "cpu_percent_mean": 60.14, + "cpu_percent_peak": 76.35, + "memory_mib_peak": 253.3 + }, + "haproxy": { + "cpu_percent_mean": 21.5, + "cpu_percent_peak": 26.89, + "memory_mib_peak": 17.21 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.085, + "concurrency": 16, + "successful_requests": 3273, + "failed_requests": 4, + "error_ratio": 0.001221, + "successful_rps": 108.792, + "successful_latency_ms": { + "p50": 111.091, + "p95": 296.427, + "p99": 382.392 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-1", + "replicas": 1, + "aggregate_api_cpu_limit": 0.5, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 3273, + "docker_stats_samples": 11, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 46.58, + "cpu_percent_peak": 52.32, + "memory_mib_peak": 108.8 + }, + "postgres": { + "cpu_percent_mean": 8.83, + "cpu_percent_peak": 13.46, + "memory_mib_peak": 182.2 + }, + "haproxy": { + "cpu_percent_mean": 2.64, + "cpu_percent_peak": 3.9, + "memory_mib_peak": 17.23 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.492, + "concurrency": 16, + "successful_requests": 29552, + "failed_requests": 6, + "error_ratio": 0.000203, + "successful_rps": 969.184, + "successful_latency_ms": { + "p50": 5.533, + "p95": 46.519, + "p99": 86.227 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-10", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 29552, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 39.5, + "cpu_percent_peak": 51.94, + "memory_mib_peak": 137.3 + }, + "postgres": { + "cpu_percent_mean": 105.42, + "cpu_percent_peak": 180.76, + "memory_mib_peak": 248.3 + }, + "haproxy": { + "cpu_percent_mean": 21.2, + "cpu_percent_peak": 26.02, + "memory_mib_peak": 16.93 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.064, + "concurrency": 16, + "successful_requests": 8255, + "failed_requests": 4, + "error_ratio": 0.000484, + "successful_rps": 274.582, + "successful_latency_ms": { + "p50": 22.905, + "p95": 184.569, + "p99": 281.505 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "scaled-2", + "replicas": 2, + "aggregate_api_cpu_limit": 1.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 8255, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 37.63, + "cpu_percent_peak": 54.27, + "memory_mib_peak": 159.2 + }, + "postgres": { + "cpu_percent_mean": 34.63, + "cpu_percent_peak": 52.05, + "memory_mib_peak": 206.2 + }, + "haproxy": { + "cpu_percent_mean": 6.53, + "cpu_percent_peak": 9.27, + "memory_mib_peak": 17.05 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.758, + "concurrency": 16, + "successful_requests": 32117, + "failed_requests": 1, + "error_ratio": 3.1e-05, + "successful_rps": 1044.198, + "successful_latency_ms": { + "p50": 4.326, + "p95": 41.163, + "p99": 89.207 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-20", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 20, + "message_rows_before": 0, + "message_rows_after": 32117, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 41.67, + "cpu_percent_peak": 52.53, + "memory_mib_peak": 63.76 + }, + "postgres": { + "cpu_percent_mean": 51.46, + "cpu_percent_peak": 68.08, + "memory_mib_peak": 265.4 + }, + "haproxy": { + "cpu_percent_mean": 18.45, + "cpu_percent_peak": 26.49, + "memory_mib_peak": 17.0 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.069, + "concurrency": 16, + "successful_requests": 22527, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 749.178, + "successful_latency_ms": { + "p50": 7.089, + "p95": 59.245, + "p99": 83.988 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "fixed-2", + "replicas": 2, + "aggregate_api_cpu_limit": 2, + "api_memory_limit_per_replica": "512m", + "db_pool_max_per_replica": 10, + "message_rows_before": 0, + "message_rows_after": 22527, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 70.71, + "cpu_percent_peak": 109.51, + "memory_mib_peak": 62.89 + }, + "postgres": { + "cpu_percent_mean": 36.75, + "cpu_percent_peak": 52.21, + "memory_mib_peak": 205.8 + }, + "haproxy": { + "cpu_percent_mean": 14.39, + "cpu_percent_peak": 19.06, + "memory_mib_peak": 17.18 + } + }, + "sample": 3 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.863, + "concurrency": 16, + "successful_requests": 32754, + "failed_requests": 1, + "error_ratio": 3.1e-05, + "successful_rps": 1061.276, + "successful_latency_ms": { + "p50": 4.696, + "p95": 41.223, + "p99": 79.977 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "variant": "pool-5", + "replicas": 4, + "aggregate_api_cpu_limit": 2.0, + "api_memory_limit_per_replica": "256m", + "db_pool_max_per_replica": 5, + "message_rows_before": 0, + "message_rows_after": 32754, + "docker_stats_samples": 10, + "container_resource_usage": { + "api": { + "cpu_percent_mean": 39.95, + "cpu_percent_peak": 53.42, + "memory_mib_peak": 62.61 + }, + "postgres": { + "cpu_percent_mean": 50.52, + "cpu_percent_peak": 67.48, + "memory_mib_peak": 209.1 + }, + "haproxy": { + "cpu_percent_mean": 21.16, + "cpu_percent_peak": 26.89, + "memory_mib_peak": 17.16 + } + }, + "sample": 3 + } + ] +} diff --git a/evidence/track1-rps.svg b/evidence/track1-rps.svg new file mode 100644 index 00000000..39f258ce --- /dev/null +++ b/evidence/track1-rps.svg @@ -0,0 +1,36 @@ + + +API scaling: successful requests per second + +0.0 + +209.4 + +418.8 + +628.2 + +837.7 + +418.5 + +698.6 + +837.7 + + +103.7 + +274.6 + +680.7 + +764.2 + +1 replicas +2 replicas +4 replicas +8 replicas +Fixed aggregate budget +Increased aggregate budget + diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..f2689b94 --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,33 @@ +global + log stdout format raw local0 + maxconn 2048 + hard-stop-after 8s + +defaults + log global + mode http + option httplog + timeout connect 1s + timeout client 10s + timeout server 10s + timeout http-request 5s + retries 1 + option redispatch + option httpchk GET /health/ready + http-check expect status 200 + +resolvers docker + nameserver embedded 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 2s + +frontend public_api + bind :8080 + default_backend messenger_api + +backend messenger_api + balance roundrobin + server-template api 1-16 api:8080 check inter 1s fall 2 rise 2 resolvers docker init-addr libc,none diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..eccf09ef --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,39 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (length(btrim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (length(btrim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_by_user ON chat_members(user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL CHECK (length(btrim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX messages_history_keyset ON messages(chat_id, created_at, id); + +-- PostgreSQL is authoritative for session durability and revocation. Valkey only +-- keeps a short-lived lookup hint; a cache hit is still checked against this row. +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX sessions_expiry ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..6e9877d5 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,11 @@ +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + method varchar(8) NOT NULL CHECK (method = 'POST'), + resource_path text NOT NULL, + key varchar(128) NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash char(64) NOT NULL, + response_body jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, method, resource_path, key) +); +CREATE INDEX idempotency_expiry ON idempotency_keys(created_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..87262a1e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +argon2-cffi==23.1.0 +argon2-cffi-bindings==26.1.0 +cffi==2.1.1 +pycparser==3.12 +psycopg[binary,pool]==3.2.3 +psycopg-binary==3.2.3 +psycopg-pool==3.3.3 +redis==5.2.1 +typing_extensions==4.16.0 diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..cbc7894d --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +# Restart every stateful dependency and all API processes without deleting volumes. +docker compose --env-file .env restart api haproxy postgres valkey diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 00000000..214a8246 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu +compose="docker compose --env-file .env" +# Add one ready replica before removing any old container. +$compose up -d --scale api=3 --wait --wait-timeout 120 api +python3 - <<'PY' +import time, urllib.request +end=time.monotonic()+60 +while time.monotonic()/dev/null + docker rm "$id" >/dev/null + docker compose --env-file .env up -d --scale api=3 --wait --wait-timeout 120 api + sleep 3 +done +docker compose --env-file .env up -d --scale api=2 --wait --wait-timeout 120 api diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..8c4acd14 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +set -eu +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" +# X-Instance-Id is the container hostname (the Docker short ID). Resolve it to +# exactly one scaled Compose container and stop only that process. +for id in $(docker compose --env-file .env ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 3 "$id" >/dev/null + exit 0 + fi +done +echo "No running api container matches TARGET_INSTANCE_ID" >&2 +exit 1 diff --git a/scripts/track1_experiment.py b/scripts/track1_experiment.py new file mode 100644 index 00000000..b00e8c4b --- /dev/null +++ b/scripts/track1_experiment.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +"""Repeatable local scaling study: fixed aggregate budget, then increased budget.""" +import json, os, platform, random, subprocess, sys, time, urllib.request +from statistics import median +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +COMPOSE = ["docker", "compose", "--env-file", ".env"] +OUT = ROOT / "evidence" +LOAD = [sys.executable, "tests/load.py", "--base-url", "http://127.0.0.1:8080", "--duration", os.getenv("BENCH_SECONDS", "30"), "--concurrency", os.getenv("BENCH_CONCURRENCY", "16")] + +def run(args, env=None, check=True): + subprocess.run(args, cwd=ROOT, env=env, check=check) + +def docker_usage(): + ids=[] + for service in ("api","postgres","haproxy"): + found=subprocess.run(COMPOSE+["ps","-q",service],cwd=ROOT,text=True,capture_output=True,check=True).stdout.split() + ids.extend(found) + if not ids: return {} + raw=subprocess.run(["docker","stats","--no-stream","--format","{{json .}}",*ids],cwd=ROOT,text=True,capture_output=True,check=True).stdout + grouped={"api":{"cpu_percent":[],"memory_mib":[]},"postgres":{"cpu_percent":[],"memory_mib":[]},"haproxy":{"cpu_percent":[],"memory_mib":[]}} + for line in raw.splitlines(): + try: item=json.loads(line) + except ValueError: continue + name=item.get("Name","") + role="api" if "-api-" in name else "postgres" if "-postgres-" in name else "haproxy" if "-haproxy-" in name else None + if not role: continue + try: grouped[role]["cpu_percent"].append(float(item["CPUPerc"].rstrip("%"))) + except (KeyError,ValueError): pass + try: + size=item["MemUsage"].split("/")[0].strip() + if size.endswith("GiB"): value=float(size[:-3])*1024 + elif size.endswith("MiB"): value=float(size[:-3]) + elif size.endswith("KiB"): value=float(size[:-3])/1024 + else: value=float(size[:-1])/(1024*1024) + grouped[role]["memory_mib"].append(value) + except (KeyError,ValueError,IndexError): pass + return grouped + +def row_count(): + env={} + for line in (ROOT/".env").read_text().splitlines(): + if "=" in line: + k,v=line.split("=",1); env[k]=v + cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-Atc","SELECT count(*) FROM messages"] + result=subprocess.run(cmd,cwd=ROOT,text=True,capture_output=True,check=True) + return int(result.stdout.strip()) + +def reset_synthetic_messages(): + """Use an identical empty message/idempotency dataset for every sample.""" + env={} + for line in (ROOT/".env").read_text().splitlines(): + if "=" in line: + k,v=line.split("=",1); env[k]=v + cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-v","ON_ERROR_STOP=1","-c","TRUNCATE TABLE messages, idempotency_keys; ANALYZE messages; ANALYZE idempotency_keys;"] + subprocess.run(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=True) + +def benchmark_load(dest, env): + reset_synthetic_messages() + before=row_count(); samples=[] + proc=subprocess.Popen(LOAD+["--output",str(dest)],cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True) + while proc.poll() is None: + stats=docker_usage() + if stats: samples.append(stats) + time.sleep(1) + output=proc.communicate()[0] + print(output,end="") + if proc.returncode: raise subprocess.CalledProcessError(proc.returncode,LOAD) + after=row_count() + flat={} + for role in ("api","postgres","haproxy"): + cpus=[x for sample in samples for x in sample[role]["cpu_percent"]] + mems=[x for sample in samples for x in sample[role]["memory_mib"]] + flat[role]={"cpu_percent_mean":round(sum(cpus)/len(cpus),2) if cpus else None,"cpu_percent_peak":round(max(cpus),2) if cpus else None,"memory_mib_peak":round(max(mems),2) if mems else None} + return {"message_rows_before":before,"message_rows_after":after,"docker_stats_samples":len(samples),"container_resource_usage":flat} + +def wait_ready(timeout=90): + deadline=time.monotonic()+timeout + while time.monotonic()','','API scaling: successful requests per second'] + for tick in range(5): + y=top+ploth-tick*ploth/4; val=vmax*tick/4 + out += [f'',f'{val:.1f}'] + colors=["#2563eb","#f97316"] + for gi,(label,items) in enumerate(groups): + pts=[] + for i,r in enumerate(items): + x=left+plotw*({1:1,2:2,4:3,8:4}[r["replicas"]])/5; y=top+ploth-(r["successful_rps"]/vmax)*ploth + pts.append(f"{x:.1f},{y:.1f}"); out.append(f''); out.append(f'{r["successful_rps"]:.1f}') + out.append(f'') + positions={1:1,2:2,4:3,8:4} + for n,pos in positions.items(): + x=left+plotw*pos/5; out.append(f'{n} replicas') + out += [f'{groups[0][0]}',f'{groups[1][0]}',''] + path.write_text("\n".join(out)+"\n") + +if __name__=="__main__": main() diff --git a/scripts/track1_offered_load.py b/scripts/track1_offered_load.py new file mode 100644 index 00000000..e0e001a0 --- /dev/null +++ b/scripts/track1_offered_load.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Rate-controlled synthetic write test to locate the throughput knee.""" +from concurrent.futures import ThreadPoolExecutor, as_completed +import json +import math +import random +import statistics +import subprocess +import threading +import time +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +sys.path.insert(0, str(ROOT / "scripts")) +from client import Client +import track1_experiment as bench +OUT = ROOT / "evidence" +DURATION = 15 +RATES = (250, 500, 750, 1000) +REPEATS = 3 + + +def percentile(values, p): + values = sorted(values) + if not values: + return None + return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 3) + + +def run_sample(actor, chat, rate, repeat, reset=True): + if reset: + bench.reset_synthetic_messages() + latencies = [] + failures = [] + lock = threading.Lock() + local = threading.local() + start = time.monotonic() + deadline = start + DURATION + + def request(index, due): + client = getattr(local, "client", None) + if client is None: + client = Client("http://127.0.0.1:8080", timeout=8) + local.client = client + sent = time.monotonic() + try: + client.message(actor, chat, f"offered-{rate}-{repeat}-{index}", key=f"offered-{rate}-{repeat}-{index}") + result = (time.monotonic() - sent, None) + except Exception as exc: + result = (None, f"{type(exc).__name__}: {str(exc)[:120]}") + with lock: + if result[0] is not None: + latencies.append(result[0]) + else: + failures.append(result[1]) + + submitted = 0 + offered_slots = 0 + skipped_overload = 0 + due = start + next_prune_at = start + .05 + max_inflight = 8192 + with ThreadPoolExecutor(max_workers=512) as pool: + futures = [] + while due < deadline: + offered_slots += 1 + if time.monotonic() >= next_prune_at: + futures = [future for future in futures if not future.done()] + next_prune_at = time.monotonic() + .05 + if len(futures) < max_inflight: + futures.append(pool.submit(request, submitted, due)) + submitted += 1 + else: + skipped_overload += 1 + due += 1 / rate + remaining = due - time.monotonic() + if remaining > 0: + time.sleep(remaining) + for future in as_completed(futures): + future.result() + elapsed = time.monotonic() - start + counts = {} + for failure in failures: + prefix = failure.split(":", 1)[0] + counts[prefix] = counts.get(prefix, 0) + 1 + row_count_error = None + try: + after = bench.row_count() + except subprocess.CalledProcessError as exc: + after = None + row_count_error = (exc.stderr or "PostgreSQL row-count query failed").strip()[-300:] + return { + "variant": f"offered-{rate}", "sample": repeat, "offered_rate_rps": rate, + "duration_seconds": DURATION, "concurrency_workers": 512, + "max_inflight_requests": max_inflight, "offered_slots": offered_slots, + "scheduled_requests": submitted, "successful_requests": len(latencies), + "failed_requests": len(failures), "failure_types": counts, + "skipped_overload": skipped_overload, + "offered_rps_actual": round(offered_slots / DURATION, 3), + "submitted_rps_actual": round(submitted / DURATION, 3), + "completed_success_rps_total_elapsed": round(len(latencies) / elapsed, 3), + "success_ratio": round(len(latencies) / submitted, 6) if submitted else None, + "effective_success_ratio_offered": round(len(latencies) / offered_slots, 6) if offered_slots else None, + "latency_successful_ms": {f"p{p}": percentile(latencies, p/100) for p in (50,95,99)}, + "drain_seconds_after_offer_window": round(max(0, elapsed - DURATION), 3), + "message_rows_after": after, + "message_rows_after_error": row_count_error, + } + + +def main(): + OUT.mkdir(exist_ok=True) + env = os.environ.copy() + env.update(API_CPUS="0.5", API_MEMORY="256m", DB_POOL_MAX="10", LB_CPUS="0.25") + bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env) + bench.wait_ready() + time.sleep(5) + client = Client("http://127.0.0.1:8080") + actor, _, _, chat = client.bootstrap() + rng = random.Random(20261106) + samples = [] + for repeat in range(1, REPEATS + 1): + rates = list(RATES) + rng.shuffle(rates) + for rate in rates: + row = run_sample(actor, chat, rate, repeat) + samples.append(row) + print(json.dumps(row, ensure_ascii=False), flush=True) + summary = [] + for rate in RATES: + group = [row for row in samples if row["offered_rate_rps"] == rate] + summary.append({ + "offered_rate_rps": rate, + "completed_success_rps_total_elapsed_median": round(statistics.median(row["completed_success_rps_total_elapsed"] for row in group), 3), + "completed_success_rps_total_elapsed_min": min(row["completed_success_rps_total_elapsed"] for row in group), + "completed_success_rps_total_elapsed_max": max(row["completed_success_rps_total_elapsed"] for row in group), + "success_ratio_median": round(statistics.median(row["success_ratio"] for row in group), 6), + "p95_ms_median": round(statistics.median(row["latency_successful_ms"]["p95"] for row in group), 3), + "effective_success_ratio_offered_median": round(statistics.median(row["effective_success_ratio_offered"] for row in group), 6), + "skipped_overload_median": int(statistics.median(row["skipped_overload"] for row in group)), + }) + result = { + "method": "Rate-controlled POST sweep; 15s per sample; offered rate 250/500/750/1000 RPS; three randomized repeats; 8 API replicas at 0.5 CPU/256 MiB, DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; max in-flight 8192; per-thread HTTP client reuse; synthetic message/idempotency tables reset per sample; no Docker stats polling during load.", + "summary": summary, "samples": samples, + } + (OUT / "offered-load-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + make_plot(summary, OUT / "offered-load.svg") + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +def make_plot(rows, path): + w, h = 900, 500 + left, right, top, bottom = 90, 90, 60, 85 + pw, ph = w-left-right, h-top-bottom + max_rps, max_p95 = 1000, 2000 + rps = [r["completed_success_rps_total_elapsed_median"] for r in rows] + p95 = [r["p95_ms_median"] for r in rows] + xs = [left + i*pw/(len(rows)-1) for i in range(len(rows))] + yr = [top + ph*(1-v/max_rps) for v in rps] + yp = [top + ph*(1-v/max_p95) for v in p95] + out = [f'', + '', + 'Насыщение при росте предложенной нагрузки'] + for tick in (0,250,500,750,1000): + y=top+ph*(1-tick/max_rps) + out += [f'', + f'{tick}'] + for x, row in zip(xs, rows): + out += [f'{row["offered_rate_rps"]}'] + out += [f'', + f''] + for x,y,value in zip(xs,yr,rps): + out.append(f'{value:.0f}') + for x,y,value in zip(xs,yp,p95): + out.append(f'{value:.0f}ms') + out += [f'Предложенная нагрузка, RPS', + f'Успешные RPS за offer+drain', + f'p95, ms', + f'Успешные RPS', + f'p95 успешных запросов', + ''] + path.write_text('\n'.join(out)+'\n') + + +if __name__ == "__main__": + main() diff --git a/scripts/track1_offered_scale.py b/scripts/track1_offered_scale.py new file mode 100644 index 00000000..af408b5a --- /dev/null +++ b/scripts/track1_offered_scale.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +"""Rate-controlled paired scale test for 8 versus 12 API replicas.""" +import json +import os +import random +import statistics +import time +from pathlib import Path + +import track1_experiment as bench +from track1_offered_load import run_sample +from client import Client + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" +REPEATS = 3 +RATES = (1000,) +REPLICAS = (8, 12) + + +def main(): + OUT.mkdir(exist_ok=True) + env = os.environ.copy() + env.update(API_CPUS="0.5", API_MEMORY="256m", DB_POOL_MAX="10", LB_CPUS="0.25") + bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env) + bench.wait_ready() + time.sleep(5) + client = Client("http://127.0.0.1:8080") + actor, _, _, chat = client.bootstrap() + rng = random.Random(20261107) + samples = [] + last_replica_count = 8 + for repeat in range(1, REPEATS + 1): + block = [(replicas, rate) for replicas in REPLICAS for rate in RATES] + rng.shuffle(block) + for replicas, rate in block: + if last_replica_count != 8: + bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env) + bench.wait_ready() + time.sleep(5) + last_replica_count = 8 + bench.reset_synthetic_messages() + if replicas != last_replica_count: + bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", f"api={replicas}", "--wait", "--wait-timeout", "180"], env) + bench.wait_ready() + time.sleep(5) + last_replica_count = replicas + row = run_sample(actor, chat, rate, repeat, reset=False) + row["replicas"] = replicas + row["sample"] = repeat + samples.append(row) + print(json.dumps(row, ensure_ascii=False), flush=True) + summary = [] + for replicas in REPLICAS: + for rate in RATES: + group = [r for r in samples if r["replicas"] == replicas and r["offered_rate_rps"] == rate] + summary.append({ + "replicas": replicas, + "offered_rate_rps": rate, + "completed_success_rps_total_elapsed_median": round(statistics.median(r["completed_success_rps_total_elapsed"] for r in group), 3), + "completed_success_rps_min": min(r["completed_success_rps_total_elapsed"] for r in group), + "completed_success_rps_max": max(r["completed_success_rps_total_elapsed"] for r in group), + "success_ratio_median": round(statistics.median(r["success_ratio"] for r in group), 6), + "p95_ms_median": round(statistics.median(r["latency_successful_ms"]["p95"] for r in group), 3), + "error_count_total": sum(r["failed_requests"] for r in group), + "message_rows_after_median": int(statistics.median(r["message_rows_after"] for r in group if r["message_rows_after"] is not None)) if any(r["message_rows_after"] is not None for r in group) else None, + }) + result = { + "method": "Randomized paired blocks at offered 1000 RPS, 15s per sample; three repeats; compare 8 versus 12 replicas at fixed 0.5 CPU/256 MiB per API replica; DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; HAProxy server-template supports 16 backends; scale to 8 before each DB reset to release idle connections, then scale to target; synthetic messages and idempotency keys truncated per sample; HTTP client reused per worker.", + "summary": summary, + "samples": samples, + } + (OUT / "offered-scale-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n") + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/server.py b/server.py new file mode 100644 index 00000000..3c8247f3 --- /dev/null +++ b/server.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +"""PostgreSQL-backed messenger API for MTUSI lab 2.""" +from __future__ import annotations + +import base64, binascii, hashlib, json, logging, os, re, secrets, signal, socket, threading, time, uuid +from datetime import datetime, timedelta, timezone +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +import redis +from argon2 import PasswordHasher, exceptions as argon_exc + +LOG = logging.getLogger("messenger") +USERNAME_RE = re.compile(r"\A[a-z0-9_]{3,32}\Z") +POOL: ConnectionPool | None = None +CACHE: redis.Redis | None = None +PASSWORDS = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +MAX_BODY_BYTES = 1024 * 1024 +CURSOR_SECRET = os.getenv("CURSOR_SECRET", "local-only-change-this-cursor-secret").encode("utf-8") +INSTANCE_ID = os.getenv("API_INSTANCE_ID") or socket.gethostname() or str(uuid.uuid4()) +IDEMPOTENCY_KEY_RE = re.compile(r"\A[A-Za-z0-9._:-]{1,128}\Z") + +def utcnow() -> datetime: return datetime.now(timezone.utc) +def stamp(value: datetime) -> str: return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z") +def new_id() -> str: return str(uuid.uuid4()) + +class ApiError(Exception): + def __init__(self, status, code, message): self.status, self.code, self.message = status, code, message + +def invalid(message="Некорректный запрос"): raise ApiError(400, "invalid_request", message) +def unavailable(): raise ApiError(503, "unavailable", "Зависимость временно недоступна") + +def db(): + assert POOL is not None + return POOL.connection() + +def migrate(): + migrations = Path(__file__).parent / "migrations" + with db() as conn: + conn.execute("SELECT pg_advisory_xact_lock(2026102802)") + conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())") + for migration in sorted(migrations.glob("*.sql")): + version = migration.stem + if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone() is None: + conn.execute(migration.read_text(encoding="utf-8")) + conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (version,)) + +def as_user(row): return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])} +def as_chat(row, members): return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in members], "created_at": stamp(row["created_at"])} +def as_message(row): return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": stamp(row["created_at"])} + +def require_text(value, field, maximum): + if not isinstance(value, str) or not value.strip() or len(value) > maximum: invalid(f"Поле {field} должно содержать от 1 до {maximum} символов и не быть пустым") + try: value.encode("utf-8") + except UnicodeEncodeError: invalid(f"Поле {field} должно содержать корректный Unicode") + return value + +def strict_object(value, allowed, required): + if not isinstance(value, dict) or set(value)-allowed or required-set(value): invalid("Тело запроса содержит неверные поля") + return value + +def parse_uuid(value, label="UUID"): + try: return str(uuid.UUID(value)) + except (ValueError, TypeError, AttributeError): invalid(f"Некорректный {label}") + +def cursor_encode(scope, row): + obj = {"scope":scope,"created_at":stamp(row["created_at"]),"id":str(row["id"])} + raw = json.dumps(obj,separators=(",",":"),ensure_ascii=True).encode() + sig = hashlib.sha256(CURSOR_SECRET+raw).digest()[:12] + return base64.urlsafe_b64encode(raw+sig).decode().rstrip("=") + +def cursor_decode(value, scope): + try: + raw = base64.urlsafe_b64decode(value+"="*(-len(value)%4)); payload,sig=raw[:-12],raw[-12:] + if not secrets.compare_digest(sig,hashlib.sha256(CURSOR_SECRET+payload).digest()[:12]): invalid("Некорректный cursor") + obj=json.loads(payload) + if set(obj)!={"scope","created_at","id"} or obj["scope"]!=scope: invalid("Cursor относится к другому списку") + dt=datetime.fromisoformat(obj["created_at"].replace("Z","+00:00")); return dt,parse_uuid(obj["id"]) + except (ValueError,TypeError,KeyError,binascii.Error): invalid("Некорректный cursor") + +class Handler(BaseHTTPRequestHandler): + protocol_version="HTTP/1.0" + def setup(self): super().setup(); self.connection.settimeout(3) + def log_message(self, fmt,*args): LOG.info("%s %s %s",self.command,urlsplit(self.path).path,args[1] if len(args)>1 else "") + def do_GET(self): self.dispatch() + def do_POST(self): self.dispatch() + def do_DELETE(self): self.dispatch() + + def send_json(self,status,payload,headers=None): + body=json.dumps(payload,ensure_ascii=False,separators=(",",":")).encode() + self.send_response(status); self.send_header("Content-Type","application/json; charset=utf-8"); self.send_header("Content-Length",str(len(body))) + self.send_header("X-Request-Id",self.request_id); self.send_header("X-Instance-Id",INSTANCE_ID) + for k,v in (headers or {}).items(): self.send_header(k,v) + self.end_headers(); self.wfile.write(body) + def empty(self,status,headers=None): + self.send_response(status); self.send_header("Content-Length","0"); self.send_header("X-Request-Id",self.request_id); self.send_header("X-Instance-Id",INSTANCE_ID) + for k,v in (headers or {}).items(): self.send_header(k,v) + self.end_headers() + def error(self,e): self.send_json(e.status,{"error":{"code":e.code,"message":e.message,"request_id":self.request_id}},getattr(e,"headers",None)) + def read_json(self): + if self.headers.get_content_type()!="application/json": raise ApiError(415,"unsupported_media_type","Ожидается Content-Type: application/json") + try: n=int(self.headers.get("Content-Length","")) + except ValueError: invalid("Некорректный Content-Length") + if n<0: invalid("Некорректный Content-Length") + if n>MAX_BODY_BYTES: raise ApiError(413,"payload_too_large","Тело запроса слишком велико") + raw=self.rfile.read(n) + if len(raw)!=n: invalid("Тело запроса передано не полностью") + try: val=json.loads(raw.decode("utf-8")) + except (UnicodeError,json.JSONDecodeError): invalid("Тело запроса должно быть корректным JSON UTF-8") + if not isinstance(val,dict): invalid("Тело запроса должно быть JSON-объектом") + return val + def challenge(self,scheme): + e=ApiError(401,"unauthorized","Нет действительной сессии"); e.headers={"WWW-Authenticate":scheme}; raise e + + def authenticate(self): + header=self.headers.get("Authorization","") + if not header.startswith("Bearer "): self.challenge('Bearer realm="messenger"') + token=header[7:].strip() + if not token or len(token)>256: self.challenge('Bearer realm="messenger"') + digest=hashlib.sha256(token.encode()).hexdigest(); cache_key="sess:"+digest + uid=None + if CACHE: + try: uid=CACHE.get(cache_key) + except redis.RedisError: pass + with db() as conn: + # Valkey's expiring user ID is a lookup hint. PostgreSQL validates it + # against the live session on every request, so missed cache deletion + # during an outage can never keep a logged-out token valid. + row=None + if uid: + row=conn.execute("SELECT u.id,u.username,u.display_name,u.created_at,s.expires_at FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=%s AND s.user_id=%s AND s.expires_at>now()",(digest,uid)).fetchone() + if not row: + row=conn.execute("SELECT u.id,u.username,u.display_name,u.created_at,s.expires_at FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=%s AND s.expires_at>now()",(digest,)).fetchone() + if not row: self.challenge('Bearer realm="messenger"') + if CACHE and str(row["id"])!=uid: + try: CACHE.setex(cache_key,max(1,int((row["expires_at"]-utcnow()).total_seconds())),str(row["id"])) + except redis.RedisError: pass + return as_user(row),digest + + def dispatch(self): + self.request_id=secrets.token_hex(16) + try: + split=urlsplit(self.path); path=split.path; q=parse_qs(split.query,keep_blank_values=True) + if path in ("/health/live","/health/ready"): + if self.command!="GET" or q: invalid("Health endpoint принимает только GET без параметров") + if path.endswith("ready"): + try: + with db() as c: c.execute("SELECT 1") + except Exception: unavailable() + self.send_json(200,{"status":"ok"}); return + if path=="/api/v1/users" and self.command=="POST": self.register(); return + if path=="/api/v1/auth/sessions" and self.command=="POST": self.login(); return + if path=="/api/v1/auth/sessions/current" and self.command=="DELETE": + actor,digest=self.authenticate() + with db() as c: c.execute("DELETE FROM sessions WHERE token_hash=%s",(digest,)) + if CACHE: + try: CACHE.delete("sess:"+digest) + except redis.RedisError: pass + self.empty(204); return + actor,_=self.authenticate() + if path=="/api/v1/users/me" and self.command=="GET": self.no_query(q); self.send_json(200,actor); return + if path=="/api/v1/chats": + if self.command=="POST": self.no_query(q); self.create_chat(actor); return + if self.command=="GET": self.list_chats(actor,q); return + m=re.fullmatch(r"/api/v1/chats/([^/]+)(?:/(messages))?",path) + if m: + cid=parse_uuid(m.group(1),"chat_id") + if m.group(2)=="messages": + if self.command=="POST": self.create_message(actor,cid); return + if self.command=="GET": self.list_messages(actor,cid,q); return + if self.command=="GET": + self.no_query(q) + with db() as c: + row=c.execute("SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s",(cid,actor["id"])).fetchone() + if not row: raise ApiError(404,"not_found","Объект не найден") + ids=c.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id",(cid,)).fetchall() + self.send_json(200,as_chat(row,[x["user_id"] for x in ids])); return + raise ApiError(404,"not_found","Объект не найден") + except ApiError as e: self.error(e) + except (psycopg.Error,PoolTimeout,redis.RedisError) as e: LOG.warning("dependency error request_id=%s type=%s",self.request_id,type(e).__name__); self.error(ApiError(503,"unavailable","Зависимость временно недоступна")) + except (BrokenPipeError,ConnectionResetError): pass + except Exception: LOG.exception("request failed request_id=%s",self.request_id); self.error(ApiError(500,"internal_error","Внутренняя ошибка сервера")) + + def no_query(self,q): + if q: invalid("Этот endpoint не принимает query-параметры") + def register(self): + b=strict_object(self.read_json(),{"username","display_name","password"},{"username","display_name","password"}) + if not isinstance(b["username"],str) or not USERNAME_RE.fullmatch(b["username"]): invalid("username должен соответствовать [a-z0-9_]{3,32}") + name=require_text(b["display_name"],"display_name",100); password=b["password"] + if not isinstance(password,str) or not 12<=len(password)<=128: invalid("password должен содержать от 12 до 128 символов") + uid=new_id(); hashed=PASSWORDS.hash(password) + try: + with db() as c: row=c.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at",(uid,b["username"],name,hashed)).fetchone() + except psycopg.errors.UniqueViolation: raise ApiError(409,"conflict","username уже занят") + self.send_json(201,as_user(row)) + def login(self): + if self.headers.get("Content-Length") not in (None, "0", ""): invalid("POST /auth/sessions не принимает body") + h=self.headers.get("Authorization","") + try: + if not h.startswith("Basic "): raise ValueError() + userpass=base64.b64decode(h[6:],validate=True).decode("utf-8"); username,password=userpass.split(":",1) + except (ValueError,UnicodeError,binascii.Error): self.challenge('Basic realm="messenger", charset="UTF-8"') + with db() as c: row=c.execute("SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s",(username,)).fetchone() + # Verify a fixed dummy hash for unknown users to reduce timing-based enumeration. + try: ok=PASSWORDS.verify(row["password_hash"] if row else DUMMY_HASH,password) + except (argon_exc.VerificationError,ValueError): ok=False + if not row or not ok: self.challenge('Basic realm="messenger", charset="UTF-8"') + token=secrets.token_urlsafe(32); digest=hashlib.sha256(token.encode()).hexdigest(); ttl=int(os.getenv("SESSION_TTL_SECONDS","3600")); expires=utcnow()+timedelta(seconds=ttl) + with db() as c: c.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)",(digest,row["id"],expires)) + if CACHE: + try: CACHE.setex("sess:"+digest,ttl,str(row["id"])) + except redis.RedisError: pass + self.send_json(201,{"token":token,"token_type":"Bearer","expires_at":stamp(expires),"user":as_user(row)}) + + def create_chat(self,actor): + b=strict_object(self.read_json(),{"title","member_ids"},{"title","member_ids"}); title=require_text(b["title"],"title",100); ids=b["member_ids"] + if not isinstance(ids,list) or not 1<=len(ids)<=100: invalid("member_ids должен содержать от 1 до 100 UUID") + ids=[parse_uuid(v,"member_id") for v in ids] + if len(set(ids))!=len(ids): invalid("member_ids не должен содержать повторов") + if actor["id"] not in ids: invalid("Создатель чата должен входить в member_ids") + cid=new_id() + try: + with db() as c: + known=c.execute("SELECT id FROM users WHERE id=ANY(%s)",(ids,)).fetchall() + if len(known)!=len(ids): invalid("Все участники чата должны быть зарегистрированы") + row=c.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at",(cid,title)).fetchone() + with c.cursor() as cursor: + cursor.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)",[(cid,u) for u in ids]) + except psycopg.errors.ForeignKeyViolation: invalid("Все участники чата должны быть зарегистрированы") + self.send_json(201,as_chat(row,ids)) + def parse_page(self,q,scope): + if set(q)-{"limit","cursor"} or any(len(v)!=1 for v in q.values()): invalid("Неверные параметры пагинации") + raw=q.get("limit",["50"])[0] + if not re.fullmatch(r"[0-9]{1,3}",raw): invalid("limit должен быть целым числом от 1 до 100") + limit=int(raw) + if not 1<=limit<=100: invalid("limit должен быть целым числом от 1 до 100") + cur=cursor_decode(q["cursor"][0],scope) if "cursor" in q and q["cursor"][0] else None + if "cursor" in q and not q["cursor"][0]: invalid("cursor не должен быть пустым") + return limit,cur + def list_chats(self,actor,q): + limit,cur=self.parse_page(q,"chats"); params=[actor["id"]]; clause="" + if cur: clause=" AND (c.created_at,c.id)>(%s,%s)"; params.extend(cur) + with db() as c: + rows=c.execute("SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s"+clause+" ORDER BY c.created_at,c.id LIMIT %s",(*params,limit+1)).fetchall() + items=[] + for row in rows[:limit]: + members=c.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id",(row["id"],)).fetchall(); items.append(as_chat(row,[m["user_id"] for m in members])) + self.send_json(200,{"items":items,"next_cursor":cursor_encode("chats",rows[limit-1]) if len(rows)>limit else None}) + def create_message(self,actor,cid): + self.no_query(parse_qs(urlsplit(self.path).query,keep_blank_values=True)) + key=self.headers.get("Idempotency-Key", "") + if not IDEMPOTENCY_KEY_RE.fullmatch(key): invalid("Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}") + with db() as c: + if not c.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",(cid,actor["id"])).fetchone(): raise ApiError(404,"not_found","Объект не найден") + b=strict_object(self.read_json(),{"text","attachment_ids"},{"text"}); text=require_text(b["text"],"text",4000) + attachment_ids=b.get("attachment_ids", []) + if attachment_ids != []: invalid("Вложения не поддерживаются в лабораторной 3") + canonical=json.dumps({"text":text,"attachment_ids":[]},ensure_ascii=False,sort_keys=True,separators=(",",":")) + request_hash=hashlib.sha256(canonical.encode("utf-8")).hexdigest() + resource_path=f"/api/v1/chats/{cid}/messages" + scope=f"{actor['id']}\nPOST\n{resource_path}\n{key}" + try: + with db() as c: + # One transaction-scoped lock serializes equal keys across API replicas. + c.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))",(scope,)) + old=c.execute("SELECT request_hash,response_body FROM idempotency_keys WHERE user_id=%s AND method='POST' AND resource_path=%s AND key=%s AND created_at>now()-interval '24 hours'",(actor["id"],resource_path,key)).fetchone() + if old: + if old["request_hash"]!=request_hash: raise ApiError(409,"conflict","Idempotency-Key уже использован с другим содержимым") + payload=old["response_body"] + else: + c.execute("DELETE FROM idempotency_keys WHERE user_id=%s AND method='POST' AND resource_path=%s AND key=%s",(actor["id"],resource_path,key)) + row=c.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING id,chat_id,sender_id,text,created_at",(new_id(),cid,actor["id"],text)).fetchone() + payload=as_message(row) + c.execute("INSERT INTO idempotency_keys(user_id,method,resource_path,key,request_hash,response_body) VALUES (%s,'POST',%s,%s,%s,%s::jsonb)",(actor["id"],resource_path,key,request_hash,json.dumps(payload,ensure_ascii=False,separators=(",",":")))) + except psycopg.errors.ForeignKeyViolation: raise ApiError(404,"not_found","Объект не найден") + self.send_json(201,payload) + def list_messages(self,actor,cid,q): + with db() as c: + if not c.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",(cid,actor["id"])).fetchone(): raise ApiError(404,"not_found","Объект не найден") + limit,cur=self.parse_page(q,"messages:"+cid); params=[cid]; clause="" + if cur: clause=" AND (created_at,id)>(%s,%s)"; params.extend(cur) + with db() as c: rows=c.execute("SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s"+clause+" ORDER BY created_at,id LIMIT %s",(*params,limit+1)).fetchall() + self.send_json(200,{"items":[as_message(r) for r in rows[:limit]],"next_cursor":cursor_encode("messages:"+cid,rows[limit-1]) if len(rows)>limit else None}) + +class Server(ThreadingHTTPServer): + daemon_threads=True; allow_reuse_address=True + +def main(): + global POOL,CACHE,DUMMY_HASH + logging.basicConfig(level=os.getenv("LOG_LEVEL","INFO").upper(),format="%(asctime)s %(levelname)s %(message)s") + pool_max=int(os.getenv("DB_POOL_MAX","10")) + if not 1<=pool_max<=50: raise SystemExit("DB_POOL_MAX must be between 1 and 50") + POOL=ConnectionPool(os.getenv("DATABASE_URL","postgresql://messenger:messenger@postgres:5432/messenger"),min_size=int(os.getenv("DB_POOL_MIN","1")),max_size=pool_max,timeout=3,kwargs={"row_factory":dict_row},open=True) + CACHE=redis.Redis.from_url(os.getenv("CACHE_URL","redis://valkey:6379/0"),socket_connect_timeout=0.5,socket_timeout=0.5,decode_responses=True) + DUMMY_HASH=PASSWORDS.hash("dummy-password-do-not-use-123") + deadline=time.monotonic()+60 + while True: + try: migrate(); break + except (psycopg.Error, PoolTimeout): + if time.monotonic()>deadline: raise + LOG.info("waiting for PostgreSQL"); time.sleep(1) + if os.getenv("RUN_MIGRATIONS_ONLY") == "1": + POOL.close() + return + try: CACHE.ping() + except redis.RedisError: LOG.warning("Valkey unavailable; session validation will use PostgreSQL") + host=os.getenv("APP_HOST","0.0.0.0"); port=int(os.getenv("APP_PORT","8080")); server=Server((host,port),Handler) + def stop(signum,_frame): LOG.info("received signal %s; stopping",signum); threading.Thread(target=server.shutdown,daemon=True).start() + signal.signal(signal.SIGTERM,stop); signal.signal(signal.SIGINT,stop) + try: LOG.info("server listening on %s:%s",host,port); server.serve_forever(poll_interval=.2) + finally: server.server_close(); POOL.close(); LOG.info("server stopped") + +if __name__=="__main__": main() diff --git a/student-tests/dependencies.py b/student-tests/dependencies.py new file mode 100644 index 00000000..4b6f782f --- /dev/null +++ b/student-tests/dependencies.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Stop/start Valkey and PostgreSQL with volumes preserved; verify bounded failures and recovery.""" +import subprocess +import time + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) + +from client import Client, Failure, arguments, check, run + +ROOT = Path(__file__).resolve().parents[1] + +def compose(action, service): + subprocess.run(["docker", "compose", "--env-file", ".env", action, service], cwd=ROOT, + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + +def main(): + args=arguments("Check session fallback and database dependency failure") + c=Client(args.base_url,args.ca_file,timeout=8) + actor=c.user("dependency") + + compose("stop","valkey") + try: + c.call("GET","/health/ready",200,schema="Health") + me=c.call("GET","/api/v1/users/me",200,actor=actor,schema="User")["json"] + check(me["id"]==actor["id"],"session failed while Valkey was down") + finally: + compose("start","valkey") + + compose("stop","postgres") + try: + c.call("GET","/health/ready",503) + c.call("GET","/api/v1/users/me",503,actor=actor) + finally: + compose("start","postgres") + + deadline=time.monotonic()+60 + while True: + try: + c.call("GET","/health/ready",200,schema="Health") + break + except Failure: + check(time.monotonic()=2,'test did not observe requests served by two API replicas') + history=base.call('GET',f"/api/v1/chats/{chat['id']}/messages?limit=100",actor=bob,schema='MessagePage')['json']['items'] + check(sum(m['id'] in ids for m in history)==1,'message is not visible exactly once through LB') + print(f'PASS: 32 concurrent retries, one message, {len(instances)} API instances observed') + +if __name__=='__main__': run(main) diff --git a/student-tests/quality.py b/student-tests/quality.py new file mode 100644 index 00000000..021a1ee0 --- /dev/null +++ b/student-tests/quality.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Level-4 checks: concurrent unique registration and atomic chat creation.""" +import concurrent.futures +import json +import os +import subprocess +import uuid + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) + +from client import Client, Failure, arguments, check, run + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +def db_count(table): + # Fixed table allowlist: user input is never interpolated here. + assert table in {"users", "chats", "chat_members", "messages"} + output = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", + "psql", "-X", "-U", "messenger", "-d", "messenger", "-At", "-c", + "SELECT count(*) FROM " + table], cwd=ROOT, check=True, text=True, + stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout + return int(output.strip()) + +def main(): + args = arguments("Check registration race and transaction rollback") + c = Client(args.base_url, args.ca_file) + alice, bob, _, _ = c.bootstrap() + username = "race_" + uuid.uuid4().hex[:16] + body = {"username": username, "display_name": "Гонка регистрации", "password": "Test-password-" + uuid.uuid4().hex} + def register(_): + try: + return c.call("POST", "/api/v1/users", [201, 409], body=body)["status"] + except Failure: + raise + with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool: + statuses = list(pool.map(register, range(12))) + check(statuses.count(201) == 1 and statuses.count(409) == 11, + "username race must create exactly one user and return 409 to all other requests") + before = db_count("chats") + after = c.call("POST", "/api/v1/chats", 400, actor=alice, + body={"title": "rollback probe", "member_ids": [alice["id"], str(uuid.uuid4())]}) + check(after["status"] == 400 and db_count("chats") == before, + "invalid multi-step chat creation left a partial chat row") + plan = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres", + "psql", "-X", "-U", "messenger", "-d", "messenger", "-At", "-c", + "EXPLAIN (ANALYZE, BUFFERS) SELECT id FROM messages WHERE chat_id=md5('chat-0')::uuid ORDER BY created_at,id LIMIT 50"], + cwd=ROOT, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.splitlines() + print(json.dumps({"registration_race": {"201": statuses.count(201), "409": statuses.count(409)}, + "chat_rollback": "PASS", "history_index_plan": plan}, ensure_ascii=False, indent=2)) + +if __name__ == "__main__": run(main) diff --git a/student-tests/session_ttl.py b/student-tests/session_ttl.py new file mode 100644 index 00000000..d13a9f20 --- /dev/null +++ b/student-tests/session_ttl.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python3 +"""Run only with SESSION_TTL_SECONDS=2 in the Compose environment.""" +import time +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests")) + +from client import Client, Failure, arguments, check, run + +def main(): + args=arguments("Check persisted session expiry with a short TTL") + c=Client(args.base_url,args.ca_file) + actor=c.user("ttl") + c.call("GET","/api/v1/users/me",200,actor=actor,schema="User") + time.sleep(2.2) + c.call("GET","/api/v1/users/me",401,actor=actor) + print("PASS: session expired at the configured two-second TTL") + +if __name__=="__main__": run(main)