diff --git a/.dockerignore b/.dockerignore
index bc16e556..639a55ae 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,4 +1,6 @@
.git
+docs/plans
+LAB_NOTES.md
.github
.env
.env.*
diff --git a/.env.example b/.env.example
index d32b9be0..ac22be39 100644
--- a/.env.example
+++ b/.env.example
@@ -1,13 +1,3 @@
-# Copy to .env for your LOCAL synthetic test environment; never commit .env.
-# These are logical interface suggestions; map actual framework names in REPORT.md.
-APP_HOST=0.0.0.0
+# Copy to .env; create secrets/postgres_password and secrets/database_url locally.
APP_PORT=8080
-APP_ENV=lab
SESSION_TTL_SECONDS=3600
-# Choose credentials locally. File paths contain no secret values.
-DATABASE_URL_FILE=/run/secrets/database_url
-# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
-SESSION_STORE=postgres
-CACHE_URL_FILE=/run/secrets/cache_url
-API_REPLICAS=2
-IDEMPOTENCY_TTL_SECONDS=86400
diff --git a/.gitignore b/.gitignore
index b5a71896..f101a4f8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
# Local/private state
+/docs/plans/
+/LAB_NOTES.md
.env
.env.*
!.env.example
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..1ecad6dc
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,12 @@
+FROM python:3.12.12-slim-bookworm
+
+ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1
+WORKDIR /app
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt && useradd --system --uid 10001 --home /app app
+COPY --chown=app:app src/ src/
+COPY --chown=app:app migrations/ migrations/
+COPY --chown=app:app scripts/migrate.py scripts/migrate.py
+USER app
+EXPOSE 8080
+CMD ["python", "-m", "src.server"]
diff --git a/REPORT.md b/REPORT.md
index c2679005..8e17dadb 100644
--- a/REPORT.md
+++ b/REPORT.md
@@ -1,37 +1,118 @@
# Отчёт — лабораторная 3
-- Автор, группа: TODO
-- Целевая оценка; выбранный трек 5, если нужен: TODO
-- Commit/PR: TODO после публикации
-- Использованные источники, библиотеки и помощь ИИ: TODO
+- Автор: Самохвалов Андрей Ремович, БПИ-2302.
+- Цель: оценка 5, трек 1 (цена масштабирования). Уровень 4 и выбранный трек проверены ниже.
+- Ветка: `solution/bpi2302-samohvalov`; commit/PR после публикации.
+- Источники: `README.md`, `COURSE.md`, `contracts/`, публичные тесты репозитория, [HAProxy 3.0 configuration](https://docs.haproxy.org/3.0/configuration.html), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/). Код и текст подготовлены с помощью Codex; результаты ниже получены реальным запуском.
## Запуск из чистого clone
-TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
+Нужны Docker с Compose, Python 3.9+ и `make`. Dockerfile собирает исходники в `python:3.12.12-slim-bookworm`; БД — `postgres:17.6-bookworm`, балансировщик — `haproxy:3.0.11-alpine`. Python-пакеты закреплены в `requirements.txt`. Стенд рассчитан на localhost и синтетические данные.
+
+```sh
+cp .env.example .env
+mkdir -p secrets
+python3 - <<'PY'
+from pathlib import Path
+from secrets import token_urlsafe
+password = token_urlsafe(32)
+Path('secrets/postgres_password').write_text(password)
+Path('secrets/database_url').write_text(f'postgresql://messenger:{password}@postgres:5432/messenger')
+PY
+chmod 600 .env secrets/*
+docker compose --env-file .env config --quiet
+docker compose --env-file .env up -d --build --wait --scale api=2
+make check
+make test
+make test-persistence ACTION_SCRIPT=scripts/restart.sh
+make test-failover ACTION_SCRIPT=scripts/stop-one.sh
+docker compose --env-file .env up -d --wait --scale api=2 # восстановить остановленную реплику
+```
+
+Для повторного failover после восстановления балансировщик должен обслуживать обе реплики; повторите команду и сверьте, какой контейнер остановлен. `docker compose --env-file .env ps -a` показывает реальный процесс. `migrate` — одноразовый сервис, который под advisory lock применяет SQL-миграции до запуска API; повторный `up` безопасно проверяет уже применённые версии. `docker compose --env-file .env down` сохраняет volume. Только для намеренного удаления учебных данных: `docker compose --env-file .env down -v`.
## Архитектура и решения
-TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
+Клиент → HAProxy (`lb`, единственный опубликованный порт) → две FastAPI-реплики (`api`) → общий PostgreSQL (`postgres`). Сессии, сообщения и записи идемпотентности хранятся в PostgreSQL. Каждая реплика принимает любой действующий Bearer без sticky session. `X-Instance-Id` — случайный alias, постоянный в пределах процесса. Внешний порт PostgreSQL и API не опубликован.
-## Доказательства критериев
+Балансировка round-robin использует `server-template` на четыре слота, Docker DNS `127.0.0.11`, повторное разрешение адресов и `/health/ready` раз в 1 секунду (`fall 2`, `rise 1`). Сбой backend обнаруживается примерно за 2 проверки; проверка через публичный endpoint важнее расчётного интервала. HAProxy имеет connect timeout 2 с и client/server timeout 15 с. Compose ждёт готовности БД и однократной миграции; API имеет 10-секундный `stop_grace_period`, Uvicorn получает SIGTERM как главный процесс контейнера. На API установлены ограничения 0.5 CPU/256 MiB на реплику, на LB — 0.5 CPU/128 MiB; на PostgreSQL — 1 CPU/512 MiB, на миграцию — 0.5 CPU/128 MiB. Дополнительное исследование фиксированного суммарного бюджета выполняется позже.
-| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
-| --- | --- | --- | --- |
-| Публичный smoke | make test | TODO | TODO |
-| Остальные пункты уровня 3 | TODO | TODO | TODO |
-| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
-| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
+API и миграция получают только `database_url`, PostgreSQL — только пароль БД, LB — только конфигурацию HAProxy в read-only mount. `student-tests/test_operations.py` проверяет опубликованные порты, mounts, secrets и лимиты из раскрытого Compose-конфига; фактические лимиты контейнеров сняты через `docker inspect` в `evidence/topology-u4.txt`.
-TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
+Создание чата и членов атомарно. POST сообщения требует ключ `[A-Za-z0-9._:-]{1,128}`; PostgreSQL unique key `(user_id, chat_id, request_key)` резервируется в транзакции вместе с сообщением и ссылкой на результат. Конкурентный повтор ждёт исход первой транзакции. При одинаковом тексте возвращается тот же `201 Message`, при другом — `409`. Ключ другой пары пользователь/чат независим. Повтор проверяет Bearer и членство до чтения сохранённого результата. Записи старше 24 часов по тому же ключу удаляются перед новым запросом; массовая очистка исторических ключей пока не автоматизирована.
-## Эксперименты и отказы
+## Проверки уровня 3
-TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
+| Критерий | Команда/наблюдение | Результат |
+| --- | --- | --- |
+| Контракт | `make check`; локальный `make test BASE_URL=http://127.0.0.1:8083` через LB | PASS; публичный lab3 smoke |
+| Реальные реплики | 12 GET `/health/live` через LB, `docker compose ps -a` | Два разных `X-Instance-Id`, два healthy API-контейнера; только LB опубликован наружу |
+| Конкурентная идемпотентность | `python -m unittest discover -s student-tests` на временной PostgreSQL | 7 тестов PASS; два конкурентных вызова, одна запись `messages` |
+| Полный рестарт | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS, прежние данные и токен; времена `StartedAt` обеих API, LB и PostgreSQL изменились |
+| Отказ API-1 | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh`; `docker compose ps -a` | PASS, 2 реплики до отказа, 112 записей, 1 переходная ошибка, первая успешная запись через 2.935 с; API-1 `Exited (0)` |
+| Отказ API-2 | Восстановить API-1, сдвинуть round-robin одним GET, повторить failover | PASS, 2 реплики до отказа, 110 записей, 1 переходная ошибка, первая успешная запись через 3.985 с; API-2 `Exited (0)` |
-| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
-| --- | --- | --- | --- |
-| TODO | TODO | TODO | TODO |
+Сырые безопасные агрегаты и времена контейнеров: `evidence/failover-2026-10-02.txt`. После опытов обе реплики восстановлены. Проверки выполнялись на macOS/Colima arm64 с локальным портом 8083, поскольку 8080 занят стендом lab2. На чистом clone `.env.example` использует 8080. `make validate` локально не запускался: зависимость `openapi-spec-validator` недоступна из-за DNS к `files.pythonhosted.org`; обязательный контракт не менялся, штатный CI job `contracts.yml` выполняет validate.
-## Ограничения и следующие шаги
+## Уровень 4: эксплуатация двух реплик
-TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
+На macOS arm64 с Colima/Docker Engine 29.5.2 и Compose 5.3.1 использован Docker VM с 2 CPU и 2.05 GB RAM. HAProxy опрашивает `/health/ready` каждую 1 с, исключает backend после двух неудачных проверок и возвращает после одной удачной; API healthcheck выполняется каждые 2 с с timeout 3 с. После SIGTERM Uvicorn прекращает приём, `docker stop -t 10` и `stop_grace_period: 10s` дают процессу одинаковый срок для завершения текущих запросов. Интервал проверок даёт ориентир около 2 с на обнаружение отказа, но не жёсткую гарантию под нагрузкой; в измерениях новые успешные чтения/записи появились за 2.4–2.7 с от начала action.
+
+Последовательность двух адресных отказов, с восстановлением первой реплики перед вторым:
+
+```sh
+python3 student-tests/test_operations.py --failover-index 1 --base-url http://127.0.0.1:8080 > evidence/failover-u4-first-target.json
+docker compose --env-file .env up -d --wait --scale api=2
+python3 student-tests/test_operations.py --failover-index 2 --base-url http://127.0.0.1:8080 > evidence/failover-u4-second-target.json
+docker compose --env-file .env up -d --wait --scale api=2
+python3 student-tests/test_operations.py --rolling --base-url http://127.0.0.1:8080 > evidence/rolling-u4.json
+```
+
+В локальном опыте указан порт `8083` вместо `8080`. Каждый адресный сценарий 15 секунд непрерывно читает и пишет с одним исходным Bearer, повторяет неопределённый POST с прежним ключом, убеждается в сохранности подтверждённых сообщений и проверяет, что выбранный Docker-контейнер действительно `exited`. Скорость намеренно ограничена паузой 50 мс на каждый из двух потоков: это проверка доступности, а не измерение предельной ёмкости.
+
+| Воздействие | Успешные чтения/записи | Ошибки чтения/POST | Успешные RPS | Первое чтение/запись после action |
+| --- | ---: | ---: | ---: | ---: |
+| Остановлен `api-5` | 214 / 211 | 1 / 1 | 28.141 | 2.555 / 2.489 с |
+| Восстановлен `api-5`, остановлен `api-6` | 210 / 209 | 1 / 1 | 27.653 | 2.660 / 2.737 с |
+
+Дополнительно `tests/load.py` с 4 клиентами и 20 с во время отказа дал 6170 успешных POST (307.662 RPS), 5 ошибок, p50/p95/p99 успешных запросов 4.797/47.602/75.875 мс. Одновременно публичный `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` зафиксировал 112 успешных записей, 1 переходную ошибку, первый успех через 3.623 с и сохранил прежнюю сессию и данные. Эти два генератора имели разные чаты и независимые токены; агрегаты лежат в `evidence/failover-u4-first-load.json` и `evidence/failover-u4-first.txt`.
+
+`scripts/rolling-update.sh` поочерёдно останавливает и удаляет две исходные API-реплики, создавая и дожидаясь готовности замены после каждого шага; `--no-recreate` сохраняет другую работающую реплику. Под параллельной нагрузкой за 24.914 с прошло 7212 чтений, 4261 обычная запись и 302 пары одновременных POST с одинаковым ключом; 151 пара прошла через разные процессы. Было 2 ошибки чтения, 2 ошибки обычных POST и 2 попытки в парах с ошибкой. Все подтверждённые сообщения прочитаны обратно, дубли после неопределённых запросов не обнаружены; наблюдались четыре разных `X-Instance-Id`. После замены `api-5` и `api-6` были healthy, а публичный `make test` повторно прошёл. Метрики и ID реально заменённых контейнеров: `evidence/rolling-u4.json` и `evidence/topology-u4.txt`.
+
+Это **поочерёдная замена той же версии образа**: ни совместимость двух разных бинарных версий, ни миграция схемы во время нагрузки не проверялись. `migrate` выполняется до старта API, а скрипт замены запускает только API без повторной миграции. Для будущей смены схемы нужна отдельная совместимая миграция; данный опыт её не доказывает.
+
+## Уровень 5: цена масштабирования
+
+**Гипотеза.** При фиксированном суммарном лимите API рост с двух до четырёх процессов не даст линейного выигрыша: каждому останется только 0.25 CPU, а конкуренция за пул и планировщик увеличит задержку. Если процессам разрешить суммарно больше CPU, пропускная способность вырастет, но на этой машине рост должен замедлиться около её двух виртуальных CPU.
+
+**Профиль и воспроизведение.** Измерения выполнены 2026-10-02 на том же macOS arm64 / Colima Docker-host с 2 CPU и 2.05 GB RAM, Docker Engine 29.5.2, Compose 5.3.1, образами и Python из раздела запуска. PostgreSQL оставался при 1 CPU/512 MiB, HAProxy при 0.5 CPU/128 MiB; pool каждой реплики ограничен восемью соединениями. Использован один синтетический чат с 1000 заранее записанными сообщениями; общий счётчик БД во время матрицы оставался 30969. Само измерение только читает последнюю страницу `/messages?limit=20` через LB с действующим Bearer. Файл `secrets/scale_fixture.json` хранит локальный тестовый токен и не включён в Git. Набор сообщений и запрос не менялись между точками и контрольным сравнением.
+
+`scripts/benchmark.sh` использует Python stdlib и задаёт *открытый* график поступления GET по монотонным часам: 64 потока, очередь максимум 256, HTTP keep-alive, timeout 4 с. У каждой точки 2 с прогрева и три независимых повтора по 8 с; заданная интенсивность 250/750/1500 запросов/с. Задержка успеха считается от запланированного момента поступления до полного ответа, включая ожидание в очереди; p50/p95/p99 относятся только к успешным ответам. Неуспешные поступления учитывают как HTTP/transport ошибки, так и переполнение очереди отдельно. Во всей матрице `errors_by_type` пуст, поэтому все потери на 1500 запросах/с — **отброшенные генератором поступления**, а не ответы API 5xx. Это доля необслуженных поступлений при заданном ограничении очереди, не точный максимум сервера. Показатель CPU самого генератора не превысил 0.115 с процессорного времени на 1 с наблюдения.
+
+Для чистого clone после настройки `.env` и secrets из раздела запуска (по умолчанию порт 8080):
+
+```sh
+./scripts/benchmark.sh --base-url http://127.0.0.1:8080 --rates 250,750,1500 --duration 8 --warmup 2 --output evidence/scale/baseline.jsonl
+python3 scripts/plot_scale.py evidence/scale/baseline.jsonl
+./scripts/benchmark.sh --base-url http://127.0.0.1:8080 --modes fixed --replicas 4,2 --rates 1500 --duration 8 --warmup 2 --output evidence/scale/confirmation-4-to-2.jsonl
+docker compose --env-file .env down
+```
+
+Скрипт отказывается дописывать существующий output: для нового опыта укажите новое имя файла. Он сам поднимает Compose с временным override лимитов и меняет число реплик. В локальном опыте использован порт 8083, так как 8080 уже занят другим учебным стендом. Матрица `fixed` делила общий лимит API 1 CPU/512 MiB между 1/2/4 репликами (соответственно 1/512, 0.5/256, 0.25/128 на процесс). Серия `growth` оставляла каждому процессу 0.5 CPU/256 MiB, поэтому сумма росла от 0.5/256 до 2 CPU/1024 MiB. Docker-host при этом **не** получил дополнительных физических ресурсов; 2 CPU у четырёх API конкурировали с БД и LB в VM на 2 CPU. Конфигурация override и фактические квоты зафиксированы в каждой JSONL-строке.
+
+Полные 54 результата с индивидуальными CPU/RAM API, PostgreSQL и LB, PostgreSQL counters/размером, pool waits, длительностью, RPS, ошибками и p50/p95/p99: [`evidence/scale/baseline.jsonl`](evidence/scale/baseline.jsonl). Медианы трёх повторов всех 18 точек: [`summary.csv`](evidence/scale/summary.csv). Графики медианных RPS и p95: [`fixed-rps.svg`](evidence/scale/fixed-rps.svg), [`fixed-p95.svg`](evidence/scale/fixed-p95.svg), [`growth-rps.svg`](evidence/scale/growth-rps.svg), [`growth-p95.svg`](evidence/scale/growth-p95.svg).
+
+| Серия, API | Успешные RPS при 750 | Успешные RPS при 1500 | Необслуженные при 1500 | p50 / p95 / p99 при 1500, мс | API / DB / LB CPU при 1500 |
+| --- | ---: | ---: | ---: | ---: | ---: |
+| fixed, 1 | 747 | 780 | 42.85% | 379 / 467 / 725 | 96 / 13 / 4% |
+| fixed, 2 | 750 | 957 | 33.88% | 315 / 468 / 516 | 99 / 14 / 5% |
+| fixed, 4 | 733 | 694 | 50.82% | 414 / 748 / 851 | 87 / 14 / 4% |
+| growth, 1 | 362 | 385 | 72.09% | 797 / 1018 / 1070 | 51 / 6 / 2% |
+| growth, 2 | 750 | 938 | 35.12% | 321 / 494 / 548 | 98 / 14 / 5% |
+| growth, 4 | 747 | 1128 | 20.58% | 231 / 457 / 547 | 143 / 20 / 7% |
+
+**Насыщение и улучшение.** При суммарном 1 CPU рост заданной интенсивности с 750 до 1500 почти не увеличивает успешный RPS у одной реплики, а у четырёх уменьшает его; p95 и отброшенные поступления резко растут. API-группа использует около 87–99% одного CPU, PostgreSQL 13–14%, LB 4–5%; в этом профиле узкое место — CPU квота API и сопутствующее ожидание пула, **не** достигнутый предел CPU БД или LB. Например, `pool_wait_ms` у fixed/1 вырос с 51465 при 750 до 182073 при 1500; это суммарное время ожидания всех запросов за окно, не длительность отдельного запроса. Добавление ресурсов до growth/4 увеличило успешный RPS, но не линейно: API заняли 143% CPU при двух доступных VM CPU, DB 20%, LB 7%, а очередь всё ещё отбрасывала 20.58% поступлений.
+
+Измеримое эксплуатационное изменение — **выбрать 2 API вместо 4 при том же общем 1 CPU/512 MiB**. Отдельный контроль после возврата к неизменённому коду и тому же чату, 3 повтора сначала с 4, затем с 2 репликами по 8 с при 1500 заданных GET/с: [`confirmation-4-to-2.jsonl`](evidence/scale/confirmation-4-to-2.jsonl). В целевом чате по-прежнему 1000 сообщений; общий счётчик БД стал 30970 из-за одного независимого синтетического сообщения в проверке ACL между сериями. Медиана успешного RPS 856.033 → 952.317 (**+11.2%**), необслуженная доля 39.18% → 33.94% (−5.24 п.п.), p95 605.467 → 489.999 мс (−19.1%). Все три RPS с 2 API (940.679–972.715) выше всех трёх с 4 API (805.135–895.216). В `compose.yaml` по умолчанию уже стоят 2 API; для применения настройки после опыта с четырьмя: `docker compose --env-file .env up -d --wait --scale api=2`. Проверка публичного API после контроля — `make test BASE_URL=http://127.0.0.1:8083`, PASS. Это настройка топологии, а не ускорение SQL или исходного кода.
+
+**Пределы вывода.** Открытый график устраняет coordinated omission замкнутого `tests/load.py`, но конечные 64 потока, очередь 256 и локальный клиент влияют на измеренную границу. Отброшенные задания не дошли до HAProxy, а HTTP 5xx в серии не наблюдались. Каждый результат описывает этот GET-профиль на одном Docker-host, не запись сообщений и не универсальную ёмкость API. Кратковременные эффекты CPU throttling и фоновая нагрузка объясняют разброс между первой матрицей и контрольной серией; поэтому эффект выбора 2 API основан на отдельном сравнении с неперекрывающимися тремя повторами. Стенд имеет один LB и один PostgreSQL, не переживает потерю хоста/БД. Текущий локальный HTTP разрешён только для изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS.
diff --git a/compose.yaml b/compose.yaml
new file mode 100644
index 00000000..de5bcff1
--- /dev/null
+++ b/compose.yaml
@@ -0,0 +1,74 @@
+name: mtusi2026lab3
+
+services:
+ postgres:
+ image: postgres:17.6-bookworm
+ environment:
+ POSTGRES_DB: messenger
+ POSTGRES_USER: messenger
+ POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
+ secrets: [postgres_password]
+ volumes: [postgres_data:/var/lib/postgresql/data]
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
+ interval: 2s
+ timeout: 2s
+ retries: 20
+ cpus: 1.0
+ mem_limit: 512m
+
+ migrate:
+ build: .
+ command: ["python", "scripts/migrate.py"]
+ environment:
+ DATABASE_URL_FILE: /run/secrets/database_url
+ secrets: [database_url]
+ depends_on:
+ postgres:
+ condition: service_healthy
+ restart: "no"
+ cpus: 0.5
+ mem_limit: 128m
+
+ api:
+ build: .
+ scale: 2
+ environment:
+ APP_HOST: 0.0.0.0
+ APP_PORT: 8080
+ DATABASE_URL_FILE: /run/secrets/database_url
+ SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
+ secrets: [database_url]
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ healthcheck:
+ test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).close()"]
+ interval: 2s
+ timeout: 3s
+ retries: 20
+ start_period: 5s
+ stop_grace_period: 10s
+ cpus: 0.5
+ mem_limit: 256m
+
+ lb:
+ image: haproxy:3.0.11-alpine
+ volumes:
+ - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
+ ports:
+ - "127.0.0.1:${APP_PORT:-8080}:8080"
+ depends_on:
+ api:
+ condition: service_healthy
+ cpus: 0.5
+ mem_limit: 128m
+
+secrets:
+ postgres_password:
+ file: ./secrets/postgres_password
+ database_url:
+ file: ./secrets/database_url
+
+volumes:
+ postgres_data:
diff --git a/evidence/failover-2026-10-02.txt b/evidence/failover-2026-10-02.txt
new file mode 100644
index 00000000..85f443e3
--- /dev/null
+++ b/evidence/failover-2026-10-02.txt
@@ -0,0 +1,40 @@
+Lab 3 local Compose evidence, 2026-10-02, macOS/Colima arm64.
+All public requests used HAProxy at http://127.0.0.1:8083; API and PostgreSQL
+had no published host ports. The local .env and secrets are ignored by Git.
+
+docker compose --env-file .env config --quiet -> PASS
+docker compose --env-file .env up -d --build --wait --scale api=2 ->
+ api-1 healthy, api-2 healthy, postgres healthy, migrate exited 0, lb running.
+make check -> PASS
+make test BASE_URL=http://127.0.0.1:8083 -> PASS: lab3 public API scenario.
+12 GET /health/live through LB observed two X-Instance-Id values.
+
+Isolated PostgreSQL integration run before Compose:
+ python -m unittest discover -s student-tests -v -> 7 tests, OK.
+ Concurrent same-key requests returned identical Message; SQL count was 1.
+
+make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8083
+ API-1 stop: observed_instances_before=2, successful_writes=112,
+ failed_attempts=1, first_recovery_seconds=2.935, PASS.
+ docker compose ps -a: api-1 Exited (0), api-2 healthy, lb/postgres running.
+ Restored api-1 using docker compose up -d --wait --scale api=2.
+ A repeat selected api-1 again; it was not counted as the second-replica proof.
+ Restored api-1, sent one GET to advance round-robin, and repeated:
+ API-2 stop: observed_instances_before=2, successful_writes=110,
+ failed_attempts=1, first_recovery_seconds=3.985, PASS.
+ docker compose ps -a: api-2 Exited (0), api-1 healthy, lb/postgres running.
+ Restored api-2; both replicas healthy.
+
+Persistence hook proof, times UTC; named postgres_data volume preserved:
+ before:
+ api-1 2026-10-02T08:46:12.886332892Z
+ api-2 2026-10-02T08:48:59.563753865Z
+ lb 2026-10-02T08:42:41.675055235Z
+ postgres 2026-10-02T08:42:32.879514014Z
+ make test-persistence ACTION_SCRIPT=scripts/restart.sh
+ -> PASS: confirmed data and existing session survived.
+ after:
+ api-1 2026-10-02T08:49:48.469504932Z
+ api-2 2026-10-02T08:49:48.407940425Z
+ lb 2026-10-02T08:49:54.047109297Z
+ postgres 2026-10-02T08:49:45.282318374Z
diff --git a/evidence/failover-u4-first-load.json b/evidence/failover-u4-first-load.json
new file mode 100644
index 00000000..c1172439
--- /dev/null
+++ b/evidence/failover-u4-first-load.json
@@ -0,0 +1,17 @@
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 20.0,
+ "elapsed_seconds": 20.054,
+ "concurrency": 4,
+ "successful_requests": 6170,
+ "failed_requests": 5,
+ "error_ratio": 0.00081,
+ "successful_rps": 307.662,
+ "successful_latency_ms": {
+ "p50": 4.797,
+ "p95": 47.602,
+ "p99": 75.875
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
diff --git a/evidence/failover-u4-first-target.json b/evidence/failover-u4-first-target.json
new file mode 100644
index 00000000..bd9244fa
--- /dev/null
+++ b/evidence/failover-u4-first-target.json
@@ -0,0 +1,16 @@
+{
+ "action": "stop one selected API replica",
+ "target_container": "mtusi2026lab3-api-5",
+ "target_container_id": "48b688fc10cbbf4a6e08c459308198ce08c200437b4da35fe4214a1cd4c65a15",
+ "target_state_after_hook": "exited",
+ "elapsed_seconds": 15.103,
+ "stop_hook_seconds": 0.855,
+ "successful_reads": 214,
+ "failed_reads": 1,
+ "successful_writes": 211,
+ "failed_write_attempts": 1,
+ "successful_rps": 28.141,
+ "first_read_after_action_seconds": 2.555,
+ "first_write_after_action_seconds": 2.489,
+ "confirmed_messages_read_back": true
+}
diff --git a/evidence/failover-u4-first.txt b/evidence/failover-u4-first.txt
new file mode 100644
index 00000000..03b2f602
--- /dev/null
+++ b/evidence/failover-u4-first.txt
@@ -0,0 +1,31 @@
+Commands: tests/load.py --duration 20 --concurrency 4, make test-failover ACTION_SCRIPT=scripts/stop-one.sh
+Failover exit: 0
+python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8083"
+Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
+{"observed_instances_before": 2, "successful_writes": 112, "failed_attempts": 1, "first_recovery_seconds": 3.623}
+PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+Load exit: 0
+{
+ "lab": 3,
+ "workload": "closed-loop POST messages; no automatic retries",
+ "requested_duration_seconds": 20.0,
+ "elapsed_seconds": 20.054,
+ "concurrency": 4,
+ "successful_requests": 6170,
+ "failed_requests": 5,
+ "error_ratio": 0.00081,
+ "successful_rps": 307.662,
+ "successful_latency_ms": {
+ "p50": 4.797,
+ "p95": 47.602,
+ "p99": 75.875
+ },
+ "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
+}
+Actual containers after hook:
+NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
+mtusi2026lab3-api-5 mtusi2026lab3-api "python -m src.server" api 10 minutes ago Up About a minute (healthy) 8080/tcp
+mtusi2026lab3-api-6 mtusi2026lab3-api "python -m src.server" api 9 minutes ago Exited (0) 16 seconds ago
+mtusi2026lab3-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb About an hour ago Up 23 minutes 127.0.0.1:8083->8080/tcp
+mtusi2026lab3-migrate-1 mtusi2026lab3-migrate "python scripts/migr…" migrate 18 minutes ago Exited (0) 57 seconds ago
+mtusi2026lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 18 minutes ago Up 18 minutes (healthy) 5432/tcp
diff --git a/evidence/failover-u4-second-target.json b/evidence/failover-u4-second-target.json
new file mode 100644
index 00000000..4bc6ca3c
--- /dev/null
+++ b/evidence/failover-u4-second-target.json
@@ -0,0 +1,16 @@
+{
+ "action": "stop one selected API replica",
+ "target_container": "mtusi2026lab3-api-6",
+ "target_container_id": "bef5f29c9a9d87f8cf60e49ecfd8edff762ba156184fb413d6008f81750b4f00",
+ "target_state_after_hook": "exited",
+ "elapsed_seconds": 15.152,
+ "stop_hook_seconds": 0.864,
+ "successful_reads": 210,
+ "failed_reads": 1,
+ "successful_writes": 209,
+ "failed_write_attempts": 1,
+ "successful_rps": 27.653,
+ "first_read_after_action_seconds": 2.66,
+ "first_write_after_action_seconds": 2.737,
+ "confirmed_messages_read_back": true
+}
diff --git a/evidence/rolling-u4.json b/evidence/rolling-u4.json
new file mode 100644
index 00000000..8fe4a2d8
--- /dev/null
+++ b/evidence/rolling-u4.json
@@ -0,0 +1,18 @@
+{
+ "action": "same-version sequential API replacement",
+ "elapsed_seconds": 24.914,
+ "successful_reads": 7212,
+ "failed_reads": 2,
+ "successful_writes": 4261,
+ "failed_write_attempts": 2,
+ "successful_idempotent_pairs": 302,
+ "failed_pair_attempts": 2,
+ "cross_replica_pairs": 151,
+ "successful_rps": 484.749,
+ "observed_instance_count": 4,
+ "replacement_output": [
+ "replaced API container bcd282fe6bdad720070e51cafc27b7084f647da7920c36e9f01e7fe56768dde7; two healthy replicas now serve traffic",
+ "replaced API container 797d654ce2c993fcdc74628443ea8deb6b2e5eafe4b54940ab06ae273976c890; two healthy replicas now serve traffic"
+ ],
+ "confirmed_messages_read_back": true
+}
diff --git a/evidence/scale/baseline.jsonl b/evidence/scale/baseline.jsonl
new file mode 100644
index 00000000..f44615e9
--- /dev/null
+++ b/evidence/scale/baseline.jsonl
@@ -0,0 +1,54 @@
+{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.95, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.831, "p95": 37.915, "p99": 211.151}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.384, "docker": {"api": {"avg_cpu_percent": 39.223, "max_memory_mib": 46.79}, "postgres": {"avg_cpu_percent": 6.993, "max_memory_mib": 53.62}, "lb": {"avg_cpu_percent": 2.253, "max_memory_mib": 41.58}}, "pool": {"observed_replicas": 1, "requests_queued": 75, "wait_ms": 600, "max_size": 8}, "postgres_delta": {"commits": 4018, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 65527, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.951, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.787, "p95": 3.429, "p99": 5.526}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.355, "docker": {"api": {"avg_cpu_percent": 26.653, "max_memory_mib": 47.33}, "postgres": {"avg_cpu_percent": 5.803, "max_memory_mib": 49.23}, "lb": {"avg_cpu_percent": 1.86, "max_memory_mib": 41.47}}, "pool": {"observed_replicas": 1, "requests_queued": 0, "wait_ms": 0, "max_size": 8}, "postgres_delta": {"commits": 4016, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61557, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.971, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.772, "p95": 3.703, "p99": 5.882}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.353, "docker": {"api": {"avg_cpu_percent": 29.56, "max_memory_mib": 45.91}, "postgres": {"avg_cpu_percent": 5.72, "max_memory_mib": 49.26}, "lb": {"avg_cpu_percent": 1.75, "max_memory_mib": 41.62}}, "pool": {"observed_replicas": 1, "requests_queued": 0, "wait_ms": 0, "max_size": 8}, "postgres_delta": {"commits": 4022, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61093, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 746.986, "elapsed_seconds": 8.032, "successful_latency_ms": {"p50": 4.909, "p95": 121.369, "p99": 137.788}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.707, "docker": {"api": {"avg_cpu_percent": 88.997, "max_memory_mib": 51.91}, "postgres": {"avg_cpu_percent": 12.417, "max_memory_mib": 49.31}, "lb": {"avg_cpu_percent": 4.683, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 1, "requests_queued": 4570, "wait_ms": 55096, "max_size": 8}, "postgres_delta": {"commits": 12605, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 191767, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.783, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.679, "p95": 85.479, "p99": 125.898}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.636, "docker": {"api": {"avg_cpu_percent": 60.08, "max_memory_mib": 52.52}, "postgres": {"avg_cpu_percent": 10.823, "max_memory_mib": 49.39}, "lb": {"avg_cpu_percent": 3.9, "max_memory_mib": 41.6}}, "pool": {"observed_replicas": 1, "requests_queued": 1631, "wait_ms": 17243, "max_size": 8}, "postgres_delta": {"commits": 11926, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 181241, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 746.631, "elapsed_seconds": 8.036, "successful_latency_ms": {"p50": 3.533, "p95": 103.334, "p99": 138.015}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.695, "docker": {"api": {"avg_cpu_percent": 84.993, "max_memory_mib": 52.01}, "postgres": {"avg_cpu_percent": 12.933, "max_memory_mib": 49.41}, "lb": {"avg_cpu_percent": 4.407, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 4243, "wait_ms": 51465, "max_size": 8}, "postgres_delta": {"commits": 11896, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 178481, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6890, "failed_requests": 5110, "error_ratio": 0.425833, "successful_rps": 823.026, "elapsed_seconds": 8.372, "successful_latency_ms": {"p50": 378.91, "p95": 447.591, "p99": 470.365}, "errors_by_type": {}, "queue_dropped": 5110, "client_cpu_seconds": 0.72, "docker": {"api": {"avg_cpu_percent": 97.463, "max_memory_mib": 64.79}, "postgres": {"avg_cpu_percent": 12.573, "max_memory_mib": 49.45}, "lb": {"avg_cpu_percent": 4.42, "max_memory_mib": 41.61}}, "pool": {"observed_replicas": 1, "requests_queued": 12335, "wait_ms": 188633, "max_size": 8}, "postgres_delta": {"commits": 14324, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 215405, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6858, "failed_requests": 5142, "error_ratio": 0.4285, "successful_rps": 780.299, "elapsed_seconds": 8.789, "successful_latency_ms": {"p50": 372.063, "p95": 466.704, "p99": 1071.218}, "errors_by_type": {}, "queue_dropped": 5142, "client_cpu_seconds": 0.757, "docker": {"api": {"avg_cpu_percent": 96.24, "max_memory_mib": 53.53}, "postgres": {"avg_cpu_percent": 13.42, "max_memory_mib": 49.68}, "lb": {"avg_cpu_percent": 4.423, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 1, "requests_queued": 11597, "wait_ms": 182073, "max_size": 8}, "postgres_delta": {"commits": 14394, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 216827, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6445, "failed_requests": 5555, "error_ratio": 0.462917, "successful_rps": 766.917, "elapsed_seconds": 8.404, "successful_latency_ms": {"p50": 382.659, "p95": 600.998, "p99": 725.297}, "errors_by_type": {}, "queue_dropped": 5555, "client_cpu_seconds": 0.75, "docker": {"api": {"avg_cpu_percent": 96.137, "max_memory_mib": 53.47}, "postgres": {"avg_cpu_percent": 15.257, "max_memory_mib": 49.48}, "lb": {"avg_cpu_percent": 4.627, "max_memory_mib": 41.67}}, "pool": {"observed_replicas": 1, "requests_queued": 11447, "wait_ms": 180076, "max_size": 8}, "postgres_delta": {"commits": 13337, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 200363, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.96, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.769, "p95": 13.026, "p99": 51.133}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.342, "docker": {"api": {"avg_cpu_percent": 38.633, "max_memory_mib": 85.02}, "postgres": {"avg_cpu_percent": 5.927, "max_memory_mib": 62.68}, "lb": {"avg_cpu_percent": 1.71, "max_memory_mib": 41.82}}, "pool": {"observed_replicas": 2, "requests_queued": 5, "wait_ms": 4, "max_size": 8}, "postgres_delta": {"commits": 4198, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 71471, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.959, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.875, "p95": 33.197, "p99": 100.083}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.356, "docker": {"api": {"avg_cpu_percent": 43.583, "max_memory_mib": 112.32}, "postgres": {"avg_cpu_percent": 5.957, "max_memory_mib": 64.63}, "lb": {"avg_cpu_percent": 1.9, "max_memory_mib": 41.56}}, "pool": {"observed_replicas": 2, "requests_queued": 3, "wait_ms": 14, "max_size": 8}, "postgres_delta": {"commits": 4072, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61681, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.937, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.873, "p95": 25.949, "p99": 138.238}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.372, "docker": {"api": {"avg_cpu_percent": 42.983, "max_memory_mib": 102.69}, "postgres": {"avg_cpu_percent": 6.063, "max_memory_mib": 64.66}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 2, "requests_queued": 31, "wait_ms": 78, "max_size": 8}, "postgres_delta": {"commits": 3777, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 57121, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.804, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.755, "p95": 108.359, "p99": 175.477}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.623, "docker": {"api": {"avg_cpu_percent": 69.88, "max_memory_mib": 108.39}, "postgres": {"avg_cpu_percent": 12.08, "max_memory_mib": 64.74}, "lb": {"avg_cpu_percent": 3.88, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 2, "requests_queued": 1294, "wait_ms": 7867, "max_size": 8}, "postgres_delta": {"commits": 12232, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 184771, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.772, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.731, "p95": 100.322, "p99": 162.856}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.61, "docker": {"api": {"avg_cpu_percent": 70.053, "max_memory_mib": 96.87}, "postgres": {"avg_cpu_percent": 12.173, "max_memory_mib": 65.0}, "lb": {"avg_cpu_percent": 3.93, "max_memory_mib": 41.97}}, "pool": {"observed_replicas": 2, "requests_queued": 1118, "wait_ms": 5573, "max_size": 8}, "postgres_delta": {"commits": 12156, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183345, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.734, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 1.816, "p95": 178.504, "p99": 224.107}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.627, "docker": {"api": {"avg_cpu_percent": 68.573, "max_memory_mib": 97.41}, "postgres": {"avg_cpu_percent": 12.37, "max_memory_mib": 64.76}, "lb": {"avg_cpu_percent": 3.767, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 2, "requests_queued": 1324, "wait_ms": 11493, "max_size": 8}, "postgres_delta": {"commits": 12142, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183157, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7863, "failed_requests": 4137, "error_ratio": 0.34475, "successful_rps": 941.177, "elapsed_seconds": 8.354, "successful_latency_ms": {"p50": 319.914, "p95": 473.58, "p99": 530.577}, "errors_by_type": {}, "queue_dropped": 4137, "client_cpu_seconds": 0.785, "docker": {"api": {"avg_cpu_percent": 98.37, "max_memory_mib": 109.51}, "postgres": {"avg_cpu_percent": 14.65, "max_memory_mib": 64.77}, "lb": {"avg_cpu_percent": 5.09, "max_memory_mib": 42.0}}, "pool": {"observed_replicas": 2, "requests_queued": 6822, "wait_ms": 77320, "max_size": 8}, "postgres_delta": {"commits": 14978, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 225169, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 8293, "failed_requests": 3707, "error_ratio": 0.308917, "successful_rps": 993.827, "elapsed_seconds": 8.345, "successful_latency_ms": {"p50": 297.841, "p95": 454.544, "p99": 505.25}, "errors_by_type": {}, "queue_dropped": 3707, "client_cpu_seconds": 0.762, "docker": {"api": {"avg_cpu_percent": 100.243, "max_memory_mib": 113.26}, "postgres": {"avg_cpu_percent": 14.147, "max_memory_mib": 65.02}, "lb": {"avg_cpu_percent": 4.87, "max_memory_mib": 42.0}}, "pool": {"observed_replicas": 2, "requests_queued": 5427, "wait_ms": 49740, "max_size": 8}, "postgres_delta": {"commits": 15964, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 240127, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7934, "failed_requests": 4066, "error_ratio": 0.338833, "successful_rps": 956.631, "elapsed_seconds": 8.294, "successful_latency_ms": {"p50": 314.99, "p95": 467.748, "p99": 516.41}, "errors_by_type": {}, "queue_dropped": 4066, "client_cpu_seconds": 0.757, "docker": {"api": {"avg_cpu_percent": 98.82, "max_memory_mib": 122.91}, "postgres": {"avg_cpu_percent": 13.203, "max_memory_mib": 64.77}, "lb": {"avg_cpu_percent": 4.807, "max_memory_mib": 41.96}}, "pool": {"observed_replicas": 2, "requests_queued": 5707, "wait_ms": 72376, "max_size": 8}, "postgres_delta": {"commits": 16857, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 253657, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.958, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 3.016, "p95": 61.385, "p99": 81.718}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.313, "docker": {"api": {"avg_cpu_percent": 64.537, "max_memory_mib": 205.98}, "postgres": {"avg_cpu_percent": 5.16, "max_memory_mib": 79.56}, "lb": {"avg_cpu_percent": 1.72, "max_memory_mib": 41.74}}, "pool": {"observed_replicas": 4, "requests_queued": 4, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4223, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 76658, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.944, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.944, "p95": 59.768, "p99": 80.401}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.323, "docker": {"api": {"avg_cpu_percent": 60.467, "max_memory_mib": 205.43}, "postgres": {"avg_cpu_percent": 5.073, "max_memory_mib": 79.71}, "lb": {"avg_cpu_percent": 1.657, "max_memory_mib": 41.74}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4020, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 60455, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.971, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 3.021, "p95": 60.482, "p99": 79.776}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.316, "docker": {"api": {"avg_cpu_percent": 51.723, "max_memory_mib": 203.98}, "postgres": {"avg_cpu_percent": 5.303, "max_memory_mib": 79.81}, "lb": {"avg_cpu_percent": 1.68, "max_memory_mib": 41.55}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4101, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61639, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 744.222, "elapsed_seconds": 8.062, "successful_latency_ms": {"p50": 3.319, "p95": 308.301, "p99": 415.861}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.613, "docker": {"api": {"avg_cpu_percent": 83.097, "max_memory_mib": 205.59}, "postgres": {"avg_cpu_percent": 13.187, "max_memory_mib": 95.02}, "lb": {"avg_cpu_percent": 3.927, "max_memory_mib": 42.05}}, "pool": {"observed_replicas": 4, "requests_queued": 952, "wait_ms": 12771, "max_size": 8}, "postgres_delta": {"commits": 11710, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 181824, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 713.504, "elapsed_seconds": 8.409, "successful_latency_ms": {"p50": 126.994, "p95": 414.136, "p99": 480.195}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.582, "docker": {"api": {"avg_cpu_percent": 92.41, "max_memory_mib": 207.79}, "postgres": {"avg_cpu_percent": 12.73, "max_memory_mib": 95.05}, "lb": {"avg_cpu_percent": 3.723, "max_memory_mib": 42.23}}, "pool": {"observed_replicas": 4, "requests_queued": 2167, "wait_ms": 26550, "max_size": 8}, "postgres_delta": {"commits": 12191, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183838, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 732.997, "elapsed_seconds": 8.186, "successful_latency_ms": {"p50": 110.445, "p95": 397.681, "p99": 486.245}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.585, "docker": {"api": {"avg_cpu_percent": 93.953, "max_memory_mib": 217.25}, "postgres": {"avg_cpu_percent": 12.887, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 3.673, "max_memory_mib": 42.28}}, "pool": {"observed_replicas": 4, "requests_queued": 1901, "wait_ms": 24258, "max_size": 8}, "postgres_delta": {"commits": 11844, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 178211, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6950, "failed_requests": 5050, "error_ratio": 0.420833, "successful_rps": 811.023, "elapsed_seconds": 8.569, "successful_latency_ms": {"p50": 315.472, "p95": 633.293, "p99": 771.213}, "errors_by_type": {}, "queue_dropped": 5050, "client_cpu_seconds": 0.698, "docker": {"api": {"avg_cpu_percent": 98.617, "max_memory_mib": 216.33}, "postgres": {"avg_cpu_percent": 15.563, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 4.53, "max_memory_mib": 42.35}}, "pool": {"observed_replicas": 4, "requests_queued": 3048, "wait_ms": 32524, "max_size": 8}, "postgres_delta": {"commits": 14083, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 211553, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 5902, "failed_requests": 6098, "error_ratio": 0.508167, "successful_rps": 693.857, "elapsed_seconds": 8.506, "successful_latency_ms": {"p50": 413.811, "p95": 769.612, "p99": 855.228}, "errors_by_type": {}, "queue_dropped": 6098, "client_cpu_seconds": 0.674, "docker": {"api": {"avg_cpu_percent": 80.183, "max_memory_mib": 204.21}, "postgres": {"avg_cpu_percent": 11.61, "max_memory_mib": 95.16}, "lb": {"avg_cpu_percent": 3.993, "max_memory_mib": 42.35}}, "pool": {"observed_replicas": 4, "requests_queued": 2088, "wait_ms": 33271, "max_size": 8}, "postgres_delta": {"commits": 12393, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 186123, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 5756, "failed_requests": 6244, "error_ratio": 0.520333, "successful_rps": 681.037, "elapsed_seconds": 8.452, "successful_latency_ms": {"p50": 420.287, "p95": 747.7, "p99": 851.226}, "errors_by_type": {}, "queue_dropped": 6244, "client_cpu_seconds": 0.604, "docker": {"api": {"avg_cpu_percent": 87.06, "max_memory_mib": 194.64}, "postgres": {"avg_cpu_percent": 13.59, "max_memory_mib": 95.16}, "lb": {"avg_cpu_percent": 3.353, "max_memory_mib": 42.36}}, "pool": {"observed_replicas": 4, "requests_queued": 2203, "wait_ms": 38980, "max_size": 8}, "postgres_delta": {"commits": 11550, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 173489, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.946, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.572, "p95": 206.839, "p99": 277.387}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.316, "docker": {"api": {"avg_cpu_percent": 33.947, "max_memory_mib": 48.01}, "postgres": {"avg_cpu_percent": 6.093, "max_memory_mib": 49.68}, "lb": {"avg_cpu_percent": 1.697, "max_memory_mib": 41.68}}, "pool": {"observed_replicas": 1, "requests_queued": 463, "wait_ms": 4901, "max_size": 8}, "postgres_delta": {"commits": 4114, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 66849, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.97, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.365, "p95": 39.928, "p99": 72.726}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.262, "docker": {"api": {"avg_cpu_percent": 28.06, "max_memory_mib": 48.11}, "postgres": {"avg_cpu_percent": 5.183, "max_memory_mib": 49.45}, "lb": {"avg_cpu_percent": 1.587, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 135, "wait_ms": 196, "max_size": 8}, "postgres_delta": {"commits": 4078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61300, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.977, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.684, "p95": 59.465, "p99": 80.414}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.345, "docker": {"api": {"avg_cpu_percent": 34.943, "max_memory_mib": 47.95}, "postgres": {"avg_cpu_percent": 6.29, "max_memory_mib": 49.47}, "lb": {"avg_cpu_percent": 2.08, "max_memory_mib": 41.66}}, "pool": {"observed_replicas": 1, "requests_queued": 210, "wait_ms": 410, "max_size": 8}, "postgres_delta": {"commits": 3956, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 60059, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3164, "failed_requests": 2836, "error_ratio": 0.472667, "successful_rps": 362.33, "elapsed_seconds": 8.732, "successful_latency_ms": {"p50": 823.553, "p95": 1071.351, "p99": 1182.579}, "errors_by_type": {}, "queue_dropped": 2836, "client_cpu_seconds": 0.398, "docker": {"api": {"avg_cpu_percent": 50.06, "max_memory_mib": 64.7}, "postgres": {"avg_cpu_percent": 6.51, "max_memory_mib": 49.75}, "lb": {"avg_cpu_percent": 2.317, "max_memory_mib": 41.69}}, "pool": {"observed_replicas": 1, "requests_queued": 5298, "wait_ms": 184862, "max_size": 8}, "postgres_delta": {"commits": 5828, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 88215, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3140, "failed_requests": 2860, "error_ratio": 0.476667, "successful_rps": 347.987, "elapsed_seconds": 9.023, "successful_latency_ms": {"p50": 809.855, "p95": 1190.388, "p99": 1238.353}, "errors_by_type": {}, "queue_dropped": 2860, "client_cpu_seconds": 0.393, "docker": {"api": {"avg_cpu_percent": 50.673, "max_memory_mib": 64.82}, "postgres": {"avg_cpu_percent": 7.463, "max_memory_mib": 50.64}, "lb": {"avg_cpu_percent": 2.083, "max_memory_mib": 41.68}}, "pool": {"observed_replicas": 1, "requests_queued": 5266, "wait_ms": 191351, "max_size": 8}, "postgres_delta": {"commits": 6465, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 97643, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3410, "failed_requests": 2590, "error_ratio": 0.431667, "successful_rps": 391.052, "elapsed_seconds": 8.72, "successful_latency_ms": {"p50": 795.636, "p95": 976.194, "p99": 988.076}, "errors_by_type": {}, "queue_dropped": 2590, "client_cpu_seconds": 0.38, "docker": {"api": {"avg_cpu_percent": 50.493, "max_memory_mib": 61.84}, "postgres": {"avg_cpu_percent": 6.57, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.033, "max_memory_mib": 41.93}}, "pool": {"observed_replicas": 1, "requests_queued": 5799, "wait_ms": 185658, "max_size": 8}, "postgres_delta": {"commits": 7011, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 105675, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3475, "failed_requests": 8525, "error_ratio": 0.710417, "successful_rps": 399.011, "elapsed_seconds": 8.709, "successful_latency_ms": {"p50": 795.926, "p95": 918.481, "p99": 980.371}, "errors_by_type": {}, "queue_dropped": 8525, "client_cpu_seconds": 0.419, "docker": {"api": {"avg_cpu_percent": 50.17, "max_memory_mib": 65.2}, "postgres": {"avg_cpu_percent": 6.02, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 1.923, "max_memory_mib": 41.7}}, "pool": {"observed_replicas": 1, "requests_queued": 6018, "wait_ms": 195259, "max_size": 8}, "postgres_delta": {"commits": 6592, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 99497, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3152, "failed_requests": 8848, "error_ratio": 0.737333, "successful_rps": 354.029, "elapsed_seconds": 8.903, "successful_latency_ms": {"p50": 878.869, "p95": 1018.139, "p99": 1070.497}, "errors_by_type": {}, "queue_dropped": 8848, "client_cpu_seconds": 0.44, "docker": {"api": {"avg_cpu_percent": 51.427, "max_memory_mib": 52.96}, "postgres": {"avg_cpu_percent": 6.83, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.257, "max_memory_mib": 41.69}}, "pool": {"observed_replicas": 1, "requests_queued": 5362, "wait_ms": 190945, "max_size": 8}, "postgres_delta": {"commits": 6204, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 93635, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3349, "failed_requests": 8651, "error_ratio": 0.720917, "successful_rps": 384.74, "elapsed_seconds": 8.705, "successful_latency_ms": {"p50": 797.336, "p95": 1034.655, "p99": 1084.912}, "errors_by_type": {}, "queue_dropped": 8651, "client_cpu_seconds": 0.442, "docker": {"api": {"avg_cpu_percent": 50.96, "max_memory_mib": 64.4}, "postgres": {"avg_cpu_percent": 5.623, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.29, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 5539, "wait_ms": 185287, "max_size": 8}, "postgres_delta": {"commits": 6874, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 103703, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.98, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.87, "p95": 10.076, "p99": 50.451}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.319, "docker": {"api": {"avg_cpu_percent": 34.92, "max_memory_mib": 85.09}, "postgres": {"avg_cpu_percent": 6.023, "max_memory_mib": 58.95}, "lb": {"avg_cpu_percent": 1.613, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 6, "wait_ms": 1, "max_size": 7}, "postgres_delta": {"commits": 4102, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 67511, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.97, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.936, "p95": 55.026, "p99": 129.173}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.398, "docker": {"api": {"avg_cpu_percent": 41.043, "max_memory_mib": 96.67}, "postgres": {"avg_cpu_percent": 8.053, "max_memory_mib": 64.64}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.7}}, "pool": {"observed_replicas": 2, "requests_queued": 15, "wait_ms": 308, "max_size": 8}, "postgres_delta": {"commits": 4081, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61880, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.982, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.886, "p95": 15.37, "p99": 54.353}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.341, "docker": {"api": {"avg_cpu_percent": 38.787, "max_memory_mib": 98.52}, "postgres": {"avg_cpu_percent": 5.4, "max_memory_mib": 64.68}, "lb": {"avg_cpu_percent": 1.707, "max_memory_mib": 41.59}}, "pool": {"observed_replicas": 2, "requests_queued": 1, "wait_ms": 2, "max_size": 8}, "postgres_delta": {"commits": 4079, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61581, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 5903, "failed_requests": 97, "error_ratio": 0.016167, "successful_rps": 728.191, "elapsed_seconds": 8.106, "successful_latency_ms": {"p50": 2.148, "p95": 468.108, "p99": 599.437}, "errors_by_type": {}, "queue_dropped": 97, "client_cpu_seconds": 0.647, "docker": {"api": {"avg_cpu_percent": 79.57, "max_memory_mib": 107.99}, "postgres": {"avg_cpu_percent": 13.603, "max_memory_mib": 64.71}, "lb": {"avg_cpu_percent": 4.013, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 1869, "wait_ms": 27621, "max_size": 8}, "postgres_delta": {"commits": 11975, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 180645, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.669, "elapsed_seconds": 8.004, "successful_latency_ms": {"p50": 2.292, "p95": 144.278, "p99": 193.517}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.716, "docker": {"api": {"avg_cpu_percent": 92.29, "max_memory_mib": 108.7}, "postgres": {"avg_cpu_percent": 13.147, "max_memory_mib": 64.73}, "lb": {"avg_cpu_percent": 4.19, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 2115, "wait_ms": 15549, "max_size": 8}, "postgres_delta": {"commits": 12108, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183127, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.762, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 2.224, "p95": 167.623, "p99": 254.554}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.731, "docker": {"api": {"avg_cpu_percent": 86.027, "max_memory_mib": 98.05}, "postgres": {"avg_cpu_percent": 13.06, "max_memory_mib": 64.73}, "lb": {"avg_cpu_percent": 4.49, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 1856, "wait_ms": 14945, "max_size": 8}, "postgres_delta": {"commits": 11997, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 179629, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7786, "failed_requests": 4214, "error_ratio": 0.351167, "successful_rps": 938.324, "elapsed_seconds": 8.298, "successful_latency_ms": {"p50": 321.188, "p95": 493.5, "p99": 548.275}, "errors_by_type": {}, "queue_dropped": 4214, "client_cpu_seconds": 0.739, "docker": {"api": {"avg_cpu_percent": 98.067, "max_memory_mib": 99.7}, "postgres": {"avg_cpu_percent": 14.437, "max_memory_mib": 64.74}, "lb": {"avg_cpu_percent": 4.373, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 7974, "wait_ms": 76567, "max_size": 8}, "postgres_delta": {"commits": 15558, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 233671, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7921, "failed_requests": 4079, "error_ratio": 0.339917, "successful_rps": 953.293, "elapsed_seconds": 8.309, "successful_latency_ms": {"p50": 321.964, "p95": 454.678, "p99": 509.653}, "errors_by_type": {}, "queue_dropped": 4079, "client_cpu_seconds": 0.758, "docker": {"api": {"avg_cpu_percent": 100.757, "max_memory_mib": 100.03}, "postgres": {"avg_cpu_percent": 15.21, "max_memory_mib": 64.75}, "lb": {"avg_cpu_percent": 4.527, "max_memory_mib": 42.23}}, "pool": {"observed_replicas": 2, "requests_queued": 9150, "wait_ms": 89177, "max_size": 8}, "postgres_delta": {"commits": 16078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 241647, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7486, "failed_requests": 4514, "error_ratio": 0.376167, "successful_rps": 900.488, "elapsed_seconds": 8.313, "successful_latency_ms": {"p50": 321.365, "p95": 527.564, "p99": 574.568}, "errors_by_type": {}, "queue_dropped": 4514, "client_cpu_seconds": 0.76, "docker": {"api": {"avg_cpu_percent": 96.313, "max_memory_mib": 114.13}, "postgres": {"avg_cpu_percent": 13.923, "max_memory_mib": 64.75}, "lb": {"avg_cpu_percent": 4.857, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 7416, "wait_ms": 85412, "max_size": 8}, "postgres_delta": {"commits": 15026, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 226057, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.9, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 2.847, "p95": 17.892, "p99": 116.852}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.335, "docker": {"api": {"avg_cpu_percent": 73.133, "max_memory_mib": 165.93}, "postgres": {"avg_cpu_percent": 5.717, "max_memory_mib": 70.38}, "lb": {"avg_cpu_percent": 1.697, "max_memory_mib": 41.96}}, "pool": {"observed_replicas": 4, "requests_queued": 45, "wait_ms": 237, "max_size": 5}, "postgres_delta": {"commits": 4090, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 71220, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.719, "elapsed_seconds": 8.009, "successful_latency_ms": {"p50": 2.907, "p95": 9.687, "p99": 21.273}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.328, "docker": {"api": {"avg_cpu_percent": 49.73, "max_memory_mib": 191.97}, "postgres": {"avg_cpu_percent": 5.503, "max_memory_mib": 70.47}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 5}, "postgres_delta": {"commits": 4110, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61812, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.939, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 3.098, "p95": 17.675, "p99": 76.487}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.402, "docker": {"api": {"avg_cpu_percent": 74.683, "max_memory_mib": 177.49}, "postgres": {"avg_cpu_percent": 7.597, "max_memory_mib": 72.36}, "lb": {"avg_cpu_percent": 1.89, "max_memory_mib": 41.57}}, "pool": {"observed_replicas": 4, "requests_queued": 1, "wait_ms": 22, "max_size": 5}, "postgres_delta": {"commits": 4078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 62186, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 747.495, "elapsed_seconds": 8.027, "successful_latency_ms": {"p50": 4.456, "p95": 312.888, "p99": 369.613}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.793, "docker": {"api": {"avg_cpu_percent": 109.91, "max_memory_mib": 213.65}, "postgres": {"avg_cpu_percent": 16.54, "max_memory_mib": 95.11}, "lb": {"avg_cpu_percent": 4.963, "max_memory_mib": 42.07}}, "pool": {"observed_replicas": 4, "requests_queued": 350, "wait_ms": 3249, "max_size": 8}, "postgres_delta": {"commits": 12366, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 193680, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 748.081, "elapsed_seconds": 8.021, "successful_latency_ms": {"p50": 2.363, "p95": 25.407, "p99": 53.156}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.616, "docker": {"api": {"avg_cpu_percent": 111.363, "max_memory_mib": 207.52}, "postgres": {"avg_cpu_percent": 13.387, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 3.747, "max_memory_mib": 41.89}}, "pool": {"observed_replicas": 4, "requests_queued": 23, "wait_ms": 57, "max_size": 8}, "postgres_delta": {"commits": 12044, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 180901, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 741.745, "elapsed_seconds": 8.089, "successful_latency_ms": {"p50": 2.595, "p95": 182.257, "p99": 375.515}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.74, "docker": {"api": {"avg_cpu_percent": 117.407, "max_memory_mib": 220.91}, "postgres": {"avg_cpu_percent": 15.23, "max_memory_mib": 95.18}, "lb": {"avg_cpu_percent": 4.083, "max_memory_mib": 42.04}}, "pool": {"observed_replicas": 4, "requests_queued": 357, "wait_ms": 3544, "max_size": 8}, "postgres_delta": {"commits": 12159, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 184013, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9320, "failed_requests": 2680, "error_ratio": 0.223333, "successful_rps": 1119.586, "elapsed_seconds": 8.325, "successful_latency_ms": {"p50": 230.705, "p95": 466.032, "p99": 733.365}, "errors_by_type": {}, "queue_dropped": 2680, "client_cpu_seconds": 0.938, "docker": {"api": {"avg_cpu_percent": 142.593, "max_memory_mib": 218.29}, "postgres": {"avg_cpu_percent": 20.28, "max_memory_mib": 96.34}, "lb": {"avg_cpu_percent": 6.71, "max_memory_mib": 42.43}}, "pool": {"observed_replicas": 4, "requests_queued": 1799, "wait_ms": 17211, "max_size": 8}, "postgres_delta": {"commits": 17785, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 267049, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9886, "failed_requests": 2114, "error_ratio": 0.176167, "successful_rps": 1170.613, "elapsed_seconds": 8.445, "successful_latency_ms": {"p50": 226.066, "p95": 413.16, "p99": 489.537}, "errors_by_type": {}, "queue_dropped": 2114, "client_cpu_seconds": 0.941, "docker": {"api": {"avg_cpu_percent": 142.217, "max_memory_mib": 221.61}, "postgres": {"avg_cpu_percent": 20.977, "max_memory_mib": 95.24}, "lb": {"avg_cpu_percent": 6.13, "max_memory_mib": 42.45}}, "pool": {"observed_replicas": 4, "requests_queued": 1972, "wait_ms": 27198, "max_size": 8}, "postgres_delta": {"commits": 20148, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 302329, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
+{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9531, "failed_requests": 2469, "error_ratio": 0.20575, "successful_rps": 1127.648, "elapsed_seconds": 8.452, "successful_latency_ms": {"p50": 246.905, "p95": 456.887, "p99": 547.176}, "errors_by_type": {}, "queue_dropped": 2469, "client_cpu_seconds": 0.97, "docker": {"api": {"avg_cpu_percent": 144.297, "max_memory_mib": 235.65}, "postgres": {"avg_cpu_percent": 19.94, "max_memory_mib": 97.48}, "lb": {"avg_cpu_percent": 6.687, "max_memory_mib": 42.46}}, "pool": {"observed_replicas": 4, "requests_queued": 1540, "wait_ms": 18841, "max_size": 8}, "postgres_delta": {"commits": 19840, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 297789, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969}
diff --git a/evidence/scale/confirmation-4-to-2.jsonl b/evidence/scale/confirmation-4-to-2.jsonl
new file mode 100644
index 00000000..9bf14525
--- /dev/null
+++ b/evidence/scale/confirmation-4-to-2.jsonl
@@ -0,0 +1,6 @@
+{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7298, "failed_requests": 4702, "error_ratio": 0.391833, "successful_rps": 856.033, "elapsed_seconds": 8.525, "successful_latency_ms": {"p50": 315.394, "p95": 605.467, "p99": 723.579}, "errors_by_type": {}, "queue_dropped": 4702, "client_cpu_seconds": 0.699, "docker": {"api": {"avg_cpu_percent": 101.373, "max_memory_mib": 230.97}, "postgres": {"avg_cpu_percent": 10.917, "max_memory_mib": 96.28}, "lb": {"avg_cpu_percent": 4.033, "max_memory_mib": 42.7}}, "pool": {"observed_replicas": 4, "requests_queued": 2961, "wait_ms": 49012, "max_size": 8}, "postgres_delta": {"commits": 14598, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 211976, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970}
+{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7569, "failed_requests": 4431, "error_ratio": 0.36925, "successful_rps": 895.216, "elapsed_seconds": 8.455, "successful_latency_ms": {"p50": 302.528, "p95": 551.032, "p99": 821.366}, "errors_by_type": {}, "queue_dropped": 4431, "client_cpu_seconds": 0.705, "docker": {"api": {"avg_cpu_percent": 99.517, "max_memory_mib": 205.15}, "postgres": {"avg_cpu_percent": 15.823, "max_memory_mib": 96.31}, "lb": {"avg_cpu_percent": 4.787, "max_memory_mib": 42.63}}, "pool": {"observed_replicas": 4, "requests_queued": 4193, "wait_ms": 33869, "max_size": 8}, "postgres_delta": {"commits": 14895, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 216728, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970}
+{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6745, "failed_requests": 5255, "error_ratio": 0.437917, "successful_rps": 805.135, "elapsed_seconds": 8.377, "successful_latency_ms": {"p50": 335.741, "p95": 691.364, "p99": 835.845}, "errors_by_type": {}, "queue_dropped": 5255, "client_cpu_seconds": 0.704, "docker": {"api": {"avg_cpu_percent": 99.893, "max_memory_mib": 202.03}, "postgres": {"avg_cpu_percent": 15.907, "max_memory_mib": 96.36}, "lb": {"avg_cpu_percent": 4.783, "max_memory_mib": 42.64}}, "pool": {"observed_replicas": 4, "requests_queued": 3062, "wait_ms": 48677, "max_size": 8}, "postgres_delta": {"commits": 13421, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 194846, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970}
+{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7927, "failed_requests": 4073, "error_ratio": 0.339417, "successful_rps": 952.317, "elapsed_seconds": 8.324, "successful_latency_ms": {"p50": 316.048, "p95": 495.87, "p99": 540.128}, "errors_by_type": {}, "queue_dropped": 4073, "client_cpu_seconds": 0.749, "docker": {"api": {"avg_cpu_percent": 96.897, "max_memory_mib": 98.05}, "postgres": {"avg_cpu_percent": 13.917, "max_memory_mib": 66.3}, "lb": {"avg_cpu_percent": 3.933, "max_memory_mib": 42.29}}, "pool": {"observed_replicas": 2, "requests_queued": 6441, "wait_ms": 59041, "max_size": 8}, "postgres_delta": {"commits": 15327, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 222787, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970}
+{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 8055, "failed_requests": 3945, "error_ratio": 0.32875, "successful_rps": 972.715, "elapsed_seconds": 8.281, "successful_latency_ms": {"p50": 309.538, "p95": 465.748, "p99": 575.029}, "errors_by_type": {}, "queue_dropped": 3945, "client_cpu_seconds": 0.738, "docker": {"api": {"avg_cpu_percent": 94.727, "max_memory_mib": 100.38}, "postgres": {"avg_cpu_percent": 15.02, "max_memory_mib": 65.29}, "lb": {"avg_cpu_percent": 4.33, "max_memory_mib": 42.54}}, "pool": {"observed_replicas": 2, "requests_queued": 5546, "wait_ms": 59665, "max_size": 8}, "postgres_delta": {"commits": 17391, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 253125, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970}
+{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7862, "failed_requests": 4138, "error_ratio": 0.344833, "successful_rps": 940.679, "elapsed_seconds": 8.358, "successful_latency_ms": {"p50": 319.584, "p95": 489.999, "p99": 595.768}, "errors_by_type": {}, "queue_dropped": 4138, "client_cpu_seconds": 0.725, "docker": {"api": {"avg_cpu_percent": 99.007, "max_memory_mib": 113.54}, "postgres": {"avg_cpu_percent": 15.377, "max_memory_mib": 68.17}, "lb": {"avg_cpu_percent": 4.427, "max_memory_mib": 42.29}}, "pool": {"observed_replicas": 2, "requests_queued": 7232, "wait_ms": 85079, "max_size": 8}, "postgres_delta": {"commits": 15296, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 222495, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970}
diff --git a/evidence/scale/fixed-p95.svg b/evidence/scale/fixed-p95.svg
new file mode 100644
index 00000000..2b887a48
--- /dev/null
+++ b/evidence/scale/fixed-p95.svg
@@ -0,0 +1,33 @@
+
diff --git a/evidence/scale/fixed-rps.svg b/evidence/scale/fixed-rps.svg
new file mode 100644
index 00000000..98ab80b2
--- /dev/null
+++ b/evidence/scale/fixed-rps.svg
@@ -0,0 +1,33 @@
+
diff --git a/evidence/scale/growth-p95.svg b/evidence/scale/growth-p95.svg
new file mode 100644
index 00000000..0d9e327a
--- /dev/null
+++ b/evidence/scale/growth-p95.svg
@@ -0,0 +1,33 @@
+
diff --git a/evidence/scale/growth-rps.svg b/evidence/scale/growth-rps.svg
new file mode 100644
index 00000000..153d0b28
--- /dev/null
+++ b/evidence/scale/growth-rps.svg
@@ -0,0 +1,33 @@
+
diff --git a/evidence/scale/summary.csv b/evidence/scale/summary.csv
new file mode 100644
index 00000000..9209371f
--- /dev/null
+++ b/evidence/scale/summary.csv
@@ -0,0 +1,19 @@
+mode,replicas,rate,successful_rps,error_ratio,p50_ms,p95_ms,p99_ms,api_cpu_percent,postgres_cpu_percent,lb_cpu_percent,api_memory_mib,postgres_memory_mib,lb_memory_mib,pool_wait_ms,queue_dropped
+fixed,1,250,249.951,0.0,2.787,3.703,5.882,29.56,5.803,1.86,46.79,49.26,41.58,0,0
+fixed,1,750,746.986,0.0,3.533,103.334,137.788,84.993,12.417,4.407,52.01,49.39,41.64,51465,0
+fixed,1,1500,780.299,0.4285,378.91,466.704,725.297,96.24,13.42,4.423,53.53,49.48,41.64,182073,5142
+fixed,2,250,249.959,0.0,2.873,25.949,100.083,42.983,5.957,1.753,102.69,64.63,41.64,14,0
+fixed,2,750,749.772,0.0,1.755,108.359,175.477,69.88,12.173,3.88,97.41,64.76,41.91,7867,0
+fixed,2,1500,956.631,0.338833,314.99,467.748,516.41,98.82,14.147,4.87,113.26,64.77,42.0,72376,4066
+fixed,4,250,249.958,0.0,3.016,60.482,80.401,60.467,5.16,1.68,205.43,79.71,41.74,0,0
+fixed,4,750,732.997,0.0,110.445,397.681,480.195,92.41,12.887,3.723,207.79,95.05,42.23,24258,0
+fixed,4,1500,693.857,0.508167,413.811,747.7,851.226,87.06,13.59,3.993,204.21,95.16,42.35,33271,6098
+growth,1,250,249.97,0.0,2.572,59.465,80.414,33.947,6.093,1.697,48.01,49.47,41.68,410,0
+growth,1,750,362.33,0.472667,809.855,1071.351,1182.579,50.493,6.57,2.083,64.7,49.75,41.69,185658,2836
+growth,1,1500,384.74,0.720917,797.336,1018.139,1070.497,50.96,6.02,2.257,64.4,49.52,41.7,190945,8651
+growth,2,250,249.98,0.0,2.886,15.37,54.353,38.787,6.023,1.707,96.67,64.64,41.7,2,0
+growth,2,750,749.669,0.0,2.224,167.623,254.554,86.027,13.147,4.19,107.99,64.73,41.94,15549,0
+growth,2,1500,938.324,0.351167,321.365,493.5,548.275,98.067,14.437,4.527,100.03,64.75,41.98,85412,4214
+growth,4,250,249.9,0.0,2.907,17.675,76.487,73.133,5.717,1.753,177.49,70.47,41.91,22,0
+growth,4,750,747.495,0.0,2.595,182.257,369.613,111.363,15.23,4.083,213.65,95.12,42.04,3249,0
+growth,4,1500,1127.648,0.20575,230.705,456.887,547.176,142.593,20.28,6.687,221.61,96.34,42.45,18841,2469
diff --git a/evidence/topology-u4.txt b/evidence/topology-u4.txt
new file mode 100644
index 00000000..a3eab65d
--- /dev/null
+++ b/evidence/topology-u4.txt
@@ -0,0 +1,17 @@
+Environment: Darwin arm64
+Docker Engine: 29.5.2
+Compose: 5.3.1
+Docker VM CPUs/MemTotal: 2 / 2054832128 bytes
+Actual container limits (NanoCpus, Memory bytes):
+/mtusi2026lab3-api-5 500000000 268435456 running
+/mtusi2026lab3-api-6 500000000 268435456 running
+/mtusi2026lab3-lb-1 500000000 134217728 running
+/mtusi2026lab3-postgres-1 1000000000 536870912 running
+/mtusi2026lab3-migrate-1 500000000 134217728 exited
+Network/port and health observation:
+NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
+mtusi2026lab3-api-5 mtusi2026lab3-api "python -m src.server" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
+mtusi2026lab3-api-6 mtusi2026lab3-api "python -m src.server" api 2 minutes ago Up 2 minutes (healthy) 8080/tcp
+mtusi2026lab3-lb-1 haproxy:3.0.11-alpine "docker-entrypoint.s…" lb About an hour ago Up 15 minutes 127.0.0.1:8083->8080/tcp
+mtusi2026lab3-migrate-1 mtusi2026lab3-migrate "python scripts/migr…" migrate 10 minutes ago Exited (0) 3 minutes ago
+mtusi2026lab3-postgres-1 postgres:17.6-bookworm "docker-entrypoint.s…" postgres 10 minutes ago Up 10 minutes (healthy) 5432/tcp
diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg
new file mode 100644
index 00000000..40952ab9
--- /dev/null
+++ b/infra/haproxy.cfg
@@ -0,0 +1,26 @@
+global
+ log stdout format raw local0
+
+defaults
+ mode http
+ log global
+ option httplog
+ timeout connect 2s
+ timeout client 15s
+ timeout server 15s
+
+resolvers docker
+ nameserver docker 127.0.0.11:53
+ resolve_retries 2
+ timeout retry 1s
+ hold valid 1s
+
+frontend public
+ bind *:8080
+ default_backend api
+
+backend api
+ balance roundrobin
+ option httpchk GET /health/ready
+ http-check expect status 200
+ server-template api 1-4 api:8080 check inter 1s fall 2 rise 1 resolvers docker init-addr none
diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql
new file mode 100644
index 00000000..dbfb0065
--- /dev/null
+++ b/migrations/001_initial.sql
@@ -0,0 +1,36 @@
+CREATE TABLE users (
+ id uuid PRIMARY KEY,
+ username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
+ display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
+ password_hash text NOT NULL,
+ created_at timestamptz NOT NULL DEFAULT clock_timestamp()
+);
+
+CREATE TABLE chats (
+ id uuid PRIMARY KEY,
+ title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
+ created_at timestamptz NOT NULL DEFAULT clock_timestamp()
+);
+
+CREATE INDEX chats_history_idx ON chats (created_at, id);
+
+CREATE TABLE memberships (
+ chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
+ user_id uuid NOT NULL REFERENCES users(id),
+ position smallint NOT NULL CHECK (position BETWEEN 0 AND 99),
+ PRIMARY KEY (chat_id, user_id),
+ UNIQUE (chat_id, position)
+);
+
+CREATE INDEX memberships_user_idx ON memberships (user_id, chat_id);
+
+CREATE TABLE messages (
+ id uuid PRIMARY KEY,
+ chat_id uuid NOT NULL,
+ sender_id uuid NOT NULL,
+ text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+ FOREIGN KEY (chat_id, sender_id) REFERENCES memberships(chat_id, user_id)
+);
+
+CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id);
diff --git a/migrations/002_sessions.sql b/migrations/002_sessions.sql
new file mode 100644
index 00000000..5d2f364a
--- /dev/null
+++ b/migrations/002_sessions.sql
@@ -0,0 +1,7 @@
+CREATE TABLE sessions (
+ token_hash bytea PRIMARY KEY,
+ user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
+ expires_at timestamptz NOT NULL
+);
+
+CREATE INDEX sessions_user_idx ON sessions (user_id);
diff --git a/migrations/003_message_replays.sql b/migrations/003_message_replays.sql
new file mode 100644
index 00000000..e5a86270
--- /dev/null
+++ b/migrations/003_message_replays.sql
@@ -0,0 +1,11 @@
+CREATE TABLE message_replays (
+ user_id uuid NOT NULL REFERENCES users(id),
+ chat_id uuid NOT NULL REFERENCES chats(id),
+ request_key text NOT NULL,
+ request_hash bytea NOT NULL,
+ message_id uuid REFERENCES messages(id),
+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+ PRIMARY KEY (user_id, chat_id, request_key)
+);
+
+CREATE INDEX message_replays_expiry_idx ON message_replays (created_at);
diff --git a/migrations/009_message_text_utf8.sql b/migrations/009_message_text_utf8.sql
new file mode 100644
index 00000000..86196ee6
--- /dev/null
+++ b/migrations/009_message_text_utf8.sql
@@ -0,0 +1,6 @@
+-- PostgreSQL text cannot contain U+0000; bytea preserves the complete UTF-8 payload.
+-- API validation enforces the 4000-character limit (up to 16000 UTF-8 bytes).
+ALTER TABLE messages DROP CONSTRAINT messages_text_check;
+ALTER TABLE messages ALTER COLUMN text TYPE bytea USING convert_to(text, 'UTF8');
+ALTER TABLE messages ADD CONSTRAINT messages_text_check
+ CHECK (octet_length(text) <= 16000);
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 00000000..58563cf5
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,6 @@
+fastapi==0.118.3
+pydantic==2.12.4
+uvicorn==0.38.0
+psycopg[binary,pool]==3.2.12
+psycopg-pool==3.3.3
+argon2-cffi==25.1.0
diff --git a/scripts/benchmark.py b/scripts/benchmark.py
new file mode 100644
index 00000000..cdcdf2eb
--- /dev/null
+++ b/scripts/benchmark.py
@@ -0,0 +1,322 @@
+#!/usr/bin/env python3
+"""Reproducible, rate-controlled GET benchmark for the lab3 scaling track."""
+
+import argparse
+from collections import defaultdict
+import http.client
+import json
+import math
+import os
+from pathlib import Path
+from queue import Full, Queue
+import statistics
+import subprocess
+import sys
+import tempfile
+import threading
+import time
+from urllib.parse import urlsplit
+from urllib.request import urlopen
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tests"))
+from client import Client # noqa: E402 - the repository's public bootstrap client
+
+
+def topology(mode, replicas):
+ if mode not in ("fixed", "growth") or replicas not in (1, 2, 4):
+ raise ValueError("mode must be fixed/growth and replicas must be 1/2/4")
+ return replicas, (1 / replicas if mode == "fixed" else 0.5), (512 // replicas if mode == "fixed" else 256)
+
+
+def validate_matrix(rows):
+ if not rows:
+ raise ValueError("no benchmark rows")
+ rates = {row["rate"] for row in rows}
+ if len(rates) < 3:
+ raise ValueError("need at least three offered rates")
+ profiles = {(row["duration"], row["warmup"], row["dataset_messages"]) for row in rows}
+ if len(profiles) != 1:
+ raise ValueError("workload profile changed across measurements")
+ for mode in ("fixed", "growth"):
+ for replicas in (1, 2, 4):
+ _, cpu, memory = topology(mode, replicas)
+ for rate in rates:
+ repeats = [row["repeat"] for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)]
+ if sorted(repeats) != [1, 2, 3]:
+ raise ValueError(f"need three repeats for {mode}/{replicas}/{rate}")
+ for row in rows:
+ if (row["mode"], row["replicas"]) == (mode, replicas) and (row["api_cpu_each"], row["api_memory_mib_each"]) != (cpu, memory):
+ raise ValueError("incorrect API resource quota")
+
+
+def percentile(values, fraction):
+ if not values:
+ return None
+ ordered = sorted(values)
+ return round(ordered[math.ceil(fraction * len(ordered)) - 1], 3)
+
+
+def command(*args):
+ return subprocess.run(args, cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip()
+
+
+def compose(*args, override=None):
+ prefix = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"]
+ if override:
+ prefix += ["-f", str(override)]
+ return command(*prefix, *args)
+
+
+def pool_snapshot(base_url, replicas):
+ found = {}
+ for _ in range(8 * replicas):
+ with urlopen(base_url + "/health/ready", timeout=4) as response:
+ headers = response.headers
+ found[headers["X-Instance-Id"]] = {
+ "requests": int(headers["X-Pool-Requests"]),
+ "queued": int(headers["X-Pool-Queued"]),
+ "wait_ms": int(headers["X-Pool-Wait-Ms"]),
+ "size": int(headers["X-Pool-Size"]),
+ "available": int(headers["X-Pool-Available"]),
+ }
+ if len(found) == replicas:
+ break
+ return found
+
+
+def db_snapshot(override):
+ sql = ("SELECT pg_database_size(current_database()),xact_commit,xact_rollback,"
+ "blks_read,blks_hit,temp_files,deadlocks,(SELECT count(*) FROM messages) "
+ "FROM pg_stat_database WHERE datname=current_database()")
+ values = command("docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", str(override),
+ "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql)
+ return dict(zip(("database_bytes", "commits", "rollbacks", "blocks_read", "blocks_hit", "temp_files", "deadlocks", "messages"),
+ map(int, values.split("|"))))
+
+
+def docker_ids(override):
+ ids = {}
+ for service in ("api", "postgres", "lb"):
+ ids[service] = compose("ps", "-q", service, override=override).splitlines()
+ return ids
+
+
+def stats_sample(ids):
+ all_ids = [id for service_ids in ids.values() for id in service_ids]
+ if not all_ids:
+ return {}
+ output = command("docker", "stats", "--no-stream", "--format", "{{json .}}", *all_ids)
+ by_id = {}
+ for line in output.splitlines():
+ item = json.loads(line)
+ by_id[item["ID"]] = item
+ result = {}
+ for service, service_ids in ids.items():
+ cpu = []
+ memory = []
+ for id in service_ids:
+ item = by_id.get(id[:12]) or by_id.get(id)
+ if not item:
+ raise ValueError("Docker stats omitted a benchmark container")
+ cpu.append(float(item["CPUPerc"].rstrip("%")))
+ amount = item["MemUsage"].split("/")[0].strip()
+ value, unit = float("".join(c for c in amount if c.isdigit() or c == ".")), "".join(c for c in amount if c.isalpha())
+ memory.append(round(value * {"B": 1 / 2**20, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024,
+ "kB": 1000 / 2**20, "MB": 10**6 / 2**20, "GB": 10**9 / 2**20}[unit], 3))
+ result[service] = {"cpu_percent": round(sum(cpu), 3), "memory_mib": round(sum(memory), 3)}
+ return result
+
+
+def open_loop(base_url, path, token, rate, duration, workers=64):
+ """Schedule arrivals by clock; latency includes queue wait and HTTP time."""
+ address = urlsplit(base_url)
+ if address.scheme != "http":
+ raise ValueError("this local lab3 benchmark expects HTTP")
+ jobs = Queue(maxsize=256)
+ latencies = []
+ errors = defaultdict(int)
+ lock = threading.Lock()
+ starts = time.monotonic()
+ cpu_start = time.process_time()
+ count = math.ceil(rate * duration)
+
+ def worker():
+ connection = None
+ while True:
+ due = jobs.get()
+ if due is None:
+ jobs.task_done()
+ break
+ try:
+ if connection is None:
+ connection = http.client.HTTPConnection(address.hostname, address.port or 80, timeout=4)
+ connection.request("GET", path, headers={"Authorization": "Bearer " + token, "Accept": "application/json"})
+ response = connection.getresponse()
+ payload = response.read()
+ if response.status != 200:
+ with lock:
+ errors[f"http_{response.status}"] += 1
+ elif len(json.loads(payload)["items"]) != 20:
+ with lock:
+ errors["wrong_page"] += 1
+ else:
+ with lock:
+ latencies.append((time.monotonic() - due) * 1000)
+ except (OSError, TimeoutError, ValueError, KeyError, http.client.HTTPException):
+ with lock:
+ errors["transport_or_response"] += 1
+ if connection:
+ connection.close()
+ connection = None
+ finally:
+ jobs.task_done()
+ if connection:
+ connection.close()
+
+ threads = [threading.Thread(target=worker, daemon=True) for _ in range(workers)]
+ for thread in threads:
+ thread.start()
+ dropped = 0
+ for index in range(count):
+ due = starts + index / rate
+ delay = due - time.monotonic()
+ if delay > 0:
+ time.sleep(delay)
+ try:
+ jobs.put_nowait(due)
+ except Full:
+ dropped += 1
+ jobs.join()
+ for _ in threads:
+ jobs.put(None)
+ for thread in threads:
+ thread.join()
+ elapsed = time.monotonic() - starts
+ error_count = sum(errors.values()) + dropped
+ return {
+ "offered_rate": rate, "scheduled_requests": count, "successful_requests": len(latencies),
+ "failed_requests": error_count, "error_ratio": round(error_count / count, 6),
+ "successful_rps": round(len(latencies) / elapsed, 3), "elapsed_seconds": round(elapsed, 3),
+ "successful_latency_ms": {f"p{p}": percentile(latencies, p / 100) for p in (50, 95, 99)},
+ "errors_by_type": dict(errors), "queue_dropped": dropped,
+ "client_cpu_seconds": round(time.process_time() - cpu_start, 3),
+ }
+
+
+def fixture(base_url, size, fixture_file):
+ if fixture_file.exists():
+ saved = json.loads(fixture_file.read_text())
+ if saved["base_url"] != base_url or saved["dataset_messages"] != size:
+ raise ValueError("saved fixture URL or size differs from requested profile")
+ Client(base_url).call("GET", saved["path"], actor={"token": saved["token"]})
+ return saved["token"], saved["path"]
+ client = Client(base_url)
+ alice, _, _, chat = client.bootstrap()
+ for index in range(size):
+ client.message(alice, chat, f"synthetic-scale-{index}")
+ path = "/api/v1/chats/" + chat["id"] + "/messages?limit=20"
+ fixture_file.parent.mkdir(parents=True, exist_ok=True)
+ with os.fdopen(os.open(fixture_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as file:
+ file.write(json.dumps({"base_url": base_url, "dataset_messages": size,
+ "token": alice["token"], "path": path}) + "\n")
+ return alice["token"], path
+
+
+def run(args):
+ full_matrix = args.modes == ["fixed", "growth"] and args.replicas == [1, 2, 4]
+ if args.repeats != 3 or (full_matrix and len(args.rates) < 3) or args.dataset_messages < 20:
+ raise ValueError("require 3 repeats, at least 3 rates for a full matrix, and at least 20 fixture messages")
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ if args.output.exists():
+ raise ValueError(f"output already exists: {args.output}")
+ token = path = None
+ rows = []
+ with tempfile.TemporaryDirectory() as temporary:
+ override = Path(temporary) / "scale.yaml"
+ for mode in args.modes:
+ for replicas in args.replicas:
+ _, cpu, memory = topology(mode, replicas)
+ override.write_text("services:\n api:\n cpus: " + str(cpu) + "\n mem_limit: " + str(memory) + "m\n")
+ compose("up", "-d", "--build", "--wait", "--scale", f"api={replicas}", override=override)
+ if token is None:
+ token, path = fixture(args.base_url, args.dataset_messages, args.fixture)
+ ids = docker_ids(override)
+ for rate in args.rates:
+ for repeat in range(1, args.repeats + 1):
+ open_loop(args.base_url, path, token, rate, args.warmup)
+ pool_before = pool_snapshot(args.base_url, replicas)
+ db_before = db_snapshot(override)
+ samples = []
+ sampling_errors = []
+ stop = threading.Event()
+
+ def sample():
+ while not stop.is_set():
+ try:
+ samples.append(stats_sample(ids))
+ except (subprocess.CalledProcessError, ValueError, KeyError) as error:
+ sampling_errors.append(type(error).__name__)
+ return
+ stop.wait(1)
+
+ sampler = threading.Thread(target=sample, daemon=True)
+ sampler.start()
+ result = open_loop(args.base_url, path, token, rate, args.duration)
+ stop.set()
+ sampler.join()
+ if sampling_errors or not samples:
+ raise RuntimeError(f"Docker CPU/RAM sampler failed: {sampling_errors}")
+ db_after = db_snapshot(override)
+ if db_after["messages"] != db_before["messages"]:
+ raise RuntimeError("message dataset changed during read-only benchmark")
+ pool_after = pool_snapshot(args.base_url, replicas)
+ pool_delta = {
+ "observed_replicas": len(set(pool_before) & set(pool_after)),
+ "requests_queued": sum(pool_after[i]["queued"] - pool_before[i]["queued"] for i in pool_before.keys() & pool_after.keys()),
+ "wait_ms": sum(pool_after[i]["wait_ms"] - pool_before[i]["wait_ms"] for i in pool_before.keys() & pool_after.keys()),
+ "max_size": max((v["size"] for v in pool_after.values()), default=0),
+ }
+ services = {name: {
+ "avg_cpu_percent": round(statistics.mean(s[name]["cpu_percent"] for s in samples if name in s), 3),
+ "max_memory_mib": round(max(s[name]["memory_mib"] for s in samples if name in s), 3),
+ } for name in ("api", "postgres", "lb") if any(name in s for s in samples)}
+ row = {
+ "mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat,
+ "duration": args.duration, "warmup": args.warmup, "dataset_messages": args.dataset_messages,
+ "api_cpu_each": cpu, "api_memory_mib_each": memory, "pool_max_size": 8,
+ **result, "docker": services, "pool": pool_delta,
+ "postgres_delta": {key: db_after[key] - db_before[key] for key in db_before if key != "database_bytes"},
+ "database_bytes": db_after["database_bytes"], "database_messages": db_after["messages"],
+ }
+ rows.append(row)
+ with args.output.open("a") as output:
+ output.write(json.dumps(row, ensure_ascii=False) + "\n")
+ print(f"{mode} replicas={replicas} rate={rate} repeat={repeat} rps={result['successful_rps']} errors={result['error_ratio']}", flush=True)
+ if full_matrix:
+ validate_matrix(rows)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--base-url", default="http://127.0.0.1:8080")
+ parser.add_argument("--rates", type=lambda value: [int(x) for x in value.split(",")], default=[100, 250, 500])
+ parser.add_argument("--modes", type=lambda value: value.split(","), default=["fixed", "growth"])
+ parser.add_argument("--replicas", type=lambda value: [int(x) for x in value.split(",")], default=[1, 2, 4])
+ parser.add_argument("--duration", type=int, default=8)
+ parser.add_argument("--warmup", type=int, default=2)
+ parser.add_argument("--repeats", type=int, default=3)
+ parser.add_argument("--dataset-messages", type=int, default=1000)
+ parser.add_argument("--fixture", type=Path, default=ROOT / "secrets/scale_fixture.json")
+ parser.add_argument("--output", type=Path, default=ROOT / "evidence/scale/baseline.jsonl")
+ args = parser.parse_args()
+ if (not args.rates or min(args.rates) <= 0 or not args.modes or not args.replicas or
+ any(mode not in ("fixed", "growth") for mode in args.modes) or
+ any(replica not in (1, 2, 4) for replica in args.replicas) or
+ args.duration <= 0 or args.warmup <= 0):
+ raise ValueError("rates and durations must be positive")
+ run(args)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh
new file mode 100755
index 00000000..d9c755cb
--- /dev/null
+++ b/scripts/benchmark.sh
@@ -0,0 +1,4 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+exec python3 scripts/benchmark.py "$@"
diff --git a/scripts/migrate.py b/scripts/migrate.py
new file mode 100644
index 00000000..e346d4f1
--- /dev/null
+++ b/scripts/migrate.py
@@ -0,0 +1,30 @@
+"""Apply numbered SQL migrations once, atomically and under a DB lock."""
+
+from pathlib import Path
+import sys
+
+import psycopg
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+from src.db import database_url
+
+
+def main() -> None:
+ migration_dir = Path(__file__).resolve().parents[1] / "migrations"
+ with psycopg.connect(database_url(), connect_timeout=5, options="-c statement_timeout=30000") as conn:
+ with conn.transaction():
+ conn.execute("SELECT pg_advisory_xact_lock(2026, 2)")
+ conn.execute(
+ "CREATE TABLE IF NOT EXISTS schema_migrations ("
+ "version integer PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT clock_timestamp())"
+ )
+ for path in sorted(migration_dir.glob("[0-9][0-9][0-9]_*.sql")):
+ version = int(path.name.split("_", 1)[0])
+ if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone():
+ continue
+ conn.execute(path.read_text(encoding="utf-8"))
+ conn.execute("INSERT INTO schema_migrations (version) VALUES (%s)", (version,))
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/plot_scale.py b/scripts/plot_scale.py
new file mode 100644
index 00000000..87e58612
--- /dev/null
+++ b/scripts/plot_scale.py
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Render the three-repeat scale matrix as small, dependency-free SVG plots."""
+
+import argparse
+import csv
+import json
+from pathlib import Path
+import statistics
+
+from benchmark import validate_matrix
+
+
+COLORS = {1: "#2563eb", 2: "#16a34a", 4: "#dc2626"}
+
+
+def svg(mode, metric, rows, output):
+ width, height = 720, 420
+ left, right, top, bottom = 75, 25, 35, 65
+ plot_w, plot_h = width - left - right, height - top - bottom
+ rates = sorted({row["rate"] for row in rows})
+ max_y = max((row[metric] or 0 for row in rows), default=0) * 1.1 or 1
+ x = lambda rate: left + (rate - rates[0]) / (rates[-1] - rates[0]) * plot_w
+ y = lambda value: top + plot_h * (1 - value / max_y)
+ label = "Успешные запросы/с" if metric == "successful_rps" else "p95, мс"
+ parts = [f'']
+ output.write_text("\n".join(parts) + "\n", encoding="utf-8")
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("input", type=Path)
+ args = parser.parse_args()
+ rows = [json.loads(line) for line in args.input.read_text().splitlines() if line.strip()]
+ validate_matrix(rows)
+ summary = []
+ for mode in ("fixed", "growth"):
+ for replicas in (1, 2, 4):
+ for rate in sorted({row["rate"] for row in rows}):
+ group = [row for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)]
+ def latency(p):
+ values = [row["successful_latency_ms"][p] for row in group if row["successful_latency_ms"][p] is not None]
+ return round(statistics.median(values), 3) if values else None
+
+ summary.append({"mode": mode, "replicas": replicas, "rate": rate,
+ "successful_rps": round(statistics.median(row["successful_rps"] for row in group), 3),
+ "error_ratio": round(statistics.median(row["error_ratio"] for row in group), 6),
+ "p50_ms": latency("p50"), "p95_ms": latency("p95"), "p99_ms": latency("p99"),
+ "api_cpu_percent": round(statistics.median(row["docker"]["api"]["avg_cpu_percent"] for row in group), 3),
+ "postgres_cpu_percent": round(statistics.median(row["docker"]["postgres"]["avg_cpu_percent"] for row in group), 3),
+ "lb_cpu_percent": round(statistics.median(row["docker"]["lb"]["avg_cpu_percent"] for row in group), 3),
+ "api_memory_mib": round(statistics.median(row["docker"]["api"]["max_memory_mib"] for row in group), 3),
+ "postgres_memory_mib": round(statistics.median(row["docker"]["postgres"]["max_memory_mib"] for row in group), 3),
+ "lb_memory_mib": round(statistics.median(row["docker"]["lb"]["max_memory_mib"] for row in group), 3),
+ "pool_wait_ms": round(statistics.median(row["pool"]["wait_ms"] for row in group), 3),
+ "queue_dropped": statistics.median(row["queue_dropped"] for row in group)})
+ with (args.input.parent / "summary.csv").open("w", newline="") as file:
+ writer = csv.DictWriter(file, fieldnames=summary[0].keys(), lineterminator="\n")
+ writer.writeheader()
+ writer.writerows(summary)
+ for mode in ("fixed", "growth"):
+ group = [row for row in summary if row["mode"] == mode]
+ svg(mode, "successful_rps", group, args.input.parent / f"{mode}-rps.svg")
+ svg(mode, "p95_ms", group, args.input.parent / f"{mode}-p95.svg")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/restart.sh b/scripts/restart.sh
new file mode 100755
index 00000000..c9148ca0
--- /dev/null
+++ b/scripts/restart.sh
@@ -0,0 +1,5 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+docker compose --env-file .env stop api lb postgres
+docker compose --env-file .env up -d --wait --scale api=2
diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh
new file mode 100755
index 00000000..09ca7d2b
--- /dev/null
+++ b/scripts/rolling-update.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+# Replace the two original API processes one at a time. No schema changes here.
+set -eu
+cd "$(dirname "$0")/.."
+
+original=$(docker compose --env-file .env ps -q api)
+set -- $original
+if [ "$#" -ne 2 ]; then
+ echo "expected exactly two running API replicas" >&2
+ exit 1
+fi
+
+for container in "$@"; do
+ if [ "$(docker inspect -f '{{.State.Health.Status}}' "$container")" != healthy ]; then
+ echo "API replica is not healthy before replacement" >&2
+ exit 1
+ fi
+done
+
+for container in "$@"; do
+ docker stop -t 10 "$container" >/dev/null
+ docker rm "$container" >/dev/null
+ # --no-recreate preserves the other old replica while Compose fills the gap.
+ docker compose --env-file .env up -d --no-deps --no-recreate --wait --scale api=2 api >/dev/null
+ current=$(docker compose --env-file .env ps -q api)
+ set -- $current
+ if [ "$#" -ne 2 ]; then
+ echo "replacement did not restore two API replicas" >&2
+ exit 1
+ fi
+ for running in "$@"; do
+ if [ "$(docker inspect -f '{{.State.Health.Status}}' "$running")" != healthy ]; then
+ echo "replacement API is not healthy" >&2
+ exit 1
+ fi
+ done
+ echo "replaced API container $container; two healthy replicas now serve traffic"
+done
diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh
new file mode 100755
index 00000000..86a70dac
--- /dev/null
+++ b/scripts/stop-one.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+set -eu
+cd "$(dirname "$0")/.."
+: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
+for container in $(docker compose --env-file .env ps -q api); do
+ instance=$(docker exec "$container" python -c "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live', timeout=2).headers['X-Instance-Id'])")
+ if [ "$instance" = "$TARGET_INSTANCE_ID" ]; then
+ docker stop -t 10 "$container" >/dev/null
+ exit 0
+ fi
+done
+echo "target API instance not found" >&2
+exit 1
diff --git a/src/api.py b/src/api.py
new file mode 100644
index 00000000..b4418118
--- /dev/null
+++ b/src/api.py
@@ -0,0 +1,103 @@
+"""Messenger HTTP routes."""
+
+import re
+from uuid import UUID
+
+from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response
+
+from .auth import basic_credentials, bearer_token
+from . import db
+from .models import CreateChat, CreateMessage, CreateUser
+from .store import store
+
+router = APIRouter()
+
+
+@router.get("/health/live")
+def live():
+ return {"status": "ok"}
+
+
+@router.get("/health/ready")
+def ready(response: Response):
+ result = store.ready()
+ stats = db.pool().get_stats()
+ for header, key in (("X-Pool-Requests", "requests_num"), ("X-Pool-Queued", "requests_queued"),
+ ("X-Pool-Wait-Ms", "requests_wait_ms"), ("X-Pool-Size", "pool_size"),
+ ("X-Pool-Available", "pool_available")):
+ response.headers[header] = str(stats.get(key, 0))
+ return result
+
+
+@router.post("/api/v1/users", status_code=201)
+def register_user(body: CreateUser):
+ return store.register(body.username, body.display_name, body.password)
+
+
+@router.post("/api/v1/auth/sessions", status_code=201)
+async def create_session(request: Request, response: Response):
+ if await request.body():
+ raise HTTPException(400)
+ username, password = basic_credentials(request.headers.get("authorization"))
+ response.headers["Cache-Control"] = "no-store"
+ return store.create_session(username, password)
+
+
+def current_user(request: Request) -> dict:
+ return store.authenticated(bearer_token(request.headers.get("authorization")))
+
+
+def parse_limit(value: str) -> int:
+ try:
+ number = int(value) if value.isdecimal() else 0
+ except ValueError:
+ raise HTTPException(400) from None
+ if not 1 <= number <= 100:
+ raise HTTPException(400)
+ return number
+
+
+@router.delete("/api/v1/auth/sessions/current", status_code=204)
+def revoke_session(request: Request):
+ store.revoke_session(bearer_token(request.headers.get("authorization")))
+ return Response(status_code=204)
+
+
+@router.get("/api/v1/users/me")
+def user_me(actor: dict = Depends(current_user)):
+ return actor
+
+
+@router.post("/api/v1/chats", status_code=201)
+def create_chat(body: CreateChat, actor: dict = Depends(current_user)):
+ return store.create_chat(actor["id"], body.title, body.member_ids)
+
+
+@router.get("/api/v1/chats")
+def list_chats(
+ limit: str = Query("50"), cursor: str | None = Query(None, min_length=1), actor: dict = Depends(current_user)
+):
+ return store.list_chats(actor["id"], parse_limit(limit), cursor)
+
+
+@router.get("/api/v1/chats/{chat_id}")
+def get_chat(chat_id: UUID, actor: dict = Depends(current_user)):
+ return store.chat(actor["id"], str(chat_id))
+
+
+@router.post("/api/v1/chats/{chat_id}/messages", status_code=201)
+def create_message(chat_id: UUID, body: CreateMessage, request: Request, actor: dict = Depends(current_user)):
+ key = request.headers.get("Idempotency-Key")
+ if key is None or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", key):
+ raise HTTPException(400)
+ return store.create_message(actor["id"], str(chat_id), body.text, key)
+
+
+@router.get("/api/v1/chats/{chat_id}/messages")
+def list_messages(
+ chat_id: UUID,
+ limit: str = Query("50"),
+ cursor: str | None = Query(None, min_length=1),
+ actor: dict = Depends(current_user),
+):
+ return store.list_messages(actor["id"], str(chat_id), parse_limit(limit), cursor)
diff --git a/src/auth.py b/src/auth.py
new file mode 100644
index 00000000..5714b173
--- /dev/null
+++ b/src/auth.py
@@ -0,0 +1,43 @@
+"""Parse HTTP credentials without exposing them in errors or logs."""
+
+import base64
+import binascii
+import hashlib
+import re
+
+from fastapi import HTTPException
+
+
+BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
+BEARER_CHALLENGE = 'Bearer realm="messenger"'
+USERNAME = re.compile(r"[a-z0-9_]{3,32}\Z")
+
+
+def unauthorized(challenge: str) -> HTTPException:
+ return HTTPException(401, headers={"WWW-Authenticate": challenge})
+
+
+def basic_credentials(header: str | None) -> tuple[str, str]:
+ if not header or not header.startswith("Basic "):
+ raise unauthorized(BASIC_CHALLENGE)
+ try:
+ raw = base64.b64decode(header[6:], validate=True).decode("utf-8")
+ username, password = raw.split(":", 1)
+ except (binascii.Error, UnicodeError, ValueError):
+ raise unauthorized(BASIC_CHALLENGE) from None
+ if not USERNAME.fullmatch(username) or not 12 <= len(password) <= 128:
+ raise unauthorized(BASIC_CHALLENGE)
+ return username, password
+
+
+def bearer_token(header: str | None) -> str:
+ if not header or not header.startswith("Bearer "):
+ raise unauthorized(BEARER_CHALLENGE)
+ token = header[7:]
+ if not 32 <= len(token) <= 256 or not token.isascii() or not re.fullmatch(r"[A-Za-z0-9_-]+", token):
+ raise unauthorized(BEARER_CHALLENGE)
+ return token
+
+
+def token_hash(token: str) -> bytes:
+ return hashlib.sha256(token.encode("ascii")).digest()
diff --git a/src/db.py b/src/db.py
new file mode 100644
index 00000000..832ff074
--- /dev/null
+++ b/src/db.py
@@ -0,0 +1,38 @@
+"""Bounded PostgreSQL pool shared by API storage and health checks."""
+
+import os
+import atexit
+from functools import lru_cache
+from pathlib import Path
+
+import psycopg
+from psycopg.rows import dict_row
+from psycopg_pool import ConnectionPool
+from psycopg_pool import PoolTimeout
+
+
+def database_url() -> str:
+ path = os.environ.get("DATABASE_URL_FILE")
+ if path:
+ return Path(path).read_text(encoding="utf-8").strip()
+ return os.environ.get("DATABASE_URL", "postgresql:///messenger")
+
+
+@lru_cache(maxsize=1)
+def pool() -> ConnectionPool:
+ result = ConnectionPool(
+ database_url(), min_size=0, max_size=8, timeout=5,
+ kwargs={"connect_timeout": 5, "options": "-c statement_timeout=5000", "row_factory": dict_row},
+ open=True,
+ )
+ atexit.register(result.close)
+ return result
+
+
+def ready() -> bool:
+ try:
+ with pool().connection() as conn:
+ conn.execute("SELECT 1 FROM users LIMIT 0")
+ return True
+ except (psycopg.Error, PoolTimeout, OSError):
+ return False
diff --git a/src/main.py b/src/main.py
new file mode 100644
index 00000000..bcac75fd
--- /dev/null
+++ b/src/main.py
@@ -0,0 +1,66 @@
+"""FastAPI entry point and contract-wide response handling."""
+
+from uuid import uuid4
+
+from fastapi import FastAPI, HTTPException, Request
+from fastapi.exceptions import RequestValidationError
+from fastapi.responses import JSONResponse
+
+from .api import router
+
+app = FastAPI()
+app.include_router(router)
+INSTANCE_ID = uuid4().hex
+
+ERRORS = {
+ 400: ("invalid_request", "Неверный запрос"),
+ 401: ("unauthorized", "Требуется действительный пользователь"),
+ 404: ("not_found", "Объект не найден"),
+ 409: ("conflict", "Конфликт данных"),
+ 413: ("payload_too_large", "Слишком большой запрос"),
+ 415: ("unsupported_media_type", "Неподдерживаемый тип данных"),
+ 429: ("rate_limited", "Слишком много запросов"),
+ 503: ("unavailable", "Сервис недоступен"),
+}
+
+
+def error_response(request: Request, status: int, extra_headers: dict | None = None) -> JSONResponse:
+ code, message = ERRORS.get(status, ("internal_error", "Внутренняя ошибка"))
+ request_id = request.state.request_id
+ return JSONResponse(
+ status_code=status,
+ content={"error": {"code": code, "message": message, "request_id": request_id}},
+ headers={"X-Request-Id": request_id, "X-Instance-Id": INSTANCE_ID, **(extra_headers or {})},
+ )
+
+
+@app.middleware("http")
+async def request_contract(request: Request, call_next):
+ request.state.request_id = str(uuid4())
+ if request.method == "POST" and request.url.path.startswith("/api/v1/") and request.url.path != "/api/v1/auth/sessions":
+ media_type, _, charset = request.headers.get("content-type", "").lower().partition(";")
+ if media_type.strip() != "application/json" or (charset and charset.strip() != "charset=utf-8"):
+ return error_response(request, 415)
+ try:
+ (await request.body()).decode("utf-8")
+ except UnicodeDecodeError:
+ return error_response(request, 400)
+ response = await call_next(request)
+ response.headers["X-Request-Id"] = request.state.request_id
+ response.headers["X-Instance-Id"] = INSTANCE_ID
+ return response
+
+
+@app.exception_handler(RequestValidationError)
+async def validation_error(request: Request, exc: RequestValidationError):
+ return error_response(request, 400)
+
+
+@app.exception_handler(HTTPException)
+async def http_error(request: Request, exc: HTTPException):
+ return error_response(request, exc.status_code, exc.headers)
+
+
+@app.exception_handler(Exception)
+async def unexpected_error(request: Request, exc: Exception):
+ return error_response(request, 500)
diff --git a/src/models.py b/src/models.py
new file mode 100644
index 00000000..30f5706a
--- /dev/null
+++ b/src/models.py
@@ -0,0 +1,60 @@
+"""Request bodies from the lab 1 contract."""
+
+from uuid import UUID
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+
+class StrictBody(BaseModel):
+ model_config = ConfigDict(extra="forbid", strict=True)
+
+
+class CreateUser(StrictBody):
+ username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
+ display_name: str = Field(min_length=1, max_length=100)
+ password: str = Field(min_length=12, max_length=128)
+
+ @field_validator("display_name")
+ @classmethod
+ def nonblank_name(cls, value: str) -> str:
+ if not value.strip():
+ raise ValueError("blank display name")
+ return value
+
+ @field_validator("password")
+ @classmethod
+ def nonblank_password(cls, value: str) -> str:
+ if not value.strip():
+ raise ValueError("blank password")
+ return value
+
+
+class CreateChat(StrictBody):
+ title: str = Field(min_length=1, max_length=100)
+ member_ids: list[str] = Field(min_length=1, max_length=100)
+
+ @field_validator("title")
+ @classmethod
+ def nonblank_title(cls, value: str) -> str:
+ if not value.strip():
+ raise ValueError("blank title")
+ return value
+
+ @field_validator("member_ids")
+ @classmethod
+ def valid_members(cls, value: list[str]) -> list[str]:
+ members = [str(UUID(member_id)) for member_id in value]
+ if len(set(members)) != len(members):
+ raise ValueError("duplicate member")
+ return members
+
+
+class CreateMessage(StrictBody):
+ text: str = Field(min_length=1, max_length=4000)
+
+ @field_validator("text")
+ @classmethod
+ def nonblank_text(cls, value: str) -> str:
+ if not value.strip():
+ raise ValueError("blank text")
+ return value
diff --git a/src/server.py b/src/server.py
new file mode 100644
index 00000000..20fb9c9c
--- /dev/null
+++ b/src/server.py
@@ -0,0 +1,14 @@
+"""Run the ASGI worker as the container's main process."""
+
+import os
+
+import uvicorn
+
+
+if __name__ == "__main__":
+ uvicorn.run(
+ "src.main:app",
+ host=os.environ.get("APP_HOST", "0.0.0.0"),
+ port=int(os.environ.get("APP_PORT", "8080")),
+ workers=1,
+ )
diff --git a/src/store.py b/src/store.py
new file mode 100644
index 00000000..0db641b1
--- /dev/null
+++ b/src/store.py
@@ -0,0 +1,272 @@
+"""PostgreSQL-backed users, chats and message history."""
+
+import base64
+import binascii
+import hashlib
+import json
+import os
+import secrets
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from uuid import UUID, uuid4
+
+import psycopg
+from argon2 import PasswordHasher
+from argon2.exceptions import InvalidHashError, VerificationError
+from fastapi import HTTPException
+from psycopg.errors import UniqueViolation
+from psycopg_pool import PoolTimeout
+
+from . import db
+from .auth import BASIC_CHALLENGE, BEARER_CHALLENGE, token_hash, unauthorized
+
+password_hasher = PasswordHasher()
+
+
+def timestamp(value: datetime) -> str:
+ return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
+
+
+def encode_cursor(resource: str, item: dict) -> str:
+ value = [resource, item["created_at"], item["id"]]
+ return base64.urlsafe_b64encode(json.dumps(value, separators=(",", ":")).encode()).decode().rstrip("=")
+
+
+def decode_cursor(cursor: str | None, resource: str) -> tuple[datetime, UUID] | None:
+ if cursor is None:
+ return None
+ try:
+ raw = base64.b64decode(cursor + "=" * (-len(cursor) % 4), altchars=b"-_", validate=True)
+ value = json.loads(raw)
+ if not isinstance(value, list) or len(value) != 3 or value[0] != resource:
+ raise ValueError
+ date_text, item_id = value[1:]
+ if not isinstance(date_text, str) or not date_text.endswith("Z") or not isinstance(item_id, str):
+ raise ValueError
+ date = datetime.fromisoformat(date_text[:-1] + "+00:00")
+ if date.utcoffset().total_seconds() != 0:
+ raise ValueError
+ return date, UUID(item_id)
+ except (binascii.Error, UnicodeError, ValueError, TypeError, AttributeError):
+ raise HTTPException(400) from None
+
+
+@contextmanager
+def connection():
+ try:
+ with db.pool().connection() as conn:
+ yield conn
+ except (psycopg.Error, PoolTimeout, OSError):
+ raise HTTPException(503) from None
+
+
+def user_view(row: dict) -> dict:
+ return {
+ "id": str(row["id"]), "username": row["username"],
+ "display_name": row["display_name"], "created_at": timestamp(row["created_at"]),
+ }
+
+
+def chat_view(row: dict) -> dict:
+ return {
+ "id": str(row["id"]), "title": row["title"],
+ "member_ids": [str(member_id) for member_id in row["member_ids"]],
+ "created_at": timestamp(row["created_at"]),
+ }
+
+
+def message_view(row: dict) -> dict:
+ return {
+ "id": str(row["id"]), "chat_id": str(row["chat_id"]),
+ "sender_id": str(row["sender_id"]), "text": row["text"].decode("utf-8"),
+ "created_at": timestamp(row["created_at"]),
+ }
+
+
+CHAT_COLUMNS = """c.id, c.title, c.created_at,
+ ARRAY(SELECT m.user_id FROM memberships m WHERE m.chat_id=c.id ORDER BY m.position) AS member_ids"""
+
+
+class Store:
+ def ready(self) -> dict:
+ if not db.ready():
+ raise HTTPException(503)
+ return {"status": "ok"}
+
+ def register(self, username: str, display_name: str, password: str) -> dict:
+ password_hash = password_hasher.hash(password)
+ with connection() as conn:
+ try:
+ row = conn.execute(
+ "INSERT INTO users (id, username, display_name, password_hash) VALUES (%s,%s,%s,%s) "
+ "RETURNING id,username,display_name,created_at",
+ (uuid4(), username, display_name, password_hash),
+ ).fetchone()
+ except UniqueViolation:
+ raise HTTPException(409) from None
+ return user_view(row)
+
+ def user(self, user_id: str | None) -> dict:
+ if not user_id:
+ raise HTTPException(401)
+ try:
+ user_uuid = UUID(user_id)
+ except (ValueError, TypeError):
+ raise HTTPException(401) from None
+ with connection() as conn:
+ row = conn.execute(
+ "SELECT id,username,display_name,created_at FROM users WHERE id=%s", (user_uuid,)
+ ).fetchone()
+ if row is None:
+ raise HTTPException(401)
+ return user_view(row)
+
+ def create_session(self, username: str, password: str) -> dict:
+ with connection() as conn:
+ row = conn.execute(
+ "SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s",
+ (username,),
+ ).fetchone()
+ try:
+ valid = row is not None and password_hasher.verify(row["password_hash"], password)
+ except (VerificationError, InvalidHashError):
+ valid = False
+ if not valid:
+ raise unauthorized(BASIC_CHALLENGE)
+ token = secrets.token_urlsafe(32)
+ ttl = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
+ expiry = conn.execute(
+ "INSERT INTO sessions (token_hash,user_id,expires_at) "
+ "VALUES (%s,%s,clock_timestamp()+make_interval(secs => %s)) RETURNING expires_at",
+ (token_hash(token), row["id"], ttl),
+ ).fetchone()["expires_at"]
+ return {"token": token, "token_type": "Bearer", "expires_at": timestamp(expiry), "user": user_view(row)}
+
+ def authenticated(self, token: str) -> dict:
+ with connection() as conn:
+ row = conn.execute(
+ "SELECT u.id,u.username,u.display_name,u.created_at FROM sessions s "
+ "JOIN users u ON u.id=s.user_id "
+ "WHERE s.token_hash=%s AND s.expires_at>clock_timestamp()",
+ (token_hash(token),),
+ ).fetchone()
+ if row is None:
+ raise unauthorized(BEARER_CHALLENGE)
+ return user_view(row)
+
+ def revoke_session(self, token: str) -> None:
+ with connection() as conn:
+ row = conn.execute(
+ "DELETE FROM sessions WHERE token_hash=%s AND expires_at>clock_timestamp() RETURNING token_hash",
+ (token_hash(token),),
+ ).fetchone()
+ if row is None:
+ raise unauthorized(BEARER_CHALLENGE)
+
+ def create_chat(self, actor_id: str, title: str, member_ids: list[str]) -> dict:
+ if actor_id not in member_ids:
+ raise HTTPException(400)
+ members = [UUID(member_id) for member_id in member_ids]
+ with connection() as conn:
+ count = conn.execute("SELECT count(*) AS n FROM users WHERE id=ANY(%s)", (members,)).fetchone()["n"]
+ if count != len(members):
+ raise HTTPException(400)
+ row = conn.execute("INSERT INTO chats (id,title) VALUES (%s,%s) RETURNING id,title,created_at", (uuid4(), title)).fetchone()
+ for position, member_id in enumerate(members):
+ conn.execute(
+ "INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,%s)",
+ (row["id"], member_id, position),
+ )
+ return chat_view({**row, "member_ids": members})
+
+ def chat(self, actor_id: str, chat_id: str) -> dict:
+ with connection() as conn:
+ row = conn.execute(
+ f"SELECT {CHAT_COLUMNS} FROM chats c "
+ "JOIN memberships access ON access.chat_id=c.id "
+ "WHERE access.user_id=%s AND c.id=%s",
+ (UUID(actor_id), UUID(chat_id)),
+ ).fetchone()
+ if row is None:
+ raise HTTPException(404)
+ return chat_view(row)
+
+ def list_chats(self, actor_id: str, limit: int, cursor: str | None) -> dict:
+ after = decode_cursor(cursor, f"chats:{actor_id}")
+ query = (f"SELECT {CHAT_COLUMNS} FROM chats c "
+ "JOIN memberships access ON access.chat_id=c.id WHERE access.user_id=%s")
+ params = [UUID(actor_id)]
+ if after:
+ query += " AND (c.created_at,c.id)>(%s,%s)"
+ params.extend(after)
+ query += " ORDER BY c.created_at,c.id LIMIT %s"
+ params.append(limit + 1)
+ with connection() as conn:
+ rows = conn.execute(query, params).fetchall()
+ items = [chat_view(row) for row in rows[:limit]]
+ return {"items": items, "next_cursor": encode_cursor(f"chats:{actor_id}", items[-1]) if len(rows) > limit else None}
+
+ def create_message(self, actor_id: str, chat_id: str, text: str, key: str) -> dict:
+ chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id)
+ request_hash = hashlib.sha256(json.dumps({"text": text}, ensure_ascii=False, sort_keys=True).encode()).digest()
+ with connection() as conn:
+ member = conn.execute(
+ "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid)
+ ).fetchone()
+ if member is None:
+ raise HTTPException(404)
+ conn.execute(
+ "DELETE FROM message_replays WHERE user_id=%s AND chat_id=%s AND request_key=%s "
+ "AND created_at dict:
+ after = decode_cursor(cursor, f"messages:{chat_id}")
+ chat_uuid, actor_uuid = UUID(chat_id), UUID(actor_id)
+ query = "SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s"
+ params = [chat_uuid]
+ if after:
+ query += " AND (created_at,id)>(%s,%s)"
+ params.extend(after)
+ query += " ORDER BY created_at,id LIMIT %s"
+ params.append(limit + 1)
+ with connection() as conn:
+ member = conn.execute(
+ "SELECT 1 FROM memberships WHERE chat_id=%s AND user_id=%s", (chat_uuid, actor_uuid)
+ ).fetchone()
+ if member is None:
+ raise HTTPException(404)
+ rows = conn.execute(query, params).fetchall()
+ items = [message_view(row) for row in rows[:limit]]
+ return {"items": items, "next_cursor": encode_cursor(f"messages:{chat_id}", items[-1]) if len(rows) > limit else None}
+
+
+store = Store()
diff --git a/student-tests/test_api_limits.py b/student-tests/test_api_limits.py
new file mode 100644
index 00000000..4936d6f3
--- /dev/null
+++ b/student-tests/test_api_limits.py
@@ -0,0 +1,20 @@
+"""Invalid pagination limits must remain client errors."""
+
+import unittest
+
+from fastapi import HTTPException
+
+from src.api import parse_limit
+
+
+class LimitTests(unittest.TestCase):
+ def test_valid_and_extremely_long_limits(self):
+ self.assertEqual(parse_limit("100"), 100)
+ for value in ("0", "101", "-1", "x", "9" * 4301):
+ with self.assertRaises(HTTPException) as failure:
+ parse_limit(value)
+ self.assertEqual(failure.exception.status_code, 400)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/student-tests/test_auth.py b/student-tests/test_auth.py
new file mode 100644
index 00000000..e9fcdf09
--- /dev/null
+++ b/student-tests/test_auth.py
@@ -0,0 +1,174 @@
+"""HTTP and database checks for durable Basic/Bearer sessions."""
+
+import base64
+import hashlib
+import json
+import os
+import socket
+import subprocess
+import sys
+import time
+import unittest
+import urllib.error
+import urllib.request
+from uuid import uuid4
+
+import psycopg
+
+
+URL = os.environ.get("DATABASE_URL")
+
+
+@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database")
+class AuthTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ subprocess.run([sys.executable, "scripts/migrate.py"], check=True)
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ cls.port = sock.getsockname()[1]
+ cls.start_server()
+
+ @classmethod
+ def start_server(cls, ttl="3600"):
+ cls.instance_id = None
+ env = {**os.environ, "SESSION_TTL_SECONDS": ttl}
+ cls.server = subprocess.Popen(
+ [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)],
+ env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+ )
+ for _ in range(100):
+ if cls.server.poll() is not None:
+ raise AssertionError("API server exited during startup")
+ try:
+ cls.call("GET", "/health/ready", expected=200)
+ return
+ except (AssertionError, urllib.error.URLError):
+ time.sleep(0.05)
+ raise AssertionError("API server did not become ready")
+
+ @classmethod
+ def stop_server(cls):
+ cls.server.terminate()
+ try:
+ cls.server.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ cls.server.kill()
+ cls.server.wait(timeout=5)
+
+ @classmethod
+ def tearDownClass(cls):
+ cls.stop_server()
+
+ @classmethod
+ def call(cls, method, path, *, body=None, authorization=None, user_id=None, expected=200):
+ headers = {"Accept": "application/json"}
+ data = None
+ if body is not None:
+ data = json.dumps(body, ensure_ascii=False).encode()
+ headers["Content-Type"] = "application/json"
+ if authorization is not None:
+ headers["Authorization"] = authorization
+ if user_id is not None:
+ headers["X-User-Id"] = user_id
+ request = urllib.request.Request(
+ f"http://127.0.0.1:{cls.port}{path}", data=data, headers=headers, method=method,
+ )
+ try:
+ response = urllib.request.urlopen(request, timeout=5)
+ except urllib.error.HTTPError as error:
+ response = error
+ with response:
+ status = response.status
+ response_headers = response.headers
+ data = response.read()
+ assert status == expected, (method, path, expected, status)
+ request_id = response_headers.get("X-Request-Id")
+ assert request_id
+ instance_id = response_headers.get("X-Instance-Id")
+ assert instance_id
+ if cls.instance_id is None:
+ cls.instance_id = instance_id
+ else:
+ assert instance_id == cls.instance_id
+ payload = json.loads(data) if data else None
+ if status >= 400:
+ assert payload["error"]["request_id"] == request_id
+ assert "password" not in data.decode().lower()
+ assert "token" not in data.decode().lower()
+ return payload, response_headers
+
+ def user(self, password="Пароль:очень-длинный-123"):
+ username = "u_" + uuid4().hex[:16]
+ body = {"username": username, "display_name": "Алиса", "password": password}
+ user, _ = self.call("POST", "/api/v1/users", body=body, expected=201)
+ self.assertNotIn("password", user)
+ return user, password
+
+ def login(self, username, password, expected=201):
+ encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
+ return self.call("POST", "/api/v1/auth/sessions", authorization="Basic " + encoded, expected=expected)
+
+ def test_basic_bearer_logout_and_storage(self):
+ alice, password = self.user()
+ bob, _ = self.user()
+ session, headers = self.login(alice["username"], password)
+ self.assertEqual(session["token_type"], "Bearer")
+ self.assertEqual(session["user"], alice)
+ self.assertGreaterEqual(len(base64.urlsafe_b64decode(session["token"] + "=" * (-len(session["token"]) % 4))), 32)
+ self.assertEqual(headers.get("Cache-Control"), "no-store")
+ bearer = "Bearer " + session["token"]
+ me, _ = self.call("GET", "/api/v1/users/me", authorization=bearer, user_id=bob["id"])
+ self.assertEqual(me, alice)
+ with psycopg.connect(URL) as conn:
+ row = conn.execute("SELECT token_hash,expires_at FROM sessions WHERE user_id=%s", (alice["id"],)).fetchone()
+ password_hash = conn.execute("SELECT password_hash FROM users WHERE id=%s", (alice["id"],)).fetchone()[0]
+ self.assertEqual(row[0], hashlib.sha256(session["token"].encode()).digest())
+ self.assertNotEqual(password_hash, password)
+ self.assertTrue(password_hash.startswith("$argon2id$"))
+ self.call("DELETE", "/api/v1/auth/sessions/current", authorization=bearer, expected=204)
+ for method, path in (("GET", "/api/v1/users/me"), ("DELETE", "/api/v1/auth/sessions/current")):
+ _, headers = self.call(method, path, authorization=bearer, expected=401)
+ self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"')
+
+ def test_challenges_and_uniform_credentials(self):
+ user, password = self.user()
+ failures = []
+ for username, candidate in ((user["username"], "wrong-password-123"), ("missing_user_123", password)):
+ payload, headers = self.login(username, candidate, expected=401)
+ failures.append((payload["error"]["code"], payload["error"]["message"]))
+ self.assertEqual(headers.get("WWW-Authenticate"), 'Basic realm="messenger", charset="UTF-8"')
+ self.assertEqual(failures[0], failures[1])
+ encoded = base64.b64encode(f"{user['username']}:{password}".encode()).decode()
+ for authorization in (None, "Basic " + encoded, "Bearer garbage"):
+ _, headers = self.call("GET", "/api/v1/users/me", authorization=authorization, user_id=user["id"], expected=401)
+ self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"')
+ _, headers = self.call("POST", "/api/v1/auth/sessions", authorization="Bearer garbage", expected=401)
+ self.assertEqual(headers.get("WWW-Authenticate"), 'Basic realm="messenger", charset="UTF-8"')
+ self.call("POST", "/api/v1/auth/sessions", authorization="Basic " + encoded, body={"unexpected": True}, expected=400)
+
+ def test_z_short_ttl_survives_server_restart_without_extension(self):
+ self.stop_server()
+ self.start_server(ttl="2")
+ try:
+ user, password = self.user()
+ session, _ = self.login(user["username"], password)
+ bearer = "Bearer " + session["token"]
+ self.call("GET", "/api/v1/users/me", authorization=bearer)
+ with psycopg.connect(URL) as conn:
+ before = conn.execute("SELECT expires_at FROM sessions WHERE user_id=%s", (user["id"],)).fetchone()[0]
+ self.stop_server()
+ self.start_server(ttl="3600")
+ with psycopg.connect(URL) as conn:
+ after = conn.execute("SELECT expires_at FROM sessions WHERE user_id=%s", (user["id"],)).fetchone()[0]
+ self.assertEqual(before, after)
+ time.sleep(2.1)
+ _, headers = self.call("GET", "/api/v1/users/me", authorization=bearer, expected=401)
+ self.assertEqual(headers.get("WWW-Authenticate"), 'Bearer realm="messenger"')
+ finally:
+ self.stop_server()
+ self.start_server()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/student-tests/test_benchmark_profile.py b/student-tests/test_benchmark_profile.py
new file mode 100644
index 00000000..660dea8e
--- /dev/null
+++ b/student-tests/test_benchmark_profile.py
@@ -0,0 +1,72 @@
+"""The scaling matrix must compare the same workload and resource budgets."""
+
+import sys
+import json
+import unittest
+from unittest.mock import patch
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts"))
+
+from benchmark import open_loop, topology, validate_matrix
+
+
+class BenchmarkProfileTests(unittest.TestCase):
+ def test_fixed_budget_is_identical_across_replica_counts(self):
+ budgets = [topology("fixed", replicas) for replicas in (1, 2, 4)]
+ self.assertEqual([replicas * cpu for replicas, cpu, _ in budgets], [1.0] * 3)
+ self.assertEqual([replicas * memory for replicas, _, memory in budgets], [512] * 3)
+
+ def test_growth_budget_increases_with_replicas(self):
+ budgets = [topology("growth", replicas) for replicas in (1, 2, 4)]
+ self.assertEqual([replicas * cpu for replicas, cpu, _ in budgets], [0.5, 1.0, 2.0])
+ self.assertEqual([replicas * memory for replicas, _, memory in budgets], [256, 512, 1024])
+
+ def test_matrix_requires_three_repeats_and_same_workload(self):
+ rows = [
+ {"mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat,
+ "duration": 8, "warmup": 2, "dataset_messages": 1000,
+ "api_cpu_each": topology(mode, replicas)[1],
+ "api_memory_mib_each": topology(mode, replicas)[2]}
+ for mode in ("fixed", "growth")
+ for replicas in (1, 2, 4)
+ for rate in (100, 250, 500)
+ for repeat in (1, 2, 3)
+ ]
+ validate_matrix(rows)
+ with self.assertRaises(ValueError):
+ validate_matrix(rows[:-1])
+ rows[-1]["dataset_messages"] = 999
+ with self.assertRaises(ValueError):
+ validate_matrix(rows)
+
+ def test_open_loop_counts_responses_at_a_scheduled_rate(self):
+ class Response:
+ status = 200
+
+ def read(self):
+ return json.dumps({"items": [{}] * 20}).encode()
+
+ class Connection:
+ def __init__(self, *args, **kwargs):
+ pass
+
+ def request(self, *args, **kwargs):
+ pass
+
+ def getresponse(self):
+ return Response()
+
+ def close(self):
+ pass
+
+ with patch("benchmark.http.client.HTTPConnection", Connection):
+ result = open_loop("http://127.0.0.1:8080", "/history", "synthetic", 20, 0.2, workers=2)
+ self.assertEqual(result["scheduled_requests"], 4)
+ self.assertEqual(result["successful_requests"], 4)
+ self.assertEqual(result["failed_requests"], 0)
+ self.assertGreater(result["successful_latency_ms"]["p95"], 0)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/student-tests/test_database.py b/student-tests/test_database.py
new file mode 100644
index 00000000..61a282f2
--- /dev/null
+++ b/student-tests/test_database.py
@@ -0,0 +1,85 @@
+"""Integration checks against a disposable PostgreSQL database."""
+
+import os
+import subprocess
+import sys
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from uuid import uuid4
+
+import psycopg
+
+from src.store import Store
+
+
+URL = os.environ.get("DATABASE_URL")
+
+
+@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database")
+class DatabaseTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ for _ in range(2):
+ subprocess.run([sys.executable, "scripts/migrate.py"], check=True)
+ cls.store = Store()
+
+ def user(self):
+ name = "u_" + uuid4().hex[:16]
+ return self.store.register(name, "Студент", "Test-password-123")
+
+ def test_migration_and_unique_username_race(self):
+ with psycopg.connect(URL) as conn:
+ versions = conn.execute("SELECT version FROM schema_migrations").fetchall()
+ self.assertEqual(versions, [(1,), (2,), (3,), (9,)])
+ name = "u_" + uuid4().hex[:16]
+
+ def register(_):
+ try:
+ return self.store.register(name, "Студент", "Test-password-123")
+ except Exception as exc:
+ return exc
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ results = list(pool.map(register, range(2)))
+ self.assertEqual(sum(isinstance(result, dict) for result in results), 1)
+ self.assertEqual([result.status_code for result in results if not isinstance(result, dict)], [409])
+ with psycopg.connect(URL) as conn:
+ self.assertEqual(conn.execute("SELECT count(*) FROM users WHERE username=%s", (name,)).fetchone()[0], 1)
+ password_hash = conn.execute("SELECT password_hash FROM users WHERE username=%s", (name,)).fetchone()[0]
+ self.assertTrue(password_hash.startswith("$argon2id$"))
+ self.assertNotIn("password", str(next(result for result in results if isinstance(result, dict))))
+ subprocess.run([sys.executable, "scripts/migrate.py"], check=True)
+ with psycopg.connect(URL) as conn:
+ self.assertEqual(conn.execute("SELECT count(*) FROM users WHERE username=%s", (name,)).fetchone()[0], 1)
+
+ def test_chat_rollback_and_keyset_history(self):
+ alice, bob, eve = self.user(), self.user(), self.user()
+ with psycopg.connect(URL) as conn:
+ before = conn.execute("SELECT count(*) FROM chats").fetchone()[0]
+ with self.assertRaises(Exception) as failure:
+ self.store.create_chat(alice["id"], "invalid", [alice["id"], str(uuid4())])
+ self.assertEqual(failure.exception.status_code, 400)
+ with psycopg.connect(URL) as conn:
+ self.assertEqual(conn.execute("SELECT count(*) FROM chats").fetchone()[0], before)
+
+ chat = self.store.create_chat(alice["id"], "Valid", [alice["id"], bob["id"]])
+ chat2 = self.store.create_chat(alice["id"], "Second", [alice["id"]])
+ chat_page = self.store.list_chats(alice["id"], 1, None)
+ self.assertEqual(chat_page["items"], [chat])
+ self.assertEqual(self.store.list_chats(alice["id"], 1, chat_page["next_cursor"]), {"items": [chat2], "next_cursor": None})
+ first = self.store.create_message(alice["id"], chat["id"], "первое", uuid4().hex)
+ second = self.store.create_message(bob["id"], chat["id"], "второе", uuid4().hex)
+ page = self.store.list_messages(alice["id"], chat["id"], 1, None)
+ self.assertEqual(page["items"], [first])
+ self.assertEqual(self.store.list_messages(bob["id"], chat["id"], 1, page["next_cursor"]), {"items": [second], "next_cursor": None})
+ self.assertEqual(Store().chat(bob["id"], chat["id"]), chat)
+ with self.assertRaises(Exception) as wrong_cursor:
+ self.store.list_messages(alice["id"], chat2["id"], 1, page["next_cursor"])
+ self.assertEqual(wrong_cursor.exception.status_code, 400)
+ with self.assertRaises(Exception) as hidden:
+ self.store.chat(eve["id"], chat["id"])
+ self.assertEqual(hidden.exception.status_code, 404)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py
new file mode 100644
index 00000000..ebc1c99f
--- /dev/null
+++ b/student-tests/test_idempotency.py
@@ -0,0 +1,53 @@
+"""Concurrent PostgreSQL-backed message retries create one row."""
+
+import os
+import subprocess
+import sys
+import unittest
+from concurrent.futures import ThreadPoolExecutor
+from uuid import uuid4
+
+import psycopg
+
+from src.store import Store
+
+
+URL = os.environ.get("DATABASE_URL")
+
+
+@unittest.skipUnless(URL, "set DATABASE_URL to a disposable PostgreSQL database")
+class IdempotencyTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ subprocess.run([sys.executable, "scripts/migrate.py"], check=True)
+ cls.store = Store()
+
+ def test_parallel_replay_conflict_and_scope(self):
+ alice = self.store.register("u_" + uuid4().hex[:16], "Алиса", "Test-password-123")
+ bob = self.store.register("u_" + uuid4().hex[:16], "Боб", "Test-password-123")
+ chat = self.store.create_chat(alice["id"], "Test", [alice["id"], bob["id"]])
+ other_chat = self.store.create_chat(alice["id"], "Other", [alice["id"]])
+ key = uuid4().hex
+
+ def send(_):
+ return Store().create_message(alice["id"], chat["id"], "same body", key)
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ first, replay = list(pool.map(send, range(2)))
+ self.assertEqual(first, replay)
+ with psycopg.connect(URL) as conn:
+ count = conn.execute(
+ "SELECT count(*) FROM messages WHERE chat_id=%s AND sender_id=%s AND text=%s",
+ (chat["id"], alice["id"], b"same body"),
+ ).fetchone()[0]
+ self.assertEqual(count, 1)
+
+ with self.assertRaises(Exception) as conflict:
+ self.store.create_message(alice["id"], chat["id"], "different body", key)
+ self.assertEqual(conflict.exception.status_code, 409)
+ self.assertNotEqual(self.store.create_message(bob["id"], chat["id"], "same body", key)["id"], first["id"])
+ self.assertNotEqual(self.store.create_message(alice["id"], other_chat["id"], "same body", key)["id"], first["id"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/student-tests/test_instance.py b/student-tests/test_instance.py
new file mode 100644
index 00000000..87e14628
--- /dev/null
+++ b/student-tests/test_instance.py
@@ -0,0 +1,54 @@
+"""Every API response identifies the live replica process."""
+
+import json
+import socket
+import subprocess
+import sys
+import time
+import unittest
+import urllib.error
+import urllib.request
+
+
+class InstanceHeaderTests(unittest.TestCase):
+ def test_process_alias_on_success_and_error(self):
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ port = sock.getsockname()[1]
+ server = subprocess.Popen(
+ [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(port)],
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+ )
+ try:
+ for _ in range(100):
+ try:
+ with urllib.request.urlopen(f"http://127.0.0.1:{port}/health/live", timeout=1) as response:
+ live_id = response.headers.get("X-Instance-Id")
+ self.assertEqual(json.load(response), {"status": "ok"})
+ break
+ except urllib.error.URLError:
+ if server.poll() is not None:
+ self.fail("API exited before readiness")
+ time.sleep(0.05)
+ else:
+ self.fail("API did not start")
+ self.assertTrue(live_id)
+ for path in ("/health/live", "/no-such-route"):
+ try:
+ response = urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=1)
+ except urllib.error.HTTPError as error:
+ response = error
+ with response:
+ self.assertEqual(response.headers.get("X-Instance-Id"), live_id)
+ self.assertTrue(response.headers.get("X-Request-Id"))
+ finally:
+ server.terminate()
+ try:
+ server.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ server.kill()
+ server.wait(timeout=5)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/student-tests/test_message_unicode.py b/student-tests/test_message_unicode.py
new file mode 100644
index 00000000..4ff79d81
--- /dev/null
+++ b/student-tests/test_message_unicode.py
@@ -0,0 +1,156 @@
+"""Run with DATABASE_URL pointing to a disposable PostgreSQL database.
+
+Checks real HTTP writes, persisted history, retries, and migration of old rows.
+"""
+
+import base64
+import json
+import os
+from pathlib import Path
+import socket
+import subprocess
+import sys
+import time
+import unittest
+import urllib.error
+import urllib.parse
+import urllib.request
+from uuid import uuid4
+
+import psycopg
+from psycopg.conninfo import make_conninfo
+
+ROOT = Path(__file__).resolve().parents[1]
+LAB = json.loads((ROOT / "lab.json").read_text())["number"]
+URL = os.environ.get("DATABASE_URL")
+
+
+@unittest.skipUnless(URL, "set DATABASE_URL to a disposable database")
+class MessageUnicodeTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True)
+ with socket.socket() as sock:
+ sock.bind(("127.0.0.1", 0))
+ cls.port = sock.getsockname()[1]
+ cls.start_server()
+ cls.addClassCleanup(cls.stop_server)
+
+ @classmethod
+ def start_server(cls):
+ cls.server = subprocess.Popen(
+ [sys.executable, "-m", "uvicorn", "src.main:app", "--host", "127.0.0.1", "--port", str(cls.port)],
+ cwd=ROOT, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
+ env={**os.environ, "VALKEY_URL": "redis://127.0.0.1:1/0", "OTEL_SDK_DISABLED": "true"},
+ )
+ for _ in range(100):
+ if cls.server.poll() is not None:
+ raise AssertionError("API exited during startup")
+ try:
+ cls.call("GET", "/health/live")
+ return
+ except (AssertionError, urllib.error.URLError):
+ time.sleep(0.05)
+ cls.stop_server()
+ raise AssertionError("API did not become ready")
+
+ @classmethod
+ def stop_server(cls):
+ cls.server.terminate()
+ try:
+ cls.server.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ cls.server.kill()
+ cls.server.wait(timeout=5)
+
+ @classmethod
+ def call(cls, method, path, body=None, headers=None, expected=200):
+ request = urllib.request.Request(
+ f"http://127.0.0.1:{cls.port}{path}", method=method,
+ data=json.dumps(body).encode() if body is not None else None,
+ headers={"Content-Type": "application/json", **(headers or {})},
+ )
+ try:
+ response = urllib.request.urlopen(request, timeout=10)
+ except urllib.error.HTTPError as exc:
+ response = exc
+ with response:
+ payload = response.read()
+ assert response.status == expected, (method, path, response.status, expected, payload)
+ return json.loads(payload) if payload else None
+
+ def test_http_round_trip_after_restart_and_replay(self):
+ username, password = "u_" + uuid4().hex[:16], "Test-password-123"
+ user = self.call("POST", "/api/v1/users", {
+ "username": username, "display_name": "Unicode", "password": password,
+ }, expected=201)
+ session = self.call("POST", "/api/v1/auth/sessions", headers={
+ "Authorization": "Basic " + base64.b64encode(f"{username}:{password}".encode()).decode(),
+ }, expected=201)
+ auth = {"Authorization": "Bearer " + session["token"]}
+ chat = self.call("POST", "/api/v1/chats", {
+ "title": "Unicode", "member_ids": [user["id"]],
+ }, auth, 201)
+ path = f"/api/v1/chats/{chat['id']}/messages"
+ texts = ["\0", "\0Привет\0🙂\0", r"literal \u0000 \0 \\", "🙂" * 3999 + "\0", "\0" * 4000]
+ messages, keys = [], []
+ for text in texts:
+ key = uuid4().hex
+ message = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201)
+ self.assertEqual(message["text"], text)
+ messages.append(message)
+ keys.append(key)
+ for text in ("", "\0" * 4001, "🙂" * 4001):
+ self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": uuid4().hex}, 400)
+ self.stop_server()
+ self.start_server()
+ found, cursor = [], None
+ while True:
+ page = self.call("GET", path + "?limit=1" + (
+ "&cursor=" + urllib.parse.quote(cursor, safe="") if cursor else ""
+ ), headers=auth)
+ found.extend(page["items"])
+ cursor = page["next_cursor"]
+ if cursor is None:
+ break
+ self.assertEqual(found, messages)
+ if LAB >= 3:
+ for text, message, key in zip(texts, messages, keys):
+ replay = self.call("POST", path, {"text": text}, {**auth, "Idempotency-Key": key}, 201)
+ self.assertEqual(replay, message)
+ self.call("POST", path, {"text": "different"}, {**auth, "Idempotency-Key": keys[0]}, 409)
+ self.assertEqual(len(self.call("GET", path, headers=auth)["items"]), len(messages))
+
+ def test_upgrade_preserves_old_text_and_is_repeatable(self):
+ schema = "unicode_" + uuid4().hex
+ with psycopg.connect(URL, autocommit=True) as conn:
+ conn.execute(f'CREATE SCHEMA "{schema}"')
+ isolated = make_conninfo(URL, options=f"-c search_path={schema}")
+ try:
+ old_text = "Старое 🙂 \\ literal \\u0000"
+ with psycopg.connect(isolated) as conn:
+ conn.execute("CREATE TABLE schema_migrations (version integer PRIMARY KEY)")
+ for migration in sorted((ROOT / "migrations").glob("*.sql")):
+ version = int(migration.name.split("_", 1)[0])
+ if version >= 9:
+ continue
+ conn.execute(migration.read_text())
+ conn.execute("INSERT INTO schema_migrations VALUES (%s)", (version,))
+ user, chat, message = uuid4(), uuid4(), uuid4()
+ conn.execute("INSERT INTO users (id,username,display_name,password_hash) VALUES (%s,'old','Old','hash')", (user,))
+ conn.execute("INSERT INTO chats (id,title) VALUES (%s,'Old')", (chat,))
+ conn.execute("INSERT INTO memberships (chat_id,user_id,position) VALUES (%s,%s,0)", (chat, user))
+ conn.execute("INSERT INTO messages (id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s)", (message, chat, user, old_text))
+ for _ in range(2):
+ subprocess.run([sys.executable, "scripts/migrate.py"], cwd=ROOT, check=True)
+ with psycopg.connect(isolated) as conn:
+ conn.execute((ROOT / "migrations/009_message_text_utf8.sql").read_text())
+ saved, = conn.execute("SELECT text FROM messages WHERE id=%s", (message,)).fetchone()
+ self.assertEqual(saved.decode("utf-8"), old_text)
+ finally:
+ with psycopg.connect(URL, autocommit=True) as conn:
+ conn.execute(f'DROP SCHEMA "{schema}" CASCADE')
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/student-tests/test_operations.py b/student-tests/test_operations.py
new file mode 100644
index 00000000..e545189e
--- /dev/null
+++ b/student-tests/test_operations.py
@@ -0,0 +1,239 @@
+"""Operator checks: topology unittest plus explicit rolling replacement exercise."""
+
+import argparse
+from concurrent.futures import ThreadPoolExecutor
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import time
+import unittest
+from uuid import uuid4
+
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "tests"))
+from client import Client, Failure # noqa: E402
+from resilience import history # noqa: E402
+
+
+class TopologyTests(unittest.TestCase):
+ def test_limits_ports_mounts_and_secrets(self):
+ proc = subprocess.run(
+ ["docker", "compose", "--env-file", ".env.example", "config", "--format", "json"],
+ cwd=ROOT, capture_output=True, text=True, check=True,
+ )
+ services = json.loads(proc.stdout)["services"]
+ self.assertEqual(set(services), {"postgres", "migrate", "api", "lb"})
+ for name, service in services.items():
+ self.assertNotIn("container_name", service, name)
+ self.assertGreater(float(service.get("cpus", 0)), 0, name)
+ self.assertGreater(int(service.get("mem_limit", 0)), 0, name)
+ if name != "lb":
+ self.assertFalse(service.get("ports"), name)
+ self.assertEqual(len(services["lb"]["ports"]), 1)
+ self.assertEqual(services["api"].get("volumes", []), [])
+ self.assertEqual(services["migrate"].get("volumes", []), [])
+ self.assertEqual(services["lb"].get("secrets", []), [])
+ self.assertEqual(len(services["lb"].get("volumes", [])), 1)
+ self.assertTrue(services["lb"]["volumes"][0]["read_only"])
+ self.assertEqual({s["source"] for s in services["api"]["secrets"]}, {"database_url"})
+ self.assertEqual({s["source"] for s in services["migrate"]["secrets"]}, {"database_url"})
+ self.assertEqual({s["source"] for s in services["postgres"]["secrets"]}, {"postgres_password"})
+
+
+def rolling(base_url):
+ client = Client(base_url, None, timeout=2)
+ alice, bob, _, chat = client.bootstrap()
+ script = ROOT / "scripts/rolling-update.sh"
+ started = time.monotonic()
+ stop = False
+
+ def writes():
+ local = Client(base_url, None, timeout=2)
+ confirmed, errors = [], 0
+ key, body = uuid4().hex, "rolling-" + uuid4().hex
+ while not stop:
+ try:
+ msg = local.message(alice, chat, body, key=key)["json"]
+ confirmed.append(msg)
+ key, body = uuid4().hex, "rolling-" + uuid4().hex
+ except Failure:
+ errors += 1 # Retry the uncertain write with its original key/body.
+ return confirmed, errors
+
+ def reads():
+ local = Client(base_url, None, timeout=2)
+ successes, errors, instances = 0, 0, set()
+ while not stop:
+ try:
+ result = local.call("GET", "/api/v1/users/me", actor=alice, schema="User")
+ successes += 1
+ instances.add(result["headers"]["X-Instance-Id"])
+ except Failure:
+ errors += 1
+ return successes, errors, instances
+
+ def duplicates():
+ local = Client(base_url, None, timeout=2)
+ ids, attempted, errors, cross_replica = [], [], 0, 0
+ with ThreadPoolExecutor(max_workers=2) as pair_pool:
+ while not stop:
+ key, body = uuid4().hex, "duplicate-" + uuid4().hex
+ attempted.append(body)
+ def send(_):
+ return local.message(alice, chat, body, key=key)
+ futures = [pair_pool.submit(send, index) for index in range(2)]
+ pair = []
+ for future in futures:
+ try:
+ pair.append(future.result())
+ except Failure:
+ errors += 1
+ if len(pair) == 2:
+ assert pair[0]["json"]["id"] == pair[1]["json"]["id"]
+ ids.append(pair[0]["json"]["id"])
+ cross_replica += pair[0]["headers"]["X-Instance-Id"] != pair[1]["headers"]["X-Instance-Id"]
+ time.sleep(0.05)
+ return ids, attempted, errors, cross_replica
+
+ with ThreadPoolExecutor(max_workers=3) as pool:
+ write_result = pool.submit(writes)
+ read_result = pool.submit(reads)
+ duplicate_result = pool.submit(duplicates)
+ proc = subprocess.Popen([str(script)], cwd=ROOT, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+ try:
+ stdout, stderr = proc.communicate(timeout=90)
+ if proc.returncode:
+ raise RuntimeError("rolling replacement failed: " + stderr.strip())
+ time.sleep(3) # Include DNS/LB recovery after the last replacement.
+ finally:
+ stop = True
+ if proc.poll() is None:
+ proc.terminate()
+ proc.wait(timeout=5)
+ confirmed, write_errors = write_result.result()
+ read_count, read_errors, instances = read_result.result()
+ duplicate_ids, attempted_duplicates, duplicate_errors, cross_replica = duplicate_result.result()
+
+ elapsed = time.monotonic() - started
+ messages = history(client, bob, chat)
+ seen = {message["id"] for message in messages}
+ assert confirmed and read_count and duplicate_ids, "load did not exercise all operations"
+ assert {message["id"] for message in confirmed}.issubset(seen), "confirmed write missing"
+ assert set(duplicate_ids).issubset(seen), "idempotent write missing"
+ duplicate_counts = {body: 0 for body in attempted_duplicates}
+ for message in messages:
+ if message["text"] in duplicate_counts:
+ duplicate_counts[message["text"]] += 1
+ assert all(count <= 1 for count in duplicate_counts.values()), "uncertain retry duplicated a message"
+ assert len(instances) >= 2 and cross_replica > 0, "rolling test did not cross replicas"
+ print(json.dumps({
+ "action": "same-version sequential API replacement", "elapsed_seconds": round(elapsed, 3),
+ "successful_reads": read_count, "failed_reads": read_errors,
+ "successful_writes": len(confirmed), "failed_write_attempts": write_errors,
+ "successful_idempotent_pairs": len(duplicate_ids), "failed_pair_attempts": duplicate_errors,
+ "cross_replica_pairs": cross_replica,
+ "successful_rps": round((read_count + len(confirmed) + 2 * len(duplicate_ids)) / elapsed, 3),
+ "observed_instance_count": len(instances), "replacement_output": stdout.strip().splitlines(),
+ "confirmed_messages_read_back": True,
+ }, ensure_ascii=False, indent=2))
+
+
+def failover(base_url, target_index):
+ containers = subprocess.run(
+ ["docker", "compose", "--env-file", ".env", "ps", "-q", "api"],
+ cwd=ROOT, capture_output=True, text=True, check=True,
+ ).stdout.splitlines()
+ assert len(containers) == 2, "expected two running API replicas"
+ target = containers[target_index - 1]
+ target_name = subprocess.run(
+ ["docker", "inspect", "-f", "{{.Name}}", target],
+ capture_output=True, text=True, check=True,
+ ).stdout.strip().lstrip("/")
+ alias = subprocess.run(
+ ["docker", "exec", target, "python", "-c",
+ "import urllib.request; print(urllib.request.urlopen('http://127.0.0.1:8080/health/live').headers['X-Instance-Id'])"],
+ capture_output=True, text=True, check=True,
+ ).stdout.strip()
+ client = Client(base_url, None, timeout=2)
+ alice, bob, _, chat = client.bootstrap()
+ started = time.monotonic()
+ stop = False
+ action_done = False
+
+ def traffic(write):
+ local = Client(base_url, None, timeout=2)
+ confirmed, errors, after_stop = [], 0, None
+ key, body = uuid4().hex, "failover-" + uuid4().hex
+ while not stop:
+ try:
+ if write:
+ result = local.message(alice, chat, body, key=key)
+ confirmed.append(result["json"])
+ key, body = uuid4().hex, "failover-" + uuid4().hex
+ else:
+ local.call("GET", "/api/v1/users/me", actor=alice, schema="User")
+ confirmed.append(1)
+ if action_done and after_stop is None:
+ after_stop = time.monotonic() - started
+ except Failure:
+ errors += 1
+ time.sleep(0.05)
+ return confirmed, errors, after_stop
+
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ writes = pool.submit(traffic, True)
+ reads = pool.submit(traffic, False)
+ try:
+ proc = subprocess.run(
+ [str(ROOT / "scripts/stop-one.sh")], cwd=ROOT,
+ env={**os.environ, "TARGET_INSTANCE_ID": alias}, capture_output=True, text=True, timeout=20,
+ )
+ action_done = True
+ stop_elapsed = time.monotonic() - started
+ assert proc.returncode == 0, "stop-one hook failed: " + proc.stderr.strip()
+ time.sleep(max(0, 15 - stop_elapsed))
+ finally:
+ stop = True
+ confirmed, write_errors, first_write = writes.result()
+ read_count, read_errors, first_read = reads.result()
+
+ actual = subprocess.run(
+ ["docker", "inspect", "-f", "{{.State.Status}}", target],
+ capture_output=True, text=True, check=True,
+ ).stdout.strip()
+ assert actual == "exited", "target container remained running"
+ messages = history(client, bob, chat)
+ seen = {message["id"] for message in messages}
+ assert confirmed and read_count, "traffic did not continue"
+ assert {message["id"] for message in confirmed}.issubset(seen), "confirmed write lost"
+ assert first_write is not None and first_write <= 10, "write recovery exceeded 10 seconds"
+ assert first_read is not None and first_read <= 10, "read recovery exceeded 10 seconds"
+ elapsed = time.monotonic() - started
+ print(json.dumps({
+ "action": "stop one selected API replica", "target_container": target_name,
+ "target_container_id": target, "target_state_after_hook": actual,
+ "elapsed_seconds": round(elapsed, 3), "stop_hook_seconds": round(stop_elapsed, 3),
+ "successful_reads": len(read_count), "failed_reads": read_errors,
+ "successful_writes": len(confirmed), "failed_write_attempts": write_errors,
+ "successful_rps": round((len(read_count) + len(confirmed)) / elapsed, 3),
+ "first_read_after_action_seconds": round(first_read, 3),
+ "first_write_after_action_seconds": round(first_write, 3),
+ "confirmed_messages_read_back": True,
+ }, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(add_help=False)
+ parser.add_argument("--rolling", action="store_true")
+ parser.add_argument("--failover-index", type=int, choices=(1, 2))
+ parser.add_argument("--base-url", default="http://127.0.0.1:8080")
+ args, remaining = parser.parse_known_args()
+ if args.rolling:
+ rolling(args.base_url)
+ elif args.failover_index:
+ failover(args.base_url, args.failover_index)
+ else:
+ unittest.main(argv=[sys.argv[0], *remaining])