From 750e56d0c61fcb8ef602661006f256423a6e6589 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=A1=D0=B0=D0=BC=D0=BE=D1=85=D0=B2=D0=B0=D0=BB=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=90=D0=BD=D0=B4=D1=80=D0=B5=D0=B9=20=D0=A0=D0=B5?= =?UTF-8?q?=D0=BC=D0=BE=D0=B2=D0=B8=D1=87?= Date: Fri, 2 Oct 2026 13:59:18 +0300 Subject: [PATCH] Measure scaling cost and confirm two-replica operating point --- REPORT.md | 38 ++- evidence/scale/baseline.jsonl | 54 ++++ evidence/scale/confirmation-4-to-2.jsonl | 6 + evidence/scale/fixed-p95.svg | 33 +++ evidence/scale/fixed-rps.svg | 33 +++ evidence/scale/growth-p95.svg | 33 +++ evidence/scale/growth-rps.svg | 33 +++ evidence/scale/summary.csv | 19 ++ scripts/benchmark.py | 322 +++++++++++++++++++++++ scripts/benchmark.sh | 4 + scripts/plot_scale.py | 86 ++++++ src/api.py | 11 +- student-tests/test_benchmark_profile.py | 72 +++++ 13 files changed, 739 insertions(+), 5 deletions(-) create mode 100644 evidence/scale/baseline.jsonl create mode 100644 evidence/scale/confirmation-4-to-2.jsonl create mode 100644 evidence/scale/fixed-p95.svg create mode 100644 evidence/scale/fixed-rps.svg create mode 100644 evidence/scale/growth-p95.svg create mode 100644 evidence/scale/growth-rps.svg create mode 100644 evidence/scale/summary.csv create mode 100644 scripts/benchmark.py create mode 100755 scripts/benchmark.sh create mode 100644 scripts/plot_scale.py create mode 100644 student-tests/test_benchmark_profile.py diff --git a/REPORT.md b/REPORT.md index 55d9252..8e17dad 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 3 - Автор: Самохвалов Андрей Ремович, БПИ-2302. -- Цель: оценка 5, трек 1 (цена масштабирования). Уровень 4 проверен ниже; исследование трека 1 выполняется отдельно. +- Цель: оценка 5, трек 1 (цена масштабирования). Уровень 4 и выбранный трек проверены ниже. - Ветка: `solution/bpi2302-samohvalov`; commit/PR после публикации. - Источники: `README.md`, `COURSE.md`, `contracts/`, публичные тесты репозитория, [HAProxy 3.0 configuration](https://docs.haproxy.org/3.0/configuration.html), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/). Код и текст подготовлены с помощью Codex; результаты ниже получены реальным запуском. @@ -81,6 +81,38 @@ python3 student-tests/test_operations.py --rolling --base-url http://127.0.0.1:8 Это **поочерёдная замена той же версии образа**: ни совместимость двух разных бинарных версий, ни миграция схемы во время нагрузки не проверялись. `migrate` выполняется до старта API, а скрипт замены запускает только API без повторной миграции. Для будущей смены схемы нужна отдельная совместимая миграция; данный опыт её не доказывает. -## Ограничения и дальнейшие измерения +## Уровень 5: цена масштабирования -Стенд имеет один Docker-host, один LB и один PostgreSQL: отказ узла или БД не переживает. `depends_on` гарантирует порядок запуска, а маршрутизацию выполняет HAProxy. Сценарии U4 показывают работоспособность и ошибки во время замены, но не предельную RPS при 1/2/4 репликах; отдельный эксперимент трека 1 будет добавлен для уровня 5. Генератор `tests/load.py` замкнутый и не устраняет coordinated omission. Текущий локальный HTTP разрешён только для изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS. +**Гипотеза.** При фиксированном суммарном лимите API рост с двух до четырёх процессов не даст линейного выигрыша: каждому останется только 0.25 CPU, а конкуренция за пул и планировщик увеличит задержку. Если процессам разрешить суммарно больше CPU, пропускная способность вырастет, но на этой машине рост должен замедлиться около её двух виртуальных CPU. + +**Профиль и воспроизведение.** Измерения выполнены 2026-10-02 на том же macOS arm64 / Colima Docker-host с 2 CPU и 2.05 GB RAM, Docker Engine 29.5.2, Compose 5.3.1, образами и Python из раздела запуска. PostgreSQL оставался при 1 CPU/512 MiB, HAProxy при 0.5 CPU/128 MiB; pool каждой реплики ограничен восемью соединениями. Использован один синтетический чат с 1000 заранее записанными сообщениями; общий счётчик БД во время матрицы оставался 30969. Само измерение только читает последнюю страницу `/messages?limit=20` через LB с действующим Bearer. Файл `secrets/scale_fixture.json` хранит локальный тестовый токен и не включён в Git. Набор сообщений и запрос не менялись между точками и контрольным сравнением. + +`scripts/benchmark.sh` использует Python stdlib и задаёт *открытый* график поступления GET по монотонным часам: 64 потока, очередь максимум 256, HTTP keep-alive, timeout 4 с. У каждой точки 2 с прогрева и три независимых повтора по 8 с; заданная интенсивность 250/750/1500 запросов/с. Задержка успеха считается от запланированного момента поступления до полного ответа, включая ожидание в очереди; p50/p95/p99 относятся только к успешным ответам. Неуспешные поступления учитывают как HTTP/transport ошибки, так и переполнение очереди отдельно. Во всей матрице `errors_by_type` пуст, поэтому все потери на 1500 запросах/с — **отброшенные генератором поступления**, а не ответы API 5xx. Это доля необслуженных поступлений при заданном ограничении очереди, не точный максимум сервера. Показатель CPU самого генератора не превысил 0.115 с процессорного времени на 1 с наблюдения. + +Для чистого clone после настройки `.env` и secrets из раздела запуска (по умолчанию порт 8080): + +```sh +./scripts/benchmark.sh --base-url http://127.0.0.1:8080 --rates 250,750,1500 --duration 8 --warmup 2 --output evidence/scale/baseline.jsonl +python3 scripts/plot_scale.py evidence/scale/baseline.jsonl +./scripts/benchmark.sh --base-url http://127.0.0.1:8080 --modes fixed --replicas 4,2 --rates 1500 --duration 8 --warmup 2 --output evidence/scale/confirmation-4-to-2.jsonl +docker compose --env-file .env down +``` + +Скрипт отказывается дописывать существующий output: для нового опыта укажите новое имя файла. Он сам поднимает Compose с временным override лимитов и меняет число реплик. В локальном опыте использован порт 8083, так как 8080 уже занят другим учебным стендом. Матрица `fixed` делила общий лимит API 1 CPU/512 MiB между 1/2/4 репликами (соответственно 1/512, 0.5/256, 0.25/128 на процесс). Серия `growth` оставляла каждому процессу 0.5 CPU/256 MiB, поэтому сумма росла от 0.5/256 до 2 CPU/1024 MiB. Docker-host при этом **не** получил дополнительных физических ресурсов; 2 CPU у четырёх API конкурировали с БД и LB в VM на 2 CPU. Конфигурация override и фактические квоты зафиксированы в каждой JSONL-строке. + +Полные 54 результата с индивидуальными CPU/RAM API, PostgreSQL и LB, PostgreSQL counters/размером, pool waits, длительностью, RPS, ошибками и p50/p95/p99: [`evidence/scale/baseline.jsonl`](evidence/scale/baseline.jsonl). Медианы трёх повторов всех 18 точек: [`summary.csv`](evidence/scale/summary.csv). Графики медианных RPS и p95: [`fixed-rps.svg`](evidence/scale/fixed-rps.svg), [`fixed-p95.svg`](evidence/scale/fixed-p95.svg), [`growth-rps.svg`](evidence/scale/growth-rps.svg), [`growth-p95.svg`](evidence/scale/growth-p95.svg). + +| Серия, API | Успешные RPS при 750 | Успешные RPS при 1500 | Необслуженные при 1500 | p50 / p95 / p99 при 1500, мс | API / DB / LB CPU при 1500 | +| --- | ---: | ---: | ---: | ---: | ---: | +| fixed, 1 | 747 | 780 | 42.85% | 379 / 467 / 725 | 96 / 13 / 4% | +| fixed, 2 | 750 | 957 | 33.88% | 315 / 468 / 516 | 99 / 14 / 5% | +| fixed, 4 | 733 | 694 | 50.82% | 414 / 748 / 851 | 87 / 14 / 4% | +| growth, 1 | 362 | 385 | 72.09% | 797 / 1018 / 1070 | 51 / 6 / 2% | +| growth, 2 | 750 | 938 | 35.12% | 321 / 494 / 548 | 98 / 14 / 5% | +| growth, 4 | 747 | 1128 | 20.58% | 231 / 457 / 547 | 143 / 20 / 7% | + +**Насыщение и улучшение.** При суммарном 1 CPU рост заданной интенсивности с 750 до 1500 почти не увеличивает успешный RPS у одной реплики, а у четырёх уменьшает его; p95 и отброшенные поступления резко растут. API-группа использует около 87–99% одного CPU, PostgreSQL 13–14%, LB 4–5%; в этом профиле узкое место — CPU квота API и сопутствующее ожидание пула, **не** достигнутый предел CPU БД или LB. Например, `pool_wait_ms` у fixed/1 вырос с 51465 при 750 до 182073 при 1500; это суммарное время ожидания всех запросов за окно, не длительность отдельного запроса. Добавление ресурсов до growth/4 увеличило успешный RPS, но не линейно: API заняли 143% CPU при двух доступных VM CPU, DB 20%, LB 7%, а очередь всё ещё отбрасывала 20.58% поступлений. + +Измеримое эксплуатационное изменение — **выбрать 2 API вместо 4 при том же общем 1 CPU/512 MiB**. Отдельный контроль после возврата к неизменённому коду и тому же чату, 3 повтора сначала с 4, затем с 2 репликами по 8 с при 1500 заданных GET/с: [`confirmation-4-to-2.jsonl`](evidence/scale/confirmation-4-to-2.jsonl). В целевом чате по-прежнему 1000 сообщений; общий счётчик БД стал 30970 из-за одного независимого синтетического сообщения в проверке ACL между сериями. Медиана успешного RPS 856.033 → 952.317 (**+11.2%**), необслуженная доля 39.18% → 33.94% (−5.24 п.п.), p95 605.467 → 489.999 мс (−19.1%). Все три RPS с 2 API (940.679–972.715) выше всех трёх с 4 API (805.135–895.216). В `compose.yaml` по умолчанию уже стоят 2 API; для применения настройки после опыта с четырьмя: `docker compose --env-file .env up -d --wait --scale api=2`. Проверка публичного API после контроля — `make test BASE_URL=http://127.0.0.1:8083`, PASS. Это настройка топологии, а не ускорение SQL или исходного кода. + +**Пределы вывода.** Открытый график устраняет coordinated omission замкнутого `tests/load.py`, но конечные 64 потока, очередь 256 и локальный клиент влияют на измеренную границу. Отброшенные задания не дошли до HAProxy, а HTTP 5xx в серии не наблюдались. Каждый результат описывает этот GET-профиль на одном Docker-host, не запись сообщений и не универсальную ёмкость API. Кратковременные эффекты CPU throttling и фоновая нагрузка объясняют разброс между первой матрицей и контрольной серией; поэтому эффект выбора 2 API основан на отдельном сравнении с неперекрывающимися тремя повторами. Стенд имеет один LB и один PostgreSQL, не переживает потерю хоста/БД. Текущий локальный HTTP разрешён только для изолированного стенда с синтетическими данными; удалённая демонстрация требует TLS. diff --git a/evidence/scale/baseline.jsonl b/evidence/scale/baseline.jsonl new file mode 100644 index 0000000..f44615e --- /dev/null +++ b/evidence/scale/baseline.jsonl @@ -0,0 +1,54 @@ +{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.95, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.831, "p95": 37.915, "p99": 211.151}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.384, "docker": {"api": {"avg_cpu_percent": 39.223, "max_memory_mib": 46.79}, "postgres": {"avg_cpu_percent": 6.993, "max_memory_mib": 53.62}, "lb": {"avg_cpu_percent": 2.253, "max_memory_mib": 41.58}}, "pool": {"observed_replicas": 1, "requests_queued": 75, "wait_ms": 600, "max_size": 8}, "postgres_delta": {"commits": 4018, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 65527, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.951, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.787, "p95": 3.429, "p99": 5.526}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.355, "docker": {"api": {"avg_cpu_percent": 26.653, "max_memory_mib": 47.33}, "postgres": {"avg_cpu_percent": 5.803, "max_memory_mib": 49.23}, "lb": {"avg_cpu_percent": 1.86, "max_memory_mib": 41.47}}, "pool": {"observed_replicas": 1, "requests_queued": 0, "wait_ms": 0, "max_size": 8}, "postgres_delta": {"commits": 4016, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61557, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.971, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.772, "p95": 3.703, "p99": 5.882}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.353, "docker": {"api": {"avg_cpu_percent": 29.56, "max_memory_mib": 45.91}, "postgres": {"avg_cpu_percent": 5.72, "max_memory_mib": 49.26}, "lb": {"avg_cpu_percent": 1.75, "max_memory_mib": 41.62}}, "pool": {"observed_replicas": 1, "requests_queued": 0, "wait_ms": 0, "max_size": 8}, "postgres_delta": {"commits": 4022, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61093, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 746.986, "elapsed_seconds": 8.032, "successful_latency_ms": {"p50": 4.909, "p95": 121.369, "p99": 137.788}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.707, "docker": {"api": {"avg_cpu_percent": 88.997, "max_memory_mib": 51.91}, "postgres": {"avg_cpu_percent": 12.417, "max_memory_mib": 49.31}, "lb": {"avg_cpu_percent": 4.683, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 1, "requests_queued": 4570, "wait_ms": 55096, "max_size": 8}, "postgres_delta": {"commits": 12605, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 191767, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.783, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.679, "p95": 85.479, "p99": 125.898}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.636, "docker": {"api": {"avg_cpu_percent": 60.08, "max_memory_mib": 52.52}, "postgres": {"avg_cpu_percent": 10.823, "max_memory_mib": 49.39}, "lb": {"avg_cpu_percent": 3.9, "max_memory_mib": 41.6}}, "pool": {"observed_replicas": 1, "requests_queued": 1631, "wait_ms": 17243, "max_size": 8}, "postgres_delta": {"commits": 11926, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 181241, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 746.631, "elapsed_seconds": 8.036, "successful_latency_ms": {"p50": 3.533, "p95": 103.334, "p99": 138.015}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.695, "docker": {"api": {"avg_cpu_percent": 84.993, "max_memory_mib": 52.01}, "postgres": {"avg_cpu_percent": 12.933, "max_memory_mib": 49.41}, "lb": {"avg_cpu_percent": 4.407, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 4243, "wait_ms": 51465, "max_size": 8}, "postgres_delta": {"commits": 11896, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 178481, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6890, "failed_requests": 5110, "error_ratio": 0.425833, "successful_rps": 823.026, "elapsed_seconds": 8.372, "successful_latency_ms": {"p50": 378.91, "p95": 447.591, "p99": 470.365}, "errors_by_type": {}, "queue_dropped": 5110, "client_cpu_seconds": 0.72, "docker": {"api": {"avg_cpu_percent": 97.463, "max_memory_mib": 64.79}, "postgres": {"avg_cpu_percent": 12.573, "max_memory_mib": 49.45}, "lb": {"avg_cpu_percent": 4.42, "max_memory_mib": 41.61}}, "pool": {"observed_replicas": 1, "requests_queued": 12335, "wait_ms": 188633, "max_size": 8}, "postgres_delta": {"commits": 14324, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 215405, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6858, "failed_requests": 5142, "error_ratio": 0.4285, "successful_rps": 780.299, "elapsed_seconds": 8.789, "successful_latency_ms": {"p50": 372.063, "p95": 466.704, "p99": 1071.218}, "errors_by_type": {}, "queue_dropped": 5142, "client_cpu_seconds": 0.757, "docker": {"api": {"avg_cpu_percent": 96.24, "max_memory_mib": 53.53}, "postgres": {"avg_cpu_percent": 13.42, "max_memory_mib": 49.68}, "lb": {"avg_cpu_percent": 4.423, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 1, "requests_queued": 11597, "wait_ms": 182073, "max_size": 8}, "postgres_delta": {"commits": 14394, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 216827, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 1, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 1.0, "api_memory_mib_each": 512, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6445, "failed_requests": 5555, "error_ratio": 0.462917, "successful_rps": 766.917, "elapsed_seconds": 8.404, "successful_latency_ms": {"p50": 382.659, "p95": 600.998, "p99": 725.297}, "errors_by_type": {}, "queue_dropped": 5555, "client_cpu_seconds": 0.75, "docker": {"api": {"avg_cpu_percent": 96.137, "max_memory_mib": 53.47}, "postgres": {"avg_cpu_percent": 15.257, "max_memory_mib": 49.48}, "lb": {"avg_cpu_percent": 4.627, "max_memory_mib": 41.67}}, "pool": {"observed_replicas": 1, "requests_queued": 11447, "wait_ms": 180076, "max_size": 8}, "postgres_delta": {"commits": 13337, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 200363, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.96, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.769, "p95": 13.026, "p99": 51.133}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.342, "docker": {"api": {"avg_cpu_percent": 38.633, "max_memory_mib": 85.02}, "postgres": {"avg_cpu_percent": 5.927, "max_memory_mib": 62.68}, "lb": {"avg_cpu_percent": 1.71, "max_memory_mib": 41.82}}, "pool": {"observed_replicas": 2, "requests_queued": 5, "wait_ms": 4, "max_size": 8}, "postgres_delta": {"commits": 4198, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 71471, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.959, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.875, "p95": 33.197, "p99": 100.083}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.356, "docker": {"api": {"avg_cpu_percent": 43.583, "max_memory_mib": 112.32}, "postgres": {"avg_cpu_percent": 5.957, "max_memory_mib": 64.63}, "lb": {"avg_cpu_percent": 1.9, "max_memory_mib": 41.56}}, "pool": {"observed_replicas": 2, "requests_queued": 3, "wait_ms": 14, "max_size": 8}, "postgres_delta": {"commits": 4072, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61681, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.937, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.873, "p95": 25.949, "p99": 138.238}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.372, "docker": {"api": {"avg_cpu_percent": 42.983, "max_memory_mib": 102.69}, "postgres": {"avg_cpu_percent": 6.063, "max_memory_mib": 64.66}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.64}}, "pool": {"observed_replicas": 2, "requests_queued": 31, "wait_ms": 78, "max_size": 8}, "postgres_delta": {"commits": 3777, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 57121, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.804, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.755, "p95": 108.359, "p99": 175.477}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.623, "docker": {"api": {"avg_cpu_percent": 69.88, "max_memory_mib": 108.39}, "postgres": {"avg_cpu_percent": 12.08, "max_memory_mib": 64.74}, "lb": {"avg_cpu_percent": 3.88, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 2, "requests_queued": 1294, "wait_ms": 7867, "max_size": 8}, "postgres_delta": {"commits": 12232, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 184771, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.772, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 1.731, "p95": 100.322, "p99": 162.856}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.61, "docker": {"api": {"avg_cpu_percent": 70.053, "max_memory_mib": 96.87}, "postgres": {"avg_cpu_percent": 12.173, "max_memory_mib": 65.0}, "lb": {"avg_cpu_percent": 3.93, "max_memory_mib": 41.97}}, "pool": {"observed_replicas": 2, "requests_queued": 1118, "wait_ms": 5573, "max_size": 8}, "postgres_delta": {"commits": 12156, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183345, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.734, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 1.816, "p95": 178.504, "p99": 224.107}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.627, "docker": {"api": {"avg_cpu_percent": 68.573, "max_memory_mib": 97.41}, "postgres": {"avg_cpu_percent": 12.37, "max_memory_mib": 64.76}, "lb": {"avg_cpu_percent": 3.767, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 2, "requests_queued": 1324, "wait_ms": 11493, "max_size": 8}, "postgres_delta": {"commits": 12142, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183157, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7863, "failed_requests": 4137, "error_ratio": 0.34475, "successful_rps": 941.177, "elapsed_seconds": 8.354, "successful_latency_ms": {"p50": 319.914, "p95": 473.58, "p99": 530.577}, "errors_by_type": {}, "queue_dropped": 4137, "client_cpu_seconds": 0.785, "docker": {"api": {"avg_cpu_percent": 98.37, "max_memory_mib": 109.51}, "postgres": {"avg_cpu_percent": 14.65, "max_memory_mib": 64.77}, "lb": {"avg_cpu_percent": 5.09, "max_memory_mib": 42.0}}, "pool": {"observed_replicas": 2, "requests_queued": 6822, "wait_ms": 77320, "max_size": 8}, "postgres_delta": {"commits": 14978, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 225169, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 8293, "failed_requests": 3707, "error_ratio": 0.308917, "successful_rps": 993.827, "elapsed_seconds": 8.345, "successful_latency_ms": {"p50": 297.841, "p95": 454.544, "p99": 505.25}, "errors_by_type": {}, "queue_dropped": 3707, "client_cpu_seconds": 0.762, "docker": {"api": {"avg_cpu_percent": 100.243, "max_memory_mib": 113.26}, "postgres": {"avg_cpu_percent": 14.147, "max_memory_mib": 65.02}, "lb": {"avg_cpu_percent": 4.87, "max_memory_mib": 42.0}}, "pool": {"observed_replicas": 2, "requests_queued": 5427, "wait_ms": 49740, "max_size": 8}, "postgres_delta": {"commits": 15964, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 240127, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7934, "failed_requests": 4066, "error_ratio": 0.338833, "successful_rps": 956.631, "elapsed_seconds": 8.294, "successful_latency_ms": {"p50": 314.99, "p95": 467.748, "p99": 516.41}, "errors_by_type": {}, "queue_dropped": 4066, "client_cpu_seconds": 0.757, "docker": {"api": {"avg_cpu_percent": 98.82, "max_memory_mib": 122.91}, "postgres": {"avg_cpu_percent": 13.203, "max_memory_mib": 64.77}, "lb": {"avg_cpu_percent": 4.807, "max_memory_mib": 41.96}}, "pool": {"observed_replicas": 2, "requests_queued": 5707, "wait_ms": 72376, "max_size": 8}, "postgres_delta": {"commits": 16857, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 253657, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.958, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 3.016, "p95": 61.385, "p99": 81.718}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.313, "docker": {"api": {"avg_cpu_percent": 64.537, "max_memory_mib": 205.98}, "postgres": {"avg_cpu_percent": 5.16, "max_memory_mib": 79.56}, "lb": {"avg_cpu_percent": 1.72, "max_memory_mib": 41.74}}, "pool": {"observed_replicas": 4, "requests_queued": 4, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4223, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 76658, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.944, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.944, "p95": 59.768, "p99": 80.401}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.323, "docker": {"api": {"avg_cpu_percent": 60.467, "max_memory_mib": 205.43}, "postgres": {"avg_cpu_percent": 5.073, "max_memory_mib": 79.71}, "lb": {"avg_cpu_percent": 1.657, "max_memory_mib": 41.74}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4020, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 60455, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.971, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 3.021, "p95": 60.482, "p99": 79.776}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.316, "docker": {"api": {"avg_cpu_percent": 51.723, "max_memory_mib": 203.98}, "postgres": {"avg_cpu_percent": 5.303, "max_memory_mib": 79.81}, "lb": {"avg_cpu_percent": 1.68, "max_memory_mib": 41.55}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 6}, "postgres_delta": {"commits": 4101, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61639, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 744.222, "elapsed_seconds": 8.062, "successful_latency_ms": {"p50": 3.319, "p95": 308.301, "p99": 415.861}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.613, "docker": {"api": {"avg_cpu_percent": 83.097, "max_memory_mib": 205.59}, "postgres": {"avg_cpu_percent": 13.187, "max_memory_mib": 95.02}, "lb": {"avg_cpu_percent": 3.927, "max_memory_mib": 42.05}}, "pool": {"observed_replicas": 4, "requests_queued": 952, "wait_ms": 12771, "max_size": 8}, "postgres_delta": {"commits": 11710, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 181824, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 713.504, "elapsed_seconds": 8.409, "successful_latency_ms": {"p50": 126.994, "p95": 414.136, "p99": 480.195}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.582, "docker": {"api": {"avg_cpu_percent": 92.41, "max_memory_mib": 207.79}, "postgres": {"avg_cpu_percent": 12.73, "max_memory_mib": 95.05}, "lb": {"avg_cpu_percent": 3.723, "max_memory_mib": 42.23}}, "pool": {"observed_replicas": 4, "requests_queued": 2167, "wait_ms": 26550, "max_size": 8}, "postgres_delta": {"commits": 12191, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183838, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 732.997, "elapsed_seconds": 8.186, "successful_latency_ms": {"p50": 110.445, "p95": 397.681, "p99": 486.245}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.585, "docker": {"api": {"avg_cpu_percent": 93.953, "max_memory_mib": 217.25}, "postgres": {"avg_cpu_percent": 12.887, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 3.673, "max_memory_mib": 42.28}}, "pool": {"observed_replicas": 4, "requests_queued": 1901, "wait_ms": 24258, "max_size": 8}, "postgres_delta": {"commits": 11844, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 178211, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6950, "failed_requests": 5050, "error_ratio": 0.420833, "successful_rps": 811.023, "elapsed_seconds": 8.569, "successful_latency_ms": {"p50": 315.472, "p95": 633.293, "p99": 771.213}, "errors_by_type": {}, "queue_dropped": 5050, "client_cpu_seconds": 0.698, "docker": {"api": {"avg_cpu_percent": 98.617, "max_memory_mib": 216.33}, "postgres": {"avg_cpu_percent": 15.563, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 4.53, "max_memory_mib": 42.35}}, "pool": {"observed_replicas": 4, "requests_queued": 3048, "wait_ms": 32524, "max_size": 8}, "postgres_delta": {"commits": 14083, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 211553, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 5902, "failed_requests": 6098, "error_ratio": 0.508167, "successful_rps": 693.857, "elapsed_seconds": 8.506, "successful_latency_ms": {"p50": 413.811, "p95": 769.612, "p99": 855.228}, "errors_by_type": {}, "queue_dropped": 6098, "client_cpu_seconds": 0.674, "docker": {"api": {"avg_cpu_percent": 80.183, "max_memory_mib": 204.21}, "postgres": {"avg_cpu_percent": 11.61, "max_memory_mib": 95.16}, "lb": {"avg_cpu_percent": 3.993, "max_memory_mib": 42.35}}, "pool": {"observed_replicas": 4, "requests_queued": 2088, "wait_ms": 33271, "max_size": 8}, "postgres_delta": {"commits": 12393, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 186123, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 5756, "failed_requests": 6244, "error_ratio": 0.520333, "successful_rps": 681.037, "elapsed_seconds": 8.452, "successful_latency_ms": {"p50": 420.287, "p95": 747.7, "p99": 851.226}, "errors_by_type": {}, "queue_dropped": 6244, "client_cpu_seconds": 0.604, "docker": {"api": {"avg_cpu_percent": 87.06, "max_memory_mib": 194.64}, "postgres": {"avg_cpu_percent": 13.59, "max_memory_mib": 95.16}, "lb": {"avg_cpu_percent": 3.353, "max_memory_mib": 42.36}}, "pool": {"observed_replicas": 4, "requests_queued": 2203, "wait_ms": 38980, "max_size": 8}, "postgres_delta": {"commits": 11550, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 173489, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.946, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 2.572, "p95": 206.839, "p99": 277.387}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.316, "docker": {"api": {"avg_cpu_percent": 33.947, "max_memory_mib": 48.01}, "postgres": {"avg_cpu_percent": 6.093, "max_memory_mib": 49.68}, "lb": {"avg_cpu_percent": 1.697, "max_memory_mib": 41.68}}, "pool": {"observed_replicas": 1, "requests_queued": 463, "wait_ms": 4901, "max_size": 8}, "postgres_delta": {"commits": 4114, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 66849, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.97, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.365, "p95": 39.928, "p99": 72.726}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.262, "docker": {"api": {"avg_cpu_percent": 28.06, "max_memory_mib": 48.11}, "postgres": {"avg_cpu_percent": 5.183, "max_memory_mib": 49.45}, "lb": {"avg_cpu_percent": 1.587, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 135, "wait_ms": 196, "max_size": 8}, "postgres_delta": {"commits": 4078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61300, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.977, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.684, "p95": 59.465, "p99": 80.414}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.345, "docker": {"api": {"avg_cpu_percent": 34.943, "max_memory_mib": 47.95}, "postgres": {"avg_cpu_percent": 6.29, "max_memory_mib": 49.47}, "lb": {"avg_cpu_percent": 2.08, "max_memory_mib": 41.66}}, "pool": {"observed_replicas": 1, "requests_queued": 210, "wait_ms": 410, "max_size": 8}, "postgres_delta": {"commits": 3956, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 60059, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3164, "failed_requests": 2836, "error_ratio": 0.472667, "successful_rps": 362.33, "elapsed_seconds": 8.732, "successful_latency_ms": {"p50": 823.553, "p95": 1071.351, "p99": 1182.579}, "errors_by_type": {}, "queue_dropped": 2836, "client_cpu_seconds": 0.398, "docker": {"api": {"avg_cpu_percent": 50.06, "max_memory_mib": 64.7}, "postgres": {"avg_cpu_percent": 6.51, "max_memory_mib": 49.75}, "lb": {"avg_cpu_percent": 2.317, "max_memory_mib": 41.69}}, "pool": {"observed_replicas": 1, "requests_queued": 5298, "wait_ms": 184862, "max_size": 8}, "postgres_delta": {"commits": 5828, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 88215, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3140, "failed_requests": 2860, "error_ratio": 0.476667, "successful_rps": 347.987, "elapsed_seconds": 9.023, "successful_latency_ms": {"p50": 809.855, "p95": 1190.388, "p99": 1238.353}, "errors_by_type": {}, "queue_dropped": 2860, "client_cpu_seconds": 0.393, "docker": {"api": {"avg_cpu_percent": 50.673, "max_memory_mib": 64.82}, "postgres": {"avg_cpu_percent": 7.463, "max_memory_mib": 50.64}, "lb": {"avg_cpu_percent": 2.083, "max_memory_mib": 41.68}}, "pool": {"observed_replicas": 1, "requests_queued": 5266, "wait_ms": 191351, "max_size": 8}, "postgres_delta": {"commits": 6465, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 97643, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 3410, "failed_requests": 2590, "error_ratio": 0.431667, "successful_rps": 391.052, "elapsed_seconds": 8.72, "successful_latency_ms": {"p50": 795.636, "p95": 976.194, "p99": 988.076}, "errors_by_type": {}, "queue_dropped": 2590, "client_cpu_seconds": 0.38, "docker": {"api": {"avg_cpu_percent": 50.493, "max_memory_mib": 61.84}, "postgres": {"avg_cpu_percent": 6.57, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.033, "max_memory_mib": 41.93}}, "pool": {"observed_replicas": 1, "requests_queued": 5799, "wait_ms": 185658, "max_size": 8}, "postgres_delta": {"commits": 7011, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 105675, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3475, "failed_requests": 8525, "error_ratio": 0.710417, "successful_rps": 399.011, "elapsed_seconds": 8.709, "successful_latency_ms": {"p50": 795.926, "p95": 918.481, "p99": 980.371}, "errors_by_type": {}, "queue_dropped": 8525, "client_cpu_seconds": 0.419, "docker": {"api": {"avg_cpu_percent": 50.17, "max_memory_mib": 65.2}, "postgres": {"avg_cpu_percent": 6.02, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 1.923, "max_memory_mib": 41.7}}, "pool": {"observed_replicas": 1, "requests_queued": 6018, "wait_ms": 195259, "max_size": 8}, "postgres_delta": {"commits": 6592, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 99497, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3152, "failed_requests": 8848, "error_ratio": 0.737333, "successful_rps": 354.029, "elapsed_seconds": 8.903, "successful_latency_ms": {"p50": 878.869, "p95": 1018.139, "p99": 1070.497}, "errors_by_type": {}, "queue_dropped": 8848, "client_cpu_seconds": 0.44, "docker": {"api": {"avg_cpu_percent": 51.427, "max_memory_mib": 52.96}, "postgres": {"avg_cpu_percent": 6.83, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.257, "max_memory_mib": 41.69}}, "pool": {"observed_replicas": 1, "requests_queued": 5362, "wait_ms": 190945, "max_size": 8}, "postgres_delta": {"commits": 6204, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 93635, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 1, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 3349, "failed_requests": 8651, "error_ratio": 0.720917, "successful_rps": 384.74, "elapsed_seconds": 8.705, "successful_latency_ms": {"p50": 797.336, "p95": 1034.655, "p99": 1084.912}, "errors_by_type": {}, "queue_dropped": 8651, "client_cpu_seconds": 0.442, "docker": {"api": {"avg_cpu_percent": 50.96, "max_memory_mib": 64.4}, "postgres": {"avg_cpu_percent": 5.623, "max_memory_mib": 49.52}, "lb": {"avg_cpu_percent": 2.29, "max_memory_mib": 41.71}}, "pool": {"observed_replicas": 1, "requests_queued": 5539, "wait_ms": 185287, "max_size": 8}, "postgres_delta": {"commits": 6874, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 103703, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.98, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.87, "p95": 10.076, "p99": 50.451}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.319, "docker": {"api": {"avg_cpu_percent": 34.92, "max_memory_mib": 85.09}, "postgres": {"avg_cpu_percent": 6.023, "max_memory_mib": 58.95}, "lb": {"avg_cpu_percent": 1.613, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 6, "wait_ms": 1, "max_size": 7}, "postgres_delta": {"commits": 4102, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 67511, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.97, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.936, "p95": 55.026, "p99": 129.173}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.398, "docker": {"api": {"avg_cpu_percent": 41.043, "max_memory_mib": 96.67}, "postgres": {"avg_cpu_percent": 8.053, "max_memory_mib": 64.64}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.7}}, "pool": {"observed_replicas": 2, "requests_queued": 15, "wait_ms": 308, "max_size": 8}, "postgres_delta": {"commits": 4081, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61880, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.982, "elapsed_seconds": 8.001, "successful_latency_ms": {"p50": 2.886, "p95": 15.37, "p99": 54.353}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.341, "docker": {"api": {"avg_cpu_percent": 38.787, "max_memory_mib": 98.52}, "postgres": {"avg_cpu_percent": 5.4, "max_memory_mib": 64.68}, "lb": {"avg_cpu_percent": 1.707, "max_memory_mib": 41.59}}, "pool": {"observed_replicas": 2, "requests_queued": 1, "wait_ms": 2, "max_size": 8}, "postgres_delta": {"commits": 4079, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61581, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 5903, "failed_requests": 97, "error_ratio": 0.016167, "successful_rps": 728.191, "elapsed_seconds": 8.106, "successful_latency_ms": {"p50": 2.148, "p95": 468.108, "p99": 599.437}, "errors_by_type": {}, "queue_dropped": 97, "client_cpu_seconds": 0.647, "docker": {"api": {"avg_cpu_percent": 79.57, "max_memory_mib": 107.99}, "postgres": {"avg_cpu_percent": 13.603, "max_memory_mib": 64.71}, "lb": {"avg_cpu_percent": 4.013, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 1869, "wait_ms": 27621, "max_size": 8}, "postgres_delta": {"commits": 11975, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 180645, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.669, "elapsed_seconds": 8.004, "successful_latency_ms": {"p50": 2.292, "p95": 144.278, "p99": 193.517}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.716, "docker": {"api": {"avg_cpu_percent": 92.29, "max_memory_mib": 108.7}, "postgres": {"avg_cpu_percent": 13.147, "max_memory_mib": 64.73}, "lb": {"avg_cpu_percent": 4.19, "max_memory_mib": 41.94}}, "pool": {"observed_replicas": 2, "requests_queued": 2115, "wait_ms": 15549, "max_size": 8}, "postgres_delta": {"commits": 12108, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 183127, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 749.762, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 2.224, "p95": 167.623, "p99": 254.554}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.731, "docker": {"api": {"avg_cpu_percent": 86.027, "max_memory_mib": 98.05}, "postgres": {"avg_cpu_percent": 13.06, "max_memory_mib": 64.73}, "lb": {"avg_cpu_percent": 4.49, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 1856, "wait_ms": 14945, "max_size": 8}, "postgres_delta": {"commits": 11997, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 179629, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7786, "failed_requests": 4214, "error_ratio": 0.351167, "successful_rps": 938.324, "elapsed_seconds": 8.298, "successful_latency_ms": {"p50": 321.188, "p95": 493.5, "p99": 548.275}, "errors_by_type": {}, "queue_dropped": 4214, "client_cpu_seconds": 0.739, "docker": {"api": {"avg_cpu_percent": 98.067, "max_memory_mib": 99.7}, "postgres": {"avg_cpu_percent": 14.437, "max_memory_mib": 64.74}, "lb": {"avg_cpu_percent": 4.373, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 7974, "wait_ms": 76567, "max_size": 8}, "postgres_delta": {"commits": 15558, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 233671, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7921, "failed_requests": 4079, "error_ratio": 0.339917, "successful_rps": 953.293, "elapsed_seconds": 8.309, "successful_latency_ms": {"p50": 321.964, "p95": 454.678, "p99": 509.653}, "errors_by_type": {}, "queue_dropped": 4079, "client_cpu_seconds": 0.758, "docker": {"api": {"avg_cpu_percent": 100.757, "max_memory_mib": 100.03}, "postgres": {"avg_cpu_percent": 15.21, "max_memory_mib": 64.75}, "lb": {"avg_cpu_percent": 4.527, "max_memory_mib": 42.23}}, "pool": {"observed_replicas": 2, "requests_queued": 9150, "wait_ms": 89177, "max_size": 8}, "postgres_delta": {"commits": 16078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 241647, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7486, "failed_requests": 4514, "error_ratio": 0.376167, "successful_rps": 900.488, "elapsed_seconds": 8.313, "successful_latency_ms": {"p50": 321.365, "p95": 527.564, "p99": 574.568}, "errors_by_type": {}, "queue_dropped": 4514, "client_cpu_seconds": 0.76, "docker": {"api": {"avg_cpu_percent": 96.313, "max_memory_mib": 114.13}, "postgres": {"avg_cpu_percent": 13.923, "max_memory_mib": 64.75}, "lb": {"avg_cpu_percent": 4.857, "max_memory_mib": 41.98}}, "pool": {"observed_replicas": 2, "requests_queued": 7416, "wait_ms": 85412, "max_size": 8}, "postgres_delta": {"commits": 15026, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 226057, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.9, "elapsed_seconds": 8.003, "successful_latency_ms": {"p50": 2.847, "p95": 17.892, "p99": 116.852}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.335, "docker": {"api": {"avg_cpu_percent": 73.133, "max_memory_mib": 165.93}, "postgres": {"avg_cpu_percent": 5.717, "max_memory_mib": 70.38}, "lb": {"avg_cpu_percent": 1.697, "max_memory_mib": 41.96}}, "pool": {"observed_replicas": 4, "requests_queued": 45, "wait_ms": 237, "max_size": 5}, "postgres_delta": {"commits": 4090, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 71220, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.719, "elapsed_seconds": 8.009, "successful_latency_ms": {"p50": 2.907, "p95": 9.687, "p99": 21.273}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.328, "docker": {"api": {"avg_cpu_percent": 49.73, "max_memory_mib": 191.97}, "postgres": {"avg_cpu_percent": 5.503, "max_memory_mib": 70.47}, "lb": {"avg_cpu_percent": 1.753, "max_memory_mib": 41.91}}, "pool": {"observed_replicas": 4, "requests_queued": 0, "wait_ms": 0, "max_size": 5}, "postgres_delta": {"commits": 4110, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 61812, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 250, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 250, "scheduled_requests": 2000, "successful_requests": 2000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 249.939, "elapsed_seconds": 8.002, "successful_latency_ms": {"p50": 3.098, "p95": 17.675, "p99": 76.487}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.402, "docker": {"api": {"avg_cpu_percent": 74.683, "max_memory_mib": 177.49}, "postgres": {"avg_cpu_percent": 7.597, "max_memory_mib": 72.36}, "lb": {"avg_cpu_percent": 1.89, "max_memory_mib": 41.57}}, "pool": {"observed_replicas": 4, "requests_queued": 1, "wait_ms": 22, "max_size": 5}, "postgres_delta": {"commits": 4078, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 62186, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 747.495, "elapsed_seconds": 8.027, "successful_latency_ms": {"p50": 4.456, "p95": 312.888, "p99": 369.613}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.793, "docker": {"api": {"avg_cpu_percent": 109.91, "max_memory_mib": 213.65}, "postgres": {"avg_cpu_percent": 16.54, "max_memory_mib": 95.11}, "lb": {"avg_cpu_percent": 4.963, "max_memory_mib": 42.07}}, "pool": {"observed_replicas": 4, "requests_queued": 350, "wait_ms": 3249, "max_size": 8}, "postgres_delta": {"commits": 12366, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 193680, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 748.081, "elapsed_seconds": 8.021, "successful_latency_ms": {"p50": 2.363, "p95": 25.407, "p99": 53.156}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.616, "docker": {"api": {"avg_cpu_percent": 111.363, "max_memory_mib": 207.52}, "postgres": {"avg_cpu_percent": 13.387, "max_memory_mib": 95.12}, "lb": {"avg_cpu_percent": 3.747, "max_memory_mib": 41.89}}, "pool": {"observed_replicas": 4, "requests_queued": 23, "wait_ms": 57, "max_size": 8}, "postgres_delta": {"commits": 12044, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 180901, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 750, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 750, "scheduled_requests": 6000, "successful_requests": 6000, "failed_requests": 0, "error_ratio": 0.0, "successful_rps": 741.745, "elapsed_seconds": 8.089, "successful_latency_ms": {"p50": 2.595, "p95": 182.257, "p99": 375.515}, "errors_by_type": {}, "queue_dropped": 0, "client_cpu_seconds": 0.74, "docker": {"api": {"avg_cpu_percent": 117.407, "max_memory_mib": 220.91}, "postgres": {"avg_cpu_percent": 15.23, "max_memory_mib": 95.18}, "lb": {"avg_cpu_percent": 4.083, "max_memory_mib": 42.04}}, "pool": {"observed_replicas": 4, "requests_queued": 357, "wait_ms": 3544, "max_size": 8}, "postgres_delta": {"commits": 12159, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 184013, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9320, "failed_requests": 2680, "error_ratio": 0.223333, "successful_rps": 1119.586, "elapsed_seconds": 8.325, "successful_latency_ms": {"p50": 230.705, "p95": 466.032, "p99": 733.365}, "errors_by_type": {}, "queue_dropped": 2680, "client_cpu_seconds": 0.938, "docker": {"api": {"avg_cpu_percent": 142.593, "max_memory_mib": 218.29}, "postgres": {"avg_cpu_percent": 20.28, "max_memory_mib": 96.34}, "lb": {"avg_cpu_percent": 6.71, "max_memory_mib": 42.43}}, "pool": {"observed_replicas": 4, "requests_queued": 1799, "wait_ms": 17211, "max_size": 8}, "postgres_delta": {"commits": 17785, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 267049, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9886, "failed_requests": 2114, "error_ratio": 0.176167, "successful_rps": 1170.613, "elapsed_seconds": 8.445, "successful_latency_ms": {"p50": 226.066, "p95": 413.16, "p99": 489.537}, "errors_by_type": {}, "queue_dropped": 2114, "client_cpu_seconds": 0.941, "docker": {"api": {"avg_cpu_percent": 142.217, "max_memory_mib": 221.61}, "postgres": {"avg_cpu_percent": 20.977, "max_memory_mib": 95.24}, "lb": {"avg_cpu_percent": 6.13, "max_memory_mib": 42.45}}, "pool": {"observed_replicas": 4, "requests_queued": 1972, "wait_ms": 27198, "max_size": 8}, "postgres_delta": {"commits": 20148, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 302329, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} +{"mode": "growth", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 9531, "failed_requests": 2469, "error_ratio": 0.20575, "successful_rps": 1127.648, "elapsed_seconds": 8.452, "successful_latency_ms": {"p50": 246.905, "p95": 456.887, "p99": 547.176}, "errors_by_type": {}, "queue_dropped": 2469, "client_cpu_seconds": 0.97, "docker": {"api": {"avg_cpu_percent": 144.297, "max_memory_mib": 235.65}, "postgres": {"avg_cpu_percent": 19.94, "max_memory_mib": 97.48}, "lb": {"avg_cpu_percent": 6.687, "max_memory_mib": 42.46}}, "pool": {"observed_replicas": 4, "requests_queued": 1540, "wait_ms": 18841, "max_size": 8}, "postgres_delta": {"commits": 19840, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 297789, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30969} diff --git a/evidence/scale/confirmation-4-to-2.jsonl b/evidence/scale/confirmation-4-to-2.jsonl new file mode 100644 index 0000000..9bf1452 --- /dev/null +++ b/evidence/scale/confirmation-4-to-2.jsonl @@ -0,0 +1,6 @@ +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7298, "failed_requests": 4702, "error_ratio": 0.391833, "successful_rps": 856.033, "elapsed_seconds": 8.525, "successful_latency_ms": {"p50": 315.394, "p95": 605.467, "p99": 723.579}, "errors_by_type": {}, "queue_dropped": 4702, "client_cpu_seconds": 0.699, "docker": {"api": {"avg_cpu_percent": 101.373, "max_memory_mib": 230.97}, "postgres": {"avg_cpu_percent": 10.917, "max_memory_mib": 96.28}, "lb": {"avg_cpu_percent": 4.033, "max_memory_mib": 42.7}}, "pool": {"observed_replicas": 4, "requests_queued": 2961, "wait_ms": 49012, "max_size": 8}, "postgres_delta": {"commits": 14598, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 211976, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7569, "failed_requests": 4431, "error_ratio": 0.36925, "successful_rps": 895.216, "elapsed_seconds": 8.455, "successful_latency_ms": {"p50": 302.528, "p95": 551.032, "p99": 821.366}, "errors_by_type": {}, "queue_dropped": 4431, "client_cpu_seconds": 0.705, "docker": {"api": {"avg_cpu_percent": 99.517, "max_memory_mib": 205.15}, "postgres": {"avg_cpu_percent": 15.823, "max_memory_mib": 96.31}, "lb": {"avg_cpu_percent": 4.787, "max_memory_mib": 42.63}}, "pool": {"observed_replicas": 4, "requests_queued": 4193, "wait_ms": 33869, "max_size": 8}, "postgres_delta": {"commits": 14895, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 216728, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 4, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.25, "api_memory_mib_each": 128, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 6745, "failed_requests": 5255, "error_ratio": 0.437917, "successful_rps": 805.135, "elapsed_seconds": 8.377, "successful_latency_ms": {"p50": 335.741, "p95": 691.364, "p99": 835.845}, "errors_by_type": {}, "queue_dropped": 5255, "client_cpu_seconds": 0.704, "docker": {"api": {"avg_cpu_percent": 99.893, "max_memory_mib": 202.03}, "postgres": {"avg_cpu_percent": 15.907, "max_memory_mib": 96.36}, "lb": {"avg_cpu_percent": 4.783, "max_memory_mib": 42.64}}, "pool": {"observed_replicas": 4, "requests_queued": 3062, "wait_ms": 48677, "max_size": 8}, "postgres_delta": {"commits": 13421, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 194846, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 1, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7927, "failed_requests": 4073, "error_ratio": 0.339417, "successful_rps": 952.317, "elapsed_seconds": 8.324, "successful_latency_ms": {"p50": 316.048, "p95": 495.87, "p99": 540.128}, "errors_by_type": {}, "queue_dropped": 4073, "client_cpu_seconds": 0.749, "docker": {"api": {"avg_cpu_percent": 96.897, "max_memory_mib": 98.05}, "postgres": {"avg_cpu_percent": 13.917, "max_memory_mib": 66.3}, "lb": {"avg_cpu_percent": 3.933, "max_memory_mib": 42.29}}, "pool": {"observed_replicas": 2, "requests_queued": 6441, "wait_ms": 59041, "max_size": 8}, "postgres_delta": {"commits": 15327, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 222787, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 2, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 8055, "failed_requests": 3945, "error_ratio": 0.32875, "successful_rps": 972.715, "elapsed_seconds": 8.281, "successful_latency_ms": {"p50": 309.538, "p95": 465.748, "p99": 575.029}, "errors_by_type": {}, "queue_dropped": 3945, "client_cpu_seconds": 0.738, "docker": {"api": {"avg_cpu_percent": 94.727, "max_memory_mib": 100.38}, "postgres": {"avg_cpu_percent": 15.02, "max_memory_mib": 65.29}, "lb": {"avg_cpu_percent": 4.33, "max_memory_mib": 42.54}}, "pool": {"observed_replicas": 2, "requests_queued": 5546, "wait_ms": 59665, "max_size": 8}, "postgres_delta": {"commits": 17391, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 253125, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} +{"mode": "fixed", "replicas": 2, "rate": 1500, "repeat": 3, "duration": 8, "warmup": 2, "dataset_messages": 1000, "api_cpu_each": 0.5, "api_memory_mib_each": 256, "pool_max_size": 8, "offered_rate": 1500, "scheduled_requests": 12000, "successful_requests": 7862, "failed_requests": 4138, "error_ratio": 0.344833, "successful_rps": 940.679, "elapsed_seconds": 8.358, "successful_latency_ms": {"p50": 319.584, "p95": 489.999, "p99": 595.768}, "errors_by_type": {}, "queue_dropped": 4138, "client_cpu_seconds": 0.725, "docker": {"api": {"avg_cpu_percent": 99.007, "max_memory_mib": 113.54}, "postgres": {"avg_cpu_percent": 15.377, "max_memory_mib": 68.17}, "lb": {"avg_cpu_percent": 4.427, "max_memory_mib": 42.29}}, "pool": {"observed_replicas": 2, "requests_queued": 7232, "wait_ms": 85079, "max_size": 8}, "postgres_delta": {"commits": 15296, "rollbacks": 0, "blocks_read": 0, "blocks_hit": 222495, "temp_files": 0, "deadlocks": 0, "messages": 0}, "database_bytes": 25767059, "database_messages": 30970} diff --git a/evidence/scale/fixed-p95.svg b/evidence/scale/fixed-p95.svg new file mode 100644 index 0000000..2b887a4 --- /dev/null +++ b/evidence/scale/fixed-p95.svg @@ -0,0 +1,33 @@ + + +fixed: p95, мс + +0 + +206 + +411 + +617 + +822 +250 +750 +1500 + + + + +1 API + + + + +2 API + + + + +4 API +Заданная интенсивность, запросов/с + diff --git a/evidence/scale/fixed-rps.svg b/evidence/scale/fixed-rps.svg new file mode 100644 index 0000000..98ab80b --- /dev/null +++ b/evidence/scale/fixed-rps.svg @@ -0,0 +1,33 @@ + + +fixed: Успешные запросы/с + +0 + +263 + +526 + +789 + +1052 +250 +750 +1500 + + + + +1 API + + + + +2 API + + + + +4 API +Заданная интенсивность, запросов/с + diff --git a/evidence/scale/growth-p95.svg b/evidence/scale/growth-p95.svg new file mode 100644 index 0000000..0d9e327 --- /dev/null +++ b/evidence/scale/growth-p95.svg @@ -0,0 +1,33 @@ + + +growth: p95, мс + +0 + +295 + +589 + +884 + +1178 +250 +750 +1500 + + + + +1 API + + + + +2 API + + + + +4 API +Заданная интенсивность, запросов/с + diff --git a/evidence/scale/growth-rps.svg b/evidence/scale/growth-rps.svg new file mode 100644 index 0000000..153d0b2 --- /dev/null +++ b/evidence/scale/growth-rps.svg @@ -0,0 +1,33 @@ + + +growth: Успешные запросы/с + +0 + +310 + +620 + +930 + +1240 +250 +750 +1500 + + + + +1 API + + + + +2 API + + + + +4 API +Заданная интенсивность, запросов/с + diff --git a/evidence/scale/summary.csv b/evidence/scale/summary.csv new file mode 100644 index 0000000..9209371 --- /dev/null +++ b/evidence/scale/summary.csv @@ -0,0 +1,19 @@ +mode,replicas,rate,successful_rps,error_ratio,p50_ms,p95_ms,p99_ms,api_cpu_percent,postgres_cpu_percent,lb_cpu_percent,api_memory_mib,postgres_memory_mib,lb_memory_mib,pool_wait_ms,queue_dropped +fixed,1,250,249.951,0.0,2.787,3.703,5.882,29.56,5.803,1.86,46.79,49.26,41.58,0,0 +fixed,1,750,746.986,0.0,3.533,103.334,137.788,84.993,12.417,4.407,52.01,49.39,41.64,51465,0 +fixed,1,1500,780.299,0.4285,378.91,466.704,725.297,96.24,13.42,4.423,53.53,49.48,41.64,182073,5142 +fixed,2,250,249.959,0.0,2.873,25.949,100.083,42.983,5.957,1.753,102.69,64.63,41.64,14,0 +fixed,2,750,749.772,0.0,1.755,108.359,175.477,69.88,12.173,3.88,97.41,64.76,41.91,7867,0 +fixed,2,1500,956.631,0.338833,314.99,467.748,516.41,98.82,14.147,4.87,113.26,64.77,42.0,72376,4066 +fixed,4,250,249.958,0.0,3.016,60.482,80.401,60.467,5.16,1.68,205.43,79.71,41.74,0,0 +fixed,4,750,732.997,0.0,110.445,397.681,480.195,92.41,12.887,3.723,207.79,95.05,42.23,24258,0 +fixed,4,1500,693.857,0.508167,413.811,747.7,851.226,87.06,13.59,3.993,204.21,95.16,42.35,33271,6098 +growth,1,250,249.97,0.0,2.572,59.465,80.414,33.947,6.093,1.697,48.01,49.47,41.68,410,0 +growth,1,750,362.33,0.472667,809.855,1071.351,1182.579,50.493,6.57,2.083,64.7,49.75,41.69,185658,2836 +growth,1,1500,384.74,0.720917,797.336,1018.139,1070.497,50.96,6.02,2.257,64.4,49.52,41.7,190945,8651 +growth,2,250,249.98,0.0,2.886,15.37,54.353,38.787,6.023,1.707,96.67,64.64,41.7,2,0 +growth,2,750,749.669,0.0,2.224,167.623,254.554,86.027,13.147,4.19,107.99,64.73,41.94,15549,0 +growth,2,1500,938.324,0.351167,321.365,493.5,548.275,98.067,14.437,4.527,100.03,64.75,41.98,85412,4214 +growth,4,250,249.9,0.0,2.907,17.675,76.487,73.133,5.717,1.753,177.49,70.47,41.91,22,0 +growth,4,750,747.495,0.0,2.595,182.257,369.613,111.363,15.23,4.083,213.65,95.12,42.04,3249,0 +growth,4,1500,1127.648,0.20575,230.705,456.887,547.176,142.593,20.28,6.687,221.61,96.34,42.45,18841,2469 diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 0000000..cdcdf2e --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +"""Reproducible, rate-controlled GET benchmark for the lab3 scaling track.""" + +import argparse +from collections import defaultdict +import http.client +import json +import math +import os +from pathlib import Path +from queue import Full, Queue +import statistics +import subprocess +import sys +import tempfile +import threading +import time +from urllib.parse import urlsplit +from urllib.request import urlopen + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) +from client import Client # noqa: E402 - the repository's public bootstrap client + + +def topology(mode, replicas): + if mode not in ("fixed", "growth") or replicas not in (1, 2, 4): + raise ValueError("mode must be fixed/growth and replicas must be 1/2/4") + return replicas, (1 / replicas if mode == "fixed" else 0.5), (512 // replicas if mode == "fixed" else 256) + + +def validate_matrix(rows): + if not rows: + raise ValueError("no benchmark rows") + rates = {row["rate"] for row in rows} + if len(rates) < 3: + raise ValueError("need at least three offered rates") + profiles = {(row["duration"], row["warmup"], row["dataset_messages"]) for row in rows} + if len(profiles) != 1: + raise ValueError("workload profile changed across measurements") + for mode in ("fixed", "growth"): + for replicas in (1, 2, 4): + _, cpu, memory = topology(mode, replicas) + for rate in rates: + repeats = [row["repeat"] for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)] + if sorted(repeats) != [1, 2, 3]: + raise ValueError(f"need three repeats for {mode}/{replicas}/{rate}") + for row in rows: + if (row["mode"], row["replicas"]) == (mode, replicas) and (row["api_cpu_each"], row["api_memory_mib_each"]) != (cpu, memory): + raise ValueError("incorrect API resource quota") + + +def percentile(values, fraction): + if not values: + return None + ordered = sorted(values) + return round(ordered[math.ceil(fraction * len(ordered)) - 1], 3) + + +def command(*args): + return subprocess.run(args, cwd=ROOT, text=True, capture_output=True, check=True).stdout.strip() + + +def compose(*args, override=None): + prefix = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"] + if override: + prefix += ["-f", str(override)] + return command(*prefix, *args) + + +def pool_snapshot(base_url, replicas): + found = {} + for _ in range(8 * replicas): + with urlopen(base_url + "/health/ready", timeout=4) as response: + headers = response.headers + found[headers["X-Instance-Id"]] = { + "requests": int(headers["X-Pool-Requests"]), + "queued": int(headers["X-Pool-Queued"]), + "wait_ms": int(headers["X-Pool-Wait-Ms"]), + "size": int(headers["X-Pool-Size"]), + "available": int(headers["X-Pool-Available"]), + } + if len(found) == replicas: + break + return found + + +def db_snapshot(override): + sql = ("SELECT pg_database_size(current_database()),xact_commit,xact_rollback," + "blks_read,blks_hit,temp_files,deadlocks,(SELECT count(*) FROM messages) " + "FROM pg_stat_database WHERE datname=current_database()") + values = command("docker", "compose", "--env-file", ".env", "-f", "compose.yaml", "-f", str(override), + "exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|", "-c", sql) + return dict(zip(("database_bytes", "commits", "rollbacks", "blocks_read", "blocks_hit", "temp_files", "deadlocks", "messages"), + map(int, values.split("|")))) + + +def docker_ids(override): + ids = {} + for service in ("api", "postgres", "lb"): + ids[service] = compose("ps", "-q", service, override=override).splitlines() + return ids + + +def stats_sample(ids): + all_ids = [id for service_ids in ids.values() for id in service_ids] + if not all_ids: + return {} + output = command("docker", "stats", "--no-stream", "--format", "{{json .}}", *all_ids) + by_id = {} + for line in output.splitlines(): + item = json.loads(line) + by_id[item["ID"]] = item + result = {} + for service, service_ids in ids.items(): + cpu = [] + memory = [] + for id in service_ids: + item = by_id.get(id[:12]) or by_id.get(id) + if not item: + raise ValueError("Docker stats omitted a benchmark container") + cpu.append(float(item["CPUPerc"].rstrip("%"))) + amount = item["MemUsage"].split("/")[0].strip() + value, unit = float("".join(c for c in amount if c.isdigit() or c == ".")), "".join(c for c in amount if c.isalpha()) + memory.append(round(value * {"B": 1 / 2**20, "KiB": 1 / 1024, "MiB": 1, "GiB": 1024, + "kB": 1000 / 2**20, "MB": 10**6 / 2**20, "GB": 10**9 / 2**20}[unit], 3)) + result[service] = {"cpu_percent": round(sum(cpu), 3), "memory_mib": round(sum(memory), 3)} + return result + + +def open_loop(base_url, path, token, rate, duration, workers=64): + """Schedule arrivals by clock; latency includes queue wait and HTTP time.""" + address = urlsplit(base_url) + if address.scheme != "http": + raise ValueError("this local lab3 benchmark expects HTTP") + jobs = Queue(maxsize=256) + latencies = [] + errors = defaultdict(int) + lock = threading.Lock() + starts = time.monotonic() + cpu_start = time.process_time() + count = math.ceil(rate * duration) + + def worker(): + connection = None + while True: + due = jobs.get() + if due is None: + jobs.task_done() + break + try: + if connection is None: + connection = http.client.HTTPConnection(address.hostname, address.port or 80, timeout=4) + connection.request("GET", path, headers={"Authorization": "Bearer " + token, "Accept": "application/json"}) + response = connection.getresponse() + payload = response.read() + if response.status != 200: + with lock: + errors[f"http_{response.status}"] += 1 + elif len(json.loads(payload)["items"]) != 20: + with lock: + errors["wrong_page"] += 1 + else: + with lock: + latencies.append((time.monotonic() - due) * 1000) + except (OSError, TimeoutError, ValueError, KeyError, http.client.HTTPException): + with lock: + errors["transport_or_response"] += 1 + if connection: + connection.close() + connection = None + finally: + jobs.task_done() + if connection: + connection.close() + + threads = [threading.Thread(target=worker, daemon=True) for _ in range(workers)] + for thread in threads: + thread.start() + dropped = 0 + for index in range(count): + due = starts + index / rate + delay = due - time.monotonic() + if delay > 0: + time.sleep(delay) + try: + jobs.put_nowait(due) + except Full: + dropped += 1 + jobs.join() + for _ in threads: + jobs.put(None) + for thread in threads: + thread.join() + elapsed = time.monotonic() - starts + error_count = sum(errors.values()) + dropped + return { + "offered_rate": rate, "scheduled_requests": count, "successful_requests": len(latencies), + "failed_requests": error_count, "error_ratio": round(error_count / count, 6), + "successful_rps": round(len(latencies) / elapsed, 3), "elapsed_seconds": round(elapsed, 3), + "successful_latency_ms": {f"p{p}": percentile(latencies, p / 100) for p in (50, 95, 99)}, + "errors_by_type": dict(errors), "queue_dropped": dropped, + "client_cpu_seconds": round(time.process_time() - cpu_start, 3), + } + + +def fixture(base_url, size, fixture_file): + if fixture_file.exists(): + saved = json.loads(fixture_file.read_text()) + if saved["base_url"] != base_url or saved["dataset_messages"] != size: + raise ValueError("saved fixture URL or size differs from requested profile") + Client(base_url).call("GET", saved["path"], actor={"token": saved["token"]}) + return saved["token"], saved["path"] + client = Client(base_url) + alice, _, _, chat = client.bootstrap() + for index in range(size): + client.message(alice, chat, f"synthetic-scale-{index}") + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=20" + fixture_file.parent.mkdir(parents=True, exist_ok=True) + with os.fdopen(os.open(fixture_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as file: + file.write(json.dumps({"base_url": base_url, "dataset_messages": size, + "token": alice["token"], "path": path}) + "\n") + return alice["token"], path + + +def run(args): + full_matrix = args.modes == ["fixed", "growth"] and args.replicas == [1, 2, 4] + if args.repeats != 3 or (full_matrix and len(args.rates) < 3) or args.dataset_messages < 20: + raise ValueError("require 3 repeats, at least 3 rates for a full matrix, and at least 20 fixture messages") + args.output.parent.mkdir(parents=True, exist_ok=True) + if args.output.exists(): + raise ValueError(f"output already exists: {args.output}") + token = path = None + rows = [] + with tempfile.TemporaryDirectory() as temporary: + override = Path(temporary) / "scale.yaml" + for mode in args.modes: + for replicas in args.replicas: + _, cpu, memory = topology(mode, replicas) + override.write_text("services:\n api:\n cpus: " + str(cpu) + "\n mem_limit: " + str(memory) + "m\n") + compose("up", "-d", "--build", "--wait", "--scale", f"api={replicas}", override=override) + if token is None: + token, path = fixture(args.base_url, args.dataset_messages, args.fixture) + ids = docker_ids(override) + for rate in args.rates: + for repeat in range(1, args.repeats + 1): + open_loop(args.base_url, path, token, rate, args.warmup) + pool_before = pool_snapshot(args.base_url, replicas) + db_before = db_snapshot(override) + samples = [] + sampling_errors = [] + stop = threading.Event() + + def sample(): + while not stop.is_set(): + try: + samples.append(stats_sample(ids)) + except (subprocess.CalledProcessError, ValueError, KeyError) as error: + sampling_errors.append(type(error).__name__) + return + stop.wait(1) + + sampler = threading.Thread(target=sample, daemon=True) + sampler.start() + result = open_loop(args.base_url, path, token, rate, args.duration) + stop.set() + sampler.join() + if sampling_errors or not samples: + raise RuntimeError(f"Docker CPU/RAM sampler failed: {sampling_errors}") + db_after = db_snapshot(override) + if db_after["messages"] != db_before["messages"]: + raise RuntimeError("message dataset changed during read-only benchmark") + pool_after = pool_snapshot(args.base_url, replicas) + pool_delta = { + "observed_replicas": len(set(pool_before) & set(pool_after)), + "requests_queued": sum(pool_after[i]["queued"] - pool_before[i]["queued"] for i in pool_before.keys() & pool_after.keys()), + "wait_ms": sum(pool_after[i]["wait_ms"] - pool_before[i]["wait_ms"] for i in pool_before.keys() & pool_after.keys()), + "max_size": max((v["size"] for v in pool_after.values()), default=0), + } + services = {name: { + "avg_cpu_percent": round(statistics.mean(s[name]["cpu_percent"] for s in samples if name in s), 3), + "max_memory_mib": round(max(s[name]["memory_mib"] for s in samples if name in s), 3), + } for name in ("api", "postgres", "lb") if any(name in s for s in samples)} + row = { + "mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat, + "duration": args.duration, "warmup": args.warmup, "dataset_messages": args.dataset_messages, + "api_cpu_each": cpu, "api_memory_mib_each": memory, "pool_max_size": 8, + **result, "docker": services, "pool": pool_delta, + "postgres_delta": {key: db_after[key] - db_before[key] for key in db_before if key != "database_bytes"}, + "database_bytes": db_after["database_bytes"], "database_messages": db_after["messages"], + } + rows.append(row) + with args.output.open("a") as output: + output.write(json.dumps(row, ensure_ascii=False) + "\n") + print(f"{mode} replicas={replicas} rate={rate} repeat={repeat} rps={result['successful_rps']} errors={result['error_ratio']}", flush=True) + if full_matrix: + validate_matrix(rows) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default="http://127.0.0.1:8080") + parser.add_argument("--rates", type=lambda value: [int(x) for x in value.split(",")], default=[100, 250, 500]) + parser.add_argument("--modes", type=lambda value: value.split(","), default=["fixed", "growth"]) + parser.add_argument("--replicas", type=lambda value: [int(x) for x in value.split(",")], default=[1, 2, 4]) + parser.add_argument("--duration", type=int, default=8) + parser.add_argument("--warmup", type=int, default=2) + parser.add_argument("--repeats", type=int, default=3) + parser.add_argument("--dataset-messages", type=int, default=1000) + parser.add_argument("--fixture", type=Path, default=ROOT / "secrets/scale_fixture.json") + parser.add_argument("--output", type=Path, default=ROOT / "evidence/scale/baseline.jsonl") + args = parser.parse_args() + if (not args.rates or min(args.rates) <= 0 or not args.modes or not args.replicas or + any(mode not in ("fixed", "growth") for mode in args.modes) or + any(replica not in (1, 2, 4) for replica in args.replicas) or + args.duration <= 0 or args.warmup <= 0): + raise ValueError("rates and durations must be positive") + run(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/benchmark.sh b/scripts/benchmark.sh new file mode 100755 index 0000000..d9c755c --- /dev/null +++ b/scripts/benchmark.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/benchmark.py "$@" diff --git a/scripts/plot_scale.py b/scripts/plot_scale.py new file mode 100644 index 0000000..87e5861 --- /dev/null +++ b/scripts/plot_scale.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Render the three-repeat scale matrix as small, dependency-free SVG plots.""" + +import argparse +import csv +import json +from pathlib import Path +import statistics + +from benchmark import validate_matrix + + +COLORS = {1: "#2563eb", 2: "#16a34a", 4: "#dc2626"} + + +def svg(mode, metric, rows, output): + width, height = 720, 420 + left, right, top, bottom = 75, 25, 35, 65 + plot_w, plot_h = width - left - right, height - top - bottom + rates = sorted({row["rate"] for row in rows}) + max_y = max((row[metric] or 0 for row in rows), default=0) * 1.1 or 1 + x = lambda rate: left + (rate - rates[0]) / (rates[-1] - rates[0]) * plot_w + y = lambda value: top + plot_h * (1 - value / max_y) + label = "Успешные запросы/с" if metric == "successful_rps" else "p95, мс" + parts = [f'', + '', + f'{mode}: {label}'] + for step in range(5): + value = max_y * step / 4 + py = y(value) + parts += [f'', + f'{value:.0f}'] + for rate in rates: + px = x(rate) + parts += [f'{rate}'] + for replicas in (1, 2, 4): + points = [(row["rate"], row[metric]) for row in rows if row["replicas"] == replicas and row[metric] is not None] + path = " ".join(f'{"M" if index == 0 else "L"}{x(rate):.1f},{y(value):.1f}' for index, (rate, value) in enumerate(points)) + parts.append(f'') + for rate, value in points: + parts.append(f'') + legend_x = left + 180 * (replicas == 2) + 360 * (replicas == 4) + parts.append(f'{replicas} API') + parts += [f'Заданная интенсивность, запросов/с', ''] + output.write_text("\n".join(parts) + "\n", encoding="utf-8") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("input", type=Path) + args = parser.parse_args() + rows = [json.loads(line) for line in args.input.read_text().splitlines() if line.strip()] + validate_matrix(rows) + summary = [] + for mode in ("fixed", "growth"): + for replicas in (1, 2, 4): + for rate in sorted({row["rate"] for row in rows}): + group = [row for row in rows if (row["mode"], row["replicas"], row["rate"]) == (mode, replicas, rate)] + def latency(p): + values = [row["successful_latency_ms"][p] for row in group if row["successful_latency_ms"][p] is not None] + return round(statistics.median(values), 3) if values else None + + summary.append({"mode": mode, "replicas": replicas, "rate": rate, + "successful_rps": round(statistics.median(row["successful_rps"] for row in group), 3), + "error_ratio": round(statistics.median(row["error_ratio"] for row in group), 6), + "p50_ms": latency("p50"), "p95_ms": latency("p95"), "p99_ms": latency("p99"), + "api_cpu_percent": round(statistics.median(row["docker"]["api"]["avg_cpu_percent"] for row in group), 3), + "postgres_cpu_percent": round(statistics.median(row["docker"]["postgres"]["avg_cpu_percent"] for row in group), 3), + "lb_cpu_percent": round(statistics.median(row["docker"]["lb"]["avg_cpu_percent"] for row in group), 3), + "api_memory_mib": round(statistics.median(row["docker"]["api"]["max_memory_mib"] for row in group), 3), + "postgres_memory_mib": round(statistics.median(row["docker"]["postgres"]["max_memory_mib"] for row in group), 3), + "lb_memory_mib": round(statistics.median(row["docker"]["lb"]["max_memory_mib"] for row in group), 3), + "pool_wait_ms": round(statistics.median(row["pool"]["wait_ms"] for row in group), 3), + "queue_dropped": statistics.median(row["queue_dropped"] for row in group)}) + with (args.input.parent / "summary.csv").open("w", newline="") as file: + writer = csv.DictWriter(file, fieldnames=summary[0].keys(), lineterminator="\n") + writer.writeheader() + writer.writerows(summary) + for mode in ("fixed", "growth"): + group = [row for row in summary if row["mode"] == mode] + svg(mode, "successful_rps", group, args.input.parent / f"{mode}-rps.svg") + svg(mode, "p95_ms", group, args.input.parent / f"{mode}-p95.svg") + + +if __name__ == "__main__": + main() diff --git a/src/api.py b/src/api.py index e9b7a8f..b441811 100644 --- a/src/api.py +++ b/src/api.py @@ -6,6 +6,7 @@ from uuid import UUID from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response from .auth import basic_credentials, bearer_token +from . import db from .models import CreateChat, CreateMessage, CreateUser from .store import store @@ -18,8 +19,14 @@ def live(): @router.get("/health/ready") -def ready(): - return store.ready() +def ready(response: Response): + result = store.ready() + stats = db.pool().get_stats() + for header, key in (("X-Pool-Requests", "requests_num"), ("X-Pool-Queued", "requests_queued"), + ("X-Pool-Wait-Ms", "requests_wait_ms"), ("X-Pool-Size", "pool_size"), + ("X-Pool-Available", "pool_available")): + response.headers[header] = str(stats.get(key, 0)) + return result @router.post("/api/v1/users", status_code=201) diff --git a/student-tests/test_benchmark_profile.py b/student-tests/test_benchmark_profile.py new file mode 100644 index 0000000..660dea8 --- /dev/null +++ b/student-tests/test_benchmark_profile.py @@ -0,0 +1,72 @@ +"""The scaling matrix must compare the same workload and resource budgets.""" + +import sys +import json +import unittest +from unittest.mock import patch +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +from benchmark import open_loop, topology, validate_matrix + + +class BenchmarkProfileTests(unittest.TestCase): + def test_fixed_budget_is_identical_across_replica_counts(self): + budgets = [topology("fixed", replicas) for replicas in (1, 2, 4)] + self.assertEqual([replicas * cpu for replicas, cpu, _ in budgets], [1.0] * 3) + self.assertEqual([replicas * memory for replicas, _, memory in budgets], [512] * 3) + + def test_growth_budget_increases_with_replicas(self): + budgets = [topology("growth", replicas) for replicas in (1, 2, 4)] + self.assertEqual([replicas * cpu for replicas, cpu, _ in budgets], [0.5, 1.0, 2.0]) + self.assertEqual([replicas * memory for replicas, _, memory in budgets], [256, 512, 1024]) + + def test_matrix_requires_three_repeats_and_same_workload(self): + rows = [ + {"mode": mode, "replicas": replicas, "rate": rate, "repeat": repeat, + "duration": 8, "warmup": 2, "dataset_messages": 1000, + "api_cpu_each": topology(mode, replicas)[1], + "api_memory_mib_each": topology(mode, replicas)[2]} + for mode in ("fixed", "growth") + for replicas in (1, 2, 4) + for rate in (100, 250, 500) + for repeat in (1, 2, 3) + ] + validate_matrix(rows) + with self.assertRaises(ValueError): + validate_matrix(rows[:-1]) + rows[-1]["dataset_messages"] = 999 + with self.assertRaises(ValueError): + validate_matrix(rows) + + def test_open_loop_counts_responses_at_a_scheduled_rate(self): + class Response: + status = 200 + + def read(self): + return json.dumps({"items": [{}] * 20}).encode() + + class Connection: + def __init__(self, *args, **kwargs): + pass + + def request(self, *args, **kwargs): + pass + + def getresponse(self): + return Response() + + def close(self): + pass + + with patch("benchmark.http.client.HTTPConnection", Connection): + result = open_loop("http://127.0.0.1:8080", "/history", "synthetic", 20, 0.2, workers=2) + self.assertEqual(result["scheduled_requests"], 4) + self.assertEqual(result["successful_requests"], 4) + self.assertEqual(result["failed_requests"], 0) + self.assertGreater(result["successful_latency_ms"]["p95"], 0) + + +if __name__ == "__main__": + unittest.main()