Lab 3: HAProxy + 2 replicas, idempotency, graceful shutdown, track 1 #157

Closed
sendormeco wants to merge 1 commits from sendormeco/mtusi2026lab3:lab3 into main
107 changed files with 24088 additions and 15 deletions
+20
View File
@@ -11,3 +11,23 @@ SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# --- реализация lab3 (см. REPORT.md, «Соответствие переменных») ---
# Порты LB на хосте (только 127.0.0.1): API, api-ttl (профиль ttl), статистика HAProxy.
API_PORT=8080
API_TTL_PORT=8081
LB_STATS_PORT=8404
# Ресурсы одной реплики API и PostgreSQL.
API_CPUS=1.0
API_MEMORY=512M
PG_CPUS=1.0
PG_MAX_CONNECTIONS=100
# Redis — read-through кэш сессий; источник истины — PostgreSQL.
SESSION_CACHE=redis
SESSION_CACHE_TTL_SECONDS=60
DB_POOL_MAX=10
KDF_CONCURRENCY=4
# Graceful shutdown: сколько секунд реплика отвечает 503 на readiness перед закрытием listener.
SHUTDOWN_DRAIN_SECONDS=3
# Ограничение конкуренции на LB (трек 5.1): одновременных запросов на реплику и ожидание в очереди LB.
LB_SERVER_MAXCONN=10
LB_QUEUE_TIMEOUT=2s
+11
View File
@@ -0,0 +1,11 @@
# Shell-скрипты и файлы, читаемые внутри Linux-контейнеров, — всегда LF (иначе ломаются на Windows с autocrlf).
*.sh text eol=lf
*.sql text eol=lf
*.py text eol=lf
*.yaml text eol=lf
*.yml text eol=lf
Dockerfile text eol=lf
.env.example text eol=lf
# Windows-обёртки
*.cmd text eol=crlf
*.ps1 text eol=crlf
+37
View File
@@ -0,0 +1,37 @@
name: Application stack
# Отдельный job решения: сборка образа API, Compose-стенд (LB + 2 реплики + migrate + PostgreSQL + Redis),
# публичные smoke/persistence/failover и собственные проверки. Секреты синтетические, генерируются на лету.
on: [push, pull_request]
permissions:
contents: read
jobs:
stack:
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@v4
- name: Secrets and env (synthetic)
run: ./scripts/init-secrets.sh
- name: Compose config
run: docker compose --env-file .env config --quiet
- name: Build and start
run: docker compose --env-file .env --profile ttl up -d --build --wait
- name: Public smoke via LB
run: make test BASE_URL=http://127.0.0.1:8080
- name: Failover (stop one replica), then restore
run: |
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080
./scripts/start-replicas.sh
cat evidence/failover-hook.txt
- name: Persistence (restart LB, replicas, PostgreSQL, Redis; volumes kept)
run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
- name: Student tests
run: make test-student
- name: Rolling update under load
run: python3 research/lb_experiments.py rolling-update
- name: Logs on failure
if: failure()
run: docker compose --env-file .env --profile ttl logs --no-color --tail=200
- name: Stop
if: always()
run: docker compose --env-file .env --profile ttl down -v
+34
View File
@@ -0,0 +1,34 @@
# API мессенджера lab3. Python 3.13, зависимости ставятся в venv в отдельной стадии.
ARG PYTHON_IMAGE=python:3.13.5-slim-bookworm
FROM ${PYTHON_IMAGE} AS deps
ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1
WORKDIR /build
COPY requirements.txt ./
RUN python -m venv /opt/venv \
&& /opt/venv/bin/pip install --require-virtualenv --no-compile -r requirements.txt
FROM ${PYTHON_IMAGE}
ARG APP_VERSION=dev
LABEL org.opencontainers.image.version=$APP_VERSION
ENV PATH=/opt/venv/bin:$PATH \
PYTHONPATH=/app/src \
PYTHONUNBUFFERED=1 \
PYTHONDONTWRITEBYTECODE=1 \
APP_VERSION=$APP_VERSION \
APP_HOST=0.0.0.0 \
APP_PORT=8080
RUN groupadd --system --gid 10001 app \
&& useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app
COPY --from=deps /opt/venv /opt/venv
WORKDIR /app
COPY migrations ./migrations
COPY src ./src
USER 10001:10001
EXPOSE 8080
STOPSIGNAL SIGTERM
HEALTHCHECK --interval=5s --timeout=3s --start-period=30s --retries=6 \
CMD ["python", "-m", "messenger", "healthcheck"]
# PID 1 — сам HTTP-процесс (exec-форма), SIGTERM даёт graceful shutdown uvicorn.
ENTRYPOINT ["python", "-m", "messenger"]
CMD ["serve"]
+73
View File
@@ -28,3 +28,76 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
# ---------------------------------------------------------------------------------------------
# Цели решения (lab3). Публичные цели выше не изменены.
# ---------------------------------------------------------------------------------------------
COMPOSE ?= docker compose --env-file .env
TRACK1 = COMPOSE_FILE=compose.yaml:research/compose.track1.yaml
.PHONY: secrets up down ps logs destroy-data restore rolling-update test-student test-api test-idempotency \
test-expiry test-cache-race test-failures lb-experiments track1 track1-report evidence
secrets:
./scripts/init-secrets.sh
# Единая команда запуска: LB + 2 реплики API + migrate (one-shot) + PostgreSQL + Redis (+ api-ttl для теста TTL).
up: secrets
$(COMPOSE) --profile ttl up -d --build --wait
ps:
$(COMPOSE) --profile ttl ps -a
logs:
$(COMPOSE) logs --tail=100 lb api
# Безопасная остановка: контейнеры удаляются, volume pgdata сохраняется.
down:
$(COMPOSE) --profile ttl --profile bench down
# НАМЕРЕННОЕ удаление учебных данных (volume pgdata). Необратимо.
destroy-data:
@echo "This deletes ALL lab3 data volumes. Type 'yes' to continue:"; read ans; [ "$$ans" = yes ]
$(COMPOSE) --profile ttl --profile bench down -v
# Вернуть реплики после make test-failover ACTION_SCRIPT=scripts/stop-one.sh
restore:
./scripts/start-replicas.sh
rolling-update:
./scripts/rolling-update.sh
test-api:
$(PYTHON) student-tests/test_api.py
test-idempotency:
$(PYTHON) student-tests/test_idempotency.py
test-expiry:
$(PYTHON) student-tests/test_expiry.py
test-cache-race:
$(COMPOSE) run --rm --no-deps -T -v "$(CURDIR)/student-tests:/app/student-tests:ro" --entrypoint python api student-tests/test_cache_race.py
# Останавливает/запускает redis и postgres (volumes не трогает).
test-failures:
$(PYTHON) student-tests/test_failures.py
test-student: test-api test-idempotency test-expiry test-cache-race test-failures
# Уровень 4: отказ каждой реплики (SIGTERM и SIGKILL) под нагрузкой + rolling update под нагрузкой.
lb-experiments:
$(PYTHON) research/lb_experiments.py failover-series --mode both
$(PYTHON) research/lb_experiments.py rolling-update
# Трек 5.1 (≈1 ч): серии нагрузки; затем сводка и графики в evidence/track1/.
track1:
$(TRACK1) $(COMPOSE) --profile bench build loadgen
$(TRACK1) $(PYTHON) research/track1.py run all
$(PYTHON) research/track1.py report
track1-report:
$(PYTHON) research/track1.py report
evidence:
./scripts/collect-evidence.sh
+271 -15
View File
@@ -1,37 +1,293 @@
# Отчёт — лабораторная 3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Автор, группа: Лебедев Дмитрий, БВТ2302
- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена масштабирования»**
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Использованные источники, библиотеки и помощь ИИ:
- библиотеки: Starlette 1.7.0, uvicorn 0.54.0, psycopg 3.2.10 + psycopg_pool 3.2.6, redis-py 5.2.1; HAProxy 3.0; генератор нагрузки — Go, только стандартная библиотека;
- документация: Docker Compose (services, deploy, networks), HAProxy 3.0 Configuration Manual (server-template, resolvers, http-check, retry-on, maxconn/queue, http-error), PostgreSQL (INSERT … ON CONFLICT, ожидание конкурентной вставки), Linux CFS bandwidth control (cpu.cfs_quota_us, nr_throttled);
- **ИИ:** реализация, тесты, эксперименты и этот отчёт подготовлены с помощью ИИ-ассистента Claude (Anthropic). Код других студентов не использовался.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
**Выбранный вариант: Docker Compose + HAProxy.** Prerequisites: Docker Engine 24+ с Compose v2, Python 3.9+ на хосте (тесты — только stdlib), `make` (Linux/macOS; для Windows — PowerShell-скрипты ниже). Свободны `127.0.0.1:8080` (API через LB), `:8081` (api-ttl через LB), `:8404` (статистика HAProxy).
**Образы (закреплены тегами):** `haproxy:3.0.5-alpine`, `python:3.13.5-slim-bookworm` (база API), `postgres:16.10-alpine`, `redis:7.4.2-alpine`, `golang:1.24.7-alpine` (только сборка генератора трека 1). API и генератор собираются из исходников репозитория.
```sh
./scripts/init-secrets.sh # secrets/* и .env (значения не печатаются)
docker compose --env-file .env config --quiet
docker compose --env-file .env --profile ttl up -d --build --wait # = make up
curl -si http://127.0.0.1:8080/health/ready | grep -i x-instance-id # запросы чередуются между репликами
make test BASE_URL=http://127.0.0.1:8080 # публичный smoke через LB
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080
./scripts/start-replicas.sh # = make restore: вернуть остановленную реплику
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
make test-student # API/auth/ACL, идемпотентность, TTL, гонка кэша, отказы БД/Redis
make lb-experiments # уровень 4: отказ каждой реплики (SIGTERM/SIGKILL) и rolling update под нагрузкой
make track1 # трек 5.1, ≈ 1 ч; затем evidence/track1/summary.md и *.svg
./scripts/collect-evidence.sh # всё, кроме трека, с выводом в evidence/
make down # безопасно: контейнеры удаляются, volume pgdata сохраняется
make destroy-data # НАМЕРЕННОЕ удаление данных (down -v), с подтверждением
```
**Windows (PowerShell 5.1, без make/WSL).** Docker Desktop запущен, Python с python.org. Используйте `127.0.0.1`, а не `localhost` (Windows сначала пробует `::1`), команды — по одной строке (в PowerShell 5.1 нет `&&`):
```powershell
powershell -NoProfile -ExecutionPolicy Bypass -File scripts\init-secrets.ps1
docker compose --env-file .env --profile ttl up -d --build --wait
$env:NO_PROXY = '127.0.0.1,localhost'
py -3 tests\smoke.py --base-url http://127.0.0.1:8080
py -3 tests\resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080
powershell -NoProfile -ExecutionPolicy Bypass -File scripts\start-replicas.ps1
py -3 tests\resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080
powershell -NoProfile -ExecutionPolicy Bypass -File scripts\collect-evidence.ps1 # всё сразу; -WithTrack1 — с треком
```
`*.cmd` — обёртки над `*.ps1`: публичный тест запускает хук напрямую, а `.ps1` Windows так не исполняет. Rolling update (`scripts/rolling-update.sh`) — POSIX sh: на Windows запускайте из Git Bash/WSL.
**Действия оператора:** только создание secrets. Миграции применяет one-shot сервис `migrate` до старта реплик; повторно — `docker compose --env-file .env run --rm --no-deps migrate` (no-op). Пароль БД записан в volume при первой инициализации — не удаляйте `secrets/` при живом `pgdata`.
> **Где собраны доказательства.** В среде разработки Docker daemon работал, но Docker Hub и PyPI были закрыты политикой сети. Поэтому `evidence/` получены на **том же `compose.yaml`** с локально собранными образами-заменителями из бинарников хоста: HAProxy 3.0.0 из исходников, PostgreSQL 16.15, Redis 7.0.15, Python 3.13 с теми же библиотеками. Подмена — только поле `image` / сборка API через override-файл, не входящий в репозиторий; топология, сети, лимиты, healthcheck, хуки и конфиг HAProxy — те же. Перед защитой выполните `make up && ./scripts/collect-evidence.sh` (Windows: `scripts\collect-evidence.ps1`), чтобы `evidence/` содержал вывод именно ваших официальных образов; тот же сценарий гоняет `.github/workflows/app.yml`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
127.0.0.1:8080 / :8081 / :8404 (stats)
клиент ──HTTP──► lb: HAProxy 3.0 ──(сеть app, internal)──► api × API_REPLICAS (без ports)
│ DNS Docker: server-template api 8 api:8080 │ uvicorn, 1 процесс, пул ≤ 10
│ GET /health/ready каждые 1 с, fall 2, rise 2 │
│ maxconn 10 на реплику, очередь ≤ 2 с ▼ (сеть data, internal)
│ retry-on conn-failure + redispatch postgres:16 [pgdata] ← источник истины:
└ panic-mode: backup = те же реплики пользователи, чаты, сообщения, сессии,
ключи идемпотентности
migrate (one-shot, до реплик) ──────────────────────────────► redis:7 — кэш сессий (без persistence)
```
| Роль | Service | Образ | Health/readiness | Volume | Сеть/порт | Credentials |
| --- | --- | --- | --- | --- | --- | --- |
| load_balancer | `lb` | `haproxy:3.0.5-alpine` + `infra/haproxy/haproxy.cfg` | `wget /stats`; бэкенды — HTTP-проверка `/health/ready` | — | edge+app; `127.0.0.1:8080/8081/8404` | — |
| api | `api` (`deploy.replicas: ${API_REPLICAS:-2}`) | сборка `Dockerfile` | `python -m messenger healthcheck` → `/health/ready` | нет (stateless) | app+data, **порты не публикуются** | secrets `database_url`, `cache_url` |
| migrations | `migrate` (one-shot, `restart: "no"`) | тот же образ, `command: migrate` | завершение с кодом 0 — условие старта `api` | — | data | `database_url` |
| postgres | `postgres` | `postgres:16.10-alpine` | `pg_isready` | `pgdata` | data | `POSTGRES_PASSWORD_FILE` |
| redis | `redis` | `redis:7.4.2-alpine` (user redis) | `redis-cli ping` | — (кэш) | data | `requirepass` из secret |
| api-ttl (тест TTL) | `api-ttl`, профиль `ttl` | образ API, `SESSION_TTL_SECONDS=2` | как у api | — | app+data; только через LB `:8081` | как у api |
Лимиты: API — `API_CPUS`/`API_MEMORY` (по умолчанию 1 CPU / 512 MiB на реплику), LB 0.5 CPU / 128 MiB, PostgreSQL 1 CPU / 1 GiB, Redis 0.25 CPU / 128 MiB, migrate 0.5 / 256 MiB. Нет `container_name` — `api` масштабируется (`--scale`, `deploy.replicas`). Контейнер API: uid 10001, `read_only`, `cap_drop: ALL`, `no-new-privileges`. Сети `app` и `data` — `internal: true`: PostgreSQL, Redis и реплики API снаружи недоступны, единственная точка входа — LB.
**Что перенесено из lab2.** Весь API, PostgreSQL как источник истины, сессии (scrypt, opaque Bearer, TTL, logout), Redis-кэш сессий с outbox-инвалидацией, keyset-пагинация, тесты lab2. **Что изменено в lab3:**
1. **Stateless-реплики.** Всё, от чего зависит корректность, — в PostgreSQL (сессии, ключи идемпотентности), Redis — общий кэш. Sticky sessions нет: `balance roundrobin`.
2. **`X-Instance-Id`** — случайный alias процесса (`api-<8 hex>`), без hostname/IP; реплика пишет его в `/tmp/instance-id` (tmpfs) — так `stop-one.sh` находит контейнер.
3. **Миграции отдельно и однократно:** сервис `migrate` выполняется до реплик (`depends_on: service_completed_successfully`), под `pg_advisory_lock`. Реплики (`MIGRATE_ON_START=false`) только проверяют, что в `schema_migrations` есть все известные им версии с теми же checksum, и до этого не становятся ready. Более новая схема допустима (expand-фаза следующей версии).
4. **Общая идемпотентность** `POST /chats/{id}/messages` (миграция `0003_idempotency_keys.sql`).
5. **Graceful shutdown** под LB и **panic mode** HAProxy (ниже).
**Идемпотентность — транзакционная граница.** Одна транзакция: удалить просроченный ключ этой тройки → `INSERT message … WHERE EXISTS(membership)` → `INSERT idempotency_keys (user, chat, key, sha256(канонический JSON), message_id) ON CONFLICT DO NOTHING`. Если ключ уже есть — `ROLLBACK` своей вставки и чтение сохранённого результата: тот же hash → тот же 201 и то же Message (`Idempotent-Replayed: true`), другой → 409. Конкурентная вставка того же ключа с **любой** реплики ждёт на уникальном индексе COMMIT/ROLLBACK первой транзакции: после COMMIT получает конфликт, после ROLLBACK вставляет сама. Поэтому одновременные повторы через разные реплики дают ровно одно сообщение, а транзиентная ошибка (откат) ключ не занимает. Повтор заново проверяет сессию (Bearer) и членство в чате. Хеш — от канонического JSON (`sort_keys`, без пробелов): порядок ключей и форматирование не важны. Срок — `IDEMPOTENCY_TTL_SECONDS` (24 ч); очистка — фоновой задачей реплик пачками по 5000.
**Health, readiness, timeouts, SIGTERM — согласованно.**
| Параметр | Значение | Зачем |
| --- | --- | --- |
| Проверка LB | `GET /health/ready`, `inter 1s`, после первой ошибки `fastinter 500ms`, `fall 2`, `rise 2`, `timeout check 300ms` | Максимальное время обнаружения отказа ≈ 1 + 0.3 + 0.5 + 0.3 ≈ **2.1 с**; возврат — 2 успешные проверки (≈ 1–2 с) |
| `timeout connect` | 300 ms | к убитому контейнеру нет RST — ждём не дольше 300 мс, затем `retry-on conn-failure` + `redispatch` на другую реплику (повтор только если запрос ещё не отправлен — безопасно и для неидемпотентных POST) |
| `timeout server` | 15 s | > `REQUEST_TIMEOUT_SECONDS` = 10 с приложения |
| SIGTERM в реплике | 3 с drain (`SHUTDOWN_DRAIN_SECONDS`): `/health/ready` = 503, ответы с `Connection: close`, запросы ещё обслуживаются; затем listener закрывается, запросы в полёте завершаются (≤ 10 с); 2-й сигнал — немедленный выход | drain (3 с) > худшего времени обнаружения LB (≈ 2.1 с) — LB выводит реплику до закрытия listener |
| `stop_grace_period` | 15 s | drain + завершение запросов в полёте |
| uvicorn keep-alive | 30 s (> idle-пула HAProxy) | соединение первым закрывает LB, а не реплика |
| `maxconn` на реплику / `timeout queue` | 10 / 2 s | ограничение конкуренции: излишек ждёт в очереди LB, а не в event loop (обоснование — трек 5.1) |
| DNS | `resolvers docker` (127.0.0.11), `hold obsolete 5s` | новые/удалённые реплики (`--scale`, rolling update) подхватываются без перезапуска LB |
`depends_on`/healthcheck Compose лишь упорядочивают старт и показывают состояние контейнера, но не распределяют трафик и не исключают больной backend из маршрута. Это делает HAProxy своими проверками.
**Panic mode.** Если readiness не проходит ни у одной реплики (например, упала БД), LB всё равно отправляет запросы в живые процессы (`backup`-слоты на те же адреса, `option allbackups`, `resolve-opts allow-dup-ip`). Клиент получает ответы приложения: анониму 401, бизнес-запросам 503 `unavailable` в формате API, `/health/live` = 200. Без этого LB отвечал бы безликим 503 на всё. Мёртвый процесс в backup-слоте даёт conn-failure и 503 LB.
**Ответы, созданные LB** (нет живых реплик, 502/504, `/internal/*` → 404), — в формате ошибок API с `request_id` = `X-Request-Id` (uuid). `X-Instance-Id` у них нет, контракт это допускает. Расширения API описаны в `contracts/EXTENSIONS.md`.
**Соответствие переменных `.env.example`.** `APP_HOST`/`APP_PORT` — адрес uvicorn (`0.0.0.0:8080`); `SESSION_TTL_SECONDS` — срок сессии; `DATABASE_URL_FILE`/`CACHE_URL_FILE` — `/run/secrets/*`; `SESSION_STORE=postgres` — источник истины сессий; `API_REPLICAS` → `deploy.replicas`; `IDEMPOTENCY_TTL_SECONDS` → срок ключа. Дополнительно: `API_PORT`, `API_TTL_PORT`, `LB_STATS_PORT`, `API_CPUS`, `API_MEMORY`, `PG_CPUS`, `PG_MAX_CONNECTIONS`, `SESSION_CACHE`, `SESSION_CACHE_TTL_SECONDS`, `DB_POOL_MAX`, `KDF_CONCURRENCY`, `SHUTDOWN_DRAIN_SECONDS`, `LB_SERVER_MAXCONN`, `LB_QUEUE_TIMEOUT` (см. комментарии в `.env.example` и `compose.yaml`).
**Rolling update и миграции.** `scripts/rolling-update.sh`: собрать образ новой версии (`APP_VERSION` → label и env); один раз применить миграции (`run --rm migrate`); для каждой старой реплики поднять одну новую (`--scale N+1 --no-recreate`), дождаться healthy, остановить старую SIGTERM-ом (drain) и удалить. Число готовых реплик не опускается ниже N. **Ограничение:** миграции обязаны быть *expand-only* (добавлять, не ломая работающую версию: новые таблицы/nullable-колонки/индексы). Удаление и переименование делаются отдельным релизом (*contract*) после того, как старых реплик не осталось. Скрипт это не проверяет: это дисциплина разработки, проверенная демонстрацией ниже.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Все файлы — в `evidence/`. Стенд описан в примечании выше; версии и контейнеры — `evidence/stack.txt` (2 vCPU Intel Xeon 2.8 GHz, 8 GiB RAM, Docker 29.9.2, Compose 5.6.1). Прогон с **пустого volume** `pgdata`.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл |
| --- | --- | --- | --- |
| Публичный smoke через LB | `tests/smoke.py --base-url http://127.0.0.1:8080` | `PASS: lab3 public API scenario` | `smoke.txt` |
| 3: стенд одной командой; ≥2 реплики, одна точка входа, общие БД и сессии | `docker compose --profile ttl up -d --build --wait` | `lb` (единственный с портами), `api-1`, `api-2`, `migrate` (exit 0), `postgres`, `redis`, `api-ttl` — healthy | `stack.txt` |
| 3: разные запросы одной сессии — разные реплики, без повторного входа | 6 запросов через LB; smoke/тесты с одним токеном | `X-Instance-Id` чередуется `api-0ac6…`/`api-ea54…`; 10 повторов POST с одним токеном обслужены 2 репликами | `instances.txt`, `test-idempotency.txt` |
| 3: остановка любой реплики; чтение/запись ≤ 10 с; подтверждённые сообщения целы | `tests/resilience.py --mode failover --action-script scripts/stop-one.sh` ×2 + `start-replicas.sh` | PASS дважды: ≥2 реплики до отказа, 0 неудачных попыток из 138–139 записей; `first_recovery_seconds` 4.7–4.8 (это время работы хука `docker stop`, записи не прерывались); остановленный контейнер (`status=exited`) в `failover-hook.txt` | `failover-1.txt`, `failover-2.txt`, `failover-hook.txt`, `restore-*.txt` |
| 3: повтор POST с тем же ключом без дублей, в т. ч. конкурентно и через другую реплику | `student-tests/test_idempotency.py` | 30 одновременных POST одного ключа: 14 + 16 ответов от двух реплик, одно тело, **1 строка**; 2 разных тела на один ключ: 1 сообщение, 10×201 + 10×409; повторы с другой реплики; 400 на неверный ключ; неуспешный запрос ключ не занимает; повтор после logout — 401; ключ старше 24 ч переиспользуется | `test-idempotency.txt` |
| 3: неподтверждённую запись можно повторить с тем же ключом | failover-тест (повтор неопределённого POST тем же ключом), серия отказов (повтор после 502/таймаута) | дублей текста 0, потерянных подтверждённых 0 во всех прогонах | `failover-*.txt`, `lb/failover-series.json` |
| 3: persistence | `tests/resilience.py --mode persistence --action-script scripts/restart.sh` | PASS тем же токеном; новые `StartedAt`/PID у LB, обеих реплик, PostgreSQL, Redis; volume сохранён | `persistence.txt`, `restart-hook.txt` |
| 3: миграции отдельно, однократно, повтор без дублей | пустой volume → сервис `migrate`; `run --rm migrate` | `applied: [0001, 0002, 0003]` один раз; повтор — `applied: []`; реплики логируют `schema compatible; replica ready` | `migrations.txt`, `migrations-rerun.txt` |
| Сохранены возможности lab2 | `test_api.py` (auth, ACL, валидация, пагинация, rollback, гонка регистрации), `test_expiry.py` (через LB :8081), `test_cache_race`, `test_failures.py` | 18/18, 3/3, all passed, 2/2 | `test-api.txt`, `test-expiry.txt`, `test-cache-race.txt`, `test-failures.txt` |
| 4: health/readiness/timeouts/SIGTERM согласованы; LB исключает недоступный backend | таблица выше; серия отказов SIGTERM | после SIGTERM реплика получает последний запрос через 0.95–1.53 с, ошибок 0 | `lb/failover-series.json` |
| 4: последовательно остановить каждую реплику; RPS, ошибки, время восстановления | `research/lb_experiments.py failover-series --mode both` | таблица ниже: SIGTERM — 0 ошибок; SIGKILL — 5–8 ответов 502 за ≤ 0.1 с; восстановление — 5.6–6.0 с до healthy | `lb/failover-series.json`, `lb-failover-series.txt` |
| 4: идемпотентность параллельными запросами с одним ключом | см. уровень 3 | 30 параллельных → 1 сообщение | `test-idempotency.txt` |
| 4: CPU/RAM-лимиты; API без портов; нет `container_name` | `compose.yaml` | у всех сервисов `deploy.resources.limits`; у `api` нет `ports`, сети `app`/`data` internal | `compose.yaml`, `stack.txt` |
| 4: rolling update под нагрузкой; миграции однократны и совместимы | `research/lb_experiments.py rolling-update` (3 обновления подряд), `research/expand_migration_demo.sh` | 0 ошибок за 27–28 с замены, образы сменились, потерь и дублей нет; новая версия без миграции не ready (`missing 0004`), миграция применена один раз, старые реплики остались ready | `lb/rolling-update-*.json`, `rolling-update.txt`, `lb/expand-migration.txt` |
| 5, трек 1 | `research/track1.py run …` (15 конфигураций, ключевые ×3) + `report` | см. раздел трека | `track1/*.json`, `track1/summary.md`, `track1/*.svg` |
| Нагрузка (профиль) | `tests/load.py --duration 30 --concurrency 4` через LB | 236.8 успешных RPS, 0 ошибок, p50/p95/p99 = 15.3/30.8/47.0 мс | `load.json` |
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
**Последовательный отказ каждой реплики под нагрузкой** (`evidence/lb/failover-series.json`). 8 потоков closed-loop через LB, у каждого свой пользователь: 1 POST (с ключом; при ошибке повтор тем же ключом и телом) на 3 GET истории; клиентский таймаут 2 с. Фазы: 5 с до отказа → действие → 10 с с одной репликой → `docker start` → 5 с. 2 реплики × 1 CPU / 512 MiB, `maxconn 10`.
| Воздействие | Реплика | Ошибки после действия | Последняя ошибка | Реплика ещё отвечала | Макс. пауза между записями | RPS до / во время / после | p99 до / во время, мс | До healthy после старта | Потери / дубли |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
| SIGTERM (`docker stop`) | api-1 | 0 | — | 0.95 с | 0.12 с | 265 / 252 / 253 | 101 / 68 | 5.6 с | 0 / 0 |
| SIGTERM | api-2 | 0 | — | 1.53 с | 0.18 с | 279 / 220 / 242 | 83 / 89 | 6.0 с | 0 / 0 |
| SIGKILL (`docker kill`) | api-1 | 5 × 502 | 0.08 с | 0.04 с | 0.32 с | 296 / 187 / 237 | 93 / 310 | 5.7 с | 0 / 0 |
| SIGKILL | api-2 | 8 × 502 | 0.10 с | 0.05 с | 0.31 с | 280 / 212 / 218 | 79 / 308 | 5.7 с | 0 / 0 |
SIGTERM: реплика 3 с отвечает 503 на readiness и дообслуживает запросы, LB выводит её за ~1 с — ни одной ошибки. SIGKILL: ошибки — только запросы «в полёте» на убитом процессе (502), дальше `retry-on conn-failure` + `redispatch` и проверка (`fall 2`) выводят реплику. Неопределённые POST повторены тем же ключом — дублей нет. Падение RPS «во время» — работа на одной реплике (её CPU и пул). «До healthy» — старт контейнера, проверка схемы и healthcheck Docker; в маршрут LB реплика возвращается через 1.3–3.6 с после старта.
**Настройка, подтверждённая измерением (до/после).** Сначала стояли `timeout connect 1s` и проверка с таймаутом по `inter` (1 с). При SIGKILL к убитому контейнеру нет RST, поэтому каждая попытка соединения и каждая проверка висели по 1 с: максимальная пауза между записями — 0.99–1.06 с, p99 ≈ 1010 мс, RPS во время отказа — 142–165. После `timeout connect 300ms` + `timeout check 300ms`: пауза 0.26–0.32 с, p99 ≈ 310 мс, RPS 187–215 (сохранены `evidence/lb/failover-series-connect1s.json` и текущий ряд).
**Rolling update под нагрузкой** (`evidence/lb/rolling-update-3.0.1.json`, `-3.0.2-expand.json`, `-3.0.3.json`): три замены подряд — 0 ошибок из 5274–5710 запросов за 27–28 с, p99 135–173 мс, потерянных/дублированных сообщений 0; image id обеих реплик сменился (`rolling-update.txt`). Вторая замена несла новую expand-миграцию 0004 (`lb/expand-migration.txt`): контейнер новой версии до миграции остался NOT-READY с логом `schema is behind … missing 0004`; `migrate` применил её один раз; старые реплики 3.0.1 остались ready.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| SIGTERM одной реплики | 0 ошибок, LB выводит её до закрытия | 0 ошибок, последний запрос на неё через ≤ 1.5 с | новые запросы не прерывались; 0 потерь |
| SIGKILL одной реплики | ошибки только у запросов в полёте | 5–8 ответов 502 за ≤ 0.1 с | ≤ 0.32 с до следующей успешной записи; 0 потерь |
| Рестарт всего стенда (restart hook) | данные и токен живы | PASS | 0 потерь |
| Остановка Redis | работа через PostgreSQL | всё 2xx, `/users/me` за 5 мс | outbox дообработан после старта; отозванный токен — 401 |
| Остановка PostgreSQL | ограниченный 503, аноним — 401 | 503 `unavailable` за 2.0 с, `/health/live` = 200 через panic mode LB | ready через 2.3 с после старта БД |
| Rolling update ×3 | без ошибок | 0 ошибок | 0 потерь |
### Трек 5.1. Цена масштабирования
**Вопрос и гипотезы.** Что даёт добавление API-реплик и где предел?
- H1. При фиксированном суммарном CPU/RAM больше реплик не дают больше пропускной способности: Python-API ограничен CPU, а у каждой реплики свои накладные расходы.
- H2. При росте ресурсов пропускная способность растёт до насыщения общего ресурса, после чего новые реплики не помогают.
- H3. При перегрузке без ограничения конкуренции полезная пропускная способность не держит плато, а проваливается (congestion collapse). Ограничение `maxconn` на LB это устраняет.
**Методика (повторяемая):** `make track1` = `research/track1.py run all` + `report`.
- **Стенд.** Тот же `compose.yaml` + `research/compose.track1.yaml`. Реплики API привязаны к ядру `API_CPUSET=0`, их суммарный бюджет задаётся квотой `API_CPUS`; LB, PostgreSQL, Redis и генератор — на ядре `AUX_CPUSET=1`. Перед каждой конфигурацией реплики и LB пересоздаются (`--force-recreate`), запуск только после того, как LB видит все реплики UP.
- **Нагрузка.** Генератор `research/loadgen` — Go, только stdlib, внутри сети стенда (без docker-proxy).
- Модель **open-loop**: запросы по расписанию, задержка — от запланированного момента, без coordinated omission.
- 48 синтетических пользователей по одному чату с историей; смесь 70% GET истории (limit 50), 20% POST сообщения (уникальный ключ), 10% GET `/users/me`.
- Ступени 40…800 req/s по 3 с прогрева + 10 с измерения; серия останавливается после двух ступеней с > 30% ошибок.
- **Метрики.**
- Ёмкость при SLO: максимальная полезная пропускная способность при p99 ≤ 250 мс и ошибках ≤ 1%.
- Максимум при ошибках ≤ 1%.
- Пик полезной пропускной способности.
- Одновременно: CPU%/RAM контейнеров (`docker stats`), очередь HAProxy (stats CSV), соединения и события ожидания PostgreSQL (одна сессия `psql \watch 1`), доля периодов CFS с троттлингом (`cpu.stat`), счётчики пула psycopg в репликах до/после.
- **Повторы.** Ключевые конфигурации (серия A, `maxconn 10`) — 3 прогона, в выводах медиана. Остальные — по 1 прогону, это отмечено.
**Исходные данные:** `evidence/track1/<config>[-rN].json` (все ступени и ресурсы), `summary.md`/`summary.json`, графики `*.svg`.
**Серия A — фиксированный бюджет API 1 CPU / 512 MiB** (scrypt-параллелизм делится так же: 4/2/1), медиана 3 прогонов:
| Конфигурация | Ёмкость при SLO, rps | Макс. при ошибках ≤ 1 %, rps | Пик, rps | CPU API при 80 rps | CFS-троттлинг при 80 rps | p99 при 80 rps |
| --- | --- | --- | --- | --- | --- | --- |
| 1 × 1.0 CPU | **240** (280/240/240) | **280** | 280 | 28 % | 0 % | 27 мс |
| 2 × 0.5 CPU | 200 (120/200/200) | 200 | 235 | 32 % | 6.7 % | 48 мс |
| 4 × 0.25 CPU | 160 (160/120/160) | 200 | 200 | 45 % | 4.7 % | 66 мс |
![Фиксированный бюджет: полезная пропускная способность](evidence/track1/fixed-cpu-goodput.svg)
![Фиксированный бюджет: p99](evidence/track1/fixed-cpu-p99.svg)
H1 подтверждена, и дробление даже вредит. Механизмы:
- **Накладные расходы на реплику.** На одной и той же нагрузке 80 rps суммарный CPU API растёт с 28 до 45%: у каждой реплики свои health-проверки LB (1/с), фоновые циклы (кэш, очистка), event loop и рантайм. Стоимость запроса на пике растёт с 3.2 до 4.4 мс CPU.
- **Квота CFS.** Реплика с квотой 0.25–0.5 ядра, выбравшая квоту за период 100 мс, замораживается до конца периода: 5–7% периодов с троттлингом уже при 80 rps и до 36% на пике. Отсюда хвост задержки при одинаковом суммарном CPU.
- **RAM.** У каждой реплики фиксированные ~30–45 MiB рантайма плюс по 32 MiB на одновременный scrypt. При 128 MiB на реплику и scrypt-параллелизме 4 первые прогоны получили **cgroup OOM kill** при регистрации пользователей (`dmesg: Memory cgroup out of memory`). Дробить RAM без пересчёта параллелизма KDF нельзя.
**Серия B — рост ресурсов** (реплики по 0.25 ядра на изолированном ядре, 1 прогон; 4×0.25 — медиана серии A) и **серия C — вся машина** (2 vCPU без изоляции, общие с LB/БД/генератором, 1 прогон):
| Конфигурация | Суммарно CPU API | Пик, rps | Макс. при ошибках ≤ 1 %, rps |
| --- | --- | --- | --- |
| 1 × 0.25 | 0.25 | 84 | 80 |
| 2 × 0.25 | 0.5 | 160 | 160 |
| 3 × 0.25 | 0.75 | 178 | 160 |
| 4 × 0.25 | 1.0 | 200 | 200 |
| 1 × 1.0 (для сравнения) | 1.0 | 280 | 280 |
| 2 × 1.0, вся машина | ≤ 2 (2 vCPU хоста) | 320 | 320 |
| 4 × 1.0, вся машина | ≤ 2 | 312 | 280 |
![Рост ресурсов](evidence/track1/growth-peak.svg)
Почему рост реплик перестаёт помогать (H2):
- До 0.5 ядра рост почти линеен (84 → 160 rps), дальше выгода падает: 0.75 → 178, 1.0 → 200. Четыре реплики по 0.25 уступают одной реплике на целом ядре (280) по причинам серии A.
- На всей машине 2 реплики дают +14% к одной (320 против 280), а 4 реплики — уже не лучше двух. На пике суммарная загрузка контейнеров — 159–171% из 200%, остальное съедают dockerd/containerd и сэмплер: **хост насыщен**.
- CPU на запрос по компонентам (machine-2×1.0, на пике):
| Компонент | мс CPU/запрос |
| --- | --- |
| API | 3.5 |
| PostgreSQL | 0.6 |
| генератор | 0.5 |
| LB | 0.23 |
| Redis | 0.12 |
API в ~6 раз дороже PostgreSQL на запрос. Значит, на этой машине первым кончается CPU для API, а БД и LB далеки от насыщения: PostgreSQL ~20% CPU; события ожидания — почти только `CPU`, изредка `IO:WALSync`, `pg_test_fsync` 0.25 мс.
- Каждая новая реплика добавляет процессы, которые конкурируют за те же 2 ядра, и пул из 10 соединений к БД: 10 → 20 → 41 соединение. Время удержания соединения на запрос (среднее по всей серии): 16 мс у 1×1.0 → 18 мс у 2×1.0 → 40 мс у 4×1.0.
- **Оценка следующего предела (экстраполяция, не измерено).** При изобилии CPU для API первым насытится PostgreSQL на одном ядре — около 1 / 0.6 мс ≈ 1600 rps на этой смеси. Затем `max_connections=100` при пуле 10 ограничивает число реплик ~9 (с учётом migrate и служебных соединений). Дальше нужен пул на стороне БД (PgBouncer) или меньший пул на реплику.
**Насыщение пула.** При 1×1.0 под перегрузкой запрос в среднем по серии ждёт свободное соединение 279–308 мс (3 прогона), а держит его 16 мс. При этом сам SQL выполняется ~1 мс: соединение «держит» корутина, которая ждёт своей очереди в перегруженном event loop. `/users/me` без БД (из кэша Redis) в тех же ступенях отвечает за ~25 мс, history/post — около 1 с. Ошибки — `PoolTimeout`/`TooManyRequests` пула, а не LB.
Серия P (1×1.0, по 1 прогону), пик / ёмкость при SLO:
| Пул | Пик, rps | Ёмкость при SLO, rps |
| --- | --- | --- |
| 5 | 320 | 280 |
| 10 | 280 | 240 |
| 20 | 280 | 200 |
| 40 | 280 | 200 |
Больший пул не помогает: ограничитель — CPU event loop, а не число соединений. Меньший пул лишь ограничивает конкуренцию. Это подводит к улучшению.
**Подтверждённое улучшение: ограничение конкуренции на LB (`maxconn 10` на реплику, `timeout queue 2s`)** — 1×1.0 CPU, пул 10, медиана 3 прогонов каждого варианта:
| Поданная нагрузка | Без лимита: полезная / ошибки / p50 | `maxconn 10`: полезная / ошибки / p50 |
| --- | --- | --- |
| 280 req/s | 280 / 0 % / 72 мс | 280 / 0 % / **12 мс** |
| 320 req/s | 276 / 13.8 % / 744 мс | **320 / 0 %** / 162 мс |
| 360 req/s | 277 / 22.9 % / 794 мс | **360 / 0.1 %** / 1323 мс |
| 400 req/s | 253 / 36.8 % / 931 мс | **354** / 11.4 % / 1670 мс |
| 460 req/s | 223 / 51.6 % / 1103 мс | **328** / 28.7 % / 2026 мс |
| Ёмкость при SLO | 240 (280/240/240) | **280** (320/280/240) |
| Макс. при ошибках ≤ 1 % | 280 (280/280/280) | **360** (360/360/320) — **+28 %**, каждый из 3 прогонов выше каждого базового |
![maxconn против перегрузки](evidence/track1/admission-goodput.svg)
H3 подтверждена. Без лимита LB отдаёт реплике всё сразу: сотни корутин в event loop борются за CPU и 10 соединений, время удержания соединения раздувается, запросы истекают по таймауту пула. CPU тратится на работу, которую потом выбрасывают, и полезная пропускная способность падает с 280 до 223 rps. С `maxconn 10` в реплике не больше 10 запросов; остальные ждут в дешёвой очереди HAProxy и при переполнении получают быстрый 503. Полезная пропускная способность держит плато 330–360 rps вместо провала.
Проверенные варианты (по 1 прогону):
| Вариант | Макс. при ошибках ≤ 1 %, rps |
| --- | --- |
| `maxconn 10`, `timeout queue 2s` | 360 (медиана) |
| `maxconn 20` | 320 |
| `maxconn 10`, `timeout queue 250ms` | 239: очередь 250 мс слишком коротка и отбрасывает всплески, которые иначе уложились бы |
| 2×0.5 с `maxconn 10` | 280 (без лимита 200) |
Значения `maxconn 10` и `timeout queue 2s` **приняты по умолчанию** в `compose.yaml`; уровни 3–4 повторно проверены уже с ними.
**Ограничения измерений:**
- Одна машина на 2 vCPU. Генератор, LB, БД и рантайм Docker делят её с API. Изоляция ядер снимает конкуренцию только для API.
- Абсолютные числа зависят от CPU хоста. Переносимы соотношения и механизмы, не rps.
- Ключевые конфигурации повторены 3 раза; разброс ёмкости при SLO — одна ступень (±40 rps), из-за этого шага сетки ёмкость определена с точностью до ступени. Остальные конфигурации — 1 прогон.
- Тёплый кэш, данные — тысячи сообщений: стоимость БД здесь минимальна; большие истории — в треке 5.2 lab2.
- Числа получены на локальных образах-заменителях: HAProxy 3.0.0 вместо 3.0.5, PostgreSQL 16.15 вместо 16.10, Redis 7.0 вместо 7.4, psycopg без C-расширения (pure-Python + системная libpq) вместо `psycopg[binary]`. Последнее завышает CPU API на запрос: в официальном образе ожидаемы более высокие абсолютные значения. Повторите `make track1`.
- Сэмплер сам потребляет CPU хоста. Первая версия с `docker exec` раз в секунду заметно занижала результаты и заменена одной сессией `psql`.
## Ответы на вопросы защиты
- **Отказ процесса vs отказ узла.** Отказ процесса (SIGKILL реплики) оставляет живыми хост, сеть, Docker и остальные контейнеры: LB видит отказ соединения, перенаправляет запросы, Docker перезапускает контейнер, volume на месте. При отказе узла пропадает всё, что на нём: все реплики, LB, PostgreSQL и его диск. На одном Docker-хосте этот стенд отказ узла не выдерживает — для этого нужны другие узлы, репликация БД и переносимый LB (трек 5.2, не выбран).
- **Почему depends_on/healthcheck — не балансировщик.** `depends_on` задаёт только порядок старта, healthcheck Docker лишь помечает контейнер unhealthy. Ни то, ни другое не распределяет запросы, не исключает больной backend из маршрута и не повторяет запрос на другой реплике. Это делает LB: активные проверки, `fall`/`rise`, `redispatch`.
- **Какие POST можно повторять после timeout и что защищает Idempotency-Key.** Без ключа повторять безопасно только то, что не создаёт новый объект, — в обязательном API таких POST нет: повтор регистрации даст 409, повтор создания чата создаст ещё один чат. `POST /messages` с тем же `Idempotency-Key` и тем же телом можно повторять сколько угодно и через любую реплику: результат первой успешной транзакции сохранён атомарно с сообщением, повтор возвращает его же, другой body — 409. LB сам повторяет только то, что не было отправлено (`retry-on conn-failure`).
- **Что сломается с сессиями в локальной памяти.** Токен, выданный репликой A, будет неизвестен реплике B: при roundrobin каждый второй запрос — 401. Logout на одной реплике не отзовёт токен на другой. Рестарт или отказ реплики разлогинит её пользователей. Sticky sessions лишь маскируют проблему и ломаются при отказе реплики. Здесь сессии в PostgreSQL, Redis — общий кэш.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- **Выдерживается:** остановка/убийство любой одной API-реплики (без ошибок при SIGTERM, ≤ 8 ошибок 502 при SIGKILL), рестарт всего стенда с volumes, rolling update под нагрузкой, отказ Redis, отказ PostgreSQL (ограниченные 503).
- **Не выдерживается:** отказ хоста; отказ единственного LB или PostgreSQL — это SPOF стенда.
- `scripts/stop-one.sh` находит реплику по alias через `docker exec` (файл `/tmp/instance-id`) — это требует работающего процесса. Убитую реплику так не найти, но этот хук останавливает живую реплику.
- Публичный failover-тест сам выбирает реплику, которая обслужила запрос. Поэтому поочерёдная остановка каждой реплики показана отдельно (`lb/failover-series.json`: api-1 и api-2, SIGTERM и SIGKILL).
- Окно устаревания кэша сессий при сетевом разделении реплики и Redis — как в lab2 (≤ `SESSION_CACHE_TTL_SECONDS`).
- Rolling update на Windows — только из Git Bash/WSL (скрипт POSIX sh).
- `evidence/` получены на образах-заменителях (см. примечание в начале). Повторите `./scripts/collect-evidence.sh` на официальных образах.
+172
View File
@@ -0,0 +1,172 @@
# Учебный стенд lab3: HAProxy (единственная точка входа) → API × N → PostgreSQL + Redis.
# Запуск: ./scripts/init-secrets.sh && docker compose --env-file .env up -d --build --wait
name: mtusi-lab3
x-api-common: &api-common
build:
context: .
dockerfile: Dockerfile
image: mtusi-lab3-api:local
environment: &api-env
APP_HOST: 0.0.0.0
APP_PORT: "8080"
APP_ENV: ${APP_ENV:-lab}
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
SESSION_STORE: postgres
SESSION_CACHE: ${SESSION_CACHE:-redis}
SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60}
IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400}
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
DB_POOL_MIN: "1"
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DB_POOL_TIMEOUT_SECONDS: "2"
KDF_CONCURRENCY: ${KDF_CONCURRENCY:-4} # каждый scrypt ≈ 32 MiB RAM: учитывайте в API_MEMORY
DB_STATEMENT_TIMEOUT_MS: "5000"
CACHE_TIMEOUT_SECONDS: "0.2"
REQUEST_TIMEOUT_SECONDS: "10"
SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:-3}
MIGRATE_ON_START: "false" # миграции — только сервис migrate
secrets: [database_url, cache_url]
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_started # Redis — кэш: реплики работают и без него
networks: [app, data] # НЕТ ports: API доступен только через LB
read_only: true
tmpfs: [/tmp]
cap_drop: [ALL]
security_opt: ["no-new-privileges:true"]
restart: unless-stopped
stop_signal: SIGTERM
stop_grace_period: 15s # drain 3 с + завершение запросов в полёте (≤10 с)
services:
lb:
image: haproxy:3.0.5-alpine
volumes:
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
environment:
LB_SERVER_MAXCONN: ${LB_SERVER_MAXCONN:-10}
LB_QUEUE_TIMEOUT: ${LB_QUEUE_TIMEOUT:-2s}
ports:
- "127.0.0.1:${API_PORT:-8080}:8080" # публичный API
- "127.0.0.1:${API_TTL_PORT:-8081}:8081" # api-ttl (профиль ttl), тоже только через LB
- "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # статистика HAProxy (только чтение)
healthcheck:
test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/stats"]
interval: 5s
timeout: 2s
retries: 3
networks: [edge, app]
restart: unless-stopped
stop_grace_period: 5s
deploy:
resources:
limits: {cpus: "0.5", memory: 128M}
# One-shot: миграции применяются ОДИН раз до запуска реплик (не гонкой реплик).
# Повторный запуск (docker compose run --rm migrate) — no-op. Реплики не становятся ready,
# пока схема не содержит все известные им миграции.
migrate:
<<: *api-common
command: ["migrate"]
depends_on:
postgres:
condition: service_healthy
networks: [data]
restart: "no"
secrets: [database_url]
environment:
<<: *api-env
SESSION_CACHE: "off"
deploy:
resources:
limits: {cpus: "0.5", memory: 256M}
api:
<<: *api-common
deploy:
replicas: ${API_REPLICAS:-2}
resources:
limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512M}"}
# Экземпляр с коротким TTL — только для теста истечения; доступен через LB на :8081.
api-ttl:
<<: *api-common
profiles: [ttl]
environment:
<<: *api-env
SESSION_TTL_SECONDS: "2"
deploy:
resources:
limits: {cpus: "0.5", memory: 256M}
postgres:
image: postgres:16.10-alpine
environment:
POSTGRES_USER: messenger
POSTGRES_DB: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --data-checksums"
command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"]
secrets: [postgres_password]
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"]
interval: 2s
timeout: 3s
retries: 30
start_period: 10s
networks: [data]
shm_size: 256m
restart: unless-stopped
stop_grace_period: 30s
deploy:
resources:
limits: {cpus: "${PG_CPUS:-1.0}", memory: 1G}
# Redis — общий кэш сессий без persistence; источник истины сессий — PostgreSQL.
redis:
image: redis:7.4.2-alpine
command:
- sh
- -c
- |
umask 077
printf 'requirepass %s\n' "$$(cat /run/secrets/redis_password)" > /tmp/redis.conf
exec redis-server /tmp/redis.conf --save '' --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru
user: redis
secrets: [redis_password]
healthcheck:
test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping | grep -q PONG"]
interval: 2s
timeout: 2s
retries: 15
networks: [data]
restart: unless-stopped
deploy:
resources:
limits: {cpus: "0.25", memory: 128M}
networks:
edge: {} # только LB: публикация портов на 127.0.0.1
app:
internal: true # LB ↔ API
data:
internal: true # API ↔ PostgreSQL/Redis
volumes:
pgdata: {}
secrets:
postgres_password:
file: ./secrets/postgres_password
redis_password:
file: ./secrets/redis_password
database_url:
file: ./secrets/database_url
cache_url:
file: ./secrets/cache_url
+13
View File
@@ -0,0 +1,13 @@
# Расширения решения (обратно совместимые)
Обязательный контракт (`openapi.json`, `README.md`) не изменён. Дополнительно реализовано:
| Что | Где | Семантика |
| --- | --- | --- |
| Заголовок ответа `Idempotent-Replayed: true` | `POST /api/v1/chats/{chat_id}/messages` | Ответ 201 — повтор ранее сохранённого результата по тому же `Idempotency-Key` (тело и код совпадают с первым ответом). Клиент может игнорировать. |
| `GET /internal/stats` | только изнутри контейнера реплики (`127.0.0.1`); LB отвечает на `/internal/*` 404 в формате ошибок API | Диагностика для исследования: alias реплики, версия, счётчики запросов, кэша сессий и пула соединений psycopg. `python -m messenger stats` в контейнере. |
| Ответы, созданные самим LB (502/503/504/404 для `/internal/*`) | HAProxy | Тело в формате ошибок API с `request_id` = `X-Request-Id` (случайный uuid). `X-Instance-Id` у них нет — контракт это допускает. |
Область ключа идемпотентности: пользователь + `POST` + `/api/v1/chats/{chat_id}/messages` + ключ. Других
идемпотентных endpoint в обязательном API нет, поэтому хранится тройка `(user_id, chat_id, key)`.
Срок хранения — `IDEMPOTENCY_TTL_SECONDS` (по умолчанию 86 400 с = 24 ч).
+6
View File
@@ -0,0 +1,6 @@
# 2026-11-03T09:51:34Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 0, "first_recovery_seconds": 4.675}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
+6
View File
@@ -0,0 +1,6 @@
# 2026-11-03T09:51:57Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 4.763}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
+32
View File
@@ -0,0 +1,32 @@
== 2026-11-03T09:51:36Z stop-one target=api-5dbb8cd6 container=mtusi-lab3-api-1
api mtusi-lab3-api-1 status=running started=2026-11-03T09:50:54.761533652Z pid=19437
api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359
api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502
lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923
migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0
postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033
redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102
== 2026-11-03T09:51:40Z after stop
api mtusi-lab3-api-1 status=exited started=2026-11-03T09:50:54.761533652Z pid=0
api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359
api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502
lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923
migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0
postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033
redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102
== 2026-11-03T09:51:59Z stop-one target=api-5fd104b6 container=mtusi-lab3-api-1
api mtusi-lab3-api-1 status=running started=2026-11-03T09:51:51.150461585Z pid=22901
api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359
api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502
lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923
migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0
postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033
redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102
== 2026-11-03T09:52:03Z after stop
api mtusi-lab3-api-1 status=exited started=2026-11-03T09:51:51.150461585Z pid=0
api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359
api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502
lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923
migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0
postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033
redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102
+9
View File
@@ -0,0 +1,9 @@
# 2026-11-03T09:51:02Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ sh -c for i in 1 2 3 4 5 6; do curl -s -o /dev/null -D - http://127.0.0.1:8080/health/live | grep -i x-instance-id; done
x-instance-id: api-7e27119d
x-instance-id: api-5dbb8cd6
x-instance-id: api-7e27119d
x-instance-id: api-5dbb8cd6
x-instance-id: api-7e27119d
x-instance-id: api-5dbb8cd6
exit=0
+8
View File
@@ -0,0 +1,8 @@
# 2026-11-03T09:53:04Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 research/lb_experiments.py failover-series --mode both
stop mtusi-lab3-api-1 errors=0 last_err= 0.00s target_last= 0.95s max_write_gap=0.12s rps(before/during)=264.6/251.9 data={'confirmed': 1928, 'lost_confirmed': 0, 'stored': 1928, 'duplicate_texts': 0}
stop mtusi-lab3-api-2 errors=0 last_err= 0.00s target_last= 1.53s max_write_gap=0.18s rps(before/during)=278.9/220.4 data={'confirmed': 1865, 'lost_confirmed': 0, 'stored': 1865, 'duplicate_texts': 0}
kill mtusi-lab3-api-1 errors=5 last_err= 0.08s target_last= 0.04s max_write_gap=0.32s rps(before/during)=295.8/186.6 data={'confirmed': 1551, 'lost_confirmed': 0, 'stored': 1551, 'duplicate_texts': 0}
kill mtusi-lab3-api-2 errors=8 last_err= 0.10s target_last= 0.05s max_write_gap=0.31s rps(before/during)=279.9/211.8 data={'confirmed': 1552, 'lost_confirmed': 0, 'stored': 1552, 'duplicate_texts': 0}
saved evidence/lb/failover-series.json
exit=0
+48
View File
@@ -0,0 +1,48 @@
# 2026-11-03T09:55:32Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 research/lb_experiments.py rolling-update
{
"script_exit": 0,
"script_seconds": 27.88,
"before_replicas": [
{
"container": "mtusi-lab3-api-1",
"instance_id": "api-471b30bc"
},
{
"container": "mtusi-lab3-api-2",
"instance_id": "api-50205d38"
}
],
"after_replicas": [
{
"container": "mtusi-lab3-api-3",
"instance_id": "api-730666b6"
},
{
"container": "mtusi-lab3-api-4",
"instance_id": "api-87d10eab"
}
],
"during": {
"seconds": 27.88,
"ok": 5696,
"errors": 0,
"rps_ok": 204.3,
"p50_ms": 31.4,
"p99_ms": 152.4,
"by_instance": {
"api-471b30bc": 896,
"api-50205d38": 2061,
"api-730666b6": 1928,
"api-87d10eab": 811
},
"error_codes": {}
},
"data": {
"confirmed": 2444,
"lost_confirmed": 0,
"stored": 2444,
"duplicate_texts": 0
}
}
exit=0
+11
View File
@@ -0,0 +1,11 @@
# 2026-11-03T09:57:04Z expand-migration demo, new version 3.0.2-expand
== 1) new-version container before migrate:
Container mtusi-lab3-api-run-b781fee6ea25 Creating
Container mtusi-lab3-api-run-b781fee6ea25 Created
health/ready inside new container: NOT-READY
log: {"ts": "2026-11-03T09:57:05Z", "level": "warning", "logger": "messenger.http", "msg": "schema is behind this code; waiting for migrate", "missing": ["0004_demo_expand"]}
== 2) migrate (one-shot):
{"applied": ["0004_demo_expand"]}
== 3) old replicas after migrate:
/mtusi-lab3-api-3 ready=yes version=3.0.1
/mtusi-lab3-api-4 ready=yes version=3.0.1
+243
View File
@@ -0,0 +1,243 @@
{
"measured_at": "2026-11-03T09:53:05+00:00",
"workers": 8,
"client_timeout_s": 2.0,
"phases_s": {
"before": 5,
"after_action": 10
},
"runs": [
{
"mode": "stop",
"container": "mtusi-lab3-api-1",
"instance_id": "api-fd0cdd87",
"action_seconds": 3.91,
"before": {
"seconds": 5.03,
"ok": 1331,
"errors": 0,
"rps_ok": 264.6,
"p50_ms": 27.1,
"p99_ms": 100.7,
"by_instance": {
"api-79e08828": 666,
"api-fd0cdd87": 665
},
"error_codes": {}
},
"during": {
"seconds": 13.91,
"ok": 3504,
"errors": 0,
"rps_ok": 251.9,
"p50_ms": 29.8,
"p99_ms": 68.3,
"by_instance": {
"api-79e08828": 3360,
"api-fd0cdd87": 144
},
"error_codes": {}
},
"after_restore": {
"seconds": 11.32,
"ok": 2858,
"errors": 0,
"rps_ok": 252.5,
"p50_ms": 27.0,
"p99_ms": 107.7,
"by_instance": {
"api-4a34d5cd": 1336,
"api-79e08828": 1522
},
"error_codes": {}
},
"errors_after_action": 0,
"last_error_after_action_s": 0,
"target_last_served_after_action_s": 0.95,
"max_gap_between_successful_writes_s": 0.12,
"replica_healthy_after_start_s": 5.6,
"restarted_replica_first_served_s": 1.28,
"data": {
"confirmed": 1928,
"lost_confirmed": 0,
"stored": 1928,
"duplicate_texts": 0
}
},
{
"mode": "stop",
"container": "mtusi-lab3-api-2",
"instance_id": "api-79e08828",
"action_seconds": 4.26,
"before": {
"seconds": 5.04,
"ok": 1407,
"errors": 0,
"rps_ok": 278.9,
"p50_ms": 24.8,
"p99_ms": 82.5,
"by_instance": {
"api-4a34d5cd": 701,
"api-79e08828": 706
},
"error_codes": {}
},
"during": {
"seconds": 14.26,
"ok": 3143,
"errors": 0,
"rps_ok": 220.4,
"p50_ms": 33.8,
"p99_ms": 88.7,
"by_instance": {
"api-4a34d5cd": 2931,
"api-79e08828": 212
},
"error_codes": {}
},
"after_restore": {
"seconds": 11.97,
"ok": 2890,
"errors": 0,
"rps_ok": 241.5,
"p50_ms": 29.0,
"p99_ms": 103.4,
"by_instance": {
"api-4a34d5cd": 1579,
"api-563af4cd": 1311
},
"error_codes": {}
},
"errors_after_action": 0,
"last_error_after_action_s": 0,
"target_last_served_after_action_s": 1.53,
"max_gap_between_successful_writes_s": 0.18,
"replica_healthy_after_start_s": 6.01,
"restarted_replica_first_served_s": 1.48,
"data": {
"confirmed": 1865,
"lost_confirmed": 0,
"stored": 1865,
"duplicate_texts": 0
}
},
{
"mode": "kill",
"container": "mtusi-lab3-api-1",
"instance_id": "api-4a34d5cd",
"action_seconds": 0.26,
"before": {
"seconds": 5.04,
"ok": 1492,
"errors": 0,
"rps_ok": 295.8,
"p50_ms": 23.1,
"p99_ms": 93.0,
"by_instance": {
"api-4a34d5cd": 747,
"api-563af4cd": 745
},
"error_codes": {}
},
"during": {
"seconds": 10.26,
"ok": 1914,
"errors": 5,
"rps_ok": 186.6,
"p50_ms": 32.8,
"p99_ms": 309.9,
"by_instance": {
"api-4a34d5cd": 6,
"api-563af4cd": 1908
},
"error_codes": {
"502": 5
}
},
"after_restore": {
"seconds": 11.75,
"ok": 2780,
"errors": 0,
"rps_ok": 236.5,
"p50_ms": 30.5,
"p99_ms": 92.7,
"by_instance": {
"api-471b30bc": 1048,
"api-563af4cd": 1732
},
"error_codes": {}
},
"errors_after_action": 5,
"last_error_after_action_s": 0.08,
"target_last_served_after_action_s": 0.04,
"max_gap_between_successful_writes_s": 0.32,
"replica_healthy_after_start_s": 5.73,
"restarted_replica_first_served_s": 3.57,
"data": {
"confirmed": 1551,
"lost_confirmed": 0,
"stored": 1551,
"duplicate_texts": 0
}
},
{
"mode": "kill",
"container": "mtusi-lab3-api-2",
"instance_id": "api-563af4cd",
"action_seconds": 0.27,
"before": {
"seconds": 5.03,
"ok": 1408,
"errors": 0,
"rps_ok": 279.9,
"p50_ms": 25.5,
"p99_ms": 78.9,
"by_instance": {
"api-471b30bc": 707,
"api-563af4cd": 701
},
"error_codes": {}
},
"during": {
"seconds": 10.27,
"ok": 2175,
"errors": 8,
"rps_ok": 211.8,
"p50_ms": 31.9,
"p99_ms": 307.9,
"by_instance": {
"api-471b30bc": 2171,
"api-563af4cd": 4
},
"error_codes": {
"502": 8
}
},
"after_restore": {
"seconds": 11.95,
"ok": 2609,
"errors": 0,
"rps_ok": 218.3,
"p50_ms": 33.3,
"p99_ms": 112.1,
"by_instance": {
"api-471b30bc": 1448,
"api-50205d38": 1161
},
"error_codes": {}
},
"errors_after_action": 8,
"last_error_after_action_s": 0.1,
"target_last_served_after_action_s": 0.05,
"max_gap_between_successful_writes_s": 0.31,
"replica_healthy_after_start_s": 5.73,
"restarted_replica_first_served_s": 1.78,
"data": {
"confirmed": 1552,
"lost_confirmed": 0,
"stored": 1552,
"duplicate_texts": 0
}
}
]
}
+79
View File
@@ -0,0 +1,79 @@
{
"measured_at": "2026-11-03T09:56:21+00:00",
"workers": 8,
"script_exit": 0,
"script_seconds": 27.88,
"before_replicas": [
{
"container": "mtusi-lab3-api-1",
"instance_id": "api-471b30bc"
},
{
"container": "mtusi-lab3-api-2",
"instance_id": "api-50205d38"
}
],
"after_replicas": [
{
"container": "mtusi-lab3-api-3",
"instance_id": "api-730666b6"
},
{
"container": "mtusi-lab3-api-4",
"instance_id": "api-87d10eab"
}
],
"images_before": [
"sha256:d9d8402d14a0"
],
"images_after": [
"sha256:fb847cbe9916"
],
"before": {
"seconds": 5.04,
"ok": 1236,
"errors": 0,
"rps_ok": 245.0,
"p50_ms": 28.1,
"p99_ms": 109.7,
"by_instance": {
"api-471b30bc": 618,
"api-50205d38": 618
},
"error_codes": {}
},
"during": {
"seconds": 27.88,
"ok": 5696,
"errors": 0,
"rps_ok": 204.3,
"p50_ms": 31.4,
"p99_ms": 152.4,
"by_instance": {
"api-471b30bc": 896,
"api-50205d38": 2061,
"api-730666b6": 1928,
"api-87d10eab": 811
},
"error_codes": {}
},
"after": {
"seconds": 10.0,
"ok": 2821,
"errors": 0,
"rps_ok": 282.1,
"p50_ms": 26.0,
"p99_ms": 74.1,
"by_instance": {
"api-730666b6": 1410,
"api-87d10eab": 1411
},
"error_codes": {}
},
"data": {
"confirmed": 2444,
"lost_confirmed": 0,
"stored": 2444,
"duplicate_texts": 0
}
}
@@ -0,0 +1,79 @@
{
"measured_at": "2026-11-03T09:58:07+00:00",
"workers": 8,
"script_exit": 0,
"script_seconds": 28.32,
"before_replicas": [
{
"container": "mtusi-lab3-api-3",
"instance_id": "api-730666b6"
},
{
"container": "mtusi-lab3-api-4",
"instance_id": "api-87d10eab"
}
],
"after_replicas": [
{
"container": "mtusi-lab3-api-5",
"instance_id": "api-ed7bd071"
},
{
"container": "mtusi-lab3-api-6",
"instance_id": "api-f4fc385b"
}
],
"images_before": [
"sha256:fb847cbe9916"
],
"images_after": [
"sha256:5c115f1efe4d"
],
"before": {
"seconds": 5.04,
"ok": 1465,
"errors": 0,
"rps_ok": 290.7,
"p50_ms": 22.7,
"p99_ms": 77.3,
"by_instance": {
"api-730666b6": 730,
"api-87d10eab": 735
},
"error_codes": {}
},
"during": {
"seconds": 28.32,
"ok": 5710,
"errors": 0,
"rps_ok": 201.6,
"p50_ms": 32.6,
"p99_ms": 134.9,
"by_instance": {
"api-730666b6": 998,
"api-87d10eab": 2125,
"api-ed7bd071": 1910,
"api-f4fc385b": 677
},
"error_codes": {}
},
"after": {
"seconds": 10.0,
"ok": 2450,
"errors": 0,
"rps_ok": 245.0,
"p50_ms": 29.8,
"p99_ms": 87.3,
"by_instance": {
"api-ed7bd071": 1219,
"api-f4fc385b": 1231
},
"error_codes": {}
},
"data": {
"confirmed": 2412,
"lost_confirmed": 0,
"stored": 2412,
"duplicate_texts": 0
}
}
+79
View File
@@ -0,0 +1,79 @@
{
"measured_at": "2026-11-03T09:58:56+00:00",
"workers": 8,
"script_exit": 0,
"script_seconds": 27.48,
"before_replicas": [
{
"container": "mtusi-lab3-api-5",
"instance_id": "api-ed7bd071"
},
{
"container": "mtusi-lab3-api-6",
"instance_id": "api-f4fc385b"
}
],
"after_replicas": [
{
"container": "mtusi-lab3-api-7",
"instance_id": "api-8e374d62"
},
{
"container": "mtusi-lab3-api-8",
"instance_id": "api-b25f5827"
}
],
"images_before": [
"sha256:5c115f1efe4d"
],
"images_after": [
"sha256:d5045137ab03"
],
"before": {
"seconds": 5.03,
"ok": 1381,
"errors": 0,
"rps_ok": 274.4,
"p50_ms": 25.7,
"p99_ms": 80.7,
"by_instance": {
"api-ed7bd071": 693,
"api-f4fc385b": 688
},
"error_codes": {}
},
"during": {
"seconds": 27.48,
"ok": 5274,
"errors": 0,
"rps_ok": 191.9,
"p50_ms": 33.8,
"p99_ms": 172.5,
"by_instance": {
"api-8e374d62": 1644,
"api-b25f5827": 756,
"api-ed7bd071": 972,
"api-f4fc385b": 1902
},
"error_codes": {}
},
"after": {
"seconds": 10.0,
"ok": 2689,
"errors": 0,
"rps_ok": 268.9,
"p50_ms": 27.2,
"p99_ms": 86.0,
"by_instance": {
"api-8e374d62": 1348,
"api-b25f5827": 1341
},
"error_codes": {}
},
"data": {
"confirmed": 2342,
"lost_confirmed": 0,
"stored": 2342,
"duplicate_texts": 0
}
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.05,
"concurrency": 4,
"successful_requests": 7116,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 236.805,
"successful_latency_ms": {
"p50": 15.312,
"p95": 30.786,
"p99": 47.039
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+20
View File
@@ -0,0 +1,20 @@
# 2026-11-03T09:56:22Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.05,
"concurrency": 4,
"successful_requests": 7116,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 236.805,
"successful_latency_ms": {
"p50": 15.312,
"p95": 30.786,
"p99": 47.039
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
exit=0
+5
View File
@@ -0,0 +1,5 @@
# 2026-11-03T09:51:02Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ make check
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
exit=0
+6
View File
@@ -0,0 +1,6 @@
# 2026-11-03T09:53:03Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ dc run --rm --no-deps migrate
Container mtusi-lab3-migrate-run-fa71ca88729f Creating
Container mtusi-lab3-migrate-run-fa71ca88729f Created
{"applied": []}
exit=0
+12
View File
@@ -0,0 +1,12 @@
# 2026-11-03T09:57:03Z empty volume pgdata -> one-shot migrate service (before replicas)
$ docker compose logs migrate
{"ts": "2026-11-03T09:50:54Z", "level": "info", "logger": "messenger.db", "msg": "migration applied", "version": "0001_init"}
{"ts": "2026-11-03T09:50:54Z", "level": "info", "logger": "messenger.db", "msg": "migration applied", "version": "0002_cache_invalidation_outbox"}
{"ts": "2026-11-03T09:50:54Z", "level": "info", "logger": "messenger.db", "msg": "migration applied", "version": "0003_idempotency_keys"}
{"applied": ["0001_init", "0002_cache_invalidation_outbox", "0003_idempotency_keys"]}
{"applied": []}
{"applied": []}
$ api replicas: schema check
{"ts": "2026-11-03T09:55:49Z", "level": "info", "logger": "messenger.http", "msg": "schema compatible; replica ready", "instance_id": "api-730666b6"}
{"ts": "2026-11-03T09:56:01Z", "level": "info", "logger": "messenger.http", "msg": "schema compatible; replica ready", "instance_id": "api-87d10eab"}
+5
View File
@@ -0,0 +1,5 @@
# 2026-11-03T09:52:21Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
exit=0
+18
View File
@@ -0,0 +1,18 @@
== 2026-11-03T09:52:23Z before restart
api mtusi-lab3-api-1 status=running started=2026-11-03T09:52:14.481518513Z pid=24423
api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359
api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502
lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923
migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0
postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033
redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102
volume mtusi-lab3_pgdata
== 2026-11-03T09:52:40Z after restart
api mtusi-lab3-api-1 status=running started=2026-11-03T09:52:37.731388393Z pid=26144
api mtusi-lab3-api-2 status=running started=2026-11-03T09:52:37.507388069Z pid=26087
api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:52:39.487250115Z pid=26364
lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:52:39.921663108Z pid=26460
migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:52:38.793556879Z pid=0
postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:52:31.371856267Z pid=25812
redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:52:31.401186275Z pid=25828
volume mtusi-lab3_pgdata
+5
View File
@@ -0,0 +1,5 @@
# 2026-11-03T09:51:50Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ ./scripts/start-replicas.sh
api mtusi-lab3-api-1 status=running started=2026-11-03T09:51:51.150461585Z pid=22901
api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359
exit=0
+5
View File
@@ -0,0 +1,5 @@
# 2026-11-03T09:52:14Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ ./scripts/start-replicas.sh
api mtusi-lab3-api-1 status=running started=2026-11-03T09:52:14.481518513Z pid=24423
api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359
exit=0
+33
View File
@@ -0,0 +1,33 @@
== 2026-11-03T09:55:43Z rolling update start: 2 replicas, new version 3.0.1
old /mtusi-lab3-api-1 image=sha256:d9d8402d14a0d254f843480ba3d1da45ab5e43b8a9a98c73283cf4205136087b
old /mtusi-lab3-api-2 image=sha256:d9d8402d14a0d254f843480ba3d1da45ab5e43b8a9a98c73283cf4205136087b
Container mtusi-lab3-migrate-run-c15ccd39df8d Creating
Container mtusi-lab3-migrate-run-c15ccd39df8d Created
{"applied": []}
2026-11-03T09:55:54Z new /mtusi-lab3-api-3 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 healthy; stopping old /mtusi-lab3-api-1
2026-11-03T09:56:06Z new /mtusi-lab3-api-4 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 healthy; stopping old /mtusi-lab3-api-2
== 2026-11-03T09:56:10Z rolling update done
now /mtusi-lab3-api-3 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 health=healthy
now /mtusi-lab3-api-4 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 health=healthy
== 2026-11-03T09:57:28Z rolling update start: 2 replicas, new version 3.0.2-expand
old /mtusi-lab3-api-3 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768
old /mtusi-lab3-api-4 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768
Container mtusi-lab3-migrate-run-97c8514a64ea Creating
Container mtusi-lab3-migrate-run-97c8514a64ea Created
{"applied": []}
2026-11-03T09:57:38Z new /mtusi-lab3-api-5 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 healthy; stopping old /mtusi-lab3-api-3
2026-11-03T09:57:51Z new /mtusi-lab3-api-6 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 healthy; stopping old /mtusi-lab3-api-4
== 2026-11-03T09:57:56Z rolling update done
now /mtusi-lab3-api-5 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 health=healthy
now /mtusi-lab3-api-6 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 health=healthy
== 2026-11-03T09:58:18Z rolling update start: 2 replicas, new version 3.0.3
old /mtusi-lab3-api-5 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57
old /mtusi-lab3-api-6 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57
Container mtusi-lab3-migrate-run-728fb26bee21 Creating
Container mtusi-lab3-migrate-run-728fb26bee21 Created
{"applied": []}
2026-11-03T09:58:28Z new /mtusi-lab3-api-7 image=sha256:d5045137ab0313c449760900fee392167eb7b6ef6c6e15cf8d598be6768bbdb3 healthy; stopping old /mtusi-lab3-api-5
2026-11-03T09:58:40Z new /mtusi-lab3-api-8 image=sha256:d5045137ab0313c449760900fee392167eb7b6ef6c6e15cf8d598be6768bbdb3 healthy; stopping old /mtusi-lab3-api-6
== 2026-11-03T09:58:44Z rolling update done
now /mtusi-lab3-api-7 image=sha256:d5045137ab0313c449760900fee392167eb7b6ef6c6e15cf8d598be6768bbdb3 health=healthy
now /mtusi-lab3-api-8 image=sha256:d5045137ab0313c449760900fee392167eb7b6ef6c6e15cf8d598be6768bbdb3 health=healthy
+4
View File
@@ -0,0 +1,4 @@
# 2026-11-03T09:51:02Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 tests/smoke.py --base-url http://127.0.0.1:8080
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
exit=0
+25
View File
@@ -0,0 +1,25 @@
# 2026-11-03T09:51:01Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
docker 29.9.2
Docker Compose version v5.6.1
CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED
mtusi-lab3-api-1 mtusi-lab3-api local linux/amd64 d9d8402d14a0 2.63GB 2 hours ago
mtusi-lab3-api-2 mtusi-lab3-api local linux/amd64 d9d8402d14a0 2.63GB 2 hours ago
mtusi-lab3-api-ttl-1 mtusi-lab3-api local linux/amd64 d9d8402d14a0 2.63GB 2 hours ago
mtusi-lab3-lb-1 mtusi-local/haproxy 3.0 linux/amd64 c93e4aeed30e 42MB 3 hours ago
mtusi-lab3-migrate-1 mtusi-lab3-api local linux/amd64 d9d8402d14a0 2.63GB 2 hours ago
mtusi-lab3-postgres-1 mtusi-local/postgres 16 linux/amd64 2f004dbde9b5 337MB 3 hours ago
mtusi-lab3-redis-1 mtusi-local/redis 7 linux/amd64 01c11b3e9bd0 37.1MB 3 hours ago
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi-lab3-api-1 mtusi-lab3-api:local "python -m messenger…" api 14 seconds ago Up 6 seconds (healthy) 8080/tcp
mtusi-lab3-api-2 mtusi-lab3-api:local "python -m messenger…" api 14 seconds ago Up 7 seconds (healthy) 8080/tcp
mtusi-lab3-api-ttl-1 mtusi-lab3-api:local "python -m messenger…" api-ttl 14 seconds ago Up 6 seconds (healthy) 8080/tcp
mtusi-lab3-lb-1 mtusi-local/haproxy:3.0 "haproxy -W -db -f /…" lb 14 seconds ago Up 14 seconds (healthy) 127.0.0.1:8080-8081->8080-8081/tcp, 127.0.0.1:8404->8404/tcp
mtusi-lab3-postgres-1 mtusi-local/postgres:16 "docker-entrypoint.s…" postgres 14 seconds ago Up 13 seconds (healthy)
mtusi-lab3-redis-1 mtusi-local/redis:7 "sh -c 'umask 077\npr…" redis 14 seconds ago Up 13 seconds (healthy)
api mtusi-lab3-api-1 status=running started=2026-11-03T09:50:54.761533652Z pid=19437
api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359
api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502
lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923
migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0
postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033
redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102
+23
View File
@@ -0,0 +1,23 @@
# 2026-11-03T09:51:04Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 student-tests/test_api.py
## API, auth, validation, ACL, transactions (http://127.0.0.1:8080)
PASS password stored only as scrypt KDF hash; User has no secret fields 0.66s
PASS wrong password and unknown user: identical 401 + Basic challenge 0.86s
PASS malformed Basic / Bearer on /auth/sessions / body on login 0.01s
PASS password with ':' and non-ASCII UTF-8 works (RFC 7617) 0.53s
PASS opaque 256-bit token, unique per login, DB stores only SHA-256 0.97s
PASS only Bearer accepted; Basic and X-User-Id never authenticate 0.02s
PASS logout revokes only the current session; repeat -> 401 1.14s
PASS strict JSON validation -> 400 0.04s
PASS message text: 4000 code points OK, 4001 / whitespace / sender_id -> 400 0.04s
PASS SQL is parameterized: injection payloads stored verbatim 0.02s
PASS ACL: outsider gets 404 everywhere (even for invalid body) 0.03s
PASS create chat validation: duplicates, caller missing, unknown member 0.01s
PASS compound create-chat rolls back on FK failure (no orphan chat) 0.56s chats before=2, failed insert left 0 rows
PASS registration race: 20 parallel same username -> exactly one 201 3.49s 201x1, 409x19, rows=1
PASS keyset pagination: 120 msgs, varying limit, no gaps/duplicates 0.94s 4 pages
PASS chat list: only own chats, ordered, cursor bound to user 0.77s
PASS read-after-write: confirmed message visible to next read 0.01s
PASS unknown route 404 / wrong method 405 in error format 0.00s
summary: 18 passed, 0 failed
exit=0
+16
View File
@@ -0,0 +1,16 @@
# 2026-11-03T09:51:31Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ make test-cache-race
docker compose --env-file .env run --rm --no-deps -T -v "/home/claude/work3/mtusi2026lab3/student-tests:/app/student-tests:ro" --entrypoint python api student-tests/test_cache_race.py
Container mtusi-lab3-api-run-bc892bef496f Creating
Container mtusi-lab3-api-run-bc892bef496f Created
PASS read-through put makes entry readable
PASS entry TTL capped by SESSION_CACHE_TTL_SECONDS pttl=60000ms
PASS invalidate removes entry
PASS late put after logout is refused by tombstone
PASS tombstone outlives any cache entry pttl=74998ms
PASS entry never outlives session expiry pttl=1199ms
PASS expired session not served from cache
cache unavailable; falling back to PostgreSQL
PASS degraded mode: cache is not read until outbox drained
summary: all passed
exit=0
+8
View File
@@ -0,0 +1,8 @@
# 2026-11-03T09:51:21Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 student-tests/test_expiry.py
## session expiry, SESSION_TTL_SECONDS=2 (http://127.0.0.1:8081)
PASS expires_at = now + SESSION_TTL_SECONDS 1.20s remaining 2.00s
PASS token valid before expiry, 401 Bearer after (cache warmed) 4.52s
PASS re-login after expiry issues a new valid session 4.82s
summary: 3 passed, 0 failed
exit=0
+7
View File
@@ -0,0 +1,7 @@
# 2026-11-03T09:52:40Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 student-tests/test_failures.py
## dependency failures (http://127.0.0.1:8080)
PASS Redis down: API bypasses cache via PostgreSQL, no anonymous trust 6.95s fallback latency 0.006s; outbox rows during outage=1, drained after recovery
PASS PostgreSQL down: bounded 503 in error format; anonymous still 401 14.78s latency(s)={'ready': 1.5, 'post_message': 2.01, 'history': 2.01, 'login': 2.01, 'register': 2.0}; ready again in 2.6s
summary: 2 passed, 0 failed
exit=0
+14
View File
@@ -0,0 +1,14 @@
# 2026-11-03T09:51:16Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2
$ python3 student-tests/test_idempotency.py
## idempotency across replicas (http://127.0.0.1:8080)
PASS sequential replays: same 201 + same Message from different replicas 0.29s replicas=2, rows=1
PASS 30 concurrent POSTs, same key + body, via LB -> exactly one message 0.63s per-replica responses {'api-7e27119d': 15, 'api-5dbb8cd6': 15}, rows=1
PASS concurrent same key, two different bodies -> one message, others 201-same or 409 0.78s 201x10, 409x10
PASS after commit: different body -> 409; reordered/whitespace JSON -> replay 0.04s
PASS scope: user + chat + key (same key elsewhere is independent) 0.03s
PASS key format: missing/empty/129 chars/invalid -> 400; 128 chars ok 0.05s
PASS failed attempt does not claim the key (400/404 not recorded) 0.02s
PASS replay re-checks session and access 0.34s
PASS key older than IDEMPOTENCY_TTL_SECONDS is reusable (24 h window) 0.49s
summary: 9 passed, 0 failed
exit=0
@@ -0,0 +1,887 @@
{
"config": "admission-1x1.00-mc10-q250",
"repeat": 1,
"series": "admission",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": 10,
"LB_QUEUE_TIMEOUT": "250ms",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T09:11:19+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 200,
"peak_ok_rps": 303.5,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 28.13,
"p50": 5.56,
"p90": 8.81,
"p99": 16.48,
"p999": 28.13
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.38,
"p99_ms": 18.47
},
"me": {
"ok": 40,
"p50_ms": 2.84,
"p99_ms": 13.55
},
"post": {
"ok": 80,
"p50_ms": 6.51,
"p99_ms": 21.29
}
},
"rate": 40,
"service_ms": {
"p50": 4.9,
"p99": 15.4
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 17.1,
"lb": 2.4,
"loadgen-run-57259902c18b": 2.9,
"postgres": 6.3,
"redis": 2.8
},
"mem_mib_max": {
"api": 31.5,
"lb": 4.4,
"loadgen-run-57259902c18b": 5.9,
"postgres": 99.1,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 70.47,
"p50": 5.47,
"p90": 8.7,
"p99": 20.74,
"p999": 70.47
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.34,
"p99_ms": 18.1
},
"me": {
"ok": 80,
"p50_ms": 2.56,
"p99_ms": 70.47
},
"post": {
"ok": 160,
"p50_ms": 6.46,
"p99_ms": 16.02
}
},
"rate": 80,
"service_ms": {
"p50": 4.88,
"p99": 20.42
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 30.7,
"lb": 3.5,
"loadgen-run-57259902c18b": 5.4,
"postgres": 8.5,
"redis": 2.8
},
"mem_mib_max": {
"api": 34.5,
"lb": 4.4,
"loadgen-run-57259902c18b": 6.2,
"postgres": 99.8,
"redis": 4.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 114.87,
"p50": 5.52,
"p90": 11.24,
"p99": 35.76,
"p999": 78.88
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.5,
"p99_ms": 43.8
},
"me": {
"ok": 120,
"p50_ms": 2.54,
"p99_ms": 7.92
},
"post": {
"ok": 240,
"p50_ms": 6.19,
"p99_ms": 22.32
}
},
"rate": 120,
"service_ms": {
"p50": 4.92,
"p99": 35.08
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 45.0,
"lb": 4.7,
"loadgen-run-57259902c18b": 7.6,
"postgres": 12.0,
"redis": 3.8
},
"mem_mib_max": {
"api": 32.4,
"lb": 4.5,
"loadgen-run-57259902c18b": 6.7,
"postgres": 107.0,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 12.4
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 85,
"p50": 5.71,
"p90": 18.4,
"p99": 68.32,
"p999": 83.44
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.79,
"p99_ms": 53.82
},
"me": {
"ok": 160,
"p50_ms": 2.38,
"p99_ms": 67.83
},
"post": {
"ok": 320,
"p50_ms": 6.16,
"p99_ms": 78.9
}
},
"rate": 160,
"service_ms": {
"p50": 5.12,
"p99": 67.77
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 57.3,
"lb": 5.3,
"loadgen-run-57259902c18b": 8.8,
"postgres": 14.3,
"redis": 3.9
},
"mem_mib_max": {
"api": 33.8,
"lb": 4.5,
"loadgen-run-57259902c18b": 6.7,
"postgres": 112.1,
"redis": 5.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.5,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.22
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14.9
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 243.98,
"p50": 8.73,
"p90": 82.82,
"p99": 216.96,
"p999": 240.66
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 11.33,
"p99_ms": 216.55
},
"me": {
"ok": 200,
"p50_ms": 2.43,
"p99_ms": 163.11
},
"post": {
"ok": 400,
"p50_ms": 6.82,
"p99_ms": 227.73
}
},
"rate": 200,
"service_ms": {
"p50": 8.1,
"p99": 216.86
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 71.7,
"lb": 6.3,
"loadgen-run-57259902c18b": 10.7,
"postgres": 16.1,
"redis": 3.8
},
"mem_mib_max": {
"api": 33.5,
"lb": 5.2,
"loadgen-run-57259902c18b": 7.2,
"postgres": 113.6,
"redis": 3.8
},
"lb_queue_max": 4,
"lb_sessions_mean": 5.2,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 6
},
"error_ratio": 0.0025,
"errors": 6,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 317.64,
"p50": 84.99,
"p90": 228.57,
"p99": 272.63,
"p999": 309.43
},
"ok": 2394,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 85.11,
"p99_ms": 269.69
},
"me": {
"ok": 240,
"p50_ms": 58.49,
"p99_ms": 239.04
},
"post": {
"ok": 474,
"p50_ms": 98.95,
"p99_ms": 301.35
}
},
"rate": 240,
"service_ms": {
"p50": 84.12,
"p99": 271.95
},
"throughput_ok_rps": 239.4,
"resources": {
"cpu_percent_mean": {
"api": 86.7,
"lb": 7.2,
"loadgen-run-57259902c18b": 13.2,
"postgres": 18.9,
"redis": 4.0
},
"mem_mib_max": {
"api": 44.4,
"lb": 5.8,
"loadgen-run-57259902c18b": 7.8,
"postgres": 114.5,
"redis": 5.1
},
"lb_queue_max": 37,
"lb_sessions_mean": 23.2,
"pg_active_mean": 0.67,
"pg_wait_events_mean": {
"CPU": 0.22,
"IO:WALSync": 0.33,
"LWLock:WALWrite": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 329
},
"error_ratio": 0.1175,
"errors": 329,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 380.71,
"p50": 256.44,
"p90": 301.41,
"p99": 337.81,
"p999": 377.64
},
"ok": 2471,
"per_op": {
"history": {
"ok": 1757,
"p50_ms": 260.64,
"p99_ms": 337.81
},
"me": {
"ok": 255,
"p50_ms": 234.22,
"p99_ms": 277.5
},
"post": {
"ok": 459,
"p50_ms": 273.47,
"p99_ms": 358.69
}
},
"rate": 280,
"service_ms": {
"p50": 256.04,
"p99": 337.24
},
"throughput_ok_rps": 247.1,
"resources": {
"cpu_percent_mean": {
"api": 89.0,
"lb": 7.8,
"loadgen-run-57259902c18b": 15.2,
"postgres": 19.1,
"redis": 3.8
},
"mem_mib_max": {
"api": 35.0,
"lb": 6.1,
"loadgen-run-57259902c18b": 8.1,
"postgres": 113.3,
"redis": 3.8
},
"lb_queue_max": 69,
"lb_sessions_mean": 62.6,
"pg_active_mean": 0.2,
"pg_wait_events_mean": {
"Client:ClientRead": 0.1,
"LWLock:WALWrite": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 320
},
"error_ratio": 0.1,
"errors": 320,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 401.65,
"p50": 257.26,
"p90": 291.54,
"p99": 325.68,
"p999": 369.28
},
"ok": 2880,
"per_op": {
"history": {
"ok": 1994,
"p50_ms": 260.09,
"p99_ms": 331.06
},
"me": {
"ok": 318,
"p50_ms": 232.19,
"p99_ms": 271.83
},
"post": {
"ok": 568,
"p50_ms": 262.35,
"p99_ms": 311.67
}
},
"rate": 320,
"service_ms": {
"p50": 256.65,
"p99": 323.66
},
"throughput_ok_rps": 288,
"resources": {
"cpu_percent_mean": {
"api": 90.4,
"lb": 7.9,
"loadgen-run-57259902c18b": 16.0,
"postgres": 20.5,
"redis": 4.1
},
"mem_mib_max": {
"api": 37.5,
"lb": 6.2,
"loadgen-run-57259902c18b": 8.3,
"postgres": 115.0,
"redis": 3.9
},
"lb_queue_max": 80,
"lb_sessions_mean": 81.6,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 565
},
"error_ratio": 0.1569,
"errors": 565,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 344.98,
"p50": 269.26,
"p90": 293.54,
"p99": 319.02,
"p999": 338.42
},
"ok": 3035,
"per_op": {
"history": {
"ok": 2134,
"p50_ms": 272.49,
"p99_ms": 322.22
},
"me": {
"ok": 338,
"p50_ms": 242.11,
"p99_ms": 270.86
},
"post": {
"ok": 563,
"p50_ms": 272.11,
"p99_ms": 314.63
}
},
"rate": 360,
"service_ms": {
"p50": 268.51,
"p99": 319
},
"throughput_ok_rps": 303.5,
"resources": {
"cpu_percent_mean": {
"api": 91.5,
"lb": 8.2,
"loadgen-run-57259902c18b": 16.9,
"postgres": 22.9,
"redis": 4.1
},
"mem_mib_max": {
"api": 33.9,
"lb": 6.6,
"loadgen-run-57259902c18b": 8.9,
"postgres": 120.7,
"redis": 3.9
},
"lb_queue_max": 90,
"lb_sessions_mean": 93.7,
"pg_active_mean": 0.78,
"pg_wait_events_mean": {
"IO:WALSync": 0.11,
"LWLock:WALWrite": 0.67,
"Timeout:VacuumDelay": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1070
},
"error_ratio": 0.2675,
"errors": 1070,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 350.75,
"p50": 276.72,
"p90": 299.81,
"p99": 325.41,
"p999": 339.11
},
"ok": 2930,
"per_op": {
"history": {
"ok": 2039,
"p50_ms": 279.88,
"p99_ms": 326.46
},
"me": {
"ok": 372,
"p50_ms": 246.41,
"p99_ms": 264.6
},
"post": {
"ok": 519,
"p50_ms": 276.53,
"p99_ms": 320.36
}
},
"rate": 400,
"service_ms": {
"p50": 276,
"p99": 324.62
},
"throughput_ok_rps": 293,
"resources": {
"cpu_percent_mean": {
"api": 91.2,
"lb": 8.3,
"loadgen-run-57259902c18b": 17.3,
"postgres": 19.7,
"redis": 4.2
},
"mem_mib_max": {
"api": 33.9,
"lb": 6.8,
"loadgen-run-57259902c18b": 10.2,
"postgres": 115.2,
"redis": 3.7
},
"lb_queue_max": 101,
"lb_sessions_mean": 108.1,
"pg_active_mean": 1.22,
"pg_wait_events_mean": {
"IO:WALSync": 0.33,
"LWLock:BufferContent": 0.11,
"LWLock:WALWrite": 0.78
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1616
},
"error_ratio": 0.3513,
"errors": 1616,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 372.94,
"p50": 279.33,
"p90": 298.59,
"p99": 337.51,
"p999": 371.1
},
"ok": 2984,
"per_op": {
"history": {
"ok": 2061,
"p50_ms": 281.81,
"p99_ms": 353.35
},
"me": {
"ok": 392,
"p50_ms": 249.67,
"p99_ms": 277.77
},
"post": {
"ok": 531,
"p50_ms": 277.99,
"p99_ms": 335.47
}
},
"rate": 460,
"service_ms": {
"p50": 278.68,
"p99": 336.83
},
"throughput_ok_rps": 298.4,
"resources": {
"cpu_percent_mean": {
"api": 89.1,
"lb": 8.8,
"loadgen-run-57259902c18b": 18.6,
"postgres": 19.8,
"redis": 3.6
},
"mem_mib_max": {
"api": 41.2,
"lb": 7.1,
"loadgen-run-57259902c18b": 10.6,
"postgres": 115.5,
"redis": 4.4
},
"lb_queue_max": 116,
"lb_sessions_mean": 123.1,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": false
},
{
"attempted": 5200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2178
},
"error_ratio": 0.4188,
"errors": 2178,
"event": "step",
"label": "admission-1x1.00-mc10-q250",
"latency_ms": {
"max": 338.87,
"p50": 279.63,
"p90": 301.71,
"p99": 326.56,
"p999": 336.07
},
"ok": 3022,
"per_op": {
"history": {
"ok": 2079,
"p50_ms": 281.76,
"p99_ms": 328.77
},
"me": {
"ok": 405,
"p50_ms": 252.07,
"p99_ms": 278.26
},
"post": {
"ok": 538,
"p50_ms": 278.28,
"p99_ms": 320.24
}
},
"rate": 520,
"service_ms": {
"p50": 278.89,
"p99": 326.13
},
"throughput_ok_rps": 302.2,
"resources": {
"cpu_percent_mean": {
"api": 89.5,
"lb": 9.3,
"loadgen-run-57259902c18b": 19.7,
"postgres": 20.1,
"redis": 3.6
},
"mem_mib_max": {
"api": 33.9,
"lb": 7.5,
"loadgen-run-57259902c18b": 11.6,
"postgres": 117.0,
"redis": 4.5
},
"lb_queue_max": 130,
"lb_sessions_mean": 137.8,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"Client:ClientRead": 0.22
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": false
}
],
"pool_before": {
"api-a6b2b034": {
"pool": {
"connections_num": 3,
"requests_num": 19,
"requests_queued": 3,
"connections_ms": 74,
"requests_wait_ms": 74,
"usage_ms": 53,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 8
}
},
"pool_after": {
"api-a6b2b034": {
"pool": {
"connections_num": 10,
"requests_num": 31049,
"requests_queued": 195,
"connections_ms": 424,
"requests_wait_ms": 1850,
"usage_ms": 570234,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 33917,
"misses": 108,
"errors": 0
},
"requests": 34351
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
@@ -0,0 +1,929 @@
{
"config": "admission-1x1.00-mc10-r2",
"repeat": 2,
"series": "admission",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": 10,
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T09:28:38+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 320,
"peak_ok_rps": 383.7,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 31.44,
"p50": 4.89,
"p90": 7.88,
"p99": 23.86,
"p999": 31.44
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 4.77,
"p99_ms": 12.48
},
"me": {
"ok": 40,
"p50_ms": 2.41,
"p99_ms": 17.55
},
"post": {
"ok": 80,
"p50_ms": 5.69,
"p99_ms": 28.51
}
},
"rate": 40,
"service_ms": {
"p50": 4.27,
"p99": 16.5
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 14.9,
"lb": 2.0,
"loadgen-run-3312d64e3026": 2.5,
"postgres": 6.3,
"redis": 2.6
},
"mem_mib_max": {
"api": 39.8,
"lb": 4.3,
"loadgen-run-3312d64e3026": 5.9,
"postgres": 135.4,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 28,
"p50": 4.34,
"p90": 6.93,
"p99": 16.14,
"p999": 28
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 4.26,
"p99_ms": 16.33
},
"me": {
"ok": 80,
"p50_ms": 2.17,
"p99_ms": 12.02
},
"post": {
"ok": 160,
"p50_ms": 5.3,
"p99_ms": 17.57
}
},
"rate": 80,
"service_ms": {
"p50": 3.72,
"p99": 15.43
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 24.6,
"lb": 2.8,
"loadgen-run-3312d64e3026": 4.1,
"postgres": 8.6,
"redis": 2.7
},
"mem_mib_max": {
"api": 31.9,
"lb": 4.4,
"loadgen-run-3312d64e3026": 6.2,
"postgres": 135.8,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 172.91,
"p50": 4.28,
"p90": 8.49,
"p99": 106.25,
"p999": 137.4
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 4.17,
"p99_ms": 25.3
},
"me": {
"ok": 120,
"p50_ms": 2.22,
"p99_ms": 22.61
},
"post": {
"ok": 240,
"p50_ms": 5.15,
"p99_ms": 126.08
}
},
"rate": 120,
"service_ms": {
"p50": 3.71,
"p99": 88.98
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 35.8,
"lb": 3.5,
"loadgen-run-3312d64e3026": 5.8,
"postgres": 10.9,
"redis": 3.2
},
"mem_mib_max": {
"api": 43.5,
"lb": 4.5,
"loadgen-run-3312d64e3026": 6.7,
"postgres": 145.0,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 361.47,
"p50": 4.41,
"p90": 17.32,
"p99": 297.48,
"p999": 359.57
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 4.53,
"p99_ms": 310.75
},
"me": {
"ok": 160,
"p50_ms": 1.77,
"p99_ms": 223.01
},
"post": {
"ok": 320,
"p50_ms": 4.63,
"p99_ms": 299.66
}
},
"rate": 160,
"service_ms": {
"p50": 3.8,
"p99": 275.17
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 49.6,
"lb": 4.3,
"loadgen-run-3312d64e3026": 7.3,
"postgres": 13.3,
"redis": 3.0
},
"mem_mib_max": {
"api": 32.9,
"lb": 5.1,
"loadgen-run-3312d64e3026": 6.9,
"postgres": 147.4,
"redis": 3.9
},
"lb_queue_max": 17,
"lb_sessions_mean": 3.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 18.2
},
"slo_ok": false
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 85.88,
"p50": 4.59,
"p90": 11.96,
"p99": 49.95,
"p999": 68.67
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 4.8,
"p99_ms": 52.14
},
"me": {
"ok": 200,
"p50_ms": 1.79,
"p99_ms": 11.98
},
"post": {
"ok": 400,
"p50_ms": 4.52,
"p99_ms": 47.34
}
},
"rate": 200,
"service_ms": {
"p50": 4,
"p99": 49.53
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 61.3,
"lb": 4.8,
"loadgen-run-3312d64e3026": 8.6,
"postgres": 14.6,
"redis": 3.3
},
"mem_mib_max": {
"api": 41.8,
"lb": 5.2,
"loadgen-run-3312d64e3026": 7.2,
"postgres": 150.1,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 1,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 160.53,
"p50": 5.22,
"p90": 26.54,
"p99": 131.48,
"p999": 152.71
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 5.58,
"p99_ms": 134.06
},
"me": {
"ok": 240,
"p50_ms": 1.82,
"p99_ms": 74.51
},
"post": {
"ok": 480,
"p50_ms": 4.95,
"p99_ms": 125.58
}
},
"rate": 240,
"service_ms": {
"p50": 4.69,
"p99": 131.27
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 71.8,
"lb": 5.5,
"loadgen-run-3312d64e3026": 9.7,
"postgres": 16.5,
"redis": 3.5
},
"mem_mib_max": {
"api": 33.5,
"lb": 5.1,
"loadgen-run-3312d64e3026": 7.2,
"postgres": 149.8,
"redis": 4.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 158.74,
"p50": 11.25,
"p90": 74.76,
"p99": 147.64,
"p999": 156.21
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 13.02,
"p99_ms": 149.26
},
"me": {
"ok": 280,
"p50_ms": 1.84,
"p99_ms": 111.47
},
"post": {
"ok": 560,
"p50_ms": 8.86,
"p99_ms": 141.55
}
},
"rate": 280,
"service_ms": {
"p50": 10.56,
"p99": 146.93
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 80.9,
"lb": 6.1,
"loadgen-run-3312d64e3026": 11.5,
"postgres": 18.6,
"redis": 3.8
},
"mem_mib_max": {
"api": 44.4,
"lb": 5.4,
"loadgen-run-3312d64e3026": 7.1,
"postgres": 151.1,
"redis": 5.2
},
"lb_queue_max": 29,
"lb_sessions_mean": 11,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 223.6,
"p50": 52.59,
"p90": 145.14,
"p99": 189.02,
"p999": 207.16
},
"ok": 3200,
"per_op": {
"history": {
"ok": 2240,
"p50_ms": 52.17,
"p99_ms": 194.35
},
"me": {
"ok": 320,
"p50_ms": 23.12,
"p99_ms": 150.98
},
"post": {
"ok": 640,
"p50_ms": 58.37,
"p99_ms": 170.51
}
},
"rate": 320,
"service_ms": {
"p50": 52.06,
"p99": 188.24
},
"throughput_ok_rps": 320,
"resources": {
"cpu_percent_mean": {
"api": 89.2,
"lb": 6.6,
"loadgen-run-3312d64e3026": 12.9,
"postgres": 19.9,
"redis": 3.6
},
"mem_mib_max": {
"api": 46.6,
"lb": 6.3,
"loadgen-run-3312d64e3026": 8.0,
"postgres": 150.3,
"redis": 3.9
},
"lb_queue_max": 30,
"lb_sessions_mean": 18.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 443.81,
"p50": 209.96,
"p90": 388.26,
"p99": 418.5,
"p999": 435.16
},
"ok": 3600,
"per_op": {
"history": {
"ok": 2520,
"p50_ms": 210.07,
"p99_ms": 418.5
},
"me": {
"ok": 360,
"p50_ms": 183.94,
"p99_ms": 393.26
},
"post": {
"ok": 720,
"p50_ms": 216.42,
"p99_ms": 421.47
}
},
"rate": 360,
"service_ms": {
"p50": 209.57,
"p99": 418.08
},
"throughput_ok_rps": 360,
"resources": {
"cpu_percent_mean": {
"api": 92.0,
"lb": 7.1,
"loadgen-run-3312d64e3026": 15.0,
"postgres": 20.5,
"redis": 3.5
},
"mem_mib_max": {
"api": 34.2,
"lb": 7.5,
"loadgen-run-3312d64e3026": 11.6,
"postgres": 150.8,
"redis": 4.0
},
"lb_queue_max": 134,
"lb_sessions_mean": 94.6,
"pg_active_mean": 0.33,
"pg_wait_events_mean": {
"CPU": 0.11,
"IO:WALSync": 0.11,
"LWLock:WALWrite": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 163
},
"error_ratio": 0.0408,
"errors": 163,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 2079.46,
"p50": 1478.7,
"p90": 2024.57,
"p99": 2046.5,
"p999": 2071.4
},
"ok": 3837,
"per_op": {
"history": {
"ok": 2685,
"p50_ms": 1475.46,
"p99_ms": 2045.91
},
"me": {
"ok": 396,
"p50_ms": 1463.34,
"p99_ms": 2016.25
},
"post": {
"ok": 756,
"p50_ms": 1484.41,
"p99_ms": 2056.84
}
},
"rate": 400,
"service_ms": {
"p50": 1478.04,
"p99": 2045.97
},
"throughput_ok_rps": 383.7,
"resources": {
"cpu_percent_mean": {
"api": 92.8,
"lb": 7.8,
"loadgen-run-3312d64e3026": 17.4,
"postgres": 19.9,
"redis": 3.2
},
"mem_mib_max": {
"api": 37.8,
"lb": 18.6,
"loadgen-run-3312d64e3026": 42.2,
"postgres": 151.3,
"redis": 4.1
},
"lb_queue_max": 799,
"lb_sessions_mean": 509.8,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1121
},
"error_ratio": 0.2437,
"errors": 1121,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 2136.81,
"p50": 2019.07,
"p90": 2038.79,
"p99": 2080.88,
"p999": 2124.48
},
"ok": 3479,
"per_op": {
"history": {
"ok": 2395,
"p50_ms": 2024.2,
"p99_ms": 2084.75
},
"me": {
"ok": 417,
"p50_ms": 1995.39,
"p99_ms": 2028.96
},
"post": {
"ok": 667,
"p50_ms": 2018.88,
"p99_ms": 2078.09
}
},
"rate": 460,
"service_ms": {
"p50": 2018.44,
"p99": 2079.6
},
"throughput_ok_rps": 347.9,
"resources": {
"cpu_percent_mean": {
"api": 91.3,
"lb": 8.0,
"loadgen-run-3312d64e3026": 17.2,
"postgres": 19.8,
"redis": 3.8
},
"mem_mib_max": {
"api": 42.0,
"lb": 23.1,
"loadgen-run-3312d64e3026": 54.8,
"postgres": 152.3,
"redis": 3.9
},
"lb_queue_max": 921,
"lb_sessions_mean": 828.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": false
},
{
"attempted": 5200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1774
},
"error_ratio": 0.3412,
"errors": 1774,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 2103.38,
"p50": 2026.19,
"p90": 2041.71,
"p99": 2068.1,
"p999": 2097.32
},
"ok": 3426,
"per_op": {
"history": {
"ok": 2351,
"p50_ms": 2029.29,
"p99_ms": 2070.26
},
"me": {
"ok": 448,
"p50_ms": 2000.47,
"p99_ms": 2016.55
},
"post": {
"ok": 627,
"p50_ms": 2022.82,
"p99_ms": 2067.05
}
},
"rate": 520,
"service_ms": {
"p50": 2025.47,
"p99": 2067.67
},
"throughput_ok_rps": 342.6,
"resources": {
"cpu_percent_mean": {
"api": 90.7,
"lb": 8.1,
"loadgen-run-3312d64e3026": 18.0,
"postgres": 19.6,
"redis": 3.4
},
"mem_mib_max": {
"api": 48.6,
"lb": 25.5,
"loadgen-run-3312d64e3026": 60.2,
"postgres": 153.8,
"redis": 4.0
},
"lb_queue_max": 1042,
"lb_sessions_mean": 993,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": false
},
{
"attempted": 6000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2623
},
"error_ratio": 0.4372,
"errors": 2623,
"event": "step",
"label": "admission-1x1.00-mc10-r2",
"latency_ms": {
"max": 2132.2,
"p50": 2027.89,
"p90": 2042.9,
"p99": 2087.73,
"p999": 2109.01
},
"ok": 3377,
"per_op": {
"history": {
"ok": 2308,
"p50_ms": 2029.99,
"p99_ms": 2087.06
},
"me": {
"ok": 467,
"p50_ms": 2002.35,
"p99_ms": 2036.52
},
"post": {
"ok": 602,
"p50_ms": 2024.5,
"p99_ms": 2100.69
}
},
"rate": 600,
"service_ms": {
"p50": 2027.28,
"p99": 2086.75
},
"throughput_ok_rps": 337.7,
"resources": {
"cpu_percent_mean": {
"api": 90.1,
"lb": 8.9,
"loadgen-run-3312d64e3026": 19.4,
"postgres": 22.7,
"redis": 3.5
},
"mem_mib_max": {
"api": 39.8,
"lb": 29.1,
"loadgen-run-3312d64e3026": 70.3,
"postgres": 164.4,
"redis": 5.2
},
"lb_queue_max": 1206,
"lb_sessions_mean": 1161.1,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1,
"Client:ClientRead": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": false
}
],
"pool_before": {
"api-87aae2e3": {
"pool": {
"connections_num": 3,
"requests_num": 19,
"requests_queued": 3,
"connections_ms": 74,
"requests_wait_ms": 86,
"usage_ms": 42,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 8
}
},
"pool_after": {
"api-87aae2e3": {
"pool": {
"connections_num": 10,
"requests_num": 40145,
"requests_queued": 196,
"connections_ms": 523,
"requests_wait_ms": 1731,
"usage_ms": 563973,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 43819,
"misses": 138,
"errors": 0
},
"requests": 44305
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
@@ -0,0 +1,931 @@
{
"config": "admission-1x1.00-mc10-r3",
"repeat": 3,
"series": "admission",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": 10,
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T09:41:30+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 280,
"peak_ok_rps": 359.7,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 34.98,
"p50": 5.3,
"p90": 8.99,
"p99": 21.81,
"p999": 34.98
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.2,
"p99_ms": 23.17
},
"me": {
"ok": 40,
"p50_ms": 2.54,
"p99_ms": 6.17
},
"post": {
"ok": 80,
"p50_ms": 6.32,
"p99_ms": 34.98
}
},
"rate": 40,
"service_ms": {
"p50": 4.65,
"p99": 21.02
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 16.0,
"lb": 2.2,
"loadgen-run-d7f8aed8393a": 2.7,
"postgres": 8.1,
"redis": 2.5
},
"mem_mib_max": {
"api": 31.4,
"lb": 4.4,
"loadgen-run-d7f8aed8393a": 5.9,
"postgres": 163.6,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 19
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 38.36,
"p50": 4.78,
"p90": 8.54,
"p99": 21.18,
"p999": 38.36
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 4.73,
"p99_ms": 21.18
},
"me": {
"ok": 80,
"p50_ms": 2.27,
"p99_ms": 4.19
},
"post": {
"ok": 160,
"p50_ms": 5.48,
"p99_ms": 31.65
}
},
"rate": 80,
"service_ms": {
"p50": 4.25,
"p99": 20.02
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 27.2,
"lb": 3.2,
"loadgen-run-d7f8aed8393a": 4.7,
"postgres": 9.4,
"redis": 2.8
},
"mem_mib_max": {
"api": 31.9,
"lb": 4.4,
"loadgen-run-d7f8aed8393a": 6.1,
"postgres": 164.0,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 19
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 35.4,
"p50": 4.88,
"p90": 9.14,
"p99": 20.6,
"p999": 34.41
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 4.93,
"p99_ms": 20.6
},
"me": {
"ok": 120,
"p50_ms": 2.3,
"p99_ms": 6.95
},
"post": {
"ok": 240,
"p50_ms": 5.24,
"p99_ms": 24.37
}
},
"rate": 120,
"service_ms": {
"p50": 4.22,
"p99": 18.43
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 39.0,
"lb": 4.2,
"loadgen-run-d7f8aed8393a": 6.5,
"postgres": 12.7,
"redis": 3.5
},
"mem_mib_max": {
"api": 32.9,
"lb": 4.5,
"loadgen-run-d7f8aed8393a": 6.6,
"postgres": 166.5,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 119.37,
"p50": 4.89,
"p90": 11.07,
"p99": 42.87,
"p999": 81.01
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.05,
"p99_ms": 54.86
},
"me": {
"ok": 160,
"p50_ms": 2,
"p99_ms": 13.04
},
"post": {
"ok": 320,
"p50_ms": 5.09,
"p99_ms": 18.23
}
},
"rate": 160,
"service_ms": {
"p50": 4.33,
"p99": 41.88
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 52.2,
"lb": 4.9,
"loadgen-run-d7f8aed8393a": 8.2,
"postgres": 15.1,
"redis": 3.5
},
"mem_mib_max": {
"api": 32.7,
"lb": 4.5,
"loadgen-run-d7f8aed8393a": 6.4,
"postgres": 173.3,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20.4
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 136.41,
"p50": 4.66,
"p90": 17.46,
"p99": 81.94,
"p999": 101.49
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 4.81,
"p99_ms": 88.62
},
"me": {
"ok": 200,
"p50_ms": 1.95,
"p99_ms": 57.93
},
"post": {
"ok": 400,
"p50_ms": 4.66,
"p99_ms": 74.58
}
},
"rate": 200,
"service_ms": {
"p50": 4.04,
"p99": 81.36
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 60.9,
"lb": 5.3,
"loadgen-run-d7f8aed8393a": 9.2,
"postgres": 16.1,
"redis": 3.8
},
"mem_mib_max": {
"api": 32.7,
"lb": 4.7,
"loadgen-run-d7f8aed8393a": 6.6,
"postgres": 177.4,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 174.67,
"p50": 6.15,
"p90": 57.49,
"p99": 154.11,
"p999": 169.4
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 6.92,
"p99_ms": 161.47
},
"me": {
"ok": 240,
"p50_ms": 1.95,
"p99_ms": 91.32
},
"post": {
"ok": 480,
"p50_ms": 6.23,
"p99_ms": 139.23
}
},
"rate": 240,
"service_ms": {
"p50": 5.59,
"p99": 153.76
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 72.9,
"lb": 5.5,
"loadgen-run-d7f8aed8393a": 9.9,
"postgres": 17.6,
"redis": 3.5
},
"mem_mib_max": {
"api": 36.0,
"lb": 5.0,
"loadgen-run-d7f8aed8393a": 7.1,
"postgres": 178.0,
"redis": 3.9
},
"lb_queue_max": 17,
"lb_sessions_mean": 6.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": true
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 214.78,
"p50": 11.51,
"p90": 94.27,
"p99": 158.26,
"p999": 206.54
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 13.2,
"p99_ms": 154.64
},
"me": {
"ok": 280,
"p50_ms": 1.83,
"p99_ms": 160.91
},
"post": {
"ok": 560,
"p50_ms": 9.32,
"p99_ms": 194.89
}
},
"rate": 280,
"service_ms": {
"p50": 10.94,
"p99": 157.84
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 81.3,
"lb": 6.1,
"loadgen-run-d7f8aed8393a": 11.3,
"postgres": 19.2,
"redis": 3.4
},
"mem_mib_max": {
"api": 33.4,
"lb": 5.5,
"loadgen-run-d7f8aed8393a": 7.6,
"postgres": 178.8,
"redis": 5.2
},
"lb_queue_max": 33,
"lb_sessions_mean": 10.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": true
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 340.17,
"p50": 161.86,
"p90": 225.6,
"p99": 297.75,
"p999": 334.72
},
"ok": 3200,
"per_op": {
"history": {
"ok": 2240,
"p50_ms": 164.01,
"p99_ms": 290.18
},
"me": {
"ok": 320,
"p50_ms": 135.46,
"p99_ms": 290.54
},
"post": {
"ok": 640,
"p50_ms": 168.49,
"p99_ms": 324.71
}
},
"rate": 320,
"service_ms": {
"p50": 161.03,
"p99": 296.81
},
"throughput_ok_rps": 320,
"resources": {
"cpu_percent_mean": {
"api": 91.0,
"lb": 7.3,
"loadgen-run-d7f8aed8393a": 14.7,
"postgres": 22.2,
"redis": 4.1
},
"mem_mib_max": {
"api": 33.9,
"lb": 6.6,
"loadgen-run-d7f8aed8393a": 8.6,
"postgres": 180.3,
"redis": 5.5
},
"lb_queue_max": 62,
"lb_sessions_mean": 49,
"pg_active_mean": 1.11,
"pg_wait_events_mean": {
"IO:WALSync": 0.11,
"LWLock:WALWrite": 1.0
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 3
},
"error_ratio": 0.0008,
"errors": 3,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 2043.41,
"p50": 1323.29,
"p90": 1777.62,
"p99": 2019.65,
"p999": 2029.59
},
"ok": 3597,
"per_op": {
"history": {
"ok": 2517,
"p50_ms": 1323.29,
"p99_ms": 2021.81
},
"me": {
"ok": 360,
"p50_ms": 1103.57,
"p99_ms": 1986.08
},
"post": {
"ok": 720,
"p50_ms": 1338.29,
"p99_ms": 2015.65
}
},
"rate": 360,
"service_ms": {
"p50": 1322.33,
"p99": 2018.72
},
"throughput_ok_rps": 359.7,
"resources": {
"cpu_percent_mean": {
"api": 90.7,
"lb": 7.7,
"loadgen-run-d7f8aed8393a": 16.4,
"postgres": 20.9,
"redis": 3.6
},
"mem_mib_max": {
"api": 46.3,
"lb": 15.6,
"loadgen-run-d7f8aed8393a": 33.0,
"postgres": 181.1,
"redis": 4.0
},
"lb_queue_max": 609,
"lb_sessions_mean": 381.5,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 457
},
"error_ratio": 0.1143,
"errors": 457,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 2103.19,
"p50": 1669.91,
"p90": 2033.99,
"p99": 2067.95,
"p999": 2097.14
},
"ok": 3543,
"per_op": {
"history": {
"ok": 2469,
"p50_ms": 1657.97,
"p99_ms": 2076.56
},
"me": {
"ok": 381,
"p50_ms": 1922.4,
"p99_ms": 2012.47
},
"post": {
"ok": 693,
"p50_ms": 1677.06,
"p99_ms": 2062.11
}
},
"rate": 400,
"service_ms": {
"p50": 1668.88,
"p99": 2067.42
},
"throughput_ok_rps": 354.3,
"resources": {
"cpu_percent_mean": {
"api": 89.8,
"lb": 7.8,
"loadgen-run-d7f8aed8393a": 16.2,
"postgres": 20.8,
"redis": 3.7
},
"mem_mib_max": {
"api": 50.8,
"lb": 20.3,
"loadgen-run-d7f8aed8393a": 44.6,
"postgres": 182.1,
"redis": 3.9
},
"lb_queue_max": 794,
"lb_sessions_mean": 576.6,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"Client:ClientRead": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1322
},
"error_ratio": 0.2874,
"errors": 1322,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 2134.47,
"p50": 2025.69,
"p90": 2042.13,
"p99": 2079.24,
"p999": 2114.4
},
"ok": 3278,
"per_op": {
"history": {
"ok": 2232,
"p50_ms": 2029.09,
"p99_ms": 2084.21
},
"me": {
"ok": 408,
"p50_ms": 1999.45,
"p99_ms": 2018.38
},
"post": {
"ok": 638,
"p50_ms": 2024.68,
"p99_ms": 2073.64
}
},
"rate": 460,
"service_ms": {
"p50": 2025,
"p99": 2079.02
},
"throughput_ok_rps": 327.8,
"resources": {
"cpu_percent_mean": {
"api": 92.0,
"lb": 8.6,
"loadgen-run-d7f8aed8393a": 17.7,
"postgres": 22.1,
"redis": 4.0
},
"mem_mib_max": {
"api": 43.3,
"lb": 23.2,
"loadgen-run-d7f8aed8393a": 55.2,
"postgres": 182.2,
"redis": 3.9
},
"lb_queue_max": 923,
"lb_sessions_mean": 851,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": false
},
{
"attempted": 5200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2107
},
"error_ratio": 0.4052,
"errors": 2107,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 2120.21,
"p50": 2029.66,
"p90": 2046.03,
"p99": 2082.03,
"p999": 2111.54
},
"ok": 3093,
"per_op": {
"history": {
"ok": 2101,
"p50_ms": 2031.92,
"p99_ms": 2083.95
},
"me": {
"ok": 431,
"p50_ms": 2002.59,
"p99_ms": 2042.16
},
"post": {
"ok": 561,
"p50_ms": 2026.16,
"p99_ms": 2069.46
}
},
"rate": 520,
"service_ms": {
"p50": 2028.9,
"p99": 2081.41
},
"throughput_ok_rps": 309.3,
"resources": {
"cpu_percent_mean": {
"api": 88.5,
"lb": 8.8,
"loadgen-run-d7f8aed8393a": 18.4,
"postgres": 21.5,
"redis": 3.7
},
"mem_mib_max": {
"api": 47.5,
"lb": 25.8,
"loadgen-run-d7f8aed8393a": 60.9,
"postgres": 184.2,
"redis": 4.0
},
"lb_queue_max": 1040,
"lb_sessions_mean": 1017.4,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": false
},
{
"attempted": 6000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 3092
},
"error_ratio": 0.5153,
"errors": 3092,
"event": "step",
"label": "admission-1x1.00-mc10-r3",
"latency_ms": {
"max": 2138.73,
"p50": 2032.96,
"p90": 2055.36,
"p99": 2094.22,
"p999": 2131.98
},
"ok": 2908,
"per_op": {
"history": {
"ok": 2009,
"p50_ms": 2034.57,
"p99_ms": 2100.68
},
"me": {
"ok": 375,
"p50_ms": 2004.51,
"p99_ms": 2037.16
},
"post": {
"ok": 524,
"p50_ms": 2032.28,
"p99_ms": 2086.16
}
},
"rate": 600,
"service_ms": {
"p50": 2032.22,
"p99": 2093.3
},
"throughput_ok_rps": 290.8,
"resources": {
"cpu_percent_mean": {
"api": 88.9,
"lb": 9.7,
"loadgen-run-d7f8aed8393a": 20.3,
"postgres": 21.6,
"redis": 3.9
},
"mem_mib_max": {
"api": 38.9,
"lb": 29.6,
"loadgen-run-d7f8aed8393a": 68.9,
"postgres": 184.0,
"redis": 5.3
},
"lb_queue_max": 1200,
"lb_sessions_mean": 1192.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24
},
"slo_ok": false
}
],
"pool_before": {
"api-7679011c": {
"pool": {
"connections_num": 3,
"requests_num": 19,
"requests_queued": 3,
"connections_ms": 73,
"requests_wait_ms": 83,
"usage_ms": 123,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 8
}
},
"pool_after": {
"api-7679011c": {
"pool": {
"connections_num": 10,
"requests_num": 38669,
"requests_queued": 214,
"connections_ms": 390,
"requests_wait_ms": 1639,
"usage_ms": 599558,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 42186,
"misses": 141,
"errors": 0
},
"requests": 42677
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+872
View File
@@ -0,0 +1,872 @@
{
"config": "admission-1x1.00-mc10",
"repeat": 1,
"series": "admission",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": 10,
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T09:03:48+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 240,
"peak_ok_rps": 339,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 26.28,
"p50": 4.92,
"p90": 7.39,
"p99": 15.4,
"p999": 26.28
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 4.8,
"p99_ms": 15.4
},
"me": {
"ok": 40,
"p50_ms": 2.35,
"p99_ms": 6.76
},
"post": {
"ok": 80,
"p50_ms": 5.71,
"p99_ms": 20.94
}
},
"rate": 40,
"service_ms": {
"p50": 4.31,
"p99": 13.37
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 15.4,
"lb": 2.1,
"loadgen-run-c28b896f7dda": 2.5,
"postgres": 5.6,
"redis": 2.6
},
"mem_mib_max": {
"api": 31.7,
"lb": 4.4,
"loadgen-run-c28b896f7dda": 5.9,
"postgres": 82.2,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 8
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 23.4,
"p50": 5.05,
"p90": 7.68,
"p99": 15.67,
"p999": 23.4
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 4.89,
"p99_ms": 15.67
},
"me": {
"ok": 80,
"p50_ms": 2.48,
"p99_ms": 17.94
},
"post": {
"ok": 160,
"p50_ms": 5.96,
"p99_ms": 15.68
}
},
"rate": 80,
"service_ms": {
"p50": 4.39,
"p99": 14.68
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 28.4,
"lb": 3.3,
"loadgen-run-c28b896f7dda": 5.2,
"postgres": 8.2,
"redis": 2.9
},
"mem_mib_max": {
"api": 32.0,
"lb": 4.3,
"loadgen-run-c28b896f7dda": 6.2,
"postgres": 82.6,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"IO:WALSync": 0.22
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 8
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 24.11,
"p50": 5,
"p90": 8.16,
"p99": 16.2,
"p999": 22.92
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 4.88,
"p99_ms": 12.84
},
"me": {
"ok": 120,
"p50_ms": 2.35,
"p99_ms": 18.12
},
"post": {
"ok": 240,
"p50_ms": 5.88,
"p99_ms": 19.11
}
},
"rate": 120,
"service_ms": {
"p50": 4.44,
"p99": 15.24
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 39.1,
"lb": 4.4,
"loadgen-run-c28b896f7dda": 6.8,
"postgres": 10.3,
"redis": 3.3
},
"mem_mib_max": {
"api": 32.2,
"lb": 4.4,
"loadgen-run-c28b896f7dda": 6.4,
"postgres": 83.6,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 8
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 208.18,
"p50": 5.64,
"p90": 24.22,
"p99": 116.98,
"p999": 188.4
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.67,
"p99_ms": 89.72
},
"me": {
"ok": 160,
"p50_ms": 2.3,
"p99_ms": 103.39
},
"post": {
"ok": 320,
"p50_ms": 6.16,
"p99_ms": 143.31
}
},
"rate": 160,
"service_ms": {
"p50": 5.04,
"p99": 116.66
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 56.5,
"lb": 5.2,
"loadgen-run-c28b896f7dda": 8.9,
"postgres": 14.3,
"redis": 3.4
},
"mem_mib_max": {
"api": 44.8,
"lb": 4.7,
"loadgen-run-c28b896f7dda": 6.8,
"postgres": 94.9,
"redis": 5.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.3,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 12.4
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 87.46,
"p50": 5.64,
"p90": 28.46,
"p99": 67.19,
"p999": 81.25
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 5.87,
"p99_ms": 71.04
},
"me": {
"ok": 200,
"p50_ms": 2.05,
"p99_ms": 20.35
},
"post": {
"ok": 400,
"p50_ms": 5.66,
"p99_ms": 58.55
}
},
"rate": 200,
"service_ms": {
"p50": 4.97,
"p99": 66.73
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 66.9,
"lb": 5.6,
"loadgen-run-c28b896f7dda": 9.6,
"postgres": 14.3,
"redis": 3.7
},
"mem_mib_max": {
"api": 33.1,
"lb": 4.6,
"loadgen-run-c28b896f7dda": 6.8,
"postgres": 95.4,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 232.48,
"p50": 11.54,
"p90": 111.14,
"p99": 223.22,
"p999": 231.73
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 14.64,
"p99_ms": 225.68
},
"me": {
"ok": 240,
"p50_ms": 2.22,
"p99_ms": 174.16
},
"post": {
"ok": 480,
"p50_ms": 7.44,
"p99_ms": 223.22
}
},
"rate": 240,
"service_ms": {
"p50": 10.89,
"p99": 222.87
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 78.0,
"lb": 6.1,
"loadgen-run-c28b896f7dda": 11.3,
"postgres": 16.6,
"redis": 3.9
},
"mem_mib_max": {
"api": 42.3,
"lb": 5.4,
"loadgen-run-c28b896f7dda": 7.2,
"postgres": 95.9,
"redis": 3.9
},
"lb_queue_max": 17,
"lb_sessions_mean": 7.2,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": true
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 449.54,
"p50": 63.09,
"p90": 274.85,
"p99": 432.81,
"p999": 445.7
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 63.04,
"p99_ms": 432.76
},
"me": {
"ok": 280,
"p50_ms": 39.06,
"p99_ms": 393.59
},
"post": {
"ok": 560,
"p50_ms": 83.47,
"p99_ms": 438.73
}
},
"rate": 280,
"service_ms": {
"p50": 62.55,
"p99": 431.42
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 88.6,
"lb": 6.9,
"loadgen-run-c28b896f7dda": 14.1,
"postgres": 18.7,
"redis": 3.7
},
"mem_mib_max": {
"api": 39.0,
"lb": 6.8,
"loadgen-run-c28b896f7dda": 9.9,
"postgres": 97.3,
"redis": 3.8
},
"lb_queue_max": 80,
"lb_sessions_mean": 24.1,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 426.08,
"p50": 233.15,
"p90": 345.38,
"p99": 400.44,
"p999": 421.35
},
"ok": 3200,
"per_op": {
"history": {
"ok": 2240,
"p50_ms": 236.18,
"p99_ms": 392.66
},
"me": {
"ok": 320,
"p50_ms": 205.69,
"p99_ms": 379.24
},
"post": {
"ok": 640,
"p50_ms": 237.5,
"p99_ms": 407.5
}
},
"rate": 320,
"service_ms": {
"p50": 232.65,
"p99": 399.91
},
"throughput_ok_rps": 320,
"resources": {
"cpu_percent_mean": {
"api": 90.6,
"lb": 7.1,
"loadgen-run-c28b896f7dda": 14.8,
"postgres": 19.7,
"redis": 3.8
},
"mem_mib_max": {
"api": 41.8,
"lb": 7.0,
"loadgen-run-c28b896f7dda": 9.6,
"postgres": 96.9,
"redis": 3.8
},
"lb_queue_max": 96,
"lb_sessions_mean": 73.4,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 210
},
"error_ratio": 0.0583,
"errors": 210,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 2093.05,
"p50": 1512.71,
"p90": 2029.09,
"p99": 2060.89,
"p999": 2089.69
},
"ok": 3390,
"per_op": {
"history": {
"ok": 2352,
"p50_ms": 1502.11,
"p99_ms": 2068.9
},
"me": {
"ok": 353,
"p50_ms": 1499.05,
"p99_ms": 2010.4
},
"post": {
"ok": 685,
"p50_ms": 1524.93,
"p99_ms": 2052.03
}
},
"rate": 360,
"service_ms": {
"p50": 1512.04,
"p99": 2060.26
},
"throughput_ok_rps": 339,
"resources": {
"cpu_percent_mean": {
"api": 90.3,
"lb": 8.1,
"loadgen-run-c28b896f7dda": 18.4,
"postgres": 19.3,
"redis": 3.5
},
"mem_mib_max": {
"api": 37.4,
"lb": 16.9,
"loadgen-run-c28b896f7dda": 36.5,
"postgres": 98.1,
"redis": 3.9
},
"lb_queue_max": 716,
"lb_sessions_mean": 404.8,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 740
},
"error_ratio": 0.185,
"errors": 740,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 2122.25,
"p50": 1990.94,
"p90": 2042.5,
"p99": 2083.5,
"p999": 2107.02
},
"ok": 3260,
"per_op": {
"history": {
"ok": 2291,
"p50_ms": 1997.39,
"p99_ms": 2084.35
},
"me": {
"ok": 363,
"p50_ms": 1989.82,
"p99_ms": 2030.34
},
"post": {
"ok": 606,
"p50_ms": 1976.43,
"p99_ms": 2078.11
}
},
"rate": 400,
"service_ms": {
"p50": 1989.83,
"p99": 2082.86
},
"throughput_ok_rps": 326,
"resources": {
"cpu_percent_mean": {
"api": 91.9,
"lb": 8.7,
"loadgen-run-c28b896f7dda": 18.3,
"postgres": 20.4,
"redis": 3.7
},
"mem_mib_max": {
"api": 38.5,
"lb": 20.8,
"loadgen-run-c28b896f7dda": 46.3,
"postgres": 98.6,
"redis": 3.8
},
"lb_queue_max": 799,
"lb_sessions_mean": 658.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1500
},
"error_ratio": 0.3261,
"errors": 1500,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 2141.66,
"p50": 2027.38,
"p90": 2048.46,
"p99": 2080.13,
"p999": 2125.89
},
"ok": 3100,
"per_op": {
"history": {
"ok": 2120,
"p50_ms": 2030.97,
"p99_ms": 2086.38
},
"me": {
"ok": 405,
"p50_ms": 2000.17,
"p99_ms": 2034.48
},
"post": {
"ok": 575,
"p50_ms": 2024.36,
"p99_ms": 2072.63
}
},
"rate": 460,
"service_ms": {
"p50": 2026.65,
"p99": 2079.89
},
"throughput_ok_rps": 310,
"resources": {
"cpu_percent_mean": {
"api": 89.6,
"lb": 8.6,
"loadgen-run-c28b896f7dda": 17.9,
"postgres": 18.9,
"redis": 3.6
},
"mem_mib_max": {
"api": 38.9,
"lb": 23.5,
"loadgen-run-c28b896f7dda": 53.2,
"postgres": 99.6,
"redis": 3.8
},
"lb_queue_max": 920,
"lb_sessions_mean": 865.2,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": false
},
{
"attempted": 5200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2368
},
"error_ratio": 0.4554,
"errors": 2368,
"event": "step",
"label": "admission-1x1.00-mc10",
"latency_ms": {
"max": 2154.91,
"p50": 2032,
"p90": 2053.04,
"p99": 2111.5,
"p999": 2145.08
},
"ok": 2832,
"per_op": {
"history": {
"ok": 1930,
"p50_ms": 2033.94,
"p99_ms": 2112.49
},
"me": {
"ok": 385,
"p50_ms": 2002.98,
"p99_ms": 2028.39
},
"post": {
"ok": 517,
"p50_ms": 2030.67,
"p99_ms": 2125.35
}
},
"rate": 520,
"service_ms": {
"p50": 2031.37,
"p99": 2111.05
},
"throughput_ok_rps": 283.2,
"resources": {
"cpu_percent_mean": {
"api": 90.6,
"lb": 9.5,
"loadgen-run-c28b896f7dda": 19.9,
"postgres": 19.7,
"redis": 3.8
},
"mem_mib_max": {
"api": 39.1,
"lb": 26.1,
"loadgen-run-c28b896f7dda": 61.8,
"postgres": 99.2,
"redis": 5.3
},
"lb_queue_max": 1040,
"lb_sessions_mean": 1014.6,
"pg_active_mean": 0.2,
"pg_wait_events_mean": {
"Client:ClientRead": 0.2
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": false
}
],
"pool_before": {
"api-2e5d1b71": {
"pool": {
"connections_num": 3,
"requests_num": 19,
"requests_queued": 3,
"connections_ms": 64,
"requests_wait_ms": 68,
"usage_ms": 37,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 8
}
},
"pool_after": {
"api-2e5d1b71": {
"pool": {
"connections_num": 10,
"requests_num": 33873,
"requests_queued": 201,
"connections_ms": 435,
"requests_wait_ms": 1656,
"usage_ms": 551219,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 36857,
"misses": 116,
"errors": 0
},
"requests": 37305
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+869
View File
@@ -0,0 +1,869 @@
{
"config": "admission-1x1.00-mc20",
"repeat": 1,
"series": "admission",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": 20,
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T09:07:55+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 200,
"peak_ok_rps": 326.3,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 72.89,
"p50": 5.74,
"p90": 10.45,
"p99": 34.26,
"p999": 72.89
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.56,
"p99_ms": 34.26
},
"me": {
"ok": 40,
"p50_ms": 2.86,
"p99_ms": 68.16
},
"post": {
"ok": 80,
"p50_ms": 6.81,
"p99_ms": 22.77
}
},
"rate": 40,
"service_ms": {
"p50": 5.09,
"p99": 33.75
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 17.4,
"lb": 2.4,
"loadgen-run-6ca2812f958b": 3.1,
"postgres": 6.0,
"redis": 2.7
},
"mem_mib_max": {
"api": 33.4,
"lb": 4.4,
"loadgen-run-6ca2812f958b": 5.9,
"postgres": 88.5,
"redis": 4.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 8
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 23.76,
"p50": 5.26,
"p90": 8.36,
"p99": 16.86,
"p999": 23.76
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.2,
"p99_ms": 14.05
},
"me": {
"ok": 80,
"p50_ms": 2.65,
"p99_ms": 8.51
},
"post": {
"ok": 160,
"p50_ms": 6.33,
"p99_ms": 23.41
}
},
"rate": 80,
"service_ms": {
"p50": 4.68,
"p99": 16.17
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 30.1,
"lb": 3.6,
"loadgen-run-6ca2812f958b": 5.3,
"postgres": 8.8,
"redis": 3.1
},
"mem_mib_max": {
"api": 31.8,
"lb": 4.5,
"loadgen-run-6ca2812f958b": 6.2,
"postgres": 90.8,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 8
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 104.11,
"p50": 5.47,
"p90": 15.4,
"p99": 59,
"p999": 97.17
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.48,
"p99_ms": 63.84
},
"me": {
"ok": 120,
"p50_ms": 2.46,
"p99_ms": 7.53
},
"post": {
"ok": 240,
"p50_ms": 6.02,
"p99_ms": 24.99
}
},
"rate": 120,
"service_ms": {
"p50": 4.92,
"p99": 58.96
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 44.3,
"lb": 4.5,
"loadgen-run-6ca2812f958b": 7.4,
"postgres": 12.3,
"redis": 4.0
},
"mem_mib_max": {
"api": 36.7,
"lb": 4.6,
"loadgen-run-6ca2812f958b": 6.7,
"postgres": 98.4,
"redis": 5.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10.4
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 117.45,
"p50": 4.86,
"p90": 12.77,
"p99": 55.34,
"p999": 94.65
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 4.93,
"p99_ms": 53.02
},
"me": {
"ok": 160,
"p50_ms": 2.05,
"p99_ms": 20.05
},
"post": {
"ok": 320,
"p50_ms": 5.22,
"p99_ms": 57.33
}
},
"rate": 160,
"service_ms": {
"p50": 4.25,
"p99": 53.68
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 51.2,
"lb": 4.9,
"loadgen-run-6ca2812f958b": 8.3,
"postgres": 12.9,
"redis": 3.6
},
"mem_mib_max": {
"api": 45.5,
"lb": 4.6,
"loadgen-run-6ca2812f958b": 6.9,
"postgres": 101.4,
"redis": 5.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.8,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.22
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 12.1
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 210.78,
"p50": 5.74,
"p90": 66.14,
"p99": 188.58,
"p999": 207.28
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 6.04,
"p99_ms": 192.47
},
"me": {
"ok": 200,
"p50_ms": 2.09,
"p99_ms": 95.94
},
"post": {
"ok": 400,
"p50_ms": 5.72,
"p99_ms": 186.99
}
},
"rate": 200,
"service_ms": {
"p50": 5.09,
"p99": 188.36
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 65.8,
"lb": 5.7,
"loadgen-run-6ca2812f958b": 9.9,
"postgres": 15.0,
"redis": 3.8
},
"mem_mib_max": {
"api": 33.4,
"lb": 5.0,
"loadgen-run-6ca2812f958b": 7.1,
"postgres": 104.4,
"redis": 3.8
},
"lb_queue_max": 17,
"lb_sessions_mean": 6.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 495.13,
"p50": 74.78,
"p90": 394.07,
"p99": 469.64,
"p999": 489.39
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 75.26,
"p99_ms": 461.44
},
"me": {
"ok": 240,
"p50_ms": 34.09,
"p99_ms": 401.2
},
"post": {
"ok": 480,
"p50_ms": 90.18,
"p99_ms": 477.79
}
},
"rate": 240,
"service_ms": {
"p50": 74.15,
"p99": 468.97
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 85.4,
"lb": 6.9,
"loadgen-run-6ca2812f958b": 13.1,
"postgres": 19.1,
"redis": 4.3
},
"mem_mib_max": {
"api": 34.2,
"lb": 6.5,
"loadgen-run-6ca2812f958b": 9.2,
"postgres": 104.5,
"redis": 3.8
},
"lb_queue_max": 87,
"lb_sessions_mean": 30.7,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 601.55,
"p50": 169.96,
"p90": 463.64,
"p99": 573.13,
"p999": 588.34
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 171.21,
"p99_ms": 566.91
},
"me": {
"ok": 280,
"p50_ms": 119.76,
"p99_ms": 522.34
},
"post": {
"ok": 560,
"p50_ms": 186.06,
"p99_ms": 585.45
}
},
"rate": 280,
"service_ms": {
"p50": 169.11,
"p99": 572.41
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 89.8,
"lb": 7.2,
"loadgen-run-6ca2812f958b": 14.4,
"postgres": 19.8,
"redis": 3.9
},
"mem_mib_max": {
"api": 41.8,
"lb": 7.6,
"loadgen-run-6ca2812f958b": 11.3,
"postgres": 105.3,
"redis": 3.9
},
"lb_queue_max": 135,
"lb_sessions_mean": 60.6,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 592.61,
"p50": 282.65,
"p90": 490.74,
"p99": 577.06,
"p999": 585.74
},
"ok": 3200,
"per_op": {
"history": {
"ok": 2240,
"p50_ms": 287.08,
"p99_ms": 576.17
},
"me": {
"ok": 320,
"p50_ms": 241.97,
"p99_ms": 521.63
},
"post": {
"ok": 640,
"p50_ms": 294.77,
"p99_ms": 580.88
}
},
"rate": 320,
"service_ms": {
"p50": 282.07,
"p99": 576.32
},
"throughput_ok_rps": 320,
"resources": {
"cpu_percent_mean": {
"api": 92.2,
"lb": 6.9,
"loadgen-run-6ca2812f958b": 14.7,
"postgres": 19.9,
"redis": 3.6
},
"mem_mib_max": {
"api": 36.1,
"lb": 7.7,
"loadgen-run-6ca2812f958b": 11.5,
"postgres": 106.7,
"redis": 5.5
},
"lb_queue_max": 145,
"lb_sessions_mean": 91.7,
"pg_active_mean": 0.2,
"pg_wait_events_mean": {
"CPU": 0.2
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 337
},
"error_ratio": 0.0936,
"errors": 337,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 2223.99,
"p50": 1971.49,
"p90": 2074.67,
"p99": 2130.15,
"p999": 2211.47
},
"ok": 3263,
"per_op": {
"history": {
"ok": 2315,
"p50_ms": 1984.72,
"p99_ms": 2152.29
},
"me": {
"ok": 334,
"p50_ms": 1915.43,
"p99_ms": 2018.04
},
"post": {
"ok": 614,
"p50_ms": 1966.23,
"p99_ms": 2106.86
}
},
"rate": 360,
"service_ms": {
"p50": 1970.5,
"p99": 2129.39
},
"throughput_ok_rps": 326.3,
"resources": {
"cpu_percent_mean": {
"api": 91.2,
"lb": 8.6,
"loadgen-run-6ca2812f958b": 18.2,
"postgres": 20.7,
"redis": 3.7
},
"mem_mib_max": {
"api": 41.7,
"lb": 18.1,
"loadgen-run-6ca2812f958b": 42.6,
"postgres": 106.5,
"redis": 4.5
},
"lb_queue_max": 719,
"lb_sessions_mean": 516.9,
"pg_active_mean": 0.4,
"pg_wait_events_mean": {
"IO:WALSync": 0.2,
"LWLock:WALWrite": 0.2
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 970
},
"error_ratio": 0.2425,
"errors": 970,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 2256.48,
"p50": 2056.27,
"p90": 2092.45,
"p99": 2166.54,
"p999": 2234.81
},
"ok": 3030,
"per_op": {
"history": {
"ok": 2188,
"p50_ms": 2059.33,
"p99_ms": 2166.54
},
"me": {
"ok": 303,
"p50_ms": 2004.44,
"p99_ms": 2041.52
},
"post": {
"ok": 539,
"p50_ms": 2062.03,
"p99_ms": 2178.87
}
},
"rate": 400,
"service_ms": {
"p50": 2054.84,
"p99": 2166.11
},
"throughput_ok_rps": 303,
"resources": {
"cpu_percent_mean": {
"api": 91.8,
"lb": 7.8,
"loadgen-run-6ca2812f958b": 16.7,
"postgres": 19.2,
"redis": 3.4
},
"mem_mib_max": {
"api": 40.1,
"lb": 21.0,
"loadgen-run-6ca2812f958b": 48.7,
"postgres": 106.7,
"redis": 4.0
},
"lb_queue_max": 800,
"lb_sessions_mean": 752.9,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1603
},
"error_ratio": 0.3485,
"errors": 1603,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 2220.71,
"p50": 2061.29,
"p90": 2099.38,
"p99": 2150.8,
"p999": 2216.83
},
"ok": 2997,
"per_op": {
"history": {
"ok": 2168,
"p50_ms": 2063.37,
"p99_ms": 2152.38
},
"me": {
"ok": 331,
"p50_ms": 2007.81,
"p99_ms": 2033.45
},
"post": {
"ok": 498,
"p50_ms": 2065.91,
"p99_ms": 2168.52
}
},
"rate": 460,
"service_ms": {
"p50": 2060.51,
"p99": 2150.43
},
"throughput_ok_rps": 299.7,
"resources": {
"cpu_percent_mean": {
"api": 91.4,
"lb": 8.6,
"loadgen-run-6ca2812f958b": 18.4,
"postgres": 19.6,
"redis": 3.4
},
"mem_mib_max": {
"api": 50.7,
"lb": 23.6,
"loadgen-run-6ca2812f958b": 53.4,
"postgres": 107.6,
"redis": 4.5
},
"lb_queue_max": 920,
"lb_sessions_mean": 893.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14
},
"slo_ok": false
},
{
"attempted": 5200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2322
},
"error_ratio": 0.4465,
"errors": 2322,
"event": "step",
"label": "admission-1x1.00-mc20",
"latency_ms": {
"max": 2194.73,
"p50": 2067.03,
"p90": 2097.92,
"p99": 2157.48,
"p999": 2187.12
},
"ok": 2878,
"per_op": {
"history": {
"ok": 2069,
"p50_ms": 2069.51,
"p99_ms": 2161.06
},
"me": {
"ok": 313,
"p50_ms": 2011.02,
"p99_ms": 2045.56
},
"post": {
"ok": 496,
"p50_ms": 2071.98,
"p99_ms": 2174.02
}
},
"rate": 520,
"service_ms": {
"p50": 2066.54,
"p99": 2157.21
},
"throughput_ok_rps": 287.8,
"resources": {
"cpu_percent_mean": {
"api": 90.4,
"lb": 8.9,
"loadgen-run-6ca2812f958b": 19.0,
"postgres": 19.3,
"redis": 3.7
},
"mem_mib_max": {
"api": 45.7,
"lb": 27.3,
"loadgen-run-6ca2812f958b": 61.5,
"postgres": 108.6,
"redis": 3.8
},
"lb_queue_max": 1041,
"lb_sessions_mean": 1040.6,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 14
},
"slo_ok": false
}
],
"pool_before": {
"api-6c123de5": {
"pool": {
"connections_num": 3,
"requests_num": 19,
"requests_queued": 3,
"connections_ms": 93,
"requests_wait_ms": 101,
"usage_ms": 53,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 8
}
},
"pool_after": {
"api-6c123de5": {
"pool": {
"connections_num": 10,
"requests_num": 33711,
"requests_queued": 23037,
"connections_ms": 438,
"requests_wait_ms": 614175,
"usage_ms": 711040,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 36469,
"misses": 116,
"errors": 0
},
"requests": 36915
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+919
View File
@@ -0,0 +1,919 @@
{
"config": "admission-2x0.50-mc10",
"repeat": 1,
"series": "admission",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": 10,
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 2,
"API_CPUS": "0.5",
"API_MEMORY": "256M",
"KDF_CONCURRENCY": 2
},
"measured_at": "2026-11-03T09:14:58+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 200,
"peak_ok_rps": 306.3,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 55.67,
"p50": 5.57,
"p90": 10.78,
"p99": 27.62,
"p999": 55.67
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.35,
"p99_ms": 22.32
},
"me": {
"ok": 40,
"p50_ms": 2.8,
"p99_ms": 19.2
},
"post": {
"ok": 80,
"p50_ms": 7.19,
"p99_ms": 28.45
}
},
"rate": 40,
"service_ms": {
"p50": 4.94,
"p99": 26.81
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 21.1,
"lb": 2.3,
"loadgen-run-65398839d17b": 3.0,
"postgres": 6.4,
"redis": 2.6
},
"mem_mib_max": {
"api": 31.1,
"lb": 4.4,
"loadgen-run-65398839d17b": 6.0,
"postgres": 110.1,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.039,
"pg_connections_mean": 12
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 73.17,
"p50": 5.26,
"p90": 12.21,
"p99": 34.4,
"p999": 73.17
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.2,
"p99_ms": 36.6
},
"me": {
"ok": 80,
"p50_ms": 2.6,
"p99_ms": 26.77
},
"post": {
"ok": 160,
"p50_ms": 6.04,
"p99_ms": 37.27
}
},
"rate": 80,
"service_ms": {
"p50": 4.67,
"p99": 31.51
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 33.6,
"lb": 3.1,
"loadgen-run-65398839d17b": 5.0,
"postgres": 8.8,
"redis": 3.0
},
"mem_mib_max": {
"api": 43.9,
"lb": 4.4,
"loadgen-run-65398839d17b": 6.3,
"postgres": 110.3,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.039,
"pg_connections_mean": 12
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 226.71,
"p50": 5.34,
"p90": 17.43,
"p99": 141.16,
"p999": 190.12
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.26,
"p99_ms": 126.11
},
"me": {
"ok": 120,
"p50_ms": 2.4,
"p99_ms": 28.24
},
"post": {
"ok": 240,
"p50_ms": 6.41,
"p99_ms": 146.28
}
},
"rate": 120,
"service_ms": {
"p50": 4.71,
"p99": 140.5
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 49.8,
"lb": 4.2,
"loadgen-run-65398839d17b": 6.7,
"postgres": 12.5,
"redis": 3.6
},
"mem_mib_max": {
"api": 44.5,
"lb": 4.6,
"loadgen-run-65398839d17b": 6.9,
"postgres": 131.8,
"redis": 5.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 18.3
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 331.57,
"p50": 6.06,
"p90": 103.28,
"p99": 260.66,
"p999": 328.14
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 6.2,
"p99_ms": 259.9
},
"me": {
"ok": 160,
"p50_ms": 2.64,
"p99_ms": 194.62
},
"post": {
"ok": 320,
"p50_ms": 6.56,
"p99_ms": 276.11
}
},
"rate": 160,
"service_ms": {
"p50": 5.6,
"p99": 260.05
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 62.8,
"lb": 5.2,
"loadgen-run-65398839d17b": 8.5,
"postgres": 14.6,
"redis": 4.0
},
"mem_mib_max": {
"api": 34.6,
"lb": 5.0,
"loadgen-run-65398839d17b": 7.5,
"postgres": 144.4,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 24.2
},
"slo_ok": false
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 228.33,
"p50": 10.68,
"p90": 119.22,
"p99": 201.85,
"p999": 215.33
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 11.69,
"p99_ms": 202.53
},
"me": {
"ok": 200,
"p50_ms": 2.29,
"p99_ms": 119.21
},
"post": {
"ok": 400,
"p50_ms": 12.03,
"p99_ms": 203.06
}
},
"rate": 200,
"service_ms": {
"p50": 9.88,
"p99": 201.58
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 77.0,
"lb": 5.7,
"loadgen-run-65398839d17b": 9.6,
"postgres": 15.8,
"redis": 3.6
},
"mem_mib_max": {
"api": 44.4,
"lb": 5.1,
"loadgen-run-65398839d17b": 7.3,
"postgres": 146.4,
"redis": 5.0
},
"lb_queue_max": 9,
"lb_sessions_mean": 6.1,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"Client:ClientRead": 0.11,
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.006,
"pg_connections_mean": 26
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 770.1,
"p50": 143.33,
"p90": 631.73,
"p99": 749.7,
"p999": 765.09
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 147.35,
"p99_ms": 751.89
},
"me": {
"ok": 240,
"p50_ms": 68.25,
"p99_ms": 685.74
},
"post": {
"ok": 480,
"p50_ms": 147.53,
"p99_ms": 749.64
}
},
"rate": 240,
"service_ms": {
"p50": 142.88,
"p99": 748.55
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 91.2,
"lb": 6.7,
"loadgen-run-65398839d17b": 13.0,
"postgres": 18.6,
"redis": 3.6
},
"mem_mib_max": {
"api": 35.0,
"lb": 7.7,
"loadgen-run-65398839d17b": 12.8,
"postgres": 145.8,
"redis": 4.3
},
"lb_queue_max": 149,
"lb_sessions_mean": 57.4,
"pg_active_mean": 0.33,
"pg_wait_events_mean": {
"CPU": 0.22,
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 26
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 1985.12,
"p50": 556.1,
"p90": 735.21,
"p99": 1921.66,
"p999": 1970.12
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 556.1,
"p99_ms": 1937.39
},
"me": {
"ok": 280,
"p50_ms": 348.18,
"p99_ms": 1869.36
},
"post": {
"ok": 560,
"p50_ms": 417.29,
"p99_ms": 1904.67
}
},
"rate": 280,
"service_ms": {
"p50": 555.55,
"p99": 1907.7
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 93.1,
"lb": 6.9,
"loadgen-run-65398839d17b": 14.2,
"postgres": 20.3,
"redis": 3.8
},
"mem_mib_max": {
"api": 46.2,
"lb": 8.4,
"loadgen-run-65398839d17b": 14.3,
"postgres": 147.5,
"redis": 3.9
},
"lb_queue_max": 176,
"lb_sessions_mean": 130.2,
"pg_active_mean": 0.2,
"pg_wait_events_mean": {
"CPU": 0.2
},
"api_cfs_throttled_period_ratio": 0.006,
"pg_connections_mean": 26
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 137
},
"error_ratio": 0.0428,
"errors": 137,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 2168.21,
"p50": 1571.72,
"p90": 2063.3,
"p99": 2115.18,
"p999": 2160.74
},
"ok": 3063,
"per_op": {
"history": {
"ok": 2165,
"p50_ms": 1583.46,
"p99_ms": 2124.16
},
"me": {
"ok": 304,
"p50_ms": 1529.66,
"p99_ms": 2022.38
},
"post": {
"ok": 594,
"p50_ms": 1529.38,
"p99_ms": 2098.44
}
},
"rate": 320,
"service_ms": {
"p50": 1571.09,
"p99": 2113.38
},
"throughput_ok_rps": 306.3,
"resources": {
"cpu_percent_mean": {
"api": 92.3,
"lb": 7.2,
"loadgen-run-65398839d17b": 15.9,
"postgres": 19.1,
"redis": 3.5
},
"mem_mib_max": {
"api": 47.8,
"lb": 16.2,
"loadgen-run-65398839d17b": 37.6,
"postgres": 147.7,
"redis": 5.5
},
"lb_queue_max": 640,
"lb_sessions_mean": 457.2,
"pg_active_mean": 0.2,
"pg_wait_events_mean": {
"CPU": 0.2
},
"api_cfs_throttled_period_ratio": 0.006,
"pg_connections_mean": 26
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 600
},
"error_ratio": 0.1667,
"errors": 600,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 2220.53,
"p50": 2031.26,
"p90": 2089.23,
"p99": 2155.93,
"p999": 2201.05
},
"ok": 3000,
"per_op": {
"history": {
"ok": 2138,
"p50_ms": 2047.34,
"p99_ms": 2163.26
},
"me": {
"ok": 317,
"p50_ms": 2000.62,
"p99_ms": 2061.42
},
"post": {
"ok": 545,
"p50_ms": 2050.82,
"p99_ms": 2150.96
}
},
"rate": 360,
"service_ms": {
"p50": 2030.2,
"p99": 2155.49
},
"throughput_ok_rps": 300,
"resources": {
"cpu_percent_mean": {
"api": 94.1,
"lb": 7.4,
"loadgen-run-65398839d17b": 15.5,
"postgres": 19.9,
"redis": 3.7
},
"mem_mib_max": {
"api": 42.1,
"lb": 19.3,
"loadgen-run-65398839d17b": 44.4,
"postgres": 147.4,
"redis": 5.3
},
"lb_queue_max": 720,
"lb_sessions_mean": 552.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.016,
"pg_connections_mean": 26
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1098
},
"error_ratio": 0.2745,
"errors": 1098,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 2296.82,
"p50": 2059.75,
"p90": 2096.99,
"p99": 2153.66,
"p999": 2268.02
},
"ok": 2902,
"per_op": {
"history": {
"ok": 2102,
"p50_ms": 2064.07,
"p99_ms": 2155.6
},
"me": {
"ok": 300,
"p50_ms": 2010.3,
"p99_ms": 2060.99
},
"post": {
"ok": 500,
"p50_ms": 2060.46,
"p99_ms": 2144.71
}
},
"rate": 400,
"service_ms": {
"p50": 2059.13,
"p99": 2153.28
},
"throughput_ok_rps": 290.2,
"resources": {
"cpu_percent_mean": {
"api": 93.9,
"lb": 8.0,
"loadgen-run-65398839d17b": 16.6,
"postgres": 19.8,
"redis": 3.6
},
"mem_mib_max": {
"api": 38.7,
"lb": 21.3,
"loadgen-run-65398839d17b": 47.0,
"postgres": 148.5,
"redis": 4.0
},
"lb_queue_max": 797,
"lb_sessions_mean": 737.6,
"pg_active_mean": 0.4,
"pg_wait_events_mean": {
"CPU": 0.2,
"IO:WALSync": 0.2
},
"api_cfs_throttled_period_ratio": 0.039,
"pg_connections_mean": 26
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1827
},
"error_ratio": 0.3972,
"errors": 1827,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 2271.42,
"p50": 2062.52,
"p90": 2114.85,
"p99": 2182.04,
"p999": 2265.54
},
"ok": 2773,
"per_op": {
"history": {
"ok": 2017,
"p50_ms": 2064.67,
"p99_ms": 2178.03
},
"me": {
"ok": 294,
"p50_ms": 2012.08,
"p99_ms": 2066.11
},
"post": {
"ok": 462,
"p50_ms": 2068.16,
"p99_ms": 2263.76
}
},
"rate": 460,
"service_ms": {
"p50": 2061.77,
"p99": 2181.92
},
"throughput_ok_rps": 277.3,
"resources": {
"cpu_percent_mean": {
"api": 91.8,
"lb": 8.5,
"loadgen-run-65398839d17b": 18.0,
"postgres": 20.3,
"redis": 3.7
},
"mem_mib_max": {
"api": 38.6,
"lb": 23.8,
"loadgen-run-65398839d17b": 54.5,
"postgres": 148.3,
"redis": 3.9
},
"lb_queue_max": 921,
"lb_sessions_mean": 929.2,
"pg_active_mean": 0.3,
"pg_wait_events_mean": {
"CPU": 0.1,
"IO:WALSync": 0.1,
"LWLock:WALWrite": 0.1
},
"api_cfs_throttled_period_ratio": 0.033,
"pg_connections_mean": 26
},
"slo_ok": false
},
{
"attempted": 5200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2594
},
"error_ratio": 0.4988,
"errors": 2594,
"event": "step",
"label": "admission-2x0.50-mc10",
"latency_ms": {
"max": 2300.66,
"p50": 2070.01,
"p90": 2115.36,
"p99": 2187.79,
"p999": 2233.89
},
"ok": 2606,
"per_op": {
"history": {
"ok": 1899,
"p50_ms": 2072.18,
"p99_ms": 2184.89
},
"me": {
"ok": 248,
"p50_ms": 2019.21,
"p99_ms": 2086.36
},
"post": {
"ok": 459,
"p50_ms": 2072.02,
"p99_ms": 2211.08
}
},
"rate": 520,
"service_ms": {
"p50": 2069.24,
"p99": 2187.39
},
"throughput_ok_rps": 260.6,
"resources": {
"cpu_percent_mean": {
"api": 92.6,
"lb": 8.5,
"loadgen-run-65398839d17b": 18.3,
"postgres": 19.5,
"redis": 3.3
},
"mem_mib_max": {
"api": 48.8,
"lb": 26.2,
"loadgen-run-65398839d17b": 61.5,
"postgres": 149.7,
"redis": 4.2
},
"lb_queue_max": 1040,
"lb_sessions_mean": 1031.1,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 26
},
"slo_ok": false
}
],
"pool_before": {
"api-a6f26611": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 121,
"requests_wait_ms": 141,
"usage_ms": 50,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-37d6ee24": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 164,
"requests_wait_ms": 184,
"usage_ms": 94,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
}
},
"pool_after": {
"api-a6f26611": {
"pool": {
"connections_num": 10,
"requests_num": 16737,
"requests_queued": 199,
"connections_ms": 827,
"requests_wait_ms": 3416,
"usage_ms": 621313,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 17286,
"misses": 55,
"errors": 2
},
"requests": 17908
},
"api-37d6ee24": {
"pool": {
"connections_num": 10,
"requests_num": 16544,
"requests_queued": 208,
"connections_ms": 842,
"requests_wait_ms": 4643,
"usage_ms": 618848,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 17540,
"misses": 73,
"errors": 0
},
"requests": 17903
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+87
View File
@@ -0,0 +1,87 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 400" font-family="system-ui,-apple-system,Segoe UI,sans-serif">
<rect width="100%" height="100%" fill="#fcfcfb"/>
<text x="64" y="26" font-size="16" font-weight="600" fill="#0b0b0b">maxconn на LB против перегрузки (1 × 1.0 CPU)</text>
<text x="64" y="46" font-size="12" fill="#52514e">Полезная пропускная способность; пул 10; без лимита и maxconn 10 — медиана 3 прогонов</text>
<line x1="64" x2="570" y1="348.0" y2="348.0" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="352.0" font-size="11" fill="#52514e" text-anchor="end">0</text>
<line x1="64" x2="570" y1="277.5" y2="277.5" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="281.5" font-size="11" fill="#52514e" text-anchor="end">100</text>
<line x1="64" x2="570" y1="207.0" y2="207.0" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="211.0" font-size="11" fill="#52514e" text-anchor="end">200</text>
<line x1="64" x2="570" y1="136.5" y2="136.5" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="140.5" font-size="11" fill="#52514e" text-anchor="end">300</text>
<line x1="64" x2="570" y1="66.0" y2="66.0" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="70.0" font-size="11" fill="#52514e" text-anchor="end">400</text>
<text x="64.0" y="366" font-size="11" fill="#52514e" text-anchor="middle">0</text>
<text x="144.3" y="366" font-size="11" fill="#52514e" text-anchor="middle">100</text>
<text x="224.6" y="366" font-size="11" fill="#52514e" text-anchor="middle">200</text>
<text x="305.0" y="366" font-size="11" fill="#52514e" text-anchor="middle">300</text>
<text x="385.3" y="366" font-size="11" fill="#52514e" text-anchor="middle">400</text>
<text x="465.6" y="366" font-size="11" fill="#52514e" text-anchor="middle">500</text>
<text x="545.9" y="366" font-size="11" fill="#52514e" text-anchor="middle">600</text>
<line x1="64" x2="570" y1="348" y2="348" stroke="#52514e" stroke-width="1"/>
<text x="317" y="388" font-size="12" fill="#52514e" text-anchor="middle">Поданная нагрузка, req/s</text>
<text x="16" y="206" font-size="12" fill="#52514e" transform="rotate(-90 16 206)" text-anchor="middle">Полезная, rps</text>
<line x1="64.0" y1="348.0" x2="387.6" y2="64.0" stroke="#52514e" stroke-width="1" stroke-dasharray="4 4"/>
<text x="353.7" y="86.7" font-size="11" fill="#52514e" text-anchor="end">пунктир: полезная = поданная</text>
<polyline fill="none" stroke="#2a78d6" stroke-width="2" stroke-linejoin="round" points="96.1,319.8 128.3,291.6 160.4,263.4 192.5,235.2 224.6,207.0 256.8,178.8 288.9,150.6 321.0,153.5 353.1,152.4 385.3,169.9 433.5,190.9"/>
<circle cx="96.1" cy="319.8" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 28 мс (медиана 3 прогонов)</title></circle>
<circle cx="128.3" cy="291.6" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 27 мс (медиана 3 прогонов)</title></circle>
<circle cx="160.4" cy="263.4" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов)</title></circle>
<circle cx="192.5" cy="235.2" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 56 мс (медиана 3 прогонов)</title></circle>
<circle cx="224.6" cy="207.0" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 106 мс (медиана 3 прогонов)</title></circle>
<circle cx="256.8" cy="178.8" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 218 мс (медиана 3 прогонов)</title></circle>
<circle cx="288.9" cy="150.6" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 322 мс (медиана 3 прогонов)</title></circle>
<circle cx="321.0" cy="153.5" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 320 req/s: полезная 276 rps, ошибок 13.8 %, p99 1518 мс (медиана 3 прогонов)</title></circle>
<circle cx="353.1" cy="152.4" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 360 req/s: полезная 277 rps, ошибок 22.9 %, p99 1671 мс (медиана 3 прогонов)</title></circle>
<circle cx="385.3" cy="169.9" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 400 req/s: полезная 253 rps, ошибок 36.8 %, p99 1340 мс (медиана 3 прогонов)</title></circle>
<circle cx="433.5" cy="190.9" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0) — 460 req/s: полезная 223 rps, ошибок 51.6 %, p99 1655 мс (медиана 3 прогонов)</title></circle>
<polyline fill="none" stroke="#eb6834" stroke-width="2" stroke-linejoin="round" points="96.1,319.8 128.3,291.6 160.4,263.4 192.5,235.2 224.6,207.0 256.8,178.8 288.9,150.6 321.0,122.4 353.1,94.4 385.3,98.2 433.5,116.9 481.7,130.0 545.9,126.5"/>
<rect x="91.6" y="315.3" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 22 мс (медиана 3 прогонов)</title></rect>
<rect x="123.8" y="287.1" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 16 мс (медиана 3 прогонов)</title></rect>
<rect x="155.9" y="258.9" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 21 мс (медиана 3 прогонов)</title></rect>
<rect x="188.0" y="230.7" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 117 мс (медиана 3 прогонов)</title></rect>
<rect x="220.1" y="202.5" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 67 мс (медиана 3 прогонов)</title></rect>
<rect x="252.3" y="174.3" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 154 мс (медиана 3 прогонов)</title></rect>
<rect x="284.4" y="146.1" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 158 мс (медиана 3 прогонов)</title></rect>
<rect x="316.5" y="117.9" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 320 req/s: полезная 320 rps, ошибок 0.0 %, p99 298 мс (медиана 3 прогонов)</title></rect>
<rect x="348.6" y="89.9" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 360 req/s: полезная 360 rps, ошибок 0.1 %, p99 2020 мс (медиана 3 прогонов)</title></rect>
<rect x="380.8" y="93.7" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 400 req/s: полезная 354 rps, ошибок 11.4 %, p99 2068 мс (медиана 3 прогонов)</title></rect>
<rect x="429.0" y="112.4" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 460 req/s: полезная 328 rps, ошибок 28.7 %, p99 2080 мс (медиана 3 прогонов)</title></rect>
<rect x="477.2" y="125.5" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 520 req/s: полезная 309 rps, ошибок 40.5 %, p99 2082 мс (медиана 3 прогонов)</title></rect>
<rect x="541.4" y="122.0" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10 — 600 req/s: полезная 314 rps, ошибок 47.6 %, p99 2091 мс (медиана 2 прогонов)</title></rect>
<polyline fill="none" stroke="#1baf7a" stroke-width="2" stroke-linejoin="round" points="96.1,319.8 128.3,291.6 160.4,263.4 192.5,235.2 224.6,207.0 256.8,178.8 288.9,150.6 321.0,122.4 353.1,118.0 385.3,134.4 433.5,136.7 481.7,145.1"/>
<path d="M96.1 314.3L101.6 324.3L90.6 324.3Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 34 мс (медиана 1 прогонов)</title></path>
<path d="M128.3 286.1L133.8 296.1L122.8 296.1Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 17 мс (медиана 1 прогонов)</title></path>
<path d="M160.4 257.9L165.9 267.9L154.9 267.9Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 59 мс (медиана 1 прогонов)</title></path>
<path d="M192.5 229.7L198.0 239.7L187.0 239.7Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 55 мс (медиана 1 прогонов)</title></path>
<path d="M224.6 201.5L230.1 211.5L219.1 211.5Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 189 мс (медиана 1 прогонов)</title></path>
<path d="M256.8 173.3L262.3 183.3L251.3 183.3Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 470 мс (медиана 1 прогонов)</title></path>
<path d="M288.9 145.1L294.4 155.1L283.4 155.1Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 573 мс (медиана 1 прогонов)</title></path>
<path d="M321.0 116.9L326.5 126.9L315.5 126.9Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 320 req/s: полезная 320 rps, ошибок 0.0 %, p99 577 мс (медиана 1 прогонов)</title></path>
<path d="M353.1 112.5L358.6 122.5L347.6 122.5Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 360 req/s: полезная 326 rps, ошибок 9.4 %, p99 2130 мс (медиана 1 прогонов)</title></path>
<path d="M385.3 128.9L390.8 138.9L379.8 138.9Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 400 req/s: полезная 303 rps, ошибок 24.2 %, p99 2167 мс (медиана 1 прогонов)</title></path>
<path d="M433.5 131.2L439.0 141.2L428.0 141.2Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 460 req/s: полезная 300 rps, ошибок 34.8 %, p99 2151 мс (медиана 1 прогонов)</title></path>
<path d="M481.7 139.6L487.2 149.6L476.2 149.6Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20 — 520 req/s: полезная 288 rps, ошибок 44.6 %, p99 2157 мс (медиана 1 прогонов)</title></path>
<polyline fill="none" stroke="#eda100" stroke-width="2" stroke-linejoin="round" points="96.1,319.8 128.3,291.6 160.4,263.4 192.5,235.2 224.6,207.0 256.8,179.2 288.9,173.8 321.0,145.0 353.1,134.0 385.3,141.4 433.5,137.6 481.7,135.0"/>
<path d="M96.1 313.8L102.1 319.8L96.1 325.8L90.1 319.8Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 16 мс (медиана 1 прогонов)</title></path>
<path d="M128.3 285.6L134.3 291.6L128.3 297.6L122.3 291.6Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 21 мс (медиана 1 прогонов)</title></path>
<path d="M160.4 257.4L166.4 263.4L160.4 269.4L154.4 263.4Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 36 мс (медиана 1 прогонов)</title></path>
<path d="M192.5 229.2L198.5 235.2L192.5 241.2L186.5 235.2Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 68 мс (медиана 1 прогонов)</title></path>
<path d="M224.6 201.0L230.6 207.0L224.6 213.0L218.6 207.0Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 217 мс (медиана 1 прогонов)</title></path>
<path d="M256.8 173.2L262.8 179.2L256.8 185.2L250.8 179.2Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 240 req/s: полезная 239 rps, ошибок 0.2 %, p99 273 мс (медиана 1 прогонов)</title></path>
<path d="M288.9 167.8L294.9 173.8L288.9 179.8L282.9 173.8Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 280 req/s: полезная 247 rps, ошибок 11.8 %, p99 338 мс (медиана 1 прогонов)</title></path>
<path d="M321.0 139.0L327.0 145.0L321.0 151.0L315.0 145.0Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 320 req/s: полезная 288 rps, ошибок 10.0 %, p99 326 мс (медиана 1 прогонов)</title></path>
<path d="M353.1 128.0L359.1 134.0L353.1 140.0L347.1 134.0Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 360 req/s: полезная 304 rps, ошибок 15.7 %, p99 319 мс (медиана 1 прогонов)</title></path>
<path d="M385.3 135.4L391.3 141.4L385.3 147.4L379.3 141.4Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 400 req/s: полезная 293 rps, ошибок 26.8 %, p99 325 мс (медиана 1 прогонов)</title></path>
<path d="M433.5 131.6L439.5 137.6L433.5 143.6L427.5 137.6Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 460 req/s: полезная 298 rps, ошибок 35.1 %, p99 338 мс (медиана 1 прогонов)</title></path>
<path d="M481.7 129.0L487.7 135.0L481.7 141.0L475.7 135.0Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс — 520 req/s: полезная 302 rps, ошибок 41.9 %, p99 327 мс (медиана 1 прогонов)</title></path>
<circle cx="586.0" cy="68.0" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>без лимита (maxconn 0)</title></circle>
<text x="596" y="72.0" font-size="11" fill="#52514e">без лимита (maxconn 0)</text>
<rect x="581.5" y="81.5" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10</title></rect>
<text x="596" y="90.0" font-size="11" fill="#52514e">maxconn 10</text>
<path d="M586.0 98.5L591.5 108.5L580.5 108.5Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>maxconn 20</title></path>
<text x="596" y="108.0" font-size="11" fill="#52514e">maxconn 20</text>
<path d="M586.0 116.0L592.0 122.0L586.0 128.0L580.0 122.0Z" fill="#eda100" stroke="#fcfcfb" stroke-width="2"><title>maxconn 10, queue 250 мс</title></path>
<text x="596" y="126.0" font-size="11" fill="#52514e">maxconn 10, queue 250 мс</text>
</svg>

After

Width:  |  Height:  |  Size: 16 KiB

+800
View File
@@ -0,0 +1,800 @@
{
"config": "fixed-1x1.00-r2",
"repeat": 2,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T09:18:58+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 280,
"peak_ok_rps": 280,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 248.53,
"p50": 5.38,
"p90": 22.13,
"p99": 213.96,
"p999": 248.53
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.21,
"p99_ms": 22.13
},
"me": {
"ok": 40,
"p50_ms": 2.67,
"p99_ms": 7.65
},
"post": {
"ok": 80,
"p50_ms": 14.1,
"p99_ms": 248.53
}
},
"rate": 40,
"service_ms": {
"p50": 4.8,
"p99": 213.63
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 17.2,
"lb": 2.2,
"loadgen-run-03af3ac2a6d0": 2.7,
"postgres": 7.3,
"redis": 2.6
},
"mem_mib_max": {
"api": 31.8,
"lb": 4.4,
"loadgen-run-03af3ac2a6d0": 6.0,
"postgres": 126.6,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.9,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"IO:WALSync": 0.11,
"LWLock:WALWrite": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 13
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 46.94,
"p50": 5.28,
"p90": 9.76,
"p99": 29.18,
"p999": 46.94
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.16,
"p99_ms": 32.25
},
"me": {
"ok": 80,
"p50_ms": 2.46,
"p99_ms": 11.2
},
"post": {
"ok": 160,
"p50_ms": 6.08,
"p99_ms": 26.84
}
},
"rate": 80,
"service_ms": {
"p50": 4.65,
"p99": 24.56
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 29.3,
"lb": 3.3,
"loadgen-run-03af3ac2a6d0": 4.9,
"postgres": 9.3,
"redis": 2.9
},
"mem_mib_max": {
"api": 32.0,
"lb": 4.4,
"loadgen-run-03af3ac2a6d0": 6.2,
"postgres": 126.9,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 85.76,
"p50": 5.25,
"p90": 9.92,
"p99": 38.83,
"p999": 79.18
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.17,
"p99_ms": 41.01
},
"me": {
"ok": 120,
"p50_ms": 2.52,
"p99_ms": 23.77
},
"post": {
"ok": 240,
"p50_ms": 6.04,
"p99_ms": 16.59
}
},
"rate": 120,
"service_ms": {
"p50": 4.63,
"p99": 37.56
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 41.0,
"lb": 4.5,
"loadgen-run-03af3ac2a6d0": 6.9,
"postgres": 11.1,
"redis": 3.1
},
"mem_mib_max": {
"api": 32.3,
"lb": 4.4,
"loadgen-run-03af3ac2a6d0": 6.2,
"postgres": 128.1,
"redis": 4.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.9,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 92.92,
"p50": 4.84,
"p90": 15.94,
"p99": 60.51,
"p999": 83.13
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 4.93,
"p99_ms": 55.7
},
"me": {
"ok": 160,
"p50_ms": 2.18,
"p99_ms": 45.8
},
"post": {
"ok": 320,
"p50_ms": 5.02,
"p99_ms": 67.91
}
},
"rate": 160,
"service_ms": {
"p50": 4.24,
"p99": 59.46
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 51.2,
"lb": 4.8,
"loadgen-run-03af3ac2a6d0": 8.2,
"postgres": 13.5,
"redis": 3.7
},
"mem_mib_max": {
"api": 42.8,
"lb": 4.6,
"loadgen-run-03af3ac2a6d0": 6.9,
"postgres": 127.7,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.9,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 137.85,
"p50": 5.46,
"p90": 21.53,
"p99": 101.42,
"p999": 131.71
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 6.04,
"p99_ms": 81.92
},
"me": {
"ok": 200,
"p50_ms": 1.94,
"p99_ms": 14.7
},
"post": {
"ok": 400,
"p50_ms": 4.75,
"p99_ms": 114.56
}
},
"rate": 200,
"service_ms": {
"p50": 4.79,
"p99": 100.58
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 63.3,
"lb": 5.3,
"loadgen-run-03af3ac2a6d0": 8.8,
"postgres": 14.3,
"redis": 3.6
},
"mem_mib_max": {
"api": 33.2,
"lb": 4.7,
"loadgen-run-03af3ac2a6d0": 6.9,
"postgres": 129.2,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 3.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 257.5,
"p50": 9.35,
"p90": 113.47,
"p99": 237.9,
"p999": 256.55
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 12.07,
"p99_ms": 217.22
},
"me": {
"ok": 240,
"p50_ms": 2.06,
"p99_ms": 73.57
},
"post": {
"ok": 480,
"p50_ms": 6.25,
"p99_ms": 248.81
}
},
"rate": 240,
"service_ms": {
"p50": 8.74,
"p99": 237.86
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 78.3,
"lb": 5.7,
"loadgen-run-03af3ac2a6d0": 10.9,
"postgres": 17.6,
"redis": 3.5
},
"mem_mib_max": {
"api": 39.4,
"lb": 5.0,
"loadgen-run-03af3ac2a6d0": 7.4,
"postgres": 128.4,
"redis": 5.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 6.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": true
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 199.51,
"p50": 19.2,
"p90": 111.53,
"p99": 190.39,
"p999": 198.45
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 22.23,
"p99_ms": 191.69
},
"me": {
"ok": 280,
"p50_ms": 2.78,
"p99_ms": 33.67
},
"post": {
"ok": 560,
"p50_ms": 23.44,
"p99_ms": 183.02
}
},
"rate": 280,
"service_ms": {
"p50": 18.61,
"p99": 189.82
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 85.2,
"lb": 6.3,
"loadgen-run-03af3ac2a6d0": 11.9,
"postgres": 18.7,
"redis": 3.7
},
"mem_mib_max": {
"api": 47.0,
"lb": 5.1,
"loadgen-run-03af3ac2a6d0": 7.7,
"postgres": 130.8,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 11.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": true
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 529
},
"error_ratio": 0.1653,
"errors": 529,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 2327.82,
"p50": 807.39,
"p90": 1125.44,
"p99": 1517.58,
"p999": 2286.06
},
"ok": 2671,
"per_op": {
"history": {
"ok": 1864,
"p50_ms": 824.16,
"p99_ms": 1522.62
},
"me": {
"ok": 318,
"p50_ms": 21.32,
"p99_ms": 780.29
},
"post": {
"ok": 489,
"p50_ms": 833.42,
"p99_ms": 2266.99
}
},
"rate": 320,
"service_ms": {
"p50": 806.93,
"p99": 1516.57
},
"throughput_ok_rps": 267.1,
"resources": {
"cpu_percent_mean": {
"api": 91.1,
"lb": 7.4,
"loadgen-run-03af3ac2a6d0": 15.0,
"postgres": 19.4,
"redis": 4.0
},
"mem_mib_max": {
"api": 40.7,
"lb": 8.1,
"loadgen-run-03af3ac2a6d0": 15.7,
"postgres": 129.4,
"redis": 5.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 207.2,
"pg_active_mean": 0.4,
"pg_wait_events_mean": {
"Client:ClientRead": 0.4
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 826
},
"error_ratio": 0.2294,
"errors": 826,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 1991.53,
"p50": 794.31,
"p90": 1047.54,
"p99": 1602.98,
"p999": 1976.35
},
"ok": 2774,
"per_op": {
"history": {
"ok": 1897,
"p50_ms": 812.89,
"p99_ms": 1653.82
},
"me": {
"ok": 360,
"p50_ms": 18.42,
"p99_ms": 788.44
},
"post": {
"ok": 517,
"p50_ms": 806.01,
"p99_ms": 1626.1
}
},
"rate": 360,
"service_ms": {
"p50": 793.71,
"p99": 1602.6
},
"throughput_ok_rps": 277.4,
"resources": {
"cpu_percent_mean": {
"api": 92.0,
"lb": 7.2,
"loadgen-run-03af3ac2a6d0": 14.6,
"postgres": 17.9,
"redis": 3.9
},
"mem_mib_max": {
"api": 52.8,
"lb": 8.1,
"loadgen-run-03af3ac2a6d0": 16.6,
"postgres": 129.9,
"redis": 5.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 214.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1732
},
"error_ratio": 0.433,
"errors": 1732,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 1590.66,
"p50": 1014.62,
"p90": 1400.84,
"p99": 1559.84,
"p999": 1584.98
},
"ok": 2268,
"per_op": {
"history": {
"ok": 1502,
"p50_ms": 1118.51,
"p99_ms": 1557.17
},
"me": {
"ok": 400,
"p50_ms": 26.08,
"p99_ms": 99.93
},
"post": {
"ok": 366,
"p50_ms": 1144.47,
"p99_ms": 1584.3
}
},
"rate": 400,
"service_ms": {
"p50": 1014.28,
"p99": 1559.56
},
"throughput_ok_rps": 226.8,
"resources": {
"cpu_percent_mean": {
"api": 88.6,
"lb": 8.7,
"loadgen-run-03af3ac2a6d0": 16.0,
"postgres": 16.7,
"redis": 4.6
},
"mem_mib_max": {
"api": 54.4,
"lb": 7.9,
"loadgen-run-03af3ac2a6d0": 16.8,
"postgres": 131.2,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 214.7,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"Client:ClientRead": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2093
},
"error_ratio": 0.455,
"errors": 2093,
"event": "step",
"label": "fixed-1x1.00-r2",
"latency_ms": {
"max": 1463.9,
"p50": 930.75,
"p90": 1322.55,
"p99": 1421.05,
"p999": 1463.03
},
"ok": 2507,
"per_op": {
"history": {
"ok": 1632,
"p50_ms": 979.78,
"p99_ms": 1424.25
},
"me": {
"ok": 460,
"p50_ms": 18.49,
"p99_ms": 107.52
},
"post": {
"ok": 415,
"p50_ms": 1011.43,
"p99_ms": 1458.17
}
},
"rate": 460,
"service_ms": {
"p50": 930.2,
"p99": 1420.76
},
"throughput_ok_rps": 250.7,
"resources": {
"cpu_percent_mean": {
"api": 89.9,
"lb": 9.1,
"loadgen-run-03af3ac2a6d0": 16.7,
"postgres": 16.5,
"redis": 4.7
},
"mem_mib_max": {
"api": 52.9,
"lb": 8.5,
"loadgen-run-03af3ac2a6d0": 17.4,
"postgres": 131.1,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 219.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 17
},
"slo_ok": false
}
],
"pool_before": {
"api-4ff42b6f": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 85,
"requests_wait_ms": 92,
"usage_ms": 90,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
}
},
"pool_after": {
"api-4ff42b6f": {
"pool": {
"connections_num": 10,
"requests_num": 32520,
"requests_queued": 13942,
"connections_ms": 233,
"requests_wait_ms": 9616407,
"usage_ms": 526852,
"requests_errors": 5825,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 34796,
"misses": 317,
"errors": 0
},
"requests": 35401
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+806
View File
@@ -0,0 +1,806 @@
{
"config": "fixed-1x1.00-r3",
"repeat": 3,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T09:31:59+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 240,
"peak_ok_rps": 316.7,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 59.69,
"p50": 4.89,
"p90": 8.33,
"p99": 28.02,
"p999": 59.69
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 4.87,
"p99_ms": 14.99
},
"me": {
"ok": 40,
"p50_ms": 2.42,
"p99_ms": 4.72
},
"post": {
"ok": 80,
"p50_ms": 5.9,
"p99_ms": 59.69
}
},
"rate": 40,
"service_ms": {
"p50": 4.32,
"p99": 27.01
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 14.0,
"lb": 1.9,
"loadgen-run-b4cc6f85f08e": 2.5,
"postgres": 7.2,
"redis": 2.2
},
"mem_mib_max": {
"api": 33.8,
"lb": 4.5,
"loadgen-run-b4cc6f85f08e": 6.0,
"postgres": 144.4,
"redis": 5.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 16
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 19.2,
"p50": 4.77,
"p90": 7.58,
"p99": 14.08,
"p999": 19.2
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 4.72,
"p99_ms": 14.64
},
"me": {
"ok": 80,
"p50_ms": 2.27,
"p99_ms": 11.67
},
"post": {
"ok": 160,
"p50_ms": 5.76,
"p99_ms": 13.21
}
},
"rate": 80,
"service_ms": {
"p50": 4.2,
"p99": 13.5
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 27.2,
"lb": 3.1,
"loadgen-run-b4cc6f85f08e": 4.8,
"postgres": 9.4,
"redis": 2.9
},
"mem_mib_max": {
"api": 31.9,
"lb": 4.5,
"loadgen-run-b4cc6f85f08e": 6.2,
"postgres": 146.1,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 16
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 82.75,
"p50": 4.82,
"p90": 8.82,
"p99": 18.5,
"p999": 67.67
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 4.87,
"p99_ms": 14.5
},
"me": {
"ok": 120,
"p50_ms": 2.22,
"p99_ms": 58.75
},
"post": {
"ok": 240,
"p50_ms": 5.14,
"p99_ms": 21.24
}
},
"rate": 120,
"service_ms": {
"p50": 4.21,
"p99": 17.05
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 38.8,
"lb": 3.9,
"loadgen-run-b4cc6f85f08e": 6.5,
"postgres": 11.4,
"redis": 3.3
},
"mem_mib_max": {
"api": 35.3,
"lb": 4.5,
"loadgen-run-b4cc6f85f08e": 6.2,
"postgres": 149.8,
"redis": 4.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 18
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 60.48,
"p50": 4.96,
"p90": 13.17,
"p99": 49.06,
"p999": 59.71
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.08,
"p99_ms": 50.46
},
"me": {
"ok": 160,
"p50_ms": 2.12,
"p99_ms": 8.97
},
"post": {
"ok": 320,
"p50_ms": 5.15,
"p99_ms": 48.86
}
},
"rate": 160,
"service_ms": {
"p50": 4.31,
"p99": 48.66
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 51.7,
"lb": 4.9,
"loadgen-run-b4cc6f85f08e": 8.0,
"postgres": 14.0,
"redis": 3.6
},
"mem_mib_max": {
"api": 32.7,
"lb": 4.5,
"loadgen-run-b4cc6f85f08e": 6.2,
"postgres": 155.8,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 180.98,
"p50": 5.25,
"p90": 24.89,
"p99": 130.76,
"p999": 147.99
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 5.78,
"p99_ms": 123.83
},
"me": {
"ok": 200,
"p50_ms": 2.06,
"p99_ms": 26.29
},
"post": {
"ok": 400,
"p50_ms": 4.98,
"p99_ms": 138.22
}
},
"rate": 200,
"service_ms": {
"p50": 4.65,
"p99": 130.34
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 63.7,
"lb": 5.3,
"loadgen-run-b4cc6f85f08e": 9.2,
"postgres": 16.0,
"redis": 3.5
},
"mem_mib_max": {
"api": 39.1,
"lb": 4.7,
"loadgen-run-b4cc6f85f08e": 7.0,
"postgres": 158.9,
"redis": 5.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 3.8,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 299.19,
"p50": 8.6,
"p90": 94.17,
"p99": 218.04,
"p999": 287.43
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 9.44,
"p99_ms": 218.12
},
"me": {
"ok": 240,
"p50_ms": 2.13,
"p99_ms": 45.38
},
"post": {
"ok": 480,
"p50_ms": 11.16,
"p99_ms": 220.45
}
},
"rate": 240,
"service_ms": {
"p50": 8.01,
"p99": 214.19
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 78.9,
"lb": 6.0,
"loadgen-run-b4cc6f85f08e": 11.0,
"postgres": 18.0,
"redis": 3.6
},
"mem_mib_max": {
"api": 46.1,
"lb": 5.0,
"loadgen-run-b4cc6f85f08e": 7.2,
"postgres": 159.0,
"redis": 4.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 9.4,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.11,
"Client:ClientRead": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": true
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 342.76,
"p50": 104.12,
"p90": 241.32,
"p99": 322.06,
"p999": 341.03
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 113.98,
"p99_ms": 319.32
},
"me": {
"ok": 280,
"p50_ms": 13.4,
"p99_ms": 45.72
},
"post": {
"ok": 560,
"p50_ms": 142.28,
"p99_ms": 337.01
}
},
"rate": 280,
"service_ms": {
"p50": 103.28,
"p99": 321.51
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 88.9,
"lb": 6.5,
"loadgen-run-b4cc6f85f08e": 12.8,
"postgres": 19.8,
"redis": 3.9
},
"mem_mib_max": {
"api": 44.3,
"lb": 5.3,
"loadgen-run-b4cc6f85f08e": 7.9,
"postgres": 158.6,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 28.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 33
},
"error_ratio": 0.0103,
"errors": 33,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 1467.42,
"p50": 477.92,
"p90": 700.51,
"p99": 1105.55,
"p999": 1354.96
},
"ok": 3167,
"per_op": {
"history": {
"ok": 2217,
"p50_ms": 494.08,
"p99_ms": 1109.71
},
"me": {
"ok": 320,
"p50_ms": 16.73,
"p99_ms": 638.61
},
"post": {
"ok": 630,
"p50_ms": 501.4,
"p99_ms": 1110.53
}
},
"rate": 320,
"service_ms": {
"p50": 477.56,
"p99": 1104.52
},
"throughput_ok_rps": 316.7,
"resources": {
"cpu_percent_mean": {
"api": 93.8,
"lb": 7.0,
"loadgen-run-b4cc6f85f08e": 14.4,
"postgres": 20.7,
"redis": 3.7
},
"mem_mib_max": {
"api": 52.8,
"lb": 7.6,
"loadgen-run-b4cc6f85f08e": 13.5,
"postgres": 160.8,
"redis": 4.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 142.7,
"pg_active_mean": 0.6,
"pg_wait_events_mean": {
"CPU": 0.1,
"IO:WALSync": 0.1,
"LWLock:WALWrite": 0.4
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 799
},
"error_ratio": 0.2219,
"errors": 799,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 1986.54,
"p50": 775.59,
"p90": 1169.18,
"p99": 1670.77,
"p999": 1869.14
},
"ok": 2801,
"per_op": {
"history": {
"ok": 1921,
"p50_ms": 795.13,
"p99_ms": 1636.24
},
"me": {
"ok": 360,
"p50_ms": 18.67,
"p99_ms": 800.91
},
"post": {
"ok": 520,
"p50_ms": 797.11,
"p99_ms": 1814.63
}
},
"rate": 360,
"service_ms": {
"p50": 774.85,
"p99": 1670.75
},
"throughput_ok_rps": 280.1,
"resources": {
"cpu_percent_mean": {
"api": 92.3,
"lb": 7.4,
"loadgen-run-b4cc6f85f08e": 15.2,
"postgres": 19.7,
"redis": 4.2
},
"mem_mib_max": {
"api": 49.8,
"lb": 7.9,
"loadgen-run-b4cc6f85f08e": 17.3,
"postgres": 161.2,
"redis": 4.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 212.3,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"IO:WALSync": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1473
},
"error_ratio": 0.3683,
"errors": 1473,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 1359.43,
"p50": 931.49,
"p90": 1160.69,
"p99": 1339.97,
"p999": 1358.42
},
"ok": 2527,
"per_op": {
"history": {
"ok": 1711,
"p50_ms": 975.71,
"p99_ms": 1342.7
},
"me": {
"ok": 400,
"p50_ms": 20.86,
"p99_ms": 89.1
},
"post": {
"ok": 416,
"p50_ms": 975.52,
"p99_ms": 1317.79
}
},
"rate": 400,
"service_ms": {
"p50": 931.07,
"p99": 1337.71
},
"throughput_ok_rps": 252.7,
"resources": {
"cpu_percent_mean": {
"api": 89.9,
"lb": 8.3,
"loadgen-run-b4cc6f85f08e": 16.2,
"postgres": 18.0,
"redis": 4.5
},
"mem_mib_max": {
"api": 53.8,
"lb": 7.6,
"loadgen-run-b4cc6f85f08e": 16.8,
"postgres": 160.1,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 215.2,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"Client:ClientRead": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2472
},
"error_ratio": 0.5374,
"errors": 2472,
"event": "step",
"label": "fixed-1x1.00-r3",
"latency_ms": {
"max": 1703.62,
"p50": 1187.41,
"p90": 1429.09,
"p99": 1676.2,
"p999": 1702.67
},
"ok": 2128,
"per_op": {
"history": {
"ok": 1366,
"p50_ms": 1293.19,
"p99_ms": 1676.2
},
"me": {
"ok": 460,
"p50_ms": 26.44,
"p99_ms": 171.21
},
"post": {
"ok": 302,
"p50_ms": 1307.55,
"p99_ms": 1684.72
}
},
"rate": 460,
"service_ms": {
"p50": 1186.63,
"p99": 1676.13
},
"throughput_ok_rps": 212.8,
"resources": {
"cpu_percent_mean": {
"api": 88.8,
"lb": 9.0,
"loadgen-run-b4cc6f85f08e": 17.0,
"postgres": 16.6,
"redis": 4.6
},
"mem_mib_max": {
"api": 53.8,
"lb": 7.7,
"loadgen-run-b4cc6f85f08e": 17.9,
"postgres": 161.3,
"redis": 4.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 220.4,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
}
],
"pool_before": {
"api-f87de045": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 80,
"requests_wait_ms": 76,
"usage_ms": 75,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
}
},
"pool_after": {
"api-f87de045": {
"pool": {
"connections_num": 10,
"requests_num": 32487,
"requests_queued": 15163,
"connections_ms": 410,
"requests_wait_ms": 9045144,
"usage_ms": 531592,
"requests_errors": 5350,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 34841,
"misses": 272,
"errors": 0
},
"requests": 35397
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+804
View File
@@ -0,0 +1,804 @@
{
"config": "fixed-1x1.00",
"repeat": 1,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T08:28:34+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 240,
"peak_ok_rps": 280,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 26.77,
"p50": 5.05,
"p90": 8.12,
"p99": 17.13,
"p999": 26.77
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 4.9,
"p99_ms": 20.07
},
"me": {
"ok": 40,
"p50_ms": 2.62,
"p99_ms": 5.13
},
"post": {
"ok": 80,
"p50_ms": 6.27,
"p99_ms": 13.07
}
},
"rate": 40,
"service_ms": {
"p50": 4.39,
"p99": 16.53
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 15.6,
"lb": 1.9,
"loadgen-run-14b3c75cfc70": 2.6,
"postgres": 4.5,
"redis": 2.2
},
"mem_mib_max": {
"api": 31.6,
"lb": 4.4,
"loadgen-run-14b3c75cfc70": 5.9,
"postgres": 51.6,
"redis": 3.5
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 57.5,
"p50": 5.04,
"p90": 8.07,
"p99": 26.76,
"p999": 57.5
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 4.95,
"p99_ms": 15.38
},
"me": {
"ok": 80,
"p50_ms": 2.55,
"p99_ms": 6.48
},
"post": {
"ok": 160,
"p50_ms": 6.41,
"p99_ms": 56.12
}
},
"rate": 80,
"service_ms": {
"p50": 4.45,
"p99": 26.32
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 28.0,
"lb": 3.4,
"loadgen-run-14b3c75cfc70": 5.1,
"postgres": 7.3,
"redis": 2.7
},
"mem_mib_max": {
"api": 31.9,
"lb": 4.4,
"loadgen-run-14b3c75cfc70": 6.2,
"postgres": 51.7,
"redis": 3.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.4,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.22
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 195.02,
"p50": 5.27,
"p90": 10.48,
"p99": 117.23,
"p999": 143.67
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.22,
"p99_ms": 60.1
},
"me": {
"ok": 120,
"p50_ms": 2.41,
"p99_ms": 59.21
},
"post": {
"ok": 240,
"p50_ms": 5.82,
"p99_ms": 139.6
}
},
"rate": 120,
"service_ms": {
"p50": 4.67,
"p99": 116.35
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 41.8,
"lb": 4.4,
"loadgen-run-14b3c75cfc70": 7.2,
"postgres": 10.0,
"redis": 3.7
},
"mem_mib_max": {
"api": 42.8,
"lb": 4.4,
"loadgen-run-14b3c75cfc70": 6.4,
"postgres": 61.0,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 6.3
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 76.33,
"p50": 4.77,
"p90": 10.67,
"p99": 56.46,
"p999": 74.06
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 4.78,
"p99_ms": 56.67
},
"me": {
"ok": 160,
"p50_ms": 2.19,
"p99_ms": 11.42
},
"post": {
"ok": 320,
"p50_ms": 5.4,
"p99_ms": 61.52
}
},
"rate": 160,
"service_ms": {
"p50": 4.17,
"p99": 55.54
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 50.6,
"lb": 5.0,
"loadgen-run-14b3c75cfc70": 8.7,
"postgres": 12.1,
"redis": 3.7
},
"mem_mib_max": {
"api": 32.8,
"lb": 4.6,
"loadgen-run-14b3c75cfc70": 6.8,
"postgres": 64.4,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 132.03,
"p50": 6.12,
"p90": 22.47,
"p99": 106.21,
"p999": 124.66
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 6.79,
"p99_ms": 91.02
},
"me": {
"ok": 200,
"p50_ms": 2.07,
"p99_ms": 51.74
},
"post": {
"ok": 400,
"p50_ms": 5.81,
"p99_ms": 118.12
}
},
"rate": 200,
"service_ms": {
"p50": 5.54,
"p99": 106
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 67.2,
"lb": 5.8,
"loadgen-run-14b3c75cfc70": 10.1,
"postgres": 14.1,
"redis": 3.8
},
"mem_mib_max": {
"api": 33.4,
"lb": 4.7,
"loadgen-run-14b3c75cfc70": 6.9,
"postgres": 65.5,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 247.31,
"p50": 7.82,
"p90": 117.69,
"p99": 213.3,
"p999": 241.35
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 8.94,
"p99_ms": 213.68
},
"me": {
"ok": 240,
"p50_ms": 2.59,
"p99_ms": 37.25
},
"post": {
"ok": 480,
"p50_ms": 7.72,
"p99_ms": 223.22
}
},
"rate": 240,
"service_ms": {
"p50": 7.22,
"p99": 211.96
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 78.8,
"lb": 6.0,
"loadgen-run-14b3c75cfc70": 11.4,
"postgres": 15.8,
"redis": 3.6
},
"mem_mib_max": {
"api": 46.5,
"lb": 4.9,
"loadgen-run-14b3c75cfc70": 7.3,
"postgres": 65.2,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 9.5,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.11,
"Client:ClientRead": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": true
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 498.37,
"p50": 72.46,
"p90": 297.04,
"p99": 408.37,
"p999": 488.08
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 80,
"p99_ms": 403.35
},
"me": {
"ok": 280,
"p50_ms": 9.14,
"p99_ms": 82.06
},
"post": {
"ok": 560,
"p50_ms": 98.9,
"p99_ms": 417.27
}
},
"rate": 280,
"service_ms": {
"p50": 71.87,
"p99": 407.71
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 89.1,
"lb": 7.0,
"loadgen-run-14b3c75cfc70": 13.8,
"postgres": 17.9,
"redis": 4.1
},
"mem_mib_max": {
"api": 45.8,
"lb": 5.7,
"loadgen-run-14b3c75cfc70": 9.1,
"postgres": 65.7,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 27.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 441
},
"error_ratio": 0.1378,
"errors": 441,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 1917.11,
"p50": 743.52,
"p90": 1047.11,
"p99": 1615.1,
"p999": 1888.72
},
"ok": 2759,
"per_op": {
"history": {
"ok": 1938,
"p50_ms": 755.54,
"p99_ms": 1614.04
},
"me": {
"ok": 319,
"p50_ms": 17.88,
"p99_ms": 542.08
},
"post": {
"ok": 502,
"p50_ms": 777.13,
"p99_ms": 1866.85
}
},
"rate": 320,
"service_ms": {
"p50": 742.61,
"p99": 1614.32
},
"throughput_ok_rps": 275.9,
"resources": {
"cpu_percent_mean": {
"api": 92.8,
"lb": 7.6,
"loadgen-run-14b3c75cfc70": 15.7,
"postgres": 18.0,
"redis": 4.2
},
"mem_mib_max": {
"api": 41.1,
"lb": 8.1,
"loadgen-run-14b3c75cfc70": 16.1,
"postgres": 66.5,
"redis": 4.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 204.1,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 965
},
"error_ratio": 0.2681,
"errors": 965,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 2078.9,
"p50": 806.35,
"p90": 1076.62,
"p99": 1707.09,
"p999": 2066.4
},
"ok": 2635,
"per_op": {
"history": {
"ok": 1795,
"p50_ms": 830.08,
"p99_ms": 1844.42
},
"me": {
"ok": 348,
"p50_ms": 18.81,
"p99_ms": 843.52
},
"post": {
"ok": 492,
"p50_ms": 833.46,
"p99_ms": 1634.82
}
},
"rate": 360,
"service_ms": {
"p50": 805.56,
"p99": 1706.51
},
"throughput_ok_rps": 263.5,
"resources": {
"cpu_percent_mean": {
"api": 93.0,
"lb": 8.2,
"loadgen-run-14b3c75cfc70": 16.6,
"postgres": 17.7,
"redis": 4.1
},
"mem_mib_max": {
"api": 53.5,
"lb": 8.0,
"loadgen-run-14b3c75cfc70": 17.8,
"postgres": 67.0,
"redis": 4.3
},
"lb_queue_max": 0,
"lb_sessions_mean": 212.9,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"Client:ClientRead": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1334
},
"error_ratio": 0.3335,
"errors": 1334,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 1155.97,
"p50": 894.03,
"p90": 1041.24,
"p99": 1136.1,
"p999": 1151.19
},
"ok": 2666,
"per_op": {
"history": {
"ok": 1827,
"p50_ms": 909.57,
"p99_ms": 1132.15
},
"me": {
"ok": 400,
"p50_ms": 20.84,
"p99_ms": 70.83
},
"post": {
"ok": 439,
"p50_ms": 904.37,
"p99_ms": 1144.54
}
},
"rate": 400,
"service_ms": {
"p50": 893.59,
"p99": 1135.1
},
"throughput_ok_rps": 266.6,
"resources": {
"cpu_percent_mean": {
"api": 90.6,
"lb": 8.3,
"loadgen-run-14b3c75cfc70": 16.2,
"postgres": 16.2,
"redis": 4.5
},
"mem_mib_max": {
"api": 41.6,
"lb": 7.6,
"loadgen-run-14b3c75cfc70": 17.0,
"postgres": 68.5,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 211.4,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2372
},
"error_ratio": 0.5157,
"errors": 2372,
"event": "step",
"label": "fixed-1x1.00",
"latency_ms": {
"max": 1742.42,
"p50": 1102.86,
"p90": 1466.56,
"p99": 1655.37,
"p999": 1696.79
},
"ok": 2228,
"per_op": {
"history": {
"ok": 1411,
"p50_ms": 1154.7,
"p99_ms": 1652
},
"me": {
"ok": 460,
"p50_ms": 25.32,
"p99_ms": 99.88
},
"post": {
"ok": 357,
"p50_ms": 1165.95,
"p99_ms": 1696.22
}
},
"rate": 460,
"service_ms": {
"p50": 1102.31,
"p99": 1654.72
},
"throughput_ok_rps": 222.8,
"resources": {
"cpu_percent_mean": {
"api": 88.0,
"lb": 9.4,
"loadgen-run-14b3c75cfc70": 18.1,
"postgres": 15.3,
"redis": 5.0
},
"mem_mib_max": {
"api": 54.2,
"lb": 8.8,
"loadgen-run-14b3c75cfc70": 17.1,
"postgres": 67.9,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 222.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10
},
"slo_ok": false
}
],
"pool_before": {
"api-a0810b44": {
"pool": {
"connections_num": 3,
"requests_num": 19,
"requests_queued": 3,
"connections_ms": 64,
"requests_wait_ms": 68,
"usage_ms": 47,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 8
}
},
"pool_after": {
"api-a0810b44": {
"pool": {
"connections_num": 10,
"requests_num": 32550,
"requests_queued": 14872,
"connections_ms": 413,
"requests_wait_ms": 10013292,
"usage_ms": 536852,
"requests_errors": 5843,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 34713,
"misses": 297,
"errors": 1
},
"requests": 35398
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+718
View File
@@ -0,0 +1,718 @@
{
"config": "fixed-2x0.50-r2",
"repeat": 2,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 2,
"API_CPUS": "0.5",
"API_MEMORY": "256M",
"KDF_CONCURRENCY": 2
},
"measured_at": "2026-11-03T09:21:48+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 120,
"peak_ok_rps": 234.7,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 61.93,
"p50": 5.16,
"p90": 9.93,
"p99": 38.56,
"p999": 61.93
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.12,
"p99_ms": 53.03
},
"me": {
"ok": 40,
"p50_ms": 2.42,
"p99_ms": 10.92
},
"post": {
"ok": 80,
"p50_ms": 5.73,
"p99_ms": 34.4
}
},
"rate": 40,
"service_ms": {
"p50": 4.57,
"p99": 37.85
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 19.3,
"lb": 2.2,
"loadgen-run-bc33f10e790a": 2.5,
"postgres": 6.5,
"redis": 2.3
},
"mem_mib_max": {
"api": 43.7,
"lb": 4.4,
"loadgen-run-bc33f10e790a": 6.1,
"postgres": 130.0,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.039,
"pg_connections_mean": 16
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 44.49,
"p50": 4.91,
"p90": 9.89,
"p99": 22.47,
"p999": 44.49
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 4.76,
"p99_ms": 16.94
},
"me": {
"ok": 80,
"p50_ms": 2.26,
"p99_ms": 16.42
},
"post": {
"ok": 160,
"p50_ms": 5.97,
"p99_ms": 24.78
}
},
"rate": 80,
"service_ms": {
"p50": 4.28,
"p99": 22.1
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 30.4,
"lb": 3.2,
"loadgen-run-bc33f10e790a": 4.6,
"postgres": 8.5,
"redis": 2.8
},
"mem_mib_max": {
"api": 42.0,
"lb": 4.4,
"loadgen-run-bc33f10e790a": 6.3,
"postgres": 130.1,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.8,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.017,
"pg_connections_mean": 16
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 206.61,
"p50": 5.1,
"p90": 18.43,
"p99": 112.38,
"p999": 146.51
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.12,
"p99_ms": 107.46
},
"me": {
"ok": 120,
"p50_ms": 2.43,
"p99_ms": 53.96
},
"post": {
"ok": 240,
"p50_ms": 5.75,
"p99_ms": 130.13
}
},
"rate": 120,
"service_ms": {
"p50": 4.49,
"p99": 111.5
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 45.5,
"lb": 4.0,
"loadgen-run-bc33f10e790a": 6.8,
"postgres": 11.5,
"redis": 3.0
},
"mem_mib_max": {
"api": 44.6,
"lb": 4.5,
"loadgen-run-bc33f10e790a": 6.8,
"postgres": 137.8,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.011,
"pg_connections_mean": 18.3
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 321.07,
"p50": 5.98,
"p90": 56.53,
"p99": 254.22,
"p999": 315.27
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.95,
"p99_ms": 202.11
},
"me": {
"ok": 160,
"p50_ms": 2.53,
"p99_ms": 147.86
},
"post": {
"ok": 320,
"p50_ms": 7.06,
"p99_ms": 307.91
}
},
"rate": 160,
"service_ms": {
"p50": 5.39,
"p99": 253.54
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 63.6,
"lb": 5.4,
"loadgen-run-bc33f10e790a": 8.7,
"postgres": 16.3,
"redis": 3.6
},
"mem_mib_max": {
"api": 42.7,
"lb": 4.8,
"loadgen-run-bc33f10e790a": 7.1,
"postgres": 159.8,
"redis": 5.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 5.1,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.011,
"pg_connections_mean": 25.3
},
"slo_ok": false
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 392.21,
"p50": 7.31,
"p90": 63.62,
"p99": 309.74,
"p999": 383.94
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 8.35,
"p99_ms": 276.99
},
"me": {
"ok": 200,
"p50_ms": 2.47,
"p99_ms": 100.7
},
"post": {
"ok": 400,
"p50_ms": 7.19,
"p99_ms": 342.89
}
},
"rate": 200,
"service_ms": {
"p50": 6.81,
"p99": 309.04
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 75.5,
"lb": 6.0,
"loadgen-run-bc33f10e790a": 10.1,
"postgres": 16.8,
"redis": 3.9
},
"mem_mib_max": {
"api": 45.9,
"lb": 5.1,
"loadgen-run-bc33f10e790a": 7.5,
"postgres": 160.6,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 5.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.011,
"pg_connections_mean": 28
},
"slo_ok": false
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 53
},
"error_ratio": 0.0221,
"errors": 53,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 1600.56,
"p50": 338.47,
"p90": 1284.39,
"p99": 1392.57,
"p999": 1590.9
},
"ok": 2347,
"per_op": {
"history": {
"ok": 1634,
"p50_ms": 385.93,
"p99_ms": 1401.71
},
"me": {
"ok": 240,
"p50_ms": 34.57,
"p99_ms": 172.6
},
"post": {
"ok": 473,
"p50_ms": 388.76,
"p99_ms": 1379.86
}
},
"rate": 240,
"service_ms": {
"p50": 338.01,
"p99": 1391.75
},
"throughput_ok_rps": 234.7,
"resources": {
"cpu_percent_mean": {
"api": 78.8,
"lb": 6.6,
"loadgen-run-bc33f10e790a": 12.4,
"postgres": 17.0,
"redis": 3.8
},
"mem_mib_max": {
"api": 49.2,
"lb": 8.4,
"loadgen-run-bc33f10e790a": 14.4,
"postgres": 163.3,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 99.7,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.223,
"pg_connections_mean": 28
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 571
},
"error_ratio": 0.2039,
"errors": 571,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 1999.62,
"p50": 1064.78,
"p90": 1647.92,
"p99": 1918.92,
"p999": 1987.41
},
"ok": 2229,
"per_op": {
"history": {
"ok": 1510,
"p50_ms": 1222.65,
"p99_ms": 1921.34
},
"me": {
"ok": 280,
"p50_ms": 44.87,
"p99_ms": 167.45
},
"post": {
"ok": 439,
"p50_ms": 1144.96,
"p99_ms": 1925.78
}
},
"rate": 280,
"service_ms": {
"p50": 1063.95,
"p99": 1918.05
},
"throughput_ok_rps": 222.9,
"resources": {
"cpu_percent_mean": {
"api": 82.5,
"lb": 7.1,
"loadgen-run-bc33f10e790a": 13.7,
"postgres": 17.8,
"redis": 4.2
},
"mem_mib_max": {
"api": 52.9,
"lb": 8.9,
"loadgen-run-bc33f10e790a": 17.8,
"postgres": 161.4,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 221.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.225,
"pg_connections_mean": 28
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1189
},
"error_ratio": 0.3716,
"errors": 1189,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 2963.48,
"p50": 1283.78,
"p90": 2024.23,
"p99": 2463.8,
"p999": 2778.97
},
"ok": 2011,
"per_op": {
"history": {
"ok": 1318,
"p50_ms": 1359.7,
"p99_ms": 2442.14
},
"me": {
"ok": 275,
"p50_ms": 48.66,
"p99_ms": 1645.93
},
"post": {
"ok": 418,
"p50_ms": 1366.84,
"p99_ms": 2481.06
}
},
"rate": 320,
"service_ms": {
"p50": 1282.87,
"p99": 2463.16
},
"throughput_ok_rps": 201.1,
"resources": {
"cpu_percent_mean": {
"api": 84.2,
"lb": 7.6,
"loadgen-run-bc33f10e790a": 13.9,
"postgres": 17.9,
"redis": 3.5
},
"mem_mib_max": {
"api": 55.8,
"lb": 9.8,
"loadgen-run-bc33f10e790a": 21.2,
"postgres": 163.2,
"redis": 5.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 247,
"pg_active_mean": 0.3,
"pg_wait_events_mean": {
"CPU": 0.3
},
"api_cfs_throttled_period_ratio": 0.172,
"pg_connections_mean": 28
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1770
},
"error_ratio": 0.4917,
"errors": 1770,
"event": "step",
"label": "fixed-2x0.50-r2",
"latency_ms": {
"max": 2899.83,
"p50": 1276.49,
"p90": 1994.35,
"p99": 2642.19,
"p999": 2865.02
},
"ok": 1830,
"per_op": {
"history": {
"ok": 1252,
"p50_ms": 1392.33,
"p99_ms": 2674.85
},
"me": {
"ok": 251,
"p50_ms": 51.72,
"p99_ms": 1485.91
},
"post": {
"ok": 327,
"p50_ms": 1421,
"p99_ms": 2151.95
}
},
"rate": 360,
"service_ms": {
"p50": 1275.94,
"p99": 2642.17
},
"throughput_ok_rps": 183,
"resources": {
"cpu_percent_mean": {
"api": 80.7,
"lb": 8.4,
"loadgen-run-bc33f10e790a": 14.9,
"postgres": 14.8,
"redis": 3.8
},
"mem_mib_max": {
"api": 58.5,
"lb": 11.3,
"loadgen-run-bc33f10e790a": 22.9,
"postgres": 162.7,
"redis": 4.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 255.7,
"pg_active_mean": 0.2,
"pg_wait_events_mean": {
"CPU": 0.2
},
"api_cfs_throttled_period_ratio": 0.207,
"pg_connections_mean": 28
},
"slo_ok": false
}
],
"pool_before": {
"api-f359a812": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 124,
"requests_wait_ms": 155,
"usage_ms": 47,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-be216ffb": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 153,
"requests_wait_ms": 206,
"usage_ms": 50,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
}
},
"pool_after": {
"api-f359a812": {
"pool": {
"connections_num": 10,
"requests_num": 12923,
"requests_queued": 5458,
"connections_ms": 837,
"requests_wait_ms": 4909083,
"usage_ms": 375953,
"requests_errors": 2881,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 11210,
"misses": 136,
"errors": 9
},
"requests": 13330
},
"api-be216ffb": {
"pool": {
"connections_num": 10,
"requests_num": 10324,
"requests_queued": 4305,
"connections_ms": 1050,
"requests_wait_ms": 3791102,
"usage_ms": 317058,
"requests_errors": 1273,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 10680,
"misses": 140,
"errors": 0
},
"requests": 11053
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+711
View File
@@ -0,0 +1,711 @@
{
"config": "fixed-2x0.50-r3",
"repeat": 3,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 2,
"API_CPUS": "0.5",
"API_MEMORY": "256M",
"KDF_CONCURRENCY": 2
},
"measured_at": "2026-11-03T09:34:50+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 200,
"peak_ok_rps": 240,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 58.6,
"p50": 5.1,
"p90": 9.91,
"p99": 32.8,
"p999": 58.6
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.01,
"p99_ms": 31.39
},
"me": {
"ok": 40,
"p50_ms": 2.82,
"p99_ms": 46.05
},
"post": {
"ok": 80,
"p50_ms": 6.29,
"p99_ms": 53.02
}
},
"rate": 40,
"service_ms": {
"p50": 4.56,
"p99": 32.16
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 18.9,
"lb": 2.1,
"loadgen-run-30ea8f468626": 2.6,
"postgres": 7.4,
"redis": 2.5
},
"mem_mib_max": {
"api": 42.3,
"lb": 4.4,
"loadgen-run-30ea8f468626": 6.1,
"postgres": 154.4,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.016,
"pg_connections_mean": 18
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 74.87,
"p50": 5.35,
"p90": 10.64,
"p99": 40.26,
"p999": 74.87
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.26,
"p99_ms": 28.64
},
"me": {
"ok": 80,
"p50_ms": 2.57,
"p99_ms": 13.94
},
"post": {
"ok": 160,
"p50_ms": 5.99,
"p99_ms": 66.31
}
},
"rate": 80,
"service_ms": {
"p50": 4.7,
"p99": 30.89
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 33.3,
"lb": 3.3,
"loadgen-run-30ea8f468626": 4.9,
"postgres": 9.8,
"redis": 2.8
},
"mem_mib_max": {
"api": 31.4,
"lb": 4.4,
"loadgen-run-30ea8f468626": 6.2,
"postgres": 155.2,
"redis": 5.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.028,
"pg_connections_mean": 18
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 134.6,
"p50": 4.92,
"p90": 9.82,
"p99": 34.96,
"p999": 127.43
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 4.81,
"p99_ms": 26.58
},
"me": {
"ok": 120,
"p50_ms": 2.27,
"p99_ms": 27.75
},
"post": {
"ok": 240,
"p50_ms": 5.78,
"p99_ms": 124.94
}
},
"rate": 120,
"service_ms": {
"p50": 4.27,
"p99": 34.87
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 45.4,
"lb": 4.1,
"loadgen-run-30ea8f468626": 6.6,
"postgres": 12.4,
"redis": 3.3
},
"mem_mib_max": {
"api": 41.6,
"lb": 4.5,
"loadgen-run-30ea8f468626": 6.6,
"postgres": 156.9,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.022,
"pg_connections_mean": 18.3
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 410.6,
"p50": 5.95,
"p90": 73.41,
"p99": 252.29,
"p999": 379.29
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 6.15,
"p99_ms": 248.75
},
"me": {
"ok": 160,
"p50_ms": 2.47,
"p99_ms": 79.63
},
"post": {
"ok": 320,
"p50_ms": 6.45,
"p99_ms": 269.86
}
},
"rate": 160,
"service_ms": {
"p50": 5.39,
"p99": 251.33
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 59.6,
"lb": 5.2,
"loadgen-run-30ea8f468626": 8.3,
"postgres": 15.7,
"redis": 3.3
},
"mem_mib_max": {
"api": 33.8,
"lb": 4.7,
"loadgen-run-30ea8f468626": 7.0,
"postgres": 183.9,
"redis": 4.3
},
"lb_queue_max": 0,
"lb_sessions_mean": 3.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.006,
"pg_connections_mean": 30
},
"slo_ok": false
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 207.88,
"p50": 7.28,
"p90": 75.29,
"p99": 168.96,
"p999": 192.93
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 8.51,
"p99_ms": 172.54
},
"me": {
"ok": 200,
"p50_ms": 2.23,
"p99_ms": 35.27
},
"post": {
"ok": 400,
"p50_ms": 6.14,
"p99_ms": 158.59
}
},
"rate": 200,
"service_ms": {
"p50": 6.7,
"p99": 168.22
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 72.5,
"lb": 5.5,
"loadgen-run-30ea8f468626": 9.4,
"postgres": 16.6,
"redis": 3.8
},
"mem_mib_max": {
"api": 44.0,
"lb": 4.9,
"loadgen-run-30ea8f468626": 7.3,
"postgres": 192.3,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 4.6,
"pg_active_mean": 0.25,
"pg_wait_events_mean": {
"IO:WALSync": 0.12,
"LWLock:WALWrite": 0.12
},
"api_cfs_throttled_period_ratio": 0.028,
"pg_connections_mean": 32
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 310.05,
"p50": 10.63,
"p90": 179.93,
"p99": 264.29,
"p999": 301.29
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 11.6,
"p99_ms": 256.9
},
"me": {
"ok": 240,
"p50_ms": 1.97,
"p99_ms": 102.26
},
"post": {
"ok": 480,
"p50_ms": 11.9,
"p99_ms": 282.57
}
},
"rate": 240,
"service_ms": {
"p50": 10.11,
"p99": 263.44
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 81.8,
"lb": 5.5,
"loadgen-run-30ea8f468626": 10.1,
"postgres": 17.9,
"redis": 3.5
},
"mem_mib_max": {
"api": 46.0,
"lb": 5.1,
"loadgen-run-30ea8f468626": 7.6,
"postgres": 192.5,
"redis": 5.3
},
"lb_queue_max": 0,
"lb_sessions_mean": 9.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.044,
"pg_connections_mean": 32
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 553
},
"error_ratio": 0.1975,
"errors": 553,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 2256.81,
"p50": 973.46,
"p90": 1724.08,
"p99": 1925.48,
"p999": 2225.79
},
"ok": 2247,
"per_op": {
"history": {
"ok": 1560,
"p50_ms": 1190.4,
"p99_ms": 1924.57
},
"me": {
"ok": 280,
"p50_ms": 38.5,
"p99_ms": 201.59
},
"post": {
"ok": 407,
"p50_ms": 1128.39,
"p99_ms": 1935.59
}
},
"rate": 280,
"service_ms": {
"p50": 972.79,
"p99": 1924.33
},
"throughput_ok_rps": 224.7,
"resources": {
"cpu_percent_mean": {
"api": 77.9,
"lb": 7.3,
"loadgen-run-30ea8f468626": 13.9,
"postgres": 17.1,
"redis": 3.7
},
"mem_mib_max": {
"api": 53.0,
"lb": 9.7,
"loadgen-run-30ea8f468626": 18.6,
"postgres": 194.3,
"redis": 5.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 207.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.31,
"pg_connections_mean": 32
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1447
},
"error_ratio": 0.4522,
"errors": 1447,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 2648.68,
"p50": 1462.09,
"p90": 2084.07,
"p99": 2498.29,
"p999": 2635.17
},
"ok": 1753,
"per_op": {
"history": {
"ok": 1151,
"p50_ms": 1550.53,
"p99_ms": 2530.79
},
"me": {
"ok": 272,
"p50_ms": 64,
"p99_ms": 1368.3
},
"post": {
"ok": 330,
"p50_ms": 1502.13,
"p99_ms": 2498.29
}
},
"rate": 320,
"service_ms": {
"p50": 1461.29,
"p99": 2497.51
},
"throughput_ok_rps": 175.3,
"resources": {
"cpu_percent_mean": {
"api": 80.8,
"lb": 7.7,
"loadgen-run-30ea8f468626": 14.0,
"postgres": 18.0,
"redis": 3.8
},
"mem_mib_max": {
"api": 56.6,
"lb": 10.5,
"loadgen-run-30ea8f468626": 21.2,
"postgres": 194.5,
"redis": 4.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 259.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.196,
"pg_connections_mean": 32
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1776
},
"error_ratio": 0.4933,
"errors": 1776,
"event": "step",
"label": "fixed-2x0.50-r3",
"latency_ms": {
"max": 2349.03,
"p50": 1325.25,
"p90": 2067.91,
"p99": 2292.96,
"p999": 2348.53
},
"ok": 1824,
"per_op": {
"history": {
"ok": 1186,
"p50_ms": 1513.93,
"p99_ms": 2300.52
},
"me": {
"ok": 331,
"p50_ms": 62.48,
"p99_ms": 271.8
},
"post": {
"ok": 307,
"p50_ms": 1439.83,
"p99_ms": 2208.61
}
},
"rate": 360,
"service_ms": {
"p50": 1324.88,
"p99": 2292.84
},
"throughput_ok_rps": 182.4,
"resources": {
"cpu_percent_mean": {
"api": 82.5,
"lb": 8.5,
"loadgen-run-30ea8f468626": 15.1,
"postgres": 16.3,
"redis": 4.1
},
"mem_mib_max": {
"api": 49.3,
"lb": 10.9,
"loadgen-run-30ea8f468626": 21.2,
"postgres": 194.8,
"redis": 4.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 253.8,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"Client:ClientRead": 0.1
},
"api_cfs_throttled_period_ratio": 0.214,
"pg_connections_mean": 32
},
"slo_ok": false
}
],
"pool_before": {
"api-474b696e": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 166,
"requests_wait_ms": 186,
"usage_ms": 130,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-c88fcc5c": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 188,
"requests_wait_ms": 180,
"usage_ms": 57,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
}
},
"pool_after": {
"api-474b696e": {
"pool": {
"connections_num": 10,
"requests_num": 12681,
"requests_queued": 4249,
"connections_ms": 952,
"requests_wait_ms": 4348360,
"usage_ms": 321100,
"requests_errors": 2849,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 12578,
"misses": 58,
"errors": 1
},
"requests": 13363
},
"api-c88fcc5c": {
"pool": {
"connections_num": 10,
"requests_num": 10515,
"requests_queued": 3617,
"connections_ms": 888,
"requests_wait_ms": 3055712,
"usage_ms": 267700,
"requests_errors": 1231,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 9978,
"misses": 68,
"errors": 19
},
"requests": 11029
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+644
View File
@@ -0,0 +1,644 @@
{
"config": "fixed-2x0.50",
"repeat": 1,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 2,
"API_CPUS": "0.5",
"API_MEMORY": "256M",
"KDF_CONCURRENCY": 2
},
"measured_at": "2026-11-03T08:31:11+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 200,
"peak_ok_rps": 234.3,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50",
"latency_ms": {
"max": 74.46,
"p50": 5.6,
"p90": 11.01,
"p99": 54.84,
"p999": 74.46
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.52,
"p99_ms": 58.87
},
"me": {
"ok": 40,
"p50_ms": 3.04,
"p99_ms": 74.46
},
"post": {
"ok": 80,
"p50_ms": 6.14,
"p99_ms": 35.31
}
},
"rate": 40,
"service_ms": {
"p50": 4.96,
"p99": 54.76
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 21.4,
"lb": 2.3,
"loadgen-run-c0edec5e3a16": 2.9,
"postgres": 5.2,
"redis": 2.4
},
"mem_mib_max": {
"api": 43.7,
"lb": 4.4,
"loadgen-run-c0edec5e3a16": 6.1,
"postgres": 64.8,
"redis": 5.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.056,
"pg_connections_mean": 8
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50",
"latency_ms": {
"max": 66.59,
"p50": 5.32,
"p90": 10.33,
"p99": 48.05,
"p999": 66.59
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.17,
"p99_ms": 48.45
},
"me": {
"ok": 80,
"p50_ms": 2.58,
"p99_ms": 19.69
},
"post": {
"ok": 160,
"p50_ms": 6.03,
"p99_ms": 50.48
}
},
"rate": 80,
"service_ms": {
"p50": 4.66,
"p99": 46.2
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 31.6,
"lb": 3.4,
"loadgen-run-c0edec5e3a16": 5.2,
"postgres": 7.5,
"redis": 2.7
},
"mem_mib_max": {
"api": 37.6,
"lb": 4.4,
"loadgen-run-c0edec5e3a16": 6.6,
"postgres": 65.1,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.067,
"pg_connections_mean": 8
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50",
"latency_ms": {
"max": 482.53,
"p50": 5.63,
"p90": 30.45,
"p99": 234.28,
"p999": 422.39
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.62,
"p99_ms": 226.5
},
"me": {
"ok": 120,
"p50_ms": 2.5,
"p99_ms": 59.5
},
"post": {
"ok": 240,
"p50_ms": 6.36,
"p99_ms": 237.83
}
},
"rate": 120,
"service_ms": {
"p50": 5,
"p99": 233.75
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 50.7,
"lb": 4.5,
"loadgen-run-c0edec5e3a16": 7.1,
"postgres": 11.2,
"redis": 3.2
},
"mem_mib_max": {
"api": 44.6,
"lb": 4.6,
"loadgen-run-c0edec5e3a16": 6.9,
"postgres": 81.9,
"redis": 5.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.9,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.033,
"pg_connections_mean": 13.4
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50",
"latency_ms": {
"max": 207.81,
"p50": 5.44,
"p90": 22.5,
"p99": 102.12,
"p999": 186.8
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.46,
"p99_ms": 101.19
},
"me": {
"ok": 160,
"p50_ms": 2.43,
"p99_ms": 55.34
},
"post": {
"ok": 320,
"p50_ms": 6.11,
"p99_ms": 180.2
}
},
"rate": 160,
"service_ms": {
"p50": 4.82,
"p99": 101.34
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 62.1,
"lb": 5.2,
"loadgen-run-c0edec5e3a16": 8.4,
"postgres": 14.4,
"redis": 3.7
},
"mem_mib_max": {
"api": 37.0,
"lb": 4.7,
"loadgen-run-c0edec5e3a16": 7.0,
"postgres": 94.9,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.9,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-2x0.50",
"latency_ms": {
"max": 334.19,
"p50": 7.53,
"p90": 99.13,
"p99": 231.85,
"p999": 261.78
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 9.15,
"p99_ms": 227.68
},
"me": {
"ok": 200,
"p50_ms": 2.38,
"p99_ms": 88.53
},
"post": {
"ok": 400,
"p50_ms": 7.55,
"p99_ms": 233.77
}
},
"rate": 200,
"service_ms": {
"p50": 6.9,
"p99": 231.05
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 74.5,
"lb": 5.8,
"loadgen-run-c0edec5e3a16": 10.1,
"postgres": 14.8,
"redis": 3.9
},
"mem_mib_max": {
"api": 35.7,
"lb": 5.0,
"loadgen-run-c0edec5e3a16": 7.6,
"postgres": 98.1,
"redis": 5.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 3.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.005,
"pg_connections_mean": 21
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 57
},
"error_ratio": 0.0238,
"errors": 57,
"event": "step",
"label": "fixed-2x0.50",
"latency_ms": {
"max": 2013.87,
"p50": 433.83,
"p90": 1311.32,
"p99": 1788.86,
"p999": 1846.65
},
"ok": 2343,
"per_op": {
"history": {
"ok": 1632,
"p50_ms": 476.98,
"p99_ms": 1809.07
},
"me": {
"ok": 240,
"p50_ms": 42.22,
"p99_ms": 1699.54
},
"post": {
"ok": 471,
"p50_ms": 470.16,
"p99_ms": 1658.32
}
},
"rate": 240,
"service_ms": {
"p50": 433.33,
"p99": 1787.71
},
"throughput_ok_rps": 234.3,
"resources": {
"cpu_percent_mean": {
"api": 80.0,
"lb": 7.1,
"loadgen-run-c0edec5e3a16": 13.5,
"postgres": 15.6,
"redis": 3.7
},
"mem_mib_max": {
"api": 50.6,
"lb": 8.5,
"loadgen-run-c0edec5e3a16": 15.3,
"postgres": 98.4,
"redis": 4.5
},
"lb_queue_max": 0,
"lb_sessions_mean": 115.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.189,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1168
},
"error_ratio": 0.4171,
"errors": 1168,
"event": "step",
"label": "fixed-2x0.50",
"latency_ms": {
"max": 2716.46,
"p50": 1237.32,
"p90": 2215.78,
"p99": 2521.89,
"p999": 2713.48
},
"ok": 1632,
"per_op": {
"history": {
"ok": 1136,
"p50_ms": 1383.52,
"p99_ms": 2538.44
},
"me": {
"ok": 222,
"p50_ms": 51.78,
"p99_ms": 1233.59
},
"post": {
"ok": 274,
"p50_ms": 1390.59,
"p99_ms": 2490.89
}
},
"rate": 280,
"service_ms": {
"p50": 1235.83,
"p99": 2521.65
},
"throughput_ok_rps": 163.2,
"resources": {
"cpu_percent_mean": {
"api": 82.9,
"lb": 8.1,
"loadgen-run-c0edec5e3a16": 14.7,
"postgres": 15.5,
"redis": 3.5
},
"mem_mib_max": {
"api": 55.8,
"lb": 10.8,
"loadgen-run-c0edec5e3a16": 23.5,
"postgres": 98.9,
"redis": 5.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 258.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.175,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1286
},
"error_ratio": 0.4019,
"errors": 1286,
"event": "step",
"label": "fixed-2x0.50",
"latency_ms": {
"max": 2542.69,
"p50": 1228.48,
"p90": 1925.6,
"p99": 2445.3,
"p999": 2534.12
},
"ok": 1914,
"per_op": {
"history": {
"ok": 1263,
"p50_ms": 1315.98,
"p99_ms": 2268.86
},
"me": {
"ok": 276,
"p50_ms": 50.38,
"p99_ms": 2227.43
},
"post": {
"ok": 375,
"p50_ms": 1328.73,
"p99_ms": 2531.62
}
},
"rate": 320,
"service_ms": {
"p50": 1227.62,
"p99": 2444.99
},
"throughput_ok_rps": 191.4,
"resources": {
"cpu_percent_mean": {
"api": 81.9,
"lb": 7.7,
"loadgen-run-c0edec5e3a16": 14.0,
"postgres": 14.2,
"redis": 3.5
},
"mem_mib_max": {
"api": 51.5,
"lb": 10.8,
"loadgen-run-c0edec5e3a16": 22.1,
"postgres": 100.5,
"redis": 4.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 229.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.228,
"pg_connections_mean": 21
},
"slo_ok": false
}
],
"pool_before": {
"api-2c6df209": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 137,
"requests_wait_ms": 133,
"usage_ms": 76,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-d9869430": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 204,
"requests_wait_ms": 204,
"usage_ms": 113,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
}
},
"pool_after": {
"api-2c6df209": {
"pool": {
"connections_num": 10,
"requests_num": 10526,
"requests_queued": 3822,
"connections_ms": 991,
"requests_wait_ms": 3557844,
"usage_ms": 282106,
"requests_errors": 2082,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 8994,
"misses": 23,
"errors": 46
},
"requests": 10702
},
"api-d9869430": {
"pool": {
"connections_num": 10,
"requests_num": 8580,
"requests_queued": 3170,
"connections_ms": 1170,
"requests_wait_ms": 2813537,
"usage_ms": 252813,
"requests_errors": 644,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 8349,
"misses": 26,
"errors": 10
},
"requests": 8982
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+816
View File
@@ -0,0 +1,816 @@
{
"config": "fixed-4x0.25-r2",
"repeat": 2,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 4,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"KDF_CONCURRENCY": 1
},
"measured_at": "2026-11-03T09:24:56+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 160,
"peak_ok_rps": 240.6,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 72.83,
"p50": 5.6,
"p90": 18.79,
"p99": 60.61,
"p999": 72.83
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.51,
"p99_ms": 60.38
},
"me": {
"ok": 40,
"p50_ms": 2.27,
"p99_ms": 54.64
},
"post": {
"ok": 80,
"p50_ms": 6.29,
"p99_ms": 72.83
}
},
"rate": 40,
"service_ms": {
"p50": 5.02,
"p99": 60.19
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 27.1,
"lb": 2.3,
"loadgen-run-36ba2530fef5": 2.6,
"postgres": 7.3,
"redis": 2.1
},
"mem_mib_max": {
"api": 43.6,
"lb": 4.4,
"loadgen-run-36ba2530fef5": 6.1,
"postgres": 146.4,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0.44,
"pg_wait_events_mean": {
"CPU": 0.44
},
"api_cfs_throttled_period_ratio": 0.133,
"pg_connections_mean": 21
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 91.42,
"p50": 5.15,
"p90": 15.24,
"p99": 81.45,
"p999": 91.42
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.1,
"p99_ms": 81.93
},
"me": {
"ok": 80,
"p50_ms": 2.48,
"p99_ms": 34.89
},
"post": {
"ok": 160,
"p50_ms": 6,
"p99_ms": 81.45
}
},
"rate": 80,
"service_ms": {
"p50": 4.53,
"p99": 80.46
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 37.3,
"lb": 3.2,
"loadgen-run-36ba2530fef5": 4.6,
"postgres": 9.1,
"redis": 2.8
},
"mem_mib_max": {
"api": 43.6,
"lb": 4.4,
"loadgen-run-36ba2530fef5": 6.3,
"postgres": 147.1,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.22
},
"api_cfs_throttled_period_ratio": 0.1,
"pg_connections_mean": 21
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 794.96,
"p50": 6.02,
"p90": 72,
"p99": 482.97,
"p999": 559.54
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.86,
"p99_ms": 482.97
},
"me": {
"ok": 120,
"p50_ms": 2.58,
"p99_ms": 71.7
},
"post": {
"ok": 240,
"p50_ms": 8.2,
"p99_ms": 511.91
}
},
"rate": 120,
"service_ms": {
"p50": 5.37,
"p99": 459.72
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 54.9,
"lb": 4.1,
"loadgen-run-36ba2530fef5": 6.4,
"postgres": 13.1,
"redis": 3.2
},
"mem_mib_max": {
"api": 44.5,
"lb": 4.9,
"loadgen-run-36ba2530fef5": 7.3,
"postgres": 184.2,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 2.8,
"pg_active_mean": 0.33,
"pg_wait_events_mean": {
"CPU": 0.33
},
"api_cfs_throttled_period_ratio": 0.137,
"pg_connections_mean": 27.8
},
"slo_ok": false
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 176.91,
"p50": 6,
"p90": 48.44,
"p99": 135.32,
"p999": 173.31
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 6.07,
"p99_ms": 129.04
},
"me": {
"ok": 160,
"p50_ms": 2.42,
"p99_ms": 72.19
},
"post": {
"ok": 320,
"p50_ms": 7.61,
"p99_ms": 168.93
}
},
"rate": 160,
"service_ms": {
"p50": 5.45,
"p99": 134.42
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 67.0,
"lb": 4.8,
"loadgen-run-36ba2530fef5": 7.7,
"postgres": 14.0,
"redis": 3.3
},
"mem_mib_max": {
"api": 44.9,
"lb": 4.9,
"loadgen-run-36ba2530fef5": 7.3,
"postgres": 186.5,
"redis": 5.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 3,
"pg_active_mean": 0.44,
"pg_wait_events_mean": {
"CPU": 0.44
},
"api_cfs_throttled_period_ratio": 0.177,
"pg_connections_mean": 37
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 1864.69,
"p50": 129.18,
"p90": 1046.29,
"p99": 1634.55,
"p999": 1752.71
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 152.99,
"p99_ms": 1634.55
},
"me": {
"ok": 200,
"p50_ms": 37.67,
"p99_ms": 1071
},
"post": {
"ok": 400,
"p50_ms": 212.9,
"p99_ms": 1637.95
}
},
"rate": 200,
"service_ms": {
"p50": 128.83,
"p99": 1633.77
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 78.8,
"lb": 5.7,
"loadgen-run-36ba2530fef5": 10.2,
"postgres": 17.8,
"redis": 3.6
},
"mem_mib_max": {
"api": 47.5,
"lb": 7.0,
"loadgen-run-36ba2530fef5": 14.0,
"postgres": 218.9,
"redis": 4.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 64.7,
"pg_active_mean": 0.56,
"pg_wait_events_mean": {
"CPU": 0.44,
"Client:ClientRead": 0.11
},
"api_cfs_throttled_period_ratio": 0.336,
"pg_connections_mean": 47.8
},
"slo_ok": false
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 93
},
"error_ratio": 0.0388,
"errors": 93,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 2432.27,
"p50": 665.72,
"p90": 1642.63,
"p99": 2389.58,
"p999": 2418.04
},
"ok": 2307,
"per_op": {
"history": {
"ok": 1606,
"p50_ms": 735.45,
"p99_ms": 2386.92
},
"me": {
"ok": 240,
"p50_ms": 85.07,
"p99_ms": 978.1
},
"post": {
"ok": 461,
"p50_ms": 788.2,
"p99_ms": 2403.19
}
},
"rate": 240,
"service_ms": {
"p50": 665.45,
"p99": 2389.27
},
"throughput_ok_rps": 230.7,
"resources": {
"cpu_percent_mean": {
"api": 84.0,
"lb": 6.2,
"loadgen-run-36ba2530fef5": 11.6,
"postgres": 18.2,
"redis": 3.4
},
"mem_mib_max": {
"api": 50.7,
"lb": 9.3,
"loadgen-run-36ba2530fef5": 19.0,
"postgres": 220.5,
"redis": 5.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 176.5,
"pg_active_mean": 0.7,
"pg_wait_events_mean": {
"CPU": 0.5,
"IO:WALSync": 0.1,
"LWLock:WALWrite": 0.1
},
"api_cfs_throttled_period_ratio": 0.609,
"pg_connections_mean": 49
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 394
},
"error_ratio": 0.1407,
"errors": 394,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 3147.58,
"p50": 1144.24,
"p90": 2259.16,
"p99": 3033.35,
"p999": 3142.23
},
"ok": 2406,
"per_op": {
"history": {
"ok": 1691,
"p50_ms": 1299.66,
"p99_ms": 2969.38
},
"me": {
"ok": 258,
"p50_ms": 98.8,
"p99_ms": 1279.44
},
"post": {
"ok": 457,
"p50_ms": 1247.21,
"p99_ms": 3114.75
}
},
"rate": 280,
"service_ms": {
"p50": 1143.61,
"p99": 3032.74
},
"throughput_ok_rps": 240.6,
"resources": {
"cpu_percent_mean": {
"api": 87.4,
"lb": 6.6,
"loadgen-run-36ba2530fef5": 12.8,
"postgres": 17.5,
"redis": 3.5
},
"mem_mib_max": {
"api": 52.6,
"lb": 11.9,
"loadgen-run-36ba2530fef5": 26.6,
"postgres": 221.1,
"redis": 4.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 302,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.525,
"pg_connections_mean": 49
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1333
},
"error_ratio": 0.4166,
"errors": 1333,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 3445.8,
"p50": 1646.59,
"p90": 2594.25,
"p99": 3379.85,
"p999": 3411.29
},
"ok": 1867,
"per_op": {
"history": {
"ok": 1284,
"p50_ms": 1767.93,
"p99_ms": 3346.81
},
"me": {
"ok": 230,
"p50_ms": 158.03,
"p99_ms": 2152.93
},
"post": {
"ok": 353,
"p50_ms": 1858.8,
"p99_ms": 3405.89
}
},
"rate": 320,
"service_ms": {
"p50": 1646.43,
"p99": 3379.49
},
"throughput_ok_rps": 186.7,
"resources": {
"cpu_percent_mean": {
"api": 88.0,
"lb": 7.6,
"loadgen-run-36ba2530fef5": 13.5,
"postgres": 16.4,
"redis": 3.2
},
"mem_mib_max": {
"api": 53.8,
"lb": 13.2,
"loadgen-run-36ba2530fef5": 31.9,
"postgres": 222.0,
"redis": 5.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 441.3,
"pg_active_mean": 0.7,
"pg_wait_events_mean": {
"CPU": 0.3,
"Client:ClientRead": 0.4
},
"api_cfs_throttled_period_ratio": 0.424,
"pg_connections_mean": 49
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1580
},
"error_ratio": 0.4389,
"errors": 1580,
"event": "step",
"label": "fixed-4x0.25-r2",
"latency_ms": {
"max": 2770.47,
"p50": 1496,
"p90": 2242.21,
"p99": 2507.19,
"p999": 2700.51
},
"ok": 2020,
"per_op": {
"history": {
"ok": 1369,
"p50_ms": 1699.72,
"p99_ms": 2460.58
},
"me": {
"ok": 299,
"p50_ms": 121.51,
"p99_ms": 343.24
},
"post": {
"ok": 352,
"p50_ms": 1567.6,
"p99_ms": 2692.62
}
},
"rate": 360,
"service_ms": {
"p50": 1495.47,
"p99": 2506.19
},
"throughput_ok_rps": 202,
"resources": {
"cpu_percent_mean": {
"api": 85.3,
"lb": 8.3,
"loadgen-run-36ba2530fef5": 13.8,
"postgres": 15.7,
"redis": 3.7
},
"mem_mib_max": {
"api": 56.5,
"lb": 13.4,
"loadgen-run-36ba2530fef5": 36.4,
"postgres": 222.2,
"redis": 5.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 415.3,
"pg_active_mean": 0.5,
"pg_wait_events_mean": {
"CPU": 0.3,
"Client:ClientRead": 0.1,
"IO:WALSync": 0.1
},
"api_cfs_throttled_period_ratio": 0.585,
"pg_connections_mean": 48.1
},
"slo_ok": false
}
],
"pool_before": {
"api-1d61817b": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 162,
"requests_wait_ms": 193,
"usage_ms": 67,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 2,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 8
},
"api-a25272a7": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 240,
"requests_wait_ms": 263,
"usage_ms": 183,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-d4ce142e": {
"pool": {
"connections_num": 3,
"requests_num": 22,
"requests_queued": 3,
"connections_ms": 213,
"requests_wait_ms": 147,
"usage_ms": 193,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
},
"api-fba3a55f": {
"pool": {
"connections_num": 3,
"requests_num": 26,
"requests_queued": 4,
"connections_ms": 294,
"requests_wait_ms": 225,
"usage_ms": 185,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 13
}
},
"pool_after": {
"api-1d61817b": {
"pool": {
"connections_num": 10,
"requests_num": 6436,
"requests_queued": 2845,
"connections_ms": 1381,
"requests_wait_ms": 3326559,
"usage_ms": 343623,
"requests_errors": 1715,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 5268,
"misses": 12,
"errors": 23
},
"requests": 6630
},
"api-a25272a7": {
"pool": {
"connections_num": 10,
"requests_num": 6380,
"requests_queued": 3327,
"connections_ms": 1579,
"requests_wait_ms": 3887774,
"usage_ms": 376655,
"requests_errors": 943,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 5519,
"misses": 13,
"errors": 21
},
"requests": 6337
},
"api-d4ce142e": {
"pool": {
"connections_num": 10,
"requests_num": 5885,
"requests_queued": 3013,
"connections_ms": 1348,
"requests_wait_ms": 3443239,
"usage_ms": 363291,
"requests_errors": 481,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 9,
"requests_waiting": 0
},
"cache": {
"hits": 5572,
"misses": 14,
"errors": 9
},
"requests": 6030
},
"api-fba3a55f": {
"pool": {
"connections_num": 10,
"requests_num": 5848,
"requests_queued": 3089,
"connections_ms": 1764,
"requests_wait_ms": 3174240,
"usage_ms": 357547,
"requests_errors": 454,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 9,
"requests_waiting": 0
},
"cache": {
"hits": 4883,
"misses": 13,
"errors": 30
},
"requests": 5812
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+734
View File
@@ -0,0 +1,734 @@
{
"config": "fixed-4x0.25-r3",
"repeat": 3,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 4,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"KDF_CONCURRENCY": 1
},
"measured_at": "2026-11-03T09:37:46+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 120,
"peak_ok_rps": 200,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r3",
"latency_ms": {
"max": 88.29,
"p50": 5.38,
"p90": 13.59,
"p99": 82.74,
"p999": 88.29
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.33,
"p99_ms": 82.74
},
"me": {
"ok": 40,
"p50_ms": 2.42,
"p99_ms": 11.79
},
"post": {
"ok": 80,
"p50_ms": 5.93,
"p99_ms": 85.85
}
},
"rate": 40,
"service_ms": {
"p50": 4.73,
"p99": 82.16
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 25.6,
"lb": 2.1,
"loadgen-run-89815b9f6838": 2.5,
"postgres": 7.7,
"redis": 2.4
},
"mem_mib_max": {
"api": 40.4,
"lb": 4.4,
"loadgen-run-89815b9f6838": 6.1,
"postgres": 175.2,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.072,
"pg_connections_mean": 25
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r3",
"latency_ms": {
"max": 92.03,
"p50": 5.28,
"p90": 22.78,
"p99": 61.16,
"p999": 92.03
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.18,
"p99_ms": 61.94
},
"me": {
"ok": 80,
"p50_ms": 2.45,
"p99_ms": 54.32
},
"post": {
"ok": 160,
"p50_ms": 6.25,
"p99_ms": 71.44
}
},
"rate": 80,
"service_ms": {
"p50": 4.65,
"p99": 60.42
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 34.9,
"lb": 3.2,
"loadgen-run-89815b9f6838": 4.7,
"postgres": 9.9,
"redis": 2.6
},
"mem_mib_max": {
"api": 43.7,
"lb": 4.4,
"loadgen-run-89815b9f6838": 6.3,
"postgres": 175.7,
"redis": 5.3
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.9,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.094,
"pg_connections_mean": 25
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r3",
"latency_ms": {
"max": 134.47,
"p50": 5.47,
"p90": 37.31,
"p99": 86.52,
"p999": 119.12
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.33,
"p99_ms": 87.77
},
"me": {
"ok": 120,
"p50_ms": 2.73,
"p99_ms": 52.33
},
"post": {
"ok": 240,
"p50_ms": 6.41,
"p99_ms": 95.71
}
},
"rate": 120,
"service_ms": {
"p50": 4.86,
"p99": 85.73
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 49.3,
"lb": 4.0,
"loadgen-run-89815b9f6838": 6.2,
"postgres": 12.2,
"redis": 3.1
},
"mem_mib_max": {
"api": 44.1,
"lb": 4.5,
"loadgen-run-89815b9f6838": 6.6,
"postgres": 178.8,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.8,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.086,
"pg_connections_mean": 25.8
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r3",
"latency_ms": {
"max": 664.67,
"p50": 7.67,
"p90": 234.63,
"p99": 471.46,
"p999": 654.99
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 8.22,
"p99_ms": 481.87
},
"me": {
"ok": 160,
"p50_ms": 2.73,
"p99_ms": 154.45
},
"post": {
"ok": 320,
"p50_ms": 8.26,
"p99_ms": 470.13
}
},
"rate": 160,
"service_ms": {
"p50": 7.1,
"p99": 471.35
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 67.1,
"lb": 4.7,
"loadgen-run-89815b9f6838": 7.8,
"postgres": 17.6,
"redis": 3.6
},
"mem_mib_max": {
"api": 44.9,
"lb": 5.1,
"loadgen-run-89815b9f6838": 7.7,
"postgres": 239.6,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 11.5,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.044,
"pg_connections_mean": 42.7
},
"slo_ok": false
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25-r3",
"latency_ms": {
"max": 2946.08,
"p50": 426.45,
"p90": 1381.86,
"p99": 1855.88,
"p999": 2676.84
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 469.38,
"p99_ms": 1789.43
},
"me": {
"ok": 200,
"p50_ms": 72.89,
"p99_ms": 448.11
},
"post": {
"ok": 400,
"p50_ms": 531.44,
"p99_ms": 2000.46
}
},
"rate": 200,
"service_ms": {
"p50": 425.95,
"p99": 1855.08
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 74.6,
"lb": 5.6,
"loadgen-run-89815b9f6838": 10.3,
"postgres": 17.5,
"redis": 3.4
},
"mem_mib_max": {
"api": 50.4,
"lb": 8.2,
"loadgen-run-89815b9f6838": 15.2,
"postgres": 250.8,
"redis": 4.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 103.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.404,
"pg_connections_mean": 52.2
},
"slo_ok": false
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 627
},
"error_ratio": 0.2613,
"errors": 627,
"event": "step",
"label": "fixed-4x0.25-r3",
"latency_ms": {
"max": 3109.98,
"p50": 614.79,
"p90": 2033.95,
"p99": 2760.99,
"p999": 3106.01
},
"ok": 1773,
"per_op": {
"history": {
"ok": 1223,
"p50_ms": 670.52,
"p99_ms": 2652.06
},
"me": {
"ok": 195,
"p50_ms": 88.16,
"p99_ms": 2495.23
},
"post": {
"ok": 355,
"p50_ms": 870.35,
"p99_ms": 2933.06
}
},
"rate": 240,
"service_ms": {
"p50": 610.97,
"p99": 2760.1
},
"throughput_ok_rps": 177.3,
"resources": {
"cpu_percent_mean": {
"api": 78.9,
"lb": 6.4,
"loadgen-run-89815b9f6838": 11.9,
"postgres": 16.7,
"redis": 3.2
},
"mem_mib_max": {
"api": 53.9,
"lb": 9.9,
"loadgen-run-89815b9f6838": 21.0,
"postgres": 250.4,
"redis": 5.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 209.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.384,
"pg_connections_mean": 53
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1102
},
"error_ratio": 0.3936,
"errors": 1102,
"event": "step",
"label": "fixed-4x0.25-r3",
"latency_ms": {
"max": 4334.56,
"p50": 1383.24,
"p90": 3095.47,
"p99": 3639.35,
"p999": 4302.63
},
"ok": 1698,
"per_op": {
"history": {
"ok": 1112,
"p50_ms": 1465.17,
"p99_ms": 3611.04
},
"me": {
"ok": 250,
"p50_ms": 120.89,
"p99_ms": 1839.54
},
"post": {
"ok": 336,
"p50_ms": 1480.37,
"p99_ms": 3671.15
}
},
"rate": 280,
"service_ms": {
"p50": 1382.39,
"p99": 3623.89
},
"throughput_ok_rps": 169.8,
"resources": {
"cpu_percent_mean": {
"api": 83.5,
"lb": 7.0,
"loadgen-run-89815b9f6838": 13.0,
"postgres": 17.5,
"redis": 4.0
},
"mem_mib_max": {
"api": 53.7,
"lb": 12.2,
"loadgen-run-89815b9f6838": 27.2,
"postgres": 251.7,
"redis": 4.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 349.4,
"pg_active_mean": 0.5,
"pg_wait_events_mean": {
"Client:ClientRead": 0.5
},
"api_cfs_throttled_period_ratio": 0.334,
"pg_connections_mean": 53
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1469
},
"error_ratio": 0.4591,
"errors": 1469,
"event": "step",
"label": "fixed-4x0.25-r3",
"latency_ms": {
"max": 4102.38,
"p50": 1496.75,
"p90": 3242.68,
"p99": 3974.51,
"p999": 4099.38
},
"ok": 1731,
"per_op": {
"history": {
"ok": 1145,
"p50_ms": 1645.44,
"p99_ms": 3926.88
},
"me": {
"ok": 255,
"p50_ms": 115.33,
"p99_ms": 2338.79
},
"post": {
"ok": 331,
"p50_ms": 1609.47,
"p99_ms": 3998.78
}
},
"rate": 320,
"service_ms": {
"p50": 1496.47,
"p99": 3973.21
},
"throughput_ok_rps": 173.1,
"resources": {
"cpu_percent_mean": {
"api": 78.7,
"lb": 8.0,
"loadgen-run-89815b9f6838": 13.2,
"postgres": 16.7,
"redis": 3.0
},
"mem_mib_max": {
"api": 58.6,
"lb": 13.2,
"loadgen-run-89815b9f6838": 30.5,
"postgres": 252.1,
"redis": 6.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 413.7,
"pg_active_mean": 0.3,
"pg_wait_events_mean": {
"Client:ClientRead": 0.3
},
"api_cfs_throttled_period_ratio": 0.375,
"pg_connections_mean": 53
},
"slo_ok": false
}
],
"pool_before": {
"api-cbf60d4e": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 212,
"requests_wait_ms": 203,
"usage_ms": 114,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 2,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-aa2feb60": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 263,
"requests_wait_ms": 285,
"usage_ms": 122,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 2,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-75fe1536": {
"pool": {
"connections_num": 3,
"requests_num": 22,
"requests_queued": 3,
"connections_ms": 218,
"requests_wait_ms": 290,
"usage_ms": 59,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
},
"api-95ddae14": {
"pool": {
"connections_num": 3,
"requests_num": 26,
"requests_queued": 3,
"connections_ms": 221,
"requests_wait_ms": 280,
"usage_ms": 90,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 13
}
},
"pool_after": {
"api-cbf60d4e": {
"pool": {
"connections_num": 10,
"requests_num": 5685,
"requests_queued": 2640,
"connections_ms": 1849,
"requests_wait_ms": 3373255,
"usage_ms": 293816,
"requests_errors": 1303,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 3738,
"misses": 16,
"errors": 17
},
"requests": 5343
},
"api-aa2feb60": {
"pool": {
"connections_num": 10,
"requests_num": 4088,
"requests_queued": 1775,
"connections_ms": 1616,
"requests_wait_ms": 1525236,
"usage_ms": 262582,
"requests_errors": 98,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 3933,
"misses": 22,
"errors": 2
},
"requests": 4204
},
"api-75fe1536": {
"pool": {
"connections_num": 10,
"requests_num": 4030,
"requests_queued": 1772,
"connections_ms": 1588,
"requests_wait_ms": 1549683,
"usage_ms": 263095,
"requests_errors": 128,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 3969,
"misses": 23,
"errors": 0
},
"requests": 4203
},
"api-95ddae14": {
"pool": {
"connections_num": 10,
"requests_num": 6393,
"requests_queued": 3048,
"connections_ms": 1829,
"requests_wait_ms": 3953309,
"usage_ms": 339101,
"requests_errors": 2079,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 4746,
"misses": 22,
"errors": 47
},
"requests": 6327
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+666
View File
@@ -0,0 +1,666 @@
{
"config": "fixed-4x0.25",
"repeat": 1,
"series": "fixed-cpu",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 4,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"KDF_CONCURRENCY": 1
},
"measured_at": "2026-11-03T08:33:59+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 160,
"peak_ok_rps": 194.4,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25",
"latency_ms": {
"max": 81.32,
"p50": 5.54,
"p90": 19.21,
"p99": 46.5,
"p999": 81.32
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.42,
"p99_ms": 46.5
},
"me": {
"ok": 40,
"p50_ms": 2.74,
"p99_ms": 37.64
},
"post": {
"ok": 80,
"p50_ms": 6.31,
"p99_ms": 81.32
}
},
"rate": 40,
"service_ms": {
"p50": 4.91,
"p99": 46.2
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 27.9,
"lb": 2.2,
"loadgen-run-12d799a1eccf": 2.7,
"postgres": 5.4,
"redis": 2.6
},
"mem_mib_max": {
"api": 41.9,
"lb": 4.4,
"loadgen-run-12d799a1eccf": 6.1,
"postgres": 86.1,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.147,
"pg_connections_mean": 15
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25",
"latency_ms": {
"max": 96.74,
"p50": 5.83,
"p90": 34.28,
"p99": 66.27,
"p999": 96.74
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.68,
"p99_ms": 72
},
"me": {
"ok": 80,
"p50_ms": 2.76,
"p99_ms": 33.16
},
"post": {
"ok": 160,
"p50_ms": 7.01,
"p99_ms": 73.64
}
},
"rate": 80,
"service_ms": {
"p50": 5.2,
"p99": 66.15
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 45.0,
"lb": 3.4,
"loadgen-run-12d799a1eccf": 5.2,
"postgres": 8.2,
"redis": 2.9
},
"mem_mib_max": {
"api": 43.9,
"lb": 4.4,
"loadgen-run-12d799a1eccf": 6.4,
"postgres": 86.5,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.9,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.047,
"pg_connections_mean": 15
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25",
"latency_ms": {
"max": 94.79,
"p50": 5.44,
"p90": 19.88,
"p99": 61.91,
"p999": 89.89
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.37,
"p99_ms": 53.57
},
"me": {
"ok": 120,
"p50_ms": 2.47,
"p99_ms": 29.03
},
"post": {
"ok": 240,
"p50_ms": 6.46,
"p99_ms": 88.89
}
},
"rate": 120,
"service_ms": {
"p50": 4.83,
"p99": 60.94
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 52.8,
"lb": 4.5,
"loadgen-run-12d799a1eccf": 6.9,
"postgres": 10.1,
"redis": 3.4
},
"mem_mib_max": {
"api": 44.2,
"lb": 4.5,
"loadgen-run-12d799a1eccf": 6.9,
"postgres": 87.2,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.6,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.22
},
"api_cfs_throttled_period_ratio": 0.08,
"pg_connections_mean": 15
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "fixed-4x0.25",
"latency_ms": {
"max": 230.21,
"p50": 7.32,
"p90": 66.78,
"p99": 183.46,
"p999": 226.59
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 7.43,
"p99_ms": 182.27
},
"me": {
"ok": 160,
"p50_ms": 2.71,
"p99_ms": 61.09
},
"post": {
"ok": 320,
"p50_ms": 7.79,
"p99_ms": 191.99
}
},
"rate": 160,
"service_ms": {
"p50": 6.7,
"p99": 181.31
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 69.2,
"lb": 5.4,
"loadgen-run-12d799a1eccf": 8.6,
"postgres": 17.0,
"redis": 3.8
},
"mem_mib_max": {
"api": 40.7,
"lb": 5.0,
"loadgen-run-12d799a1eccf": 7.3,
"postgres": 140.7,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 2.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.014,
"pg_connections_mean": 38
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 222
},
"error_ratio": 0.111,
"errors": 222,
"event": "step",
"label": "fixed-4x0.25",
"latency_ms": {
"max": 3740.56,
"p50": 77.46,
"p90": 2713.32,
"p99": 3646.61,
"p999": 3739.67
},
"ok": 1778,
"per_op": {
"history": {
"ok": 1255,
"p50_ms": 81.51,
"p99_ms": 3673.79
},
"me": {
"ok": 171,
"p50_ms": 33.14,
"p99_ms": 1846.93
},
"post": {
"ok": 352,
"p50_ms": 104.81,
"p99_ms": 3504.62
}
},
"rate": 200,
"service_ms": {
"p50": 76.72,
"p99": 3646
},
"throughput_ok_rps": 177.8,
"resources": {
"cpu_percent_mean": {
"api": 74.2,
"lb": 6.6,
"loadgen-run-12d799a1eccf": 11.4,
"postgres": 16.2,
"redis": 4.0
},
"mem_mib_max": {
"api": 46.6,
"lb": 8.0,
"loadgen-run-12d799a1eccf": 16.0,
"postgres": 152.5,
"redis": 4.3
},
"lb_queue_max": 0,
"lb_sessions_mean": 100.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.244,
"pg_connections_mean": 42
},
"slo_ok": false
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 797
},
"error_ratio": 0.3321,
"errors": 797,
"event": "step",
"label": "fixed-4x0.25",
"latency_ms": {
"max": 3706.47,
"p50": 574.82,
"p90": 2602.1,
"p99": 3632.31,
"p999": 3691.57
},
"ok": 1603,
"per_op": {
"history": {
"ok": 1093,
"p50_ms": 622.32,
"p99_ms": 3645.45
},
"me": {
"ok": 177,
"p50_ms": 89.5,
"p99_ms": 2342.5
},
"post": {
"ok": 333,
"p50_ms": 634.35,
"p99_ms": 3584.81
}
},
"rate": 240,
"service_ms": {
"p50": 573.6,
"p99": 3631.91
},
"throughput_ok_rps": 160.3,
"resources": {
"cpu_percent_mean": {
"api": 79.7,
"lb": 7.6,
"loadgen-run-12d799a1eccf": 14.0,
"postgres": 16.9,
"redis": 3.5
},
"mem_mib_max": {
"api": 53.3,
"lb": 10.4,
"loadgen-run-12d799a1eccf": 19.0,
"postgres": 153.7,
"redis": 4.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 201.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.264,
"pg_connections_mean": 42
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 856
},
"error_ratio": 0.3057,
"errors": 856,
"event": "step",
"label": "fixed-4x0.25",
"latency_ms": {
"max": 4265.81,
"p50": 1659.46,
"p90": 3147.39,
"p99": 3983.01,
"p999": 4258.79
},
"ok": 1944,
"per_op": {
"history": {
"ok": 1347,
"p50_ms": 1801.24,
"p99_ms": 4114.07
},
"me": {
"ok": 226,
"p50_ms": 173.87,
"p99_ms": 2076.44
},
"post": {
"ok": 371,
"p50_ms": 1726.89,
"p99_ms": 3875.73
}
},
"rate": 280,
"service_ms": {
"p50": 1658.8,
"p99": 3982.24
},
"throughput_ok_rps": 194.4,
"resources": {
"cpu_percent_mean": {
"api": 86.2,
"lb": 8.6,
"loadgen-run-12d799a1eccf": 15.2,
"postgres": 18.2,
"redis": 4.1
},
"mem_mib_max": {
"api": 54.4,
"lb": 13.7,
"loadgen-run-12d799a1eccf": 29.6,
"postgres": 154.7,
"redis": 5.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 367.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.358,
"pg_connections_mean": 42
},
"slo_ok": false
}
],
"pool_before": {
"api-9fb88c7c": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 267,
"requests_wait_ms": 264,
"usage_ms": 112,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-ed246c7a": {
"pool": {
"connections_num": 3,
"requests_num": 23,
"requests_queued": 3,
"connections_ms": 230,
"requests_wait_ms": 278,
"usage_ms": 87,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 11
},
"api-d4560f02": {
"pool": {
"connections_num": 3,
"requests_num": 25,
"requests_queued": 3,
"connections_ms": 215,
"requests_wait_ms": 144,
"usage_ms": 196,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 12
},
"api-06619766": {
"pool": {
"connections_num": 3,
"requests_num": 28,
"requests_queued": 3,
"connections_ms": 301,
"requests_wait_ms": 313,
"usage_ms": 229,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 14
}
},
"pool_after": {
"api-9fb88c7c": {
"pool": {
"connections_num": 10,
"requests_num": 3407,
"requests_queued": 1188,
"connections_ms": 1921,
"requests_wait_ms": 844090,
"usage_ms": 188476,
"requests_errors": 139,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 2807,
"misses": 10,
"errors": 17
},
"requests": 3298
},
"api-ed246c7a": {
"pool": {
"connections_num": 10,
"requests_num": 3754,
"requests_queued": 1329,
"connections_ms": 1879,
"requests_wait_ms": 1059850,
"usage_ms": 208745,
"requests_errors": 232,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 3128,
"misses": 12,
"errors": 34
},
"requests": 3700
},
"api-d4560f02": {
"pool": {
"connections_num": 10,
"requests_num": 4050,
"requests_queued": 1639,
"connections_ms": 1709,
"requests_wait_ms": 1645005,
"usage_ms": 253579,
"requests_errors": 327,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 3828,
"misses": 12,
"errors": 5
},
"requests": 4179
},
"api-06619766": {
"pool": {
"connections_num": 10,
"requests_num": 5175,
"requests_queued": 2168,
"connections_ms": 1886,
"requests_wait_ms": 2863319,
"usage_ms": 250661,
"requests_errors": 1187,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 2883,
"misses": 15,
"errors": 20
},
"requests": 4713
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+61
View File
@@ -0,0 +1,61 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 400" font-family="system-ui,-apple-system,Segoe UI,sans-serif">
<rect width="100%" height="100%" fill="#fcfcfb"/>
<text x="64" y="26" font-size="16" font-weight="600" fill="#0b0b0b">Фиксированный бюджет API (1 CPU, 512 MiB): дробление на реплики</text>
<text x="64" y="46" font-size="12" fill="#52514e">Полезная пропускная способность (2xx), медиана 3 прогонов; open-loop, смесь 70/20/10</text>
<line x1="64" x2="570" y1="348.0" y2="348.0" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="352.0" font-size="11" fill="#52514e" text-anchor="end">0</text>
<line x1="64" x2="570" y1="257.4" y2="257.4" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="261.4" font-size="11" fill="#52514e" text-anchor="end">100</text>
<line x1="64" x2="570" y1="166.9" y2="166.9" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="170.9" font-size="11" fill="#52514e" text-anchor="end">200</text>
<line x1="64" x2="570" y1="76.3" y2="76.3" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="80.3" font-size="11" fill="#52514e" text-anchor="end">300</text>
<text x="64.0" y="366" font-size="11" fill="#52514e" text-anchor="middle">0</text>
<text x="168.8" y="366" font-size="11" fill="#52514e" text-anchor="middle">100</text>
<text x="273.5" y="366" font-size="11" fill="#52514e" text-anchor="middle">200</text>
<text x="378.3" y="366" font-size="11" fill="#52514e" text-anchor="middle">300</text>
<text x="483.0" y="366" font-size="11" fill="#52514e" text-anchor="middle">400</text>
<line x1="64" x2="570" y1="348" y2="348" stroke="#52514e" stroke-width="1"/>
<text x="317" y="388" font-size="12" fill="#52514e" text-anchor="middle">Поданная нагрузка, req/s</text>
<text x="16" y="206" font-size="12" fill="#52514e" transform="rotate(-90 16 206)" text-anchor="middle">Полезная, rps</text>
<line x1="64.0" y1="348.0" x2="392.5" y2="64.0" stroke="#52514e" stroke-width="1" stroke-dasharray="4 4"/>
<text x="358.3" y="86.7" font-size="11" fill="#52514e" text-anchor="end">пунктир: полезная = поданная</text>
<polyline fill="none" stroke="#2a78d6" stroke-width="2" stroke-linejoin="round" points="105.9,311.8 147.8,275.6 189.7,239.3 231.6,203.1 273.5,166.9 315.4,130.7 357.3,94.4 399.2,98.1 441.1,96.8 483.0,119.2 545.9,146.2"/>
<circle cx="105.9" cy="311.8" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 28 мс (медиана 3 прогонов)</title></circle>
<circle cx="147.8" cy="275.6" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 27 мс (медиана 3 прогонов)</title></circle>
<circle cx="189.7" cy="239.3" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов)</title></circle>
<circle cx="231.6" cy="203.1" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 56 мс (медиана 3 прогонов)</title></circle>
<circle cx="273.5" cy="166.9" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 106 мс (медиана 3 прогонов)</title></circle>
<circle cx="315.4" cy="130.7" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 218 мс (медиана 3 прогонов)</title></circle>
<circle cx="357.3" cy="94.4" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 322 мс (медиана 3 прогонов)</title></circle>
<circle cx="399.2" cy="98.1" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 320 req/s: полезная 276 rps, ошибок 13.8 %, p99 1518 мс (медиана 3 прогонов)</title></circle>
<circle cx="441.1" cy="96.8" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 360 req/s: полезная 277 rps, ошибок 22.9 %, p99 1671 мс (медиана 3 прогонов)</title></circle>
<circle cx="483.0" cy="119.2" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 400 req/s: полезная 253 rps, ошибок 36.8 %, p99 1340 мс (медиана 3 прогонов)</title></circle>
<circle cx="545.9" cy="146.2" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 460 req/s: полезная 223 rps, ошибок 51.6 %, p99 1655 мс (медиана 3 прогонов)</title></circle>
<polyline fill="none" stroke="#eb6834" stroke-width="2" stroke-linejoin="round" points="105.9,311.8 147.8,275.6 189.7,239.3 231.6,203.1 273.5,166.9 315.4,135.5 357.3,146.1 399.2,174.7 441.1,182.5"/>
<rect x="101.4" y="307.3" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов)</title></rect>
<rect x="143.3" y="271.1" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 40 мс (медиана 3 прогонов)</title></rect>
<rect x="185.2" y="234.8" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 112 мс (медиана 3 прогонов)</title></rect>
<rect x="227.1" y="198.6" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 252 мс (медиана 3 прогонов)</title></rect>
<rect x="269.0" y="162.4" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 232 мс (медиана 3 прогонов)</title></rect>
<rect x="310.9" y="131.0" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 240 req/s: полезная 235 rps, ошибок 2.2 %, p99 1393 мс (медиана 3 прогонов)</title></rect>
<rect x="352.8" y="141.6" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 280 req/s: полезная 223 rps, ошибок 20.4 %, p99 1925 мс (медиана 3 прогонов)</title></rect>
<rect x="394.7" y="170.2" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 320 req/s: полезная 191 rps, ошибок 40.2 %, p99 2464 мс (медиана 3 прогонов)</title></rect>
<rect x="436.6" y="178.0" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 360 req/s: полезная 183 rps, ошибок 49.3 %, p99 2468 мс (медиана 2 прогонов)</title></rect>
<polyline fill="none" stroke="#1baf7a" stroke-width="2" stroke-linejoin="round" points="105.9,311.8 147.8,275.6 189.7,239.3 231.6,203.1 273.5,166.9 315.4,187.4 357.3,171.9 399.2,185.1 441.1,165.1"/>
<path d="M105.9 306.3L111.4 316.3L100.4 316.3Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 61 мс (медиана 3 прогонов)</title></path>
<path d="M147.8 270.1L153.3 280.1L142.3 280.1Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 66 мс (медиана 3 прогонов)</title></path>
<path d="M189.7 233.8L195.2 243.8L184.2 243.8Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 87 мс (медиана 3 прогонов)</title></path>
<path d="M231.6 197.6L237.1 207.6L226.1 207.6Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 183 мс (медиана 3 прогонов)</title></path>
<path d="M273.5 161.4L279.0 171.4L268.0 171.4Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 1856 мс (медиана 3 прогонов)</title></path>
<path d="M315.4 181.9L320.9 191.9L309.9 191.9Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 240 req/s: полезная 177 rps, ошибок 26.1 %, p99 2761 мс (медиана 3 прогонов)</title></path>
<path d="M357.3 166.4L362.8 176.4L351.8 176.4Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 280 req/s: полезная 194 rps, ошибок 30.6 %, p99 3639 мс (медиана 3 прогонов)</title></path>
<path d="M399.2 179.6L404.7 189.6L393.7 189.6Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 320 req/s: полезная 180 rps, ошибок 43.8 %, p99 3677 мс (медиана 2 прогонов)</title></path>
<path d="M441.1 159.6L446.6 169.6L435.6 169.6Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 360 req/s: полезная 202 rps, ошибок 43.9 %, p99 2507 мс (медиана 1 прогонов)</title></path>
<circle cx="586.0" cy="68.0" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU</title></circle>
<text x="596" y="72.0" font-size="11" fill="#52514e">1 × 1.0 CPU</text>
<rect x="581.5" y="81.5" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU</title></rect>
<text x="596" y="90.0" font-size="11" fill="#52514e">2 × 0.5 CPU</text>
<path d="M586.0 98.5L591.5 108.5L580.5 108.5Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU</title></path>
<text x="596" y="108.0" font-size="11" fill="#52514e">4 × 0.25 CPU</text>
</svg>

After

Width:  |  Height:  |  Size: 10 KiB

+65
View File
@@ -0,0 +1,65 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 400" font-family="system-ui,-apple-system,Segoe UI,sans-serif">
<rect width="100%" height="100%" fill="#fcfcfb"/>
<text x="64" y="26" font-size="16" font-weight="600" fill="#0b0b0b">Фиксированный бюджет API: хвост задержки</text>
<text x="64" y="46" font-size="12" fill="#52514e">p99 успешных ответов от запланированного момента, медиана 3 прогонов</text>
<line x1="64" x2="570" y1="348.0" y2="348.0" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="352.0" font-size="11" fill="#52514e" text-anchor="end">0</text>
<line x1="64" x2="570" y1="293.4" y2="293.4" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="297.4" font-size="11" fill="#52514e" text-anchor="end">500</text>
<line x1="64" x2="570" y1="238.8" y2="238.8" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="242.8" font-size="11" fill="#52514e" text-anchor="end">1000</text>
<line x1="64" x2="570" y1="184.2" y2="184.2" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="188.2" font-size="11" fill="#52514e" text-anchor="end">1500</text>
<line x1="64" x2="570" y1="129.5" y2="129.5" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="133.5" font-size="11" fill="#52514e" text-anchor="end">2000</text>
<line x1="64" x2="570" y1="74.9" y2="74.9" stroke="#e4e3df" stroke-width="1"/>
<text x="56" y="78.9" font-size="11" fill="#52514e" text-anchor="end">2500</text>
<text x="64.0" y="366" font-size="11" fill="#52514e" text-anchor="middle">0</text>
<text x="168.8" y="366" font-size="11" fill="#52514e" text-anchor="middle">100</text>
<text x="273.5" y="366" font-size="11" fill="#52514e" text-anchor="middle">200</text>
<text x="378.3" y="366" font-size="11" fill="#52514e" text-anchor="middle">300</text>
<text x="483.0" y="366" font-size="11" fill="#52514e" text-anchor="middle">400</text>
<line x1="64" x2="570" y1="348" y2="348" stroke="#52514e" stroke-width="1"/>
<text x="317" y="388" font-size="12" fill="#52514e" text-anchor="middle">Поданная нагрузка, req/s</text>
<text x="16" y="206" font-size="12" fill="#52514e" transform="rotate(-90 16 206)" text-anchor="middle">p99, мс (обрезано на 2500)</text>
<line x1="64" x2="570" y1="320.7" y2="320.7" stroke="#52514e" stroke-width="1" stroke-dasharray="4 4"/>
<text x="566" y="315.7" font-size="11" fill="#52514e" text-anchor="end">SLO 250 мс</text>
<polyline fill="none" stroke="#2a78d6" stroke-width="2" stroke-linejoin="round" points="105.9,344.9 147.8,345.1 189.7,343.8 231.6,341.8 273.5,336.4 315.4,324.2 357.3,312.8 399.2,182.2 441.1,165.5 483.0,201.6 545.9,167.2"/>
<circle cx="105.9" cy="344.9" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 28 мс (медиана 3 прогонов)</title></circle>
<circle cx="147.8" cy="345.1" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 27 мс (медиана 3 прогонов)</title></circle>
<circle cx="189.7" cy="343.8" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов)</title></circle>
<circle cx="231.6" cy="341.8" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 56 мс (медиана 3 прогонов)</title></circle>
<circle cx="273.5" cy="336.4" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 106 мс (медиана 3 прогонов)</title></circle>
<circle cx="315.4" cy="324.2" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 218 мс (медиана 3 прогонов)</title></circle>
<circle cx="357.3" cy="312.8" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 322 мс (медиана 3 прогонов)</title></circle>
<circle cx="399.2" cy="182.2" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 320 req/s: полезная 276 rps, ошибок 13.8 %, p99 1518 мс (медиана 3 прогонов)</title></circle>
<circle cx="441.1" cy="165.5" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 360 req/s: полезная 277 rps, ошибок 22.9 %, p99 1671 мс (медиана 3 прогонов)</title></circle>
<circle cx="483.0" cy="201.6" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 400 req/s: полезная 253 rps, ошибок 36.8 %, p99 1340 мс (медиана 3 прогонов)</title></circle>
<circle cx="545.9" cy="167.2" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU — 460 req/s: полезная 223 rps, ошибок 51.6 %, p99 1655 мс (медиана 3 прогонов)</title></circle>
<polyline fill="none" stroke="#eb6834" stroke-width="2" stroke-linejoin="round" points="105.9,343.8 147.8,343.6 189.7,335.7 231.6,320.4 273.5,322.7 315.4,195.9 357.3,137.7 399.2,78.9 441.1,78.5"/>
<rect x="101.4" y="339.3" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов)</title></rect>
<rect x="143.3" y="339.1" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 40 мс (медиана 3 прогонов)</title></rect>
<rect x="185.2" y="331.2" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 112 мс (медиана 3 прогонов)</title></rect>
<rect x="227.1" y="315.9" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 252 мс (медиана 3 прогонов)</title></rect>
<rect x="269.0" y="318.2" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 232 мс (медиана 3 прогонов)</title></rect>
<rect x="310.9" y="191.4" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 240 req/s: полезная 235 rps, ошибок 2.2 %, p99 1393 мс (медиана 3 прогонов)</title></rect>
<rect x="352.8" y="133.2" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 280 req/s: полезная 223 rps, ошибок 20.4 %, p99 1925 мс (медиана 3 прогонов)</title></rect>
<rect x="394.7" y="74.4" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 320 req/s: полезная 191 rps, ошибок 40.2 %, p99 2464 мс (медиана 3 прогонов)</title></rect>
<rect x="436.6" y="74.0" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU — 360 req/s: полезная 183 rps, ошибок 49.3 %, p99 2468 мс (медиана 2 прогонов)</title></rect>
<polyline fill="none" stroke="#1baf7a" stroke-width="2" stroke-linejoin="round" points="105.9,341.4 147.8,340.8 189.7,338.5 231.6,328.0 273.5,145.3 315.4,74.9 357.3,74.9 399.2,74.9 441.1,74.9"/>
<path d="M105.9 335.9L111.4 345.9L100.4 345.9Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 61 мс (медиана 3 прогонов)</title></path>
<path d="M147.8 335.3L153.3 345.3L142.3 345.3Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 66 мс (медиана 3 прогонов)</title></path>
<path d="M189.7 333.0L195.2 343.0L184.2 343.0Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 87 мс (медиана 3 прогонов)</title></path>
<path d="M231.6 322.5L237.1 332.5L226.1 332.5Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 183 мс (медиана 3 прогонов)</title></path>
<path d="M273.5 139.8L279.0 149.8L268.0 149.8Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 1856 мс (медиана 3 прогонов)</title></path>
<path d="M315.4 69.4L320.9 79.4L309.9 79.4Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 240 req/s: полезная 177 rps, ошибок 26.1 %, p99 2761 мс (медиана 3 прогонов)</title></path>
<path d="M357.3 69.4L362.8 79.4L351.8 79.4Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 280 req/s: полезная 194 rps, ошибок 30.6 %, p99 3639 мс (медиана 3 прогонов)</title></path>
<path d="M399.2 69.4L404.7 79.4L393.7 79.4Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 320 req/s: полезная 180 rps, ошибок 43.8 %, p99 3677 мс (медиана 2 прогонов)</title></path>
<path d="M441.1 69.4L446.6 79.4L435.6 79.4Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU — 360 req/s: полезная 202 rps, ошибок 43.9 %, p99 2507 мс (медиана 1 прогонов)</title></path>
<circle cx="586.0" cy="68.0" r="4.5" fill="#2a78d6" stroke="#fcfcfb" stroke-width="2"><title>1 × 1.0 CPU</title></circle>
<text x="596" y="72.0" font-size="11" fill="#52514e">1 × 1.0 CPU</text>
<rect x="581.5" y="81.5" width="9" height="9" rx="1.5" fill="#eb6834" stroke="#fcfcfb" stroke-width="2"><title>2 × 0.5 CPU</title></rect>
<text x="596" y="90.0" font-size="11" fill="#52514e">2 × 0.5 CPU</text>
<path d="M586.0 98.5L591.5 108.5L580.5 108.5Z" fill="#1baf7a" stroke="#fcfcfb" stroke-width="2"><title>4 × 0.25 CPU</title></path>
<text x="596" y="108.0" font-size="11" fill="#52514e">4 × 0.25 CPU</text>
</svg>

After

Width:  |  Height:  |  Size: 10 KiB

+341
View File
@@ -0,0 +1,341 @@
{
"config": "growth-1x0.25",
"repeat": 1,
"series": "growth",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"KDF_CONCURRENCY": 1
},
"measured_at": "2026-11-03T08:37:56+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 40,
"peak_ok_rps": 83.5,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-1x0.25",
"latency_ms": {
"max": 163.97,
"p50": 5.87,
"p90": 51.55,
"p99": 133.55,
"p999": 163.97
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.69,
"p99_ms": 86.2
},
"me": {
"ok": 40,
"p50_ms": 2.74,
"p99_ms": 78.28
},
"post": {
"ok": 80,
"p50_ms": 6.87,
"p99_ms": 163.97
}
},
"rate": 40,
"service_ms": {
"p50": 5.26,
"p99": 132.57
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 16.0,
"lb": 2.7,
"loadgen-run-083085fa5191": 3.1,
"postgres": 6.4,
"redis": 2.6
},
"mem_mib_max": {
"api": 41.7,
"lb": 4.5,
"loadgen-run-083085fa5191": 6.4,
"postgres": 83.0,
"redis": 4.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.222,
"pg_connections_mean": 12
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-1x0.25",
"latency_ms": {
"max": 1106.74,
"p50": 25.44,
"p90": 488.84,
"p99": 684.07,
"p999": 1106.74
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 27.24,
"p99_ms": 642.11
},
"me": {
"ok": 80,
"p50_ms": 9.52,
"p99_ms": 95.12
},
"post": {
"ok": 160,
"p50_ms": 22.17,
"p99_ms": 719.12
}
},
"rate": 80,
"service_ms": {
"p50": 24.84,
"p99": 683.5
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 24.8,
"lb": 3.2,
"loadgen-run-083085fa5191": 5.3,
"postgres": 7.4,
"redis": 2.7
},
"mem_mib_max": {
"api": 45.9,
"lb": 4.9,
"loadgen-run-083085fa5191": 7.2,
"postgres": 84.8,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 11.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.912,
"pg_connections_mean": 12.7
},
"slo_ok": false
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 365
},
"error_ratio": 0.3042,
"errors": 365,
"event": "step",
"label": "growth-1x0.25",
"latency_ms": {
"max": 4351.01,
"p50": 2040.56,
"p90": 2269.49,
"p99": 3927.25,
"p999": 4351.01
},
"ok": 835,
"per_op": {
"history": {
"ok": 551,
"p50_ms": 2075.27,
"p99_ms": 3934.79
},
"me": {
"ok": 120,
"p50_ms": 71.28,
"p99_ms": 1794.18
},
"post": {
"ok": 164,
"p50_ms": 2066.38,
"p99_ms": 3876.14
}
},
"rate": 120,
"service_ms": {
"p50": 2040.24,
"p99": 3926.97
},
"throughput_ok_rps": 83.5,
"resources": {
"cpu_percent_mean": {
"api": 25.7,
"lb": 4.1,
"loadgen-run-083085fa5191": 8.4,
"postgres": 7.4,
"redis": 2.6
},
"mem_mib_max": {
"api": 51.2,
"lb": 7.7,
"loadgen-run-083085fa5191": 14.9,
"postgres": 85.1,
"redis": 5.3
},
"lb_queue_max": 0,
"lb_sessions_mean": 161.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 1.0,
"pg_connections_mean": 13
},
"slo_ok": false
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1162
},
"error_ratio": 0.7262,
"errors": 1162,
"event": "step",
"label": "growth-1x0.25",
"latency_ms": {
"max": 4794.08,
"p50": 3150.27,
"p90": 3933.96,
"p99": 4099.07,
"p999": 4794.08
},
"ok": 438,
"per_op": {
"history": {
"ok": 269,
"p50_ms": 3503.56,
"p99_ms": 4344.37
},
"me": {
"ok": 88,
"p50_ms": 992.99,
"p99_ms": 2236.79
},
"post": {
"ok": 81,
"p50_ms": 2758.74,
"p99_ms": 4099.07
}
},
"rate": 160,
"service_ms": {
"p50": 3149.5,
"p99": 4098.99
},
"throughput_ok_rps": 43.8,
"resources": {
"cpu_percent_mean": {
"api": 25.5,
"lb": 5.3,
"loadgen-run-083085fa5191": 9.8,
"postgres": 6.8,
"redis": 2.6
},
"mem_mib_max": {
"api": 54.2,
"lb": 9.1,
"loadgen-run-083085fa5191": 17.7,
"postgres": 85.1,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 220.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 1.0,
"pg_connections_mean": 13
},
"slo_ok": false
}
],
"pool_before": {
"api-3d5fcadf": {
"pool": {
"connections_num": 3,
"requests_num": 19,
"requests_queued": 3,
"connections_ms": 215,
"requests_wait_ms": 296,
"usage_ms": 42,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
}
},
"pool_after": {
"api-3d5fcadf": {
"pool": {
"connections_num": 10,
"requests_num": 6470,
"requests_queued": 2801,
"connections_ms": 1357,
"requests_wait_ms": 4124254,
"usage_ms": 270202,
"requests_errors": 1682,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 4407,
"misses": 40,
"errors": 36
},
"requests": 6050
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+522
View File
@@ -0,0 +1,522 @@
{
"config": "growth-2x0.25",
"repeat": 1,
"series": "growth",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 2,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"KDF_CONCURRENCY": 1
},
"measured_at": "2026-11-03T08:40:54+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 80,
"peak_ok_rps": 159.8,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-2x0.25",
"latency_ms": {
"max": 81.86,
"p50": 5.5,
"p90": 20.81,
"p99": 72.09,
"p999": 81.86
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.36,
"p99_ms": 72.09
},
"me": {
"ok": 40,
"p50_ms": 2.94,
"p99_ms": 67.97
},
"post": {
"ok": 80,
"p50_ms": 7.25,
"p99_ms": 77.37
}
},
"rate": 40,
"service_ms": {
"p50": 4.92,
"p99": 71.44
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 17.4,
"lb": 2.6,
"loadgen-run-7ce67b9d067d": 2.9,
"postgres": 6.1,
"redis": 2.8
},
"mem_mib_max": {
"api": 37.3,
"lb": 4.6,
"loadgen-run-7ce67b9d067d": 6.1,
"postgres": 80.3,
"redis": 4.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.083,
"pg_connections_mean": 10
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-2x0.25",
"latency_ms": {
"max": 119.96,
"p50": 5.94,
"p90": 48.55,
"p99": 90.77,
"p999": 119.96
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.64,
"p99_ms": 94.32
},
"me": {
"ok": 80,
"p50_ms": 2.87,
"p99_ms": 71.43
},
"post": {
"ok": 160,
"p50_ms": 7.32,
"p99_ms": 87.38
}
},
"rate": 80,
"service_ms": {
"p50": 5.27,
"p99": 89.94
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 34.3,
"lb": 3.5,
"loadgen-run-7ce67b9d067d": 5.2,
"postgres": 10.2,
"redis": 3.1
},
"mem_mib_max": {
"api": 44.6,
"lb": 4.9,
"loadgen-run-7ce67b9d067d": 7.1,
"postgres": 101.6,
"redis": 4.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.4,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.324,
"pg_connections_mean": 19
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-2x0.25",
"latency_ms": {
"max": 1962.59,
"p50": 287.31,
"p90": 793.16,
"p99": 1436.06,
"p999": 1509.54
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 304.65,
"p99_ms": 1432.21
},
"me": {
"ok": 120,
"p50_ms": 49.86,
"p99_ms": 188.42
},
"post": {
"ok": 240,
"p50_ms": 438.94,
"p99_ms": 1470.95
}
},
"rate": 120,
"service_ms": {
"p50": 286.93,
"p99": 1435.02
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 41.4,
"lb": 4.4,
"loadgen-run-7ce67b9d067d": 7.5,
"postgres": 11.0,
"redis": 2.6
},
"mem_mib_max": {
"api": 46.9,
"lb": 5.7,
"loadgen-run-7ce67b9d067d": 8.1,
"postgres": 114.1,
"redis": 4.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 40.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.765,
"pg_connections_mean": 23
},
"slo_ok": false
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2
},
"error_ratio": 0.0013,
"errors": 2,
"event": "step",
"label": "growth-2x0.25",
"latency_ms": {
"max": 3292.73,
"p50": 979.17,
"p90": 2008.11,
"p99": 2192,
"p999": 2922.43
},
"ok": 1598,
"per_op": {
"history": {
"ok": 1119,
"p50_ms": 1070.65,
"p99_ms": 2203.93
},
"me": {
"ok": 160,
"p50_ms": 61,
"p99_ms": 507.81
},
"post": {
"ok": 319,
"p50_ms": 1205.83,
"p99_ms": 2192
}
},
"rate": 160,
"service_ms": {
"p50": 975.73,
"p99": 2191.43
},
"throughput_ok_rps": 159.8,
"resources": {
"cpu_percent_mean": {
"api": 42.6,
"lb": 5.0,
"loadgen-run-7ce67b9d067d": 9.4,
"postgres": 11.5,
"redis": 2.6
},
"mem_mib_max": {
"api": 49.6,
"lb": 7.4,
"loadgen-run-7ce67b9d067d": 13.4,
"postgres": 115.2,
"redis": 5.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 149.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.918,
"pg_connections_mean": 24
},
"slo_ok": false
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 708
},
"error_ratio": 0.354,
"errors": 708,
"event": "step",
"label": "growth-2x0.25",
"latency_ms": {
"max": 3749.16,
"p50": 1473.88,
"p90": 2265.03,
"p99": 3631.02,
"p999": 3728.39
},
"ok": 1292,
"per_op": {
"history": {
"ok": 883,
"p50_ms": 1650.32,
"p99_ms": 3631.02
},
"me": {
"ok": 179,
"p50_ms": 86.56,
"p99_ms": 2081.3
},
"post": {
"ok": 230,
"p50_ms": 1675.91,
"p99_ms": 3694.18
}
},
"rate": 200,
"service_ms": {
"p50": 1473.65,
"p99": 3630.79
},
"throughput_ok_rps": 129.2,
"resources": {
"cpu_percent_mean": {
"api": 47.2,
"lb": 5.7,
"loadgen-run-7ce67b9d067d": 10.5,
"postgres": 11.0,
"redis": 3.1
},
"mem_mib_max": {
"api": 54.0,
"lb": 9.9,
"loadgen-run-7ce67b9d067d": 19.6,
"postgres": 115.8,
"redis": 4.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 241.9,
"pg_active_mean": 0.9,
"pg_wait_events_mean": {
"CPU": 0.5,
"IO:WALSync": 0.1,
"LWLock:WALWrite": 0.3
},
"api_cfs_throttled_period_ratio": 0.944,
"pg_connections_mean": 24
},
"slo_ok": false
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1212
},
"error_ratio": 0.505,
"errors": 1212,
"event": "step",
"label": "growth-2x0.25",
"latency_ms": {
"max": 3296.52,
"p50": 1697.67,
"p90": 2757.62,
"p99": 3260.39,
"p999": 3294.14
},
"ok": 1188,
"per_op": {
"history": {
"ok": 804,
"p50_ms": 1835.07,
"p99_ms": 3262.87
},
"me": {
"ok": 181,
"p50_ms": 96.98,
"p99_ms": 2195.43
},
"post": {
"ok": 203,
"p50_ms": 1750.8,
"p99_ms": 2960.61
}
},
"rate": 240,
"service_ms": {
"p50": 1694.36,
"p99": 3259.65
},
"throughput_ok_rps": 118.8,
"resources": {
"cpu_percent_mean": {
"api": 45.9,
"lb": 6.1,
"loadgen-run-7ce67b9d067d": 10.7,
"postgres": 9.2,
"redis": 2.4
},
"mem_mib_max": {
"api": 54.4,
"lb": 9.9,
"loadgen-run-7ce67b9d067d": 20.7,
"postgres": 116.1,
"redis": 5.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 249.7,
"pg_active_mean": 0.3,
"pg_wait_events_mean": {
"CPU": 0.3
},
"api_cfs_throttled_period_ratio": 0.942,
"pg_connections_mean": 24
},
"slo_ok": false
}
],
"pool_before": {
"api-2307bb79": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 229,
"requests_wait_ms": 297,
"usage_ms": 55,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
},
"api-9f69e8f9": {
"pool": {
"connections_num": 3,
"requests_num": 21,
"requests_queued": 3,
"connections_ms": 212,
"requests_wait_ms": 290,
"usage_ms": 44,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
}
},
"pool_after": {
"api-2307bb79": {
"pool": {
"connections_num": 10,
"requests_num": 6230,
"requests_queued": 3483,
"connections_ms": 1132,
"requests_wait_ms": 3430506,
"usage_ms": 321376,
"requests_errors": 1354,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 4948,
"misses": 47,
"errors": 16
},
"requests": 6140
},
"api-9f69e8f9": {
"pool": {
"connections_num": 10,
"requests_num": 5583,
"requests_queued": 2977,
"connections_ms": 1605,
"requests_wait_ms": 3531264,
"usage_ms": 316196,
"requests_errors": 855,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 5142,
"misses": 47,
"errors": 8
},
"requests": 5767
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+704
View File
@@ -0,0 +1,704 @@
{
"config": "growth-3x0.25",
"repeat": 1,
"series": "growth",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 3,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"KDF_CONCURRENCY": 1
},
"measured_at": "2026-11-03T08:44:02+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 80,
"peak_ok_rps": 177.8,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-3x0.25",
"latency_ms": {
"max": 113.11,
"p50": 5.99,
"p90": 22.54,
"p99": 82.06,
"p999": 113.11
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.79,
"p99_ms": 82.06
},
"me": {
"ok": 40,
"p50_ms": 2.92,
"p99_ms": 66.08
},
"post": {
"ok": 80,
"p50_ms": 6.98,
"p99_ms": 113.11
}
},
"rate": 40,
"service_ms": {
"p50": 5.36,
"p99": 81.31
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 22.2,
"lb": 2.3,
"loadgen-run-69e377d75d2b": 2.8,
"postgres": 6.4,
"redis": 2.7
},
"mem_mib_max": {
"api": 38.1,
"lb": 4.4,
"loadgen-run-69e377d75d2b": 6.1,
"postgres": 97.1,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.096,
"pg_connections_mean": 15
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-3x0.25",
"latency_ms": {
"max": 140.63,
"p50": 5.78,
"p90": 41.54,
"p99": 89.95,
"p999": 140.63
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.68,
"p99_ms": 93.77
},
"me": {
"ok": 80,
"p50_ms": 2.73,
"p99_ms": 58.74
},
"post": {
"ok": 160,
"p50_ms": 6.73,
"p99_ms": 78.67
}
},
"rate": 80,
"service_ms": {
"p50": 5.15,
"p99": 87.88
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 36.9,
"lb": 3.6,
"loadgen-run-69e377d75d2b": 5.3,
"postgres": 8.8,
"redis": 2.9
},
"mem_mib_max": {
"api": 35.9,
"lb": 4.4,
"loadgen-run-69e377d75d2b": 6.3,
"postgres": 98.4,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.188,
"pg_connections_mean": 15
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-3x0.25",
"latency_ms": {
"max": 526.34,
"p50": 6.86,
"p90": 161.34,
"p99": 300.82,
"p999": 456.11
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 6.74,
"p99_ms": 300.82
},
"me": {
"ok": 120,
"p50_ms": 2.92,
"p99_ms": 125.9
},
"post": {
"ok": 240,
"p50_ms": 9.24,
"p99_ms": 327.57
}
},
"rate": 120,
"service_ms": {
"p50": 6.13,
"p99": 298.48
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 55.4,
"lb": 4.4,
"loadgen-run-69e377d75d2b": 7.2,
"postgres": 13.4,
"redis": 3.3
},
"mem_mib_max": {
"api": 44.5,
"lb": 4.7,
"loadgen-run-69e377d75d2b": 6.9,
"postgres": 125.7,
"redis": 4.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 5.1,
"pg_active_mean": 0.33,
"pg_wait_events_mean": {
"CPU": 0.11,
"IO:WALSync": 0.22
},
"api_cfs_throttled_period_ratio": 0.319,
"pg_connections_mean": 24.7
},
"slo_ok": false
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "growth-3x0.25",
"latency_ms": {
"max": 736.64,
"p50": 55.32,
"p90": 395.09,
"p99": 564.5,
"p999": 720.14
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 61.49,
"p99_ms": 567.01
},
"me": {
"ok": 160,
"p50_ms": 19.48,
"p99_ms": 324.03
},
"post": {
"ok": 320,
"p50_ms": 79.86,
"p99_ms": 564.5
}
},
"rate": 160,
"service_ms": {
"p50": 54.68,
"p99": 563.45
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 66.8,
"lb": 5.7,
"loadgen-run-69e377d75d2b": 9.5,
"postgres": 15.5,
"redis": 3.8
},
"mem_mib_max": {
"api": 45.3,
"lb": 5.3,
"loadgen-run-69e377d75d2b": 7.8,
"postgres": 145.4,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 24.4,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.11,
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.626,
"pg_connections_mean": 33.9
},
"slo_ok": false
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 411
},
"error_ratio": 0.2055,
"errors": 411,
"event": "step",
"label": "growth-3x0.25",
"latency_ms": {
"max": 3347.53,
"p50": 750.43,
"p90": 2257.96,
"p99": 2992.67,
"p999": 3341.77
},
"ok": 1589,
"per_op": {
"history": {
"ok": 1083,
"p50_ms": 784.79,
"p99_ms": 2972.94
},
"me": {
"ok": 186,
"p50_ms": 122.24,
"p99_ms": 2428.17
},
"post": {
"ok": 320,
"p50_ms": 871.16,
"p99_ms": 3259.04
}
},
"rate": 200,
"service_ms": {
"p50": 750.39,
"p99": 2991.38
},
"throughput_ok_rps": 158.9,
"resources": {
"cpu_percent_mean": {
"api": 66.3,
"lb": 6.7,
"loadgen-run-69e377d75d2b": 12.9,
"postgres": 15.0,
"redis": 3.6
},
"mem_mib_max": {
"api": 53.9,
"lb": 9.7,
"loadgen-run-69e377d75d2b": 17.2,
"postgres": 145.4,
"redis": 4.5
},
"lb_queue_max": 0,
"lb_sessions_mean": 158.8,
"pg_active_mean": 0.3,
"pg_wait_events_mean": {
"Client:ClientRead": 0.2,
"IO:WALSync": 0.1
},
"api_cfs_throttled_period_ratio": 0.884,
"pg_connections_mean": 35
},
"slo_ok": false
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 622
},
"error_ratio": 0.2592,
"errors": 622,
"event": "step",
"label": "growth-3x0.25",
"latency_ms": {
"max": 3100.61,
"p50": 1149.65,
"p90": 2244.54,
"p99": 2549.02,
"p999": 2631.99
},
"ok": 1778,
"per_op": {
"history": {
"ok": 1230,
"p50_ms": 1339.55,
"p99_ms": 2565.64
},
"me": {
"ok": 220,
"p50_ms": 100.21,
"p99_ms": 693.97
},
"post": {
"ok": 328,
"p50_ms": 1548.52,
"p99_ms": 2452.56
}
},
"rate": 240,
"service_ms": {
"p50": 1149.31,
"p99": 2548.93
},
"throughput_ok_rps": 177.8,
"resources": {
"cpu_percent_mean": {
"api": 70.6,
"lb": 6.7,
"loadgen-run-69e377d75d2b": 12.0,
"postgres": 13.5,
"redis": 3.5
},
"mem_mib_max": {
"api": 54.0,
"lb": 11.3,
"loadgen-run-69e377d75d2b": 22.6,
"postgres": 147.3,
"redis": 4.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 275,
"pg_active_mean": 0.4,
"pg_wait_events_mean": {
"CPU": 0.3,
"IO:WALSync": 0.1
},
"api_cfs_throttled_period_ratio": 0.912,
"pg_connections_mean": 35
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1612
},
"error_ratio": 0.5757,
"errors": 1612,
"event": "step",
"label": "growth-3x0.25",
"latency_ms": {
"max": 4774.18,
"p50": 1901.67,
"p90": 3072.91,
"p99": 4240.71,
"p999": 4773.95
},
"ok": 1188,
"per_op": {
"history": {
"ok": 780,
"p50_ms": 2061.96,
"p99_ms": 4241.06
},
"me": {
"ok": 182,
"p50_ms": 208.11,
"p99_ms": 2614.16
},
"post": {
"ok": 226,
"p50_ms": 1970.44,
"p99_ms": 4772.46
}
},
"rate": 280,
"service_ms": {
"p50": 1900.44,
"p99": 4239.84
},
"throughput_ok_rps": 118.8,
"resources": {
"cpu_percent_mean": {
"api": 66.8,
"lb": 8.1,
"loadgen-run-69e377d75d2b": 14.5,
"postgres": 11.9,
"redis": 3.4
},
"mem_mib_max": {
"api": 57.4,
"lb": 13.3,
"loadgen-run-69e377d75d2b": 27.1,
"postgres": 146.4,
"redis": 4.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 356.8,
"pg_active_mean": 0.4,
"pg_wait_events_mean": {
"IO:WALSync": 0.1,
"LWLock:WALWrite": 0.3
},
"api_cfs_throttled_period_ratio": 0.862,
"pg_connections_mean": 35
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2458
},
"error_ratio": 0.7681,
"errors": 2458,
"event": "step",
"label": "growth-3x0.25",
"latency_ms": {
"max": 4971.63,
"p50": 2239.43,
"p90": 4064.12,
"p99": 4614.18,
"p999": 4971.63
},
"ok": 742,
"per_op": {
"history": {
"ok": 511,
"p50_ms": 2359.66,
"p99_ms": 4614.18
},
"me": {
"ok": 83,
"p50_ms": 224.99,
"p99_ms": 3368.95
},
"post": {
"ok": 148,
"p50_ms": 2246.82,
"p99_ms": 4922.68
}
},
"rate": 320,
"service_ms": {
"p50": 2239.16,
"p99": 4613.09
},
"throughput_ok_rps": 74.2,
"resources": {
"cpu_percent_mean": {
"api": 65.2,
"lb": 8.7,
"loadgen-run-69e377d75d2b": 15.7,
"postgres": 12.4,
"redis": 3.3
},
"mem_mib_max": {
"api": 57.6,
"lb": 12.8,
"loadgen-run-69e377d75d2b": 30.6,
"postgres": 146.9,
"redis": 5.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 359.3,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"CPU": 0.1
},
"api_cfs_throttled_period_ratio": 0.891,
"pg_connections_mean": 35
},
"slo_ok": false
}
],
"pool_before": {
"api-883b7718": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 328,
"requests_wait_ms": 404,
"usage_ms": 197,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-6519d22e": {
"pool": {
"connections_num": 3,
"requests_num": 22,
"requests_queued": 3,
"connections_ms": 236,
"requests_wait_ms": 201,
"usage_ms": 135,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
},
"api-f113faa6": {
"pool": {
"connections_num": 3,
"requests_num": 22,
"requests_queued": 3,
"connections_ms": 328,
"requests_wait_ms": 491,
"usage_ms": 143,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
}
},
"pool_after": {
"api-883b7718": {
"pool": {
"connections_num": 10,
"requests_num": 5415,
"requests_queued": 2370,
"connections_ms": 1932,
"requests_wait_ms": 2757267,
"usage_ms": 326768,
"requests_errors": 1021,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 4476,
"misses": 20,
"errors": 25
},
"requests": 5489
},
"api-6519d22e": {
"pool": {
"connections_num": 10,
"requests_num": 5503,
"requests_queued": 2599,
"connections_ms": 1826,
"requests_wait_ms": 2907009,
"usage_ms": 340635,
"requests_errors": 744,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 4857,
"misses": 26,
"errors": 8
},
"requests": 5613
},
"api-f113faa6": {
"pool": {
"connections_num": 10,
"requests_num": 9185,
"requests_queued": 3748,
"connections_ms": 1941,
"requests_wait_ms": 5470804,
"usage_ms": 411062,
"requests_errors": 4012,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 4782,
"misses": 38,
"errors": 50
},
"requests": 8820
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+43
View File
@@ -0,0 +1,43 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 760 360" font-family="system-ui,-apple-system,Segoe UI,sans-serif">
<rect width="100%" height="100%" fill="#fcfcfb"/>
<text x="64" y="26" font-size="16" font-weight="600" fill="#0b0b0b">Рост ресурсов: пик полезной пропускной способности</text>
<text x="64" y="46" font-size="12" fill="#52514e">Ступени 0.25 CPU на изолированном ядре, затем вся машина (2 vCPU, общие с БД/LB/генератором)</text>
<line x1="64" x2="736" y1="290.0" y2="290.0" stroke="#e4e3df"/>
<text x="56" y="294.0" font-size="11" fill="#52514e" text-anchor="end">0</text>
<line x1="64" x2="736" y1="230.1" y2="230.1" stroke="#e4e3df"/>
<text x="56" y="234.1" font-size="11" fill="#52514e" text-anchor="end">100</text>
<line x1="64" x2="736" y1="170.3" y2="170.3" stroke="#e4e3df"/>
<text x="56" y="174.3" font-size="11" fill="#52514e" text-anchor="end">200</text>
<line x1="64" x2="736" y1="110.4" y2="110.4" stroke="#e4e3df"/>
<text x="56" y="114.4" font-size="11" fill="#52514e" text-anchor="end">300</text>
<path d="M85.1 290.0V243.7Q85.1 239.7 89.1 239.7H134.9Q138.9 239.7 138.9 243.7V290.0Z" fill="#2a78d6"><title>growth-1x0.25: пик 84 rps</title></path>
<text x="112.0" y="233.7" font-size="12" fill="#0b0b0b" text-anchor="middle">84</text>
<text x="112.0" y="306.0" font-size="11" fill="#52514e" text-anchor="middle">1×0.25</text>
<text x="112.0" y="319.0" font-size="11" fill="#52514e" text-anchor="middle">(0.25 CPU)</text>
<path d="M181.1 290.0V198.2Q181.1 194.2 185.1 194.2H230.9Q234.9 194.2 234.9 198.2V290.0Z" fill="#2a78d6"><title>growth-2x0.25: пик 160 rps</title></path>
<text x="208.0" y="188.2" font-size="12" fill="#0b0b0b" text-anchor="middle">160</text>
<text x="208.0" y="306.0" font-size="11" fill="#52514e" text-anchor="middle">2×0.25</text>
<text x="208.0" y="319.0" font-size="11" fill="#52514e" text-anchor="middle">(0.5 CPU)</text>
<path d="M277.1 290.0V187.5Q277.1 183.5 281.1 183.5H326.9Q330.9 183.5 330.9 187.5V290.0Z" fill="#2a78d6"><title>growth-3x0.25: пик 178 rps</title></path>
<text x="304.0" y="177.5" font-size="12" fill="#0b0b0b" text-anchor="middle">178</text>
<text x="304.0" y="306.0" font-size="11" fill="#52514e" text-anchor="middle">3×0.25</text>
<text x="304.0" y="319.0" font-size="11" fill="#52514e" text-anchor="middle">(0.75 CPU)</text>
<path d="M373.1 290.0V174.3Q373.1 170.3 377.1 170.3H422.9Q426.9 170.3 426.9 174.3V290.0Z" fill="#2a78d6"><title>fixed-4x0.25: пик 200 rps</title></path>
<text x="400.0" y="164.3" font-size="12" fill="#0b0b0b" text-anchor="middle">200</text>
<text x="400.0" y="306.0" font-size="11" fill="#52514e" text-anchor="middle">4×0.25</text>
<text x="400.0" y="319.0" font-size="11" fill="#52514e" text-anchor="middle">(1 CPU)</text>
<path d="M469.1 290.0V126.4Q469.1 122.4 473.1 122.4H518.9Q522.9 122.4 522.9 126.4V290.0Z" fill="#2a78d6"><title>fixed-1x1.00: пик 280 rps</title></path>
<text x="496.0" y="116.4" font-size="12" fill="#0b0b0b" text-anchor="middle">280</text>
<text x="496.0" y="306.0" font-size="11" fill="#52514e" text-anchor="middle">1×1.0</text>
<text x="496.0" y="319.0" font-size="11" fill="#52514e" text-anchor="middle">(1 CPU)</text>
<path d="M565.1 290.0V102.5Q565.1 98.5 569.1 98.5H614.9Q618.9 98.5 618.9 102.5V290.0Z" fill="#2a78d6"><title>machine-2x1.00: пик 320 rps</title></path>
<text x="592.0" y="92.5" font-size="12" fill="#0b0b0b" text-anchor="middle">320</text>
<text x="592.0" y="306.0" font-size="11" fill="#52514e" text-anchor="middle">2×1.0</text>
<text x="592.0" y="319.0" font-size="11" fill="#52514e" text-anchor="middle">(2 vCPU хоста)</text>
<path d="M661.1 290.0V107.3Q661.1 103.3 665.1 103.3H710.9Q714.9 103.3 714.9 107.3V290.0Z" fill="#2a78d6"><title>machine-4x1.00: пик 312 rps</title></path>
<text x="688.0" y="97.3" font-size="12" fill="#0b0b0b" text-anchor="middle">312</text>
<text x="688.0" y="306.0" font-size="11" fill="#52514e" text-anchor="middle">4×1.0</text>
<text x="688.0" y="319.0" font-size="11" fill="#52514e" text-anchor="middle">(2 vCPU хоста)</text>
<line x1="64" x2="736" y1="290" y2="290" stroke="#52514e"/>
<text x="16" y="177" font-size="12" fill="#52514e" transform="rotate(-90 16 177)" text-anchor="middle">Пик полезной, rps</text>
</svg>

After

Width:  |  Height:  |  Size: 4.3 KiB

+849
View File
@@ -0,0 +1,849 @@
{
"config": "machine-2x1.00",
"repeat": 1,
"series": "machine",
"params": {
"API_CPUSET": "0-1",
"AUX_CPUSET": "0-1",
"DB_POOL_MAX": "10",
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 2,
"API_CPUS": "1.0",
"API_MEMORY": "256M",
"KDF_CONCURRENCY": 2
},
"measured_at": "2026-11-03T08:47:28+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 320,
"peak_ok_rps": 320,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 21.87,
"p50": 5.44,
"p90": 7.77,
"p99": 13.42,
"p999": 21.87
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.38,
"p99_ms": 13.42
},
"me": {
"ok": 40,
"p50_ms": 2.76,
"p99_ms": 7.88
},
"post": {
"ok": 80,
"p50_ms": 6.44,
"p99_ms": 16.93
}
},
"rate": 40,
"service_ms": {
"p50": 4.86,
"p99": 12.3
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 20.2,
"lb": 2.2,
"loadgen-run-0cfe7ec82689": 3.7,
"postgres": 4.9,
"redis": 2.5
},
"mem_mib_max": {
"api": 43.9,
"lb": 4.9,
"loadgen-run-0cfe7ec82689": 6.7,
"postgres": 57.0,
"redis": 3.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.4,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"Client:ClientRead": 0.11,
"IO:DataFileRead": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 6
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 55.69,
"p50": 5.56,
"p90": 8.19,
"p99": 23.02,
"p999": 55.69
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.53,
"p99_ms": 24.68
},
"me": {
"ok": 80,
"p50_ms": 2.74,
"p99_ms": 7.51
},
"post": {
"ok": 160,
"p50_ms": 6.25,
"p99_ms": 22.85
}
},
"rate": 80,
"service_ms": {
"p50": 5.01,
"p99": 22.72
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 34.1,
"lb": 3.7,
"loadgen-run-0cfe7ec82689": 7.1,
"postgres": 7.8,
"redis": 3.1
},
"mem_mib_max": {
"api": 31.6,
"lb": 4.9,
"loadgen-run-0cfe7ec82689": 7.1,
"postgres": 57.2,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.5,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 6
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 58.19,
"p50": 5.8,
"p90": 10.61,
"p99": 36.92,
"p999": 55.22
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.81,
"p99_ms": 36.82
},
"me": {
"ok": 120,
"p50_ms": 2.68,
"p99_ms": 26.45
},
"post": {
"ok": 240,
"p50_ms": 6.3,
"p99_ms": 50.78
}
},
"rate": 120,
"service_ms": {
"p50": 5.17,
"p99": 32.4
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 51.4,
"lb": 4.9,
"loadgen-run-0cfe7ec82689": 10.3,
"postgres": 10.0,
"redis": 3.5
},
"mem_mib_max": {
"api": 31.8,
"lb": 4.9,
"loadgen-run-0cfe7ec82689": 7.1,
"postgres": 59.0,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.005,
"pg_connections_mean": 6
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 167.26,
"p50": 5.44,
"p90": 13.3,
"p99": 108.05,
"p999": 145.02
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.49,
"p99_ms": 110.96
},
"me": {
"ok": 160,
"p50_ms": 2.4,
"p99_ms": 49.53
},
"post": {
"ok": 320,
"p50_ms": 6.14,
"p99_ms": 95.27
}
},
"rate": 160,
"service_ms": {
"p50": 4.88,
"p99": 107.62
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 63.4,
"lb": 5.5,
"loadgen-run-0cfe7ec82689": 11.1,
"postgres": 12.7,
"redis": 3.7
},
"mem_mib_max": {
"api": 40.6,
"lb": 5.0,
"loadgen-run-0cfe7ec82689": 7.3,
"postgres": 77.1,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.017,
"pg_connections_mean": 10.1
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 312.38,
"p50": 6.54,
"p90": 20.02,
"p99": 130.19,
"p999": 269.03
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 6.86,
"p99_ms": 120.22
},
"me": {
"ok": 200,
"p50_ms": 2.31,
"p99_ms": 43.91
},
"post": {
"ok": 400,
"p50_ms": 7.28,
"p99_ms": 214.54
}
},
"rate": 200,
"service_ms": {
"p50": 6,
"p99": 130.09
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 78.9,
"lb": 5.9,
"loadgen-run-0cfe7ec82689": 12.9,
"postgres": 14.2,
"redis": 3.7
},
"mem_mib_max": {
"api": 32.8,
"lb": 5.1,
"loadgen-run-0cfe7ec82689": 7.6,
"postgres": 85.0,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 2.9,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.011,
"pg_connections_mean": 16.7
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 233.7,
"p50": 7.95,
"p90": 47.79,
"p99": 167.31,
"p999": 211.5
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 8.88,
"p99_ms": 170.69
},
"me": {
"ok": 240,
"p50_ms": 2.26,
"p99_ms": 41.58
},
"post": {
"ok": 480,
"p50_ms": 7.29,
"p99_ms": 168.55
}
},
"rate": 240,
"service_ms": {
"p50": 7.35,
"p99": 166.1
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 94.1,
"lb": 6.6,
"loadgen-run-0cfe7ec82689": 14.9,
"postgres": 16.5,
"redis": 3.5
},
"mem_mib_max": {
"api": 44.2,
"lb": 5.3,
"loadgen-run-0cfe7ec82689": 7.6,
"postgres": 92.4,
"redis": 4.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.1,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"IO:WALSync": 0.11,
"LWLock:WALWrite": 0.11
},
"api_cfs_throttled_period_ratio": 0.006,
"pg_connections_mean": 18.8
},
"slo_ok": true
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 240.79,
"p50": 9.58,
"p90": 59.16,
"p99": 157.6,
"p999": 200.58
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 11.79,
"p99_ms": 157.59
},
"me": {
"ok": 280,
"p50_ms": 2.11,
"p99_ms": 51.53
},
"post": {
"ok": 560,
"p50_ms": 8.33,
"p99_ms": 191.05
}
},
"rate": 280,
"service_ms": {
"p50": 8.88,
"p99": 156.09
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 103.9,
"lb": 7.0,
"loadgen-run-0cfe7ec82689": 15.6,
"postgres": 17.5,
"redis": 3.6
},
"mem_mib_max": {
"api": 44.7,
"lb": 5.4,
"loadgen-run-0cfe7ec82689": 7.8,
"postgres": 93.4,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 4.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 204.39,
"p50": 14.01,
"p90": 82.12,
"p99": 161.5,
"p999": 192.47
},
"ok": 3200,
"per_op": {
"history": {
"ok": 2240,
"p50_ms": 15.95,
"p99_ms": 156.65
},
"me": {
"ok": 320,
"p50_ms": 2.55,
"p99_ms": 56.48
},
"post": {
"ok": 640,
"p50_ms": 17.66,
"p99_ms": 174.15
}
},
"rate": 320,
"service_ms": {
"p50": 13.39,
"p99": 158.47
},
"throughput_ok_rps": 320,
"resources": {
"cpu_percent_mean": {
"api": 111.0,
"lb": 7.3,
"loadgen-run-0cfe7ec82689": 17.4,
"postgres": 19.3,
"redis": 3.8
},
"mem_mib_max": {
"api": 45.9,
"lb": 7.0,
"loadgen-run-0cfe7ec82689": 7.8,
"postgres": 93.9,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 9,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20
},
"slo_ok": true
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 720
},
"error_ratio": 0.2,
"errors": 720,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 2461.41,
"p50": 715.82,
"p90": 1287.85,
"p99": 1921.22,
"p999": 2429.42
},
"ok": 2880,
"per_op": {
"history": {
"ok": 1992,
"p50_ms": 811.39,
"p99_ms": 2041.89
},
"me": {
"ok": 353,
"p50_ms": 43.79,
"p99_ms": 1489.58
},
"post": {
"ok": 535,
"p50_ms": 805.48,
"p99_ms": 1721.12
}
},
"rate": 360,
"service_ms": {
"p50": 714.71,
"p99": 1921.18
},
"throughput_ok_rps": 288,
"resources": {
"cpu_percent_mean": {
"api": 108.5,
"lb": 9.4,
"loadgen-run-0cfe7ec82689": 20.0,
"postgres": 18.8,
"redis": 3.9
},
"mem_mib_max": {
"api": 52.7,
"lb": 9.8,
"loadgen-run-0cfe7ec82689": 21.9,
"postgres": 96.1,
"redis": 4.5
},
"lb_queue_max": 0,
"lb_sessions_mean": 192,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.012,
"pg_connections_mean": 20
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1868
},
"error_ratio": 0.467,
"errors": 1868,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 2561.02,
"p50": 1274.55,
"p90": 1694.59,
"p99": 2389.72,
"p999": 2555.42
},
"ok": 2132,
"per_op": {
"history": {
"ok": 1425,
"p50_ms": 1358.48,
"p99_ms": 2392.22
},
"me": {
"ok": 273,
"p50_ms": 82.93,
"p99_ms": 1469.12
},
"post": {
"ok": 434,
"p50_ms": 1368,
"p99_ms": 2392.65
}
},
"rate": 400,
"service_ms": {
"p50": 1270.48,
"p99": 2389.29
},
"throughput_ok_rps": 213.2,
"resources": {
"cpu_percent_mean": {
"api": 106.8,
"lb": 10.6,
"loadgen-run-0cfe7ec82689": 20.8,
"postgres": 21.4,
"redis": 3.4
},
"mem_mib_max": {
"api": 53.8,
"lb": 11.7,
"loadgen-run-0cfe7ec82689": 24.6,
"postgres": 96.4,
"redis": 5.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 273,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.22
},
"api_cfs_throttled_period_ratio": 0.018,
"pg_connections_mean": 20
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2050
},
"error_ratio": 0.4457,
"errors": 2050,
"event": "step",
"label": "machine-2x1.00",
"latency_ms": {
"max": 1864.86,
"p50": 1068.02,
"p90": 1507.62,
"p99": 1828.84,
"p999": 1855.65
},
"ok": 2550,
"per_op": {
"history": {
"ok": 1670,
"p50_ms": 1153.33,
"p99_ms": 1833.5
},
"me": {
"ok": 430,
"p50_ms": 37.12,
"p99_ms": 222.89
},
"post": {
"ok": 450,
"p50_ms": 1161.06,
"p99_ms": 1831.85
}
},
"rate": 460,
"service_ms": {
"p50": 1067.2,
"p99": 1828.63
},
"throughput_ok_rps": 255,
"resources": {
"cpu_percent_mean": {
"api": 104.4,
"lb": 10.4,
"loadgen-run-0cfe7ec82689": 21.7,
"postgres": 16.0,
"redis": 4.4
},
"mem_mib_max": {
"api": 56.3,
"lb": 11.5,
"loadgen-run-0cfe7ec82689": 22.4,
"postgres": 96.6,
"redis": 6.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 247.1,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"Client:ClientRead": 0.11,
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.006,
"pg_connections_mean": 20
},
"slo_ok": false
}
],
"pool_before": {
"api-7e1b920a": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 117,
"requests_wait_ms": 95,
"usage_ms": 64,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
},
"api-f503cd6e": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 80,
"requests_wait_ms": 85,
"usage_ms": 32,
"pool_min": 1,
"pool_max": 10,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 10
}
},
"pool_after": {
"api-7e1b920a": {
"pool": {
"connections_num": 10,
"requests_num": 18225,
"requests_queued": 5467,
"connections_ms": 674,
"requests_wait_ms": 4249639,
"usage_ms": 326862,
"requests_errors": 3391,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 16568,
"misses": 119,
"errors": 3
},
"requests": 18926
},
"api-f503cd6e": {
"pool": {
"connections_num": 10,
"requests_num": 15491,
"requests_queued": 4334,
"connections_ms": 641,
"requests_wait_ms": 3038536,
"usage_ms": 285901,
"requests_errors": 1914,
"pool_min": 1,
"pool_max": 10,
"pool_size": 10,
"pool_available": 10,
"requests_waiting": 0
},
"cache": {
"hits": 15862,
"misses": 128,
"errors": 2
},
"requests": 16676
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
File diff suppressed because it is too large Load Diff
+809
View File
@@ -0,0 +1,809 @@
{
"config": "pool-1x1.00-p20",
"repeat": 1,
"series": "pool",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": 20,
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T08:57:33+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 200,
"peak_ok_rps": 280,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 25.35,
"p50": 5.17,
"p90": 8.29,
"p99": 15.35,
"p999": 25.35
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 5.01,
"p99_ms": 15.35
},
"me": {
"ok": 40,
"p50_ms": 2.5,
"p99_ms": 7.06
},
"post": {
"ok": 80,
"p50_ms": 6.48,
"p99_ms": 25.35
}
},
"rate": 40,
"service_ms": {
"p50": 4.61,
"p99": 14.31
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 15.1,
"lb": 2.1,
"loadgen-run-d4289cdc835a": 2.7,
"postgres": 4.7,
"redis": 2.3
},
"mem_mib_max": {
"api": 31.5,
"lb": 4.3,
"loadgen-run-d4289cdc835a": 5.9,
"postgres": 66.8,
"redis": 5.3
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.11,
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 6
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 45.39,
"p50": 5.26,
"p90": 7.78,
"p99": 14.71,
"p999": 45.39
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.24,
"p99_ms": 13.2
},
"me": {
"ok": 80,
"p50_ms": 2.43,
"p99_ms": 4.55
},
"post": {
"ok": 160,
"p50_ms": 5.84,
"p99_ms": 35.4
}
},
"rate": 80,
"service_ms": {
"p50": 4.67,
"p99": 13.63
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 29.2,
"lb": 3.3,
"loadgen-run-d4289cdc835a": 5.1,
"postgres": 7.3,
"redis": 2.8
},
"mem_mib_max": {
"api": 31.8,
"lb": 4.4,
"loadgen-run-d4289cdc835a": 6.2,
"postgres": 67.0,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 6
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 27.65,
"p50": 4.82,
"p90": 8.01,
"p99": 19.27,
"p999": 26.18
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 4.76,
"p99_ms": 21.96
},
"me": {
"ok": 120,
"p50_ms": 2.21,
"p99_ms": 10.94
},
"post": {
"ok": 240,
"p50_ms": 5.52,
"p99_ms": 14.23
}
},
"rate": 120,
"service_ms": {
"p50": 4.21,
"p99": 18.71
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 39.5,
"lb": 4.2,
"loadgen-run-d4289cdc835a": 6.7,
"postgres": 9.8,
"redis": 3.2
},
"mem_mib_max": {
"api": 37.7,
"lb": 4.5,
"loadgen-run-d4289cdc835a": 6.2,
"postgres": 70.7,
"redis": 5.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 7
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 99.68,
"p50": 5.09,
"p90": 13.32,
"p99": 77.24,
"p999": 96.76
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.1,
"p99_ms": 78.51
},
"me": {
"ok": 160,
"p50_ms": 2.06,
"p99_ms": 7.9
},
"post": {
"ok": 320,
"p50_ms": 5.79,
"p99_ms": 71.27
}
},
"rate": 160,
"service_ms": {
"p50": 4.53,
"p99": 77.04
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 52.9,
"lb": 4.9,
"loadgen-run-d4289cdc835a": 8.2,
"postgres": 12.8,
"redis": 3.5
},
"mem_mib_max": {
"api": 32.7,
"lb": 4.6,
"loadgen-run-d4289cdc835a": 6.5,
"postgres": 79.6,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.9,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 10.1
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 134.92,
"p50": 6.69,
"p90": 54.8,
"p99": 116.76,
"p999": 131.97
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 7.37,
"p99_ms": 113.52
},
"me": {
"ok": 200,
"p50_ms": 2.28,
"p99_ms": 44.94
},
"post": {
"ok": 400,
"p50_ms": 6.85,
"p99_ms": 128.33
}
},
"rate": 200,
"service_ms": {
"p50": 6.24,
"p99": 115.72
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 69.8,
"lb": 5.8,
"loadgen-run-d4289cdc835a": 10.1,
"postgres": 16.5,
"redis": 3.8
},
"mem_mib_max": {
"api": 36.0,
"lb": 4.9,
"loadgen-run-d4289cdc835a": 7.0,
"postgres": 106.2,
"redis": 4.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 3.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 20.1
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 532.26,
"p50": 56.78,
"p90": 343.8,
"p99": 412.4,
"p999": 523.07
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 64.99,
"p99_ms": 417.7
},
"me": {
"ok": 240,
"p50_ms": 15.39,
"p99_ms": 159.45
},
"post": {
"ok": 480,
"p50_ms": 85.9,
"p99_ms": 412.4
}
},
"rate": 240,
"service_ms": {
"p50": 56.11,
"p99": 412.06
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 85.2,
"lb": 6.3,
"loadgen-run-d4289cdc835a": 12.6,
"postgres": 18.1,
"redis": 4.0
},
"mem_mib_max": {
"api": 48.5,
"lb": 5.5,
"loadgen-run-d4289cdc835a": 8.4,
"postgres": 104.4,
"redis": 5.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 26.2,
"pg_active_mean": 1.22,
"pg_wait_events_mean": {
"CPU": 0.11,
"IO:WALSync": 0.11,
"LWLock:WALWrite": 1.0
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 460.29,
"p50": 105.06,
"p90": 387.13,
"p99": 431.08,
"p999": 448.04
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 128.94,
"p99_ms": 432.79
},
"me": {
"ok": 280,
"p50_ms": 21.32,
"p99_ms": 112.9
},
"post": {
"ok": 560,
"p50_ms": 135.12,
"p99_ms": 419.06
}
},
"rate": 280,
"service_ms": {
"p50": 104.62,
"p99": 430.5
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 89.7,
"lb": 6.8,
"loadgen-run-d4289cdc835a": 14.2,
"postgres": 18.6,
"redis": 3.9
},
"mem_mib_max": {
"api": 49.5,
"lb": 6.2,
"loadgen-run-d4289cdc835a": 9.4,
"postgres": 105.0,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 38.7,
"pg_active_mean": 0.44,
"pg_wait_events_mean": {
"CPU": 0.33,
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 494
},
"error_ratio": 0.1544,
"errors": 494,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 2208.7,
"p50": 857.91,
"p90": 1154.5,
"p99": 1829.56,
"p999": 2202.57
},
"ok": 2706,
"per_op": {
"history": {
"ok": 1856,
"p50_ms": 899.64,
"p99_ms": 1829.56
},
"me": {
"ok": 319,
"p50_ms": 40.14,
"p99_ms": 904.79
},
"post": {
"ok": 531,
"p50_ms": 889.44,
"p99_ms": 2061.05
}
},
"rate": 320,
"service_ms": {
"p50": 857.14,
"p99": 1829.14
},
"throughput_ok_rps": 270.6,
"resources": {
"cpu_percent_mean": {
"api": 92.7,
"lb": 7.8,
"loadgen-run-d4289cdc835a": 16.8,
"postgres": 19.8,
"redis": 4.2
},
"mem_mib_max": {
"api": 54.5,
"lb": 8.1,
"loadgen-run-d4289cdc835a": 17.2,
"postgres": 106.1,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 204.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 978
},
"error_ratio": 0.2717,
"errors": 978,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 2431.92,
"p50": 851.53,
"p90": 1453.99,
"p99": 1853.51,
"p999": 2409.37
},
"ok": 2622,
"per_op": {
"history": {
"ok": 1769,
"p50_ms": 873.33,
"p99_ms": 1867.17
},
"me": {
"ok": 354,
"p50_ms": 40.19,
"p99_ms": 846.62
},
"post": {
"ok": 499,
"p50_ms": 879.56,
"p99_ms": 1983.31
}
},
"rate": 360,
"service_ms": {
"p50": 851.02,
"p99": 1853.48
},
"throughput_ok_rps": 262.2,
"resources": {
"cpu_percent_mean": {
"api": 92.5,
"lb": 7.9,
"loadgen-run-d4289cdc835a": 15.7,
"postgres": 18.0,
"redis": 4.5
},
"mem_mib_max": {
"api": 55.4,
"lb": 8.3,
"loadgen-run-d4289cdc835a": 18.1,
"postgres": 106.8,
"redis": 4.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 231.8,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1821
},
"error_ratio": 0.4553,
"errors": 1821,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 1704.67,
"p50": 1106.16,
"p90": 1558.94,
"p99": 1681.95,
"p999": 1694.46
},
"ok": 2179,
"per_op": {
"history": {
"ok": 1410,
"p50_ms": 1197.31,
"p99_ms": 1683.7
},
"me": {
"ok": 400,
"p50_ms": 47.32,
"p99_ms": 116.31
},
"post": {
"ok": 369,
"p50_ms": 1199.01,
"p99_ms": 1682.46
}
},
"rate": 400,
"service_ms": {
"p50": 1105.9,
"p99": 1681.27
},
"throughput_ok_rps": 217.9,
"resources": {
"cpu_percent_mean": {
"api": 88.6,
"lb": 8.6,
"loadgen-run-d4289cdc835a": 16.5,
"postgres": 15.0,
"redis": 4.3
},
"mem_mib_max": {
"api": 55.5,
"lb": 8.1,
"loadgen-run-d4289cdc835a": 17.9,
"postgres": 107.4,
"redis": 5.3
},
"lb_queue_max": 0,
"lb_sessions_mean": 232.7,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2817
},
"error_ratio": 0.6124,
"errors": 2817,
"event": "step",
"label": "pool-1x1.00-p20",
"latency_ms": {
"max": 2241.17,
"p50": 1298.81,
"p90": 1741.3,
"p99": 2197.71,
"p999": 2238.3
},
"ok": 1783,
"per_op": {
"history": {
"ok": 1156,
"p50_ms": 1392.51,
"p99_ms": 2197.79
},
"me": {
"ok": 295,
"p50_ms": 66.89,
"p99_ms": 1300.77
},
"post": {
"ok": 332,
"p50_ms": 1427.86,
"p99_ms": 2226.95
}
},
"rate": 460,
"service_ms": {
"p50": 1298.29,
"p99": 2197.03
},
"throughput_ok_rps": 178.3,
"resources": {
"cpu_percent_mean": {
"api": 88.7,
"lb": 10.3,
"loadgen-run-d4289cdc835a": 18.8,
"postgres": 18.7,
"redis": 3.3
},
"mem_mib_max": {
"api": 55.5,
"lb": 8.0,
"loadgen-run-d4289cdc835a": 19.9,
"postgres": 114.3,
"redis": 4.2
},
"lb_queue_max": 0,
"lb_sessions_mean": 234.3,
"pg_active_mean": 0.2,
"pg_wait_events_mean": {
"CPU": 0.1,
"Client:ClientRead": 0.1,
"IO:WALWrite": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 21
},
"slo_ok": false
}
],
"pool_before": {
"api-7df7c520": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 76,
"requests_wait_ms": 81,
"usage_ms": 53,
"pool_min": 1,
"pool_max": 20,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
}
},
"pool_after": {
"api-7df7c520": {
"pool": {
"connections_num": 20,
"requests_num": 33619,
"requests_queued": 14834,
"connections_ms": 1282,
"requests_wait_ms": 9575838,
"usage_ms": 1121363,
"requests_errors": 7049,
"pool_min": 1,
"pool_max": 20,
"pool_size": 20,
"pool_available": 20,
"requests_waiting": 0
},
"cache": {
"hits": 32179,
"misses": 316,
"errors": 1
},
"requests": 35397
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+670
View File
@@ -0,0 +1,670 @@
{
"config": "pool-1x1.00-p40",
"repeat": 1,
"series": "pool",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": 40,
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T09:00:19+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 200,
"peak_ok_rps": 280,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 24.88,
"p50": 5.11,
"p90": 7.57,
"p99": 18.06,
"p999": 24.88
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 4.96,
"p99_ms": 18.79
},
"me": {
"ok": 40,
"p50_ms": 2.38,
"p99_ms": 17.98
},
"post": {
"ok": 80,
"p50_ms": 6.21,
"p99_ms": 13.85
}
},
"rate": 40,
"service_ms": {
"p50": 4.5,
"p99": 17.45
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 16.2,
"lb": 2.2,
"loadgen-run-dfb98a6b4250": 2.7,
"postgres": 5.3,
"redis": 2.0
},
"mem_mib_max": {
"api": 31.5,
"lb": 4.4,
"loadgen-run-dfb98a6b4250": 5.9,
"postgres": 75.5,
"redis": 3.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 7
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 78.53,
"p50": 5.2,
"p90": 8.48,
"p99": 34.62,
"p999": 78.53
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 5.08,
"p99_ms": 41.45
},
"me": {
"ok": 80,
"p50_ms": 2.47,
"p99_ms": 76.29
},
"post": {
"ok": 160,
"p50_ms": 6.37,
"p99_ms": 21.92
}
},
"rate": 80,
"service_ms": {
"p50": 4.65,
"p99": 34.2
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 29.3,
"lb": 3.5,
"loadgen-run-dfb98a6b4250": 5.3,
"postgres": 8.1,
"redis": 3.1
},
"mem_mib_max": {
"api": 32.0,
"lb": 4.4,
"loadgen-run-dfb98a6b4250": 6.2,
"postgres": 79.3,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 7.2
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 46.75,
"p50": 5.16,
"p90": 10.09,
"p99": 26.59,
"p999": 45.19
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.13,
"p99_ms": 26.39
},
"me": {
"ok": 120,
"p50_ms": 2.35,
"p99_ms": 6.55
},
"post": {
"ok": 240,
"p50_ms": 5.95,
"p99_ms": 35.64
}
},
"rate": 120,
"service_ms": {
"p50": 4.56,
"p99": 26.25
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 42.7,
"lb": 4.3,
"loadgen-run-dfb98a6b4250": 7.1,
"postgres": 10.3,
"redis": 3.5
},
"mem_mib_max": {
"api": 32.5,
"lb": 4.4,
"loadgen-run-dfb98a6b4250": 6.7,
"postgres": 80.7,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.6,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"IO:WALSync": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 9
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 85.67,
"p50": 5.44,
"p90": 17.43,
"p99": 55.11,
"p999": 85.46
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.55,
"p99_ms": 58.98
},
"me": {
"ok": 160,
"p50_ms": 2.35,
"p99_ms": 12.67
},
"post": {
"ok": 320,
"p50_ms": 5.62,
"p99_ms": 51.19
}
},
"rate": 160,
"service_ms": {
"p50": 4.85,
"p99": 52.82
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 57.6,
"lb": 5.3,
"loadgen-run-dfb98a6b4250": 8.8,
"postgres": 12.7,
"redis": 3.4
},
"mem_mib_max": {
"api": 33.8,
"lb": 4.5,
"loadgen-run-dfb98a6b4250": 6.9,
"postgres": 88.2,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 1.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 11.2
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 252.61,
"p50": 6.5,
"p90": 51.08,
"p99": 190.85,
"p999": 243.4
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 7.91,
"p99_ms": 183.04
},
"me": {
"ok": 200,
"p50_ms": 2.07,
"p99_ms": 102.47
},
"post": {
"ok": 400,
"p50_ms": 6.08,
"p99_ms": 230.48
}
},
"rate": 200,
"service_ms": {
"p50": 5.99,
"p99": 190.26
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 70.2,
"lb": 5.7,
"loadgen-run-dfb98a6b4250": 10.0,
"postgres": 15.8,
"redis": 4.0
},
"mem_mib_max": {
"api": 45.9,
"lb": 5.9,
"loadgen-run-dfb98a6b4250": 7.5,
"postgres": 109.4,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 3.1,
"pg_active_mean": 0.33,
"pg_wait_events_mean": {
"CPU": 0.33
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 15.7
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 711.16,
"p50": 18.48,
"p90": 363.61,
"p99": 537.32,
"p999": 692.24
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 22.52,
"p99_ms": 667.34
},
"me": {
"ok": 240,
"p50_ms": 2.7,
"p99_ms": 393.19
},
"post": {
"ok": 480,
"p50_ms": 22.17,
"p99_ms": 520.32
}
},
"rate": 240,
"service_ms": {
"p50": 17.84,
"p99": 536.87
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 80.9,
"lb": 6.6,
"loadgen-run-dfb98a6b4250": 12.4,
"postgres": 20.1,
"redis": 4.1
},
"mem_mib_max": {
"api": 43.9,
"lb": 6.0,
"loadgen-run-dfb98a6b4250": 9.4,
"postgres": 144.8,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 17.7,
"pg_active_mean": 0.11,
"pg_wait_events_mean": {
"CPU": 0.11,
"Timeout:VacuumDelay": 0.11
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 25.8
},
"slo_ok": false
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 518.91,
"p50": 219.27,
"p90": 416.78,
"p99": 473.03,
"p999": 495.86
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 253.72,
"p99_ms": 458.22
},
"me": {
"ok": 280,
"p50_ms": 45.34,
"p99_ms": 164.64
},
"post": {
"ok": 560,
"p50_ms": 277.69,
"p99_ms": 488.27
}
},
"rate": 280,
"service_ms": {
"p50": 219.03,
"p99": 469.76
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 87.5,
"lb": 6.6,
"loadgen-run-dfb98a6b4250": 14.0,
"postgres": 20.0,
"redis": 3.9
},
"mem_mib_max": {
"api": 49.9,
"lb": 6.3,
"loadgen-run-dfb98a6b4250": 9.8,
"postgres": 160.8,
"redis": 4.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 57.2,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 39.4
},
"slo_ok": false
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 986
},
"error_ratio": 0.3081,
"errors": 986,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 3113.46,
"p50": 1173.18,
"p90": 1641.36,
"p99": 2305.47,
"p999": 3110.62
},
"ok": 2214,
"per_op": {
"history": {
"ok": 1528,
"p50_ms": 1274.1,
"p99_ms": 2854.56
},
"me": {
"ok": 320,
"p50_ms": 85.55,
"p99_ms": 1344.67
},
"post": {
"ok": 366,
"p50_ms": 1298.79,
"p99_ms": 1894.6
}
},
"rate": 320,
"service_ms": {
"p50": 1172.44,
"p99": 2304.66
},
"throughput_ok_rps": 221.4,
"resources": {
"cpu_percent_mean": {
"api": 92.3,
"lb": 8.1,
"loadgen-run-dfb98a6b4250": 15.8,
"postgres": 16.2,
"redis": 4.4
},
"mem_mib_max": {
"api": 45.6,
"lb": 8.4,
"loadgen-run-dfb98a6b4250": 19.6,
"postgres": 163.0,
"redis": 5.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 261,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 42
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1255
},
"error_ratio": 0.3486,
"errors": 1255,
"event": "step",
"label": "pool-1x1.00-p40",
"latency_ms": {
"max": 2909.46,
"p50": 1044.08,
"p90": 1533.05,
"p99": 1924.77,
"p999": 2721.03
},
"ok": 2345,
"per_op": {
"history": {
"ok": 1623,
"p50_ms": 1079.4,
"p99_ms": 1913.46
},
"me": {
"ok": 338,
"p50_ms": 74.83,
"p99_ms": 1384.79
},
"post": {
"ok": 384,
"p50_ms": 1097.97,
"p99_ms": 1953.4
}
},
"rate": 360,
"service_ms": {
"p50": 1043.49,
"p99": 1924.11
},
"throughput_ok_rps": 234.5,
"resources": {
"cpu_percent_mean": {
"api": 91.8,
"lb": 8.5,
"loadgen-run-dfb98a6b4250": 16.3,
"postgres": 18.4,
"redis": 4.5
},
"mem_mib_max": {
"api": 57.0,
"lb": 8.8,
"loadgen-run-dfb98a6b4250": 19.6,
"postgres": 163.8,
"redis": 4.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 264.7,
"pg_active_mean": 0.1,
"pg_wait_events_mean": {
"Client:ClientRead": 0.1
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 42
},
"slo_ok": false
}
],
"pool_before": {
"api-86705189": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 95,
"requests_wait_ms": 94,
"usage_ms": 64,
"pool_min": 1,
"pool_max": 40,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
}
},
"pool_after": {
"api-86705189": {
"pool": {
"connections_num": 40,
"requests_num": 22545,
"requests_queued": 7424,
"connections_ms": 4509,
"requests_wait_ms": 4677487,
"usage_ms": 1169861,
"requests_errors": 2332,
"pool_min": 1,
"pool_max": 40,
"pool_size": 40,
"pool_available": 40,
"requests_waiting": 0
},
"cache": {
"hits": 23399,
"misses": 280,
"errors": 1
},
"requests": 24198
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+793
View File
@@ -0,0 +1,793 @@
{
"config": "pool-1x1.00-p5",
"repeat": 1,
"series": "pool",
"params": {
"API_CPUSET": "0",
"AUX_CPUSET": "1",
"DB_POOL_MAX": 5,
"SESSION_CACHE": "redis",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s",
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"KDF_CONCURRENCY": 4
},
"measured_at": "2026-11-03T08:54:01+00:00",
"slo": {
"p99_ms": 250.0,
"error_ratio": 0.01
},
"capacity_at_slo_rps": 280,
"peak_ok_rps": 320,
"steps": [
{
"attempted": 400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 33.44,
"p50": 5.1,
"p90": 8.04,
"p99": 13.8,
"p999": 33.44
},
"ok": 400,
"per_op": {
"history": {
"ok": 280,
"p50_ms": 4.98,
"p99_ms": 11.95
},
"me": {
"ok": 40,
"p50_ms": 2.64,
"p99_ms": 13.8
},
"post": {
"ok": 80,
"p50_ms": 6.25,
"p99_ms": 33.44
}
},
"rate": 40,
"service_ms": {
"p50": 4.56,
"p99": 13.34
},
"throughput_ok_rps": 40,
"resources": {
"cpu_percent_mean": {
"api": 16.1,
"lb": 2.2,
"loadgen-run-b334fcc6230c": 2.6,
"postgres": 4.5,
"redis": 2.6
},
"mem_mib_max": {
"api": 31.6,
"lb": 4.4,
"loadgen-run-b334fcc6230c": 6.1,
"postgres": 53.4,
"redis": 3.5
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 4
},
"slo_ok": true
},
{
"attempted": 800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 35.89,
"p50": 4.73,
"p90": 6.84,
"p99": 16.1,
"p999": 35.89
},
"ok": 800,
"per_op": {
"history": {
"ok": 560,
"p50_ms": 4.61,
"p99_ms": 16.1
},
"me": {
"ok": 80,
"p50_ms": 2.26,
"p99_ms": 15.26
},
"post": {
"ok": 160,
"p50_ms": 5.84,
"p99_ms": 23.37
}
},
"rate": 80,
"service_ms": {
"p50": 4.11,
"p99": 15.45
},
"throughput_ok_rps": 80,
"resources": {
"cpu_percent_mean": {
"api": 26.8,
"lb": 3.3,
"loadgen-run-b334fcc6230c": 4.7,
"postgres": 6.9,
"redis": 2.8
},
"mem_mib_max": {
"api": 31.8,
"lb": 4.4,
"loadgen-run-b334fcc6230c": 6.3,
"postgres": 54.5,
"redis": 3.6
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 4
},
"slo_ok": true
},
{
"attempted": 1200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 112.91,
"p50": 5,
"p90": 8.32,
"p99": 55.97,
"p999": 99.69
},
"ok": 1200,
"per_op": {
"history": {
"ok": 840,
"p50_ms": 5.05,
"p99_ms": 66.08
},
"me": {
"ok": 120,
"p50_ms": 2.21,
"p99_ms": 24.33
},
"post": {
"ok": 240,
"p50_ms": 5.31,
"p99_ms": 43.99
}
},
"rate": 120,
"service_ms": {
"p50": 4.41,
"p99": 55.57
},
"throughput_ok_rps": 120,
"resources": {
"cpu_percent_mean": {
"api": 41.2,
"lb": 4.1,
"loadgen-run-b334fcc6230c": 6.8,
"postgres": 9.8,
"redis": 3.4
},
"mem_mib_max": {
"api": 39.5,
"lb": 4.5,
"loadgen-run-b334fcc6230c": 6.3,
"postgres": 57.6,
"redis": 4.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.3,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": true
},
{
"attempted": 1600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 127.66,
"p50": 5.19,
"p90": 12.29,
"p99": 81.13,
"p999": 115.15
},
"ok": 1600,
"per_op": {
"history": {
"ok": 1120,
"p50_ms": 5.23,
"p99_ms": 86.17
},
"me": {
"ok": 160,
"p50_ms": 2.17,
"p99_ms": 7.97
},
"post": {
"ok": 320,
"p50_ms": 5.52,
"p99_ms": 26.79
}
},
"rate": 160,
"service_ms": {
"p50": 4.64,
"p99": 80.4
},
"throughput_ok_rps": 160,
"resources": {
"cpu_percent_mean": {
"api": 54.7,
"lb": 5.2,
"loadgen-run-b334fcc6230c": 8.7,
"postgres": 12.0,
"redis": 3.7
},
"mem_mib_max": {
"api": 32.6,
"lb": 4.5,
"loadgen-run-b334fcc6230c": 6.8,
"postgres": 58.0,
"redis": 3.8
},
"lb_queue_max": 0,
"lb_sessions_mean": 0.8,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": true
},
{
"attempted": 2000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 169,
"p50": 8.92,
"p90": 45.38,
"p99": 148.97,
"p999": 162.73
},
"ok": 2000,
"per_op": {
"history": {
"ok": 1400,
"p50_ms": 11.07,
"p99_ms": 152.32
},
"me": {
"ok": 200,
"p50_ms": 2.52,
"p99_ms": 32.35
},
"post": {
"ok": 400,
"p50_ms": 6.59,
"p99_ms": 141.8
}
},
"rate": 200,
"service_ms": {
"p50": 8.36,
"p99": 148.57
},
"throughput_ok_rps": 200,
"resources": {
"cpu_percent_mean": {
"api": 74.3,
"lb": 6.2,
"loadgen-run-b334fcc6230c": 10.5,
"postgres": 18.1,
"redis": 3.7
},
"mem_mib_max": {
"api": 33.1,
"lb": 4.8,
"loadgen-run-b334fcc6230c": 7.0,
"postgres": 59.1,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 2.7,
"pg_active_mean": 0.22,
"pg_wait_events_mean": {
"CPU": 0.22
},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": true
},
{
"attempted": 2400,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 266.6,
"p50": 43.3,
"p90": 136.78,
"p99": 238.74,
"p999": 259.53
},
"ok": 2400,
"per_op": {
"history": {
"ok": 1680,
"p50_ms": 52.51,
"p99_ms": 239.03
},
"me": {
"ok": 240,
"p50_ms": 3.63,
"p99_ms": 64.38
},
"post": {
"ok": 480,
"p50_ms": 75.42,
"p99_ms": 243.05
}
},
"rate": 240,
"service_ms": {
"p50": 42.7,
"p99": 237.65
},
"throughput_ok_rps": 240,
"resources": {
"cpu_percent_mean": {
"api": 85.0,
"lb": 6.6,
"loadgen-run-b334fcc6230c": 12.0,
"postgres": 16.2,
"redis": 4.1
},
"mem_mib_max": {
"api": 46.0,
"lb": 4.8,
"loadgen-run-b334fcc6230c": 7.5,
"postgres": 59.7,
"redis": 3.9
},
"lb_queue_max": 0,
"lb_sessions_mean": 16.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": true
},
{
"attempted": 2800,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 260.24,
"p50": 50.89,
"p90": 173.45,
"p99": 240.32,
"p999": 254.25
},
"ok": 2800,
"per_op": {
"history": {
"ok": 1960,
"p50_ms": 58.88,
"p99_ms": 244.24
},
"me": {
"ok": 280,
"p50_ms": 4.89,
"p99_ms": 76.83
},
"post": {
"ok": 560,
"p50_ms": 72.52,
"p99_ms": 229.92
}
},
"rate": 280,
"service_ms": {
"p50": 49.97,
"p99": 239.93
},
"throughput_ok_rps": 280,
"resources": {
"cpu_percent_mean": {
"api": 89.0,
"lb": 6.8,
"loadgen-run-b334fcc6230c": 13.4,
"postgres": 17.8,
"redis": 4.0
},
"mem_mib_max": {
"api": 45.8,
"lb": 5.1,
"loadgen-run-b334fcc6230c": 7.7,
"postgres": 60.2,
"redis": 3.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 16.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": true
},
{
"attempted": 3200,
"dropped": 0,
"duration_s": 10,
"error_kinds": {},
"error_ratio": 0,
"errors": 0,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 417.82,
"p50": 53.71,
"p90": 209.84,
"p99": 268.52,
"p999": 395.31
},
"ok": 3200,
"per_op": {
"history": {
"ok": 2240,
"p50_ms": 59.81,
"p99_ms": 257.43
},
"me": {
"ok": 320,
"p50_ms": 4.73,
"p99_ms": 49.46
},
"post": {
"ok": 640,
"p50_ms": 74.29,
"p99_ms": 280.87
}
},
"rate": 320,
"service_ms": {
"p50": 53.03,
"p99": 267.56
},
"throughput_ok_rps": 320,
"resources": {
"cpu_percent_mean": {
"api": 90.3,
"lb": 6.5,
"loadgen-run-b334fcc6230c": 13.0,
"postgres": 17.7,
"redis": 3.5
},
"mem_mib_max": {
"api": 38.8,
"lb": 5.3,
"loadgen-run-b334fcc6230c": 7.9,
"postgres": 61.0,
"redis": 5.4
},
"lb_queue_max": 0,
"lb_sessions_mean": 22.4,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": false
},
{
"attempted": 3600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 663
},
"error_ratio": 0.1842,
"errors": 663,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 2043.4,
"p50": 685.54,
"p90": 940.61,
"p99": 1494.15,
"p999": 2032.18
},
"ok": 2937,
"per_op": {
"history": {
"ok": 2047,
"p50_ms": 699.36,
"p99_ms": 1513.61
},
"me": {
"ok": 359,
"p50_ms": 10.2,
"p99_ms": 1176.8
},
"post": {
"ok": 531,
"p50_ms": 710.38,
"p99_ms": 1384.34
}
},
"rate": 360,
"service_ms": {
"p50": 684.96,
"p99": 1493.45
},
"throughput_ok_rps": 293.7,
"resources": {
"cpu_percent_mean": {
"api": 90.9,
"lb": 7.6,
"loadgen-run-b334fcc6230c": 15.8,
"postgres": 15.7,
"redis": 4.1
},
"mem_mib_max": {
"api": 51.2,
"lb": 8.5,
"loadgen-run-b334fcc6230c": 16.8,
"postgres": 62.4,
"redis": 4.0
},
"lb_queue_max": 0,
"lb_sessions_mean": 205.1,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": false
},
{
"attempted": 4000,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 1441
},
"error_ratio": 0.3603,
"errors": 1441,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 1320.84,
"p50": 894.91,
"p90": 1148.13,
"p99": 1290.83,
"p999": 1317.19
},
"ok": 2559,
"per_op": {
"history": {
"ok": 1763,
"p50_ms": 911.45,
"p99_ms": 1290.57
},
"me": {
"ok": 400,
"p50_ms": 11.26,
"p99_ms": 68.81
},
"post": {
"ok": 396,
"p50_ms": 927.42,
"p99_ms": 1306.13
}
},
"rate": 400,
"service_ms": {
"p50": 894.37,
"p99": 1290.08
},
"throughput_ok_rps": 255.9,
"resources": {
"cpu_percent_mean": {
"api": 89.4,
"lb": 7.8,
"loadgen-run-b334fcc6230c": 15.6,
"postgres": 14.1,
"redis": 4.2
},
"mem_mib_max": {
"api": 43.8,
"lb": 8.1,
"loadgen-run-b334fcc6230c": 16.6,
"postgres": 61.7,
"redis": 4.1
},
"lb_queue_max": 0,
"lb_sessions_mean": 208.6,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": false
},
{
"attempted": 4600,
"dropped": 0,
"duration_s": 10,
"error_kinds": {
"503": 2702
},
"error_ratio": 0.5874,
"errors": 2702,
"event": "step",
"label": "pool-1x1.00-p5",
"latency_ms": {
"max": 2217.32,
"p50": 999.11,
"p90": 1518.48,
"p99": 2081.03,
"p999": 2215.87
},
"ok": 1898,
"per_op": {
"history": {
"ok": 1193,
"p50_ms": 1119.84,
"p99_ms": 2105.92
},
"me": {
"ok": 391,
"p50_ms": 13.79,
"p99_ms": 762.31
},
"post": {
"ok": 314,
"p50_ms": 1244.02,
"p99_ms": 2024.97
}
},
"rate": 460,
"service_ms": {
"p50": 998.63,
"p99": 2080.44
},
"throughput_ok_rps": 189.8,
"resources": {
"cpu_percent_mean": {
"api": 86.8,
"lb": 10.1,
"loadgen-run-b334fcc6230c": 18.0,
"postgres": 14.3,
"redis": 3.8
},
"mem_mib_max": {
"api": 51.9,
"lb": 8.3,
"loadgen-run-b334fcc6230c": 23.6,
"postgres": 62.2,
"redis": 4.7
},
"lb_queue_max": 0,
"lb_sessions_mean": 210.5,
"pg_active_mean": 0,
"pg_wait_events_mean": {},
"api_cfs_throttled_period_ratio": 0.0,
"pg_connections_mean": 5
},
"slo_ok": false
}
],
"pool_before": {
"api-b0d1c3a3": {
"pool": {
"connections_num": 3,
"requests_num": 20,
"requests_queued": 3,
"connections_ms": 63,
"requests_wait_ms": 72,
"usage_ms": 56,
"pool_min": 1,
"pool_max": 5,
"pool_size": 3,
"pool_available": 3,
"requests_waiting": 0
},
"cache": {
"hits": 0,
"misses": 0,
"errors": 0
},
"requests": 9
}
},
"pool_after": {
"api-b0d1c3a3": {
"pool": {
"connections_num": 5,
"requests_num": 32970,
"requests_queued": 16715,
"connections_ms": 199,
"requests_wait_ms": 7636714,
"usage_ms": 289126,
"requests_errors": 5343,
"pool_min": 1,
"pool_max": 5,
"pool_size": 5,
"pool_available": 5,
"requests_waiting": 0
},
"cache": {
"hits": 33637,
"misses": 218,
"errors": 1
},
"requests": 35404
}
},
"loadgen": {
"users": 48,
"seed": 10,
"duration_s": 10,
"warmup_s": 3,
"mix": "history=70,post=20,me=10",
"model": "open-loop, latency from scheduled time"
}
}
+656
View File
@@ -0,0 +1,656 @@
{
"admission-1x1.00-mc10": {
"series": "admission",
"repeats": 3,
"median": {
"capacity_slo": 280,
"capacity_err1": 359.7,
"peak": 359.7
},
"values": {
"capacity_slo": [
320,
280,
240
],
"capacity_err1": [
360,
359.7,
320
],
"peak": [
383.7,
359.7,
339
]
},
"params": {
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": 10,
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 400,
"cpu_percent": {
"api": 92.8,
"lb": 7.8,
"loadgen-run-3312d64e3026": 17.4,
"postgres": 19.9,
"redis": 3.2
},
"cfs_throttled": 0.0,
"lb_queue_max": 799,
"pg_connections": 20
}
},
"admission-1x1.00-mc10-q250": {
"series": "admission",
"repeats": 1,
"median": {
"capacity_slo": 200,
"capacity_err1": 239.4,
"peak": 303.5
},
"values": {
"capacity_slo": [
200
],
"capacity_err1": [
239.4
],
"peak": [
303.5
]
},
"params": {
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": 10,
"LB_QUEUE_TIMEOUT": "250ms"
},
"at_peak": {
"rate": 360,
"cpu_percent": {
"api": 91.5,
"lb": 8.2,
"loadgen-run-57259902c18b": 16.9,
"postgres": 22.9,
"redis": 4.1
},
"cfs_throttled": 0.0,
"lb_queue_max": 90,
"pg_connections": 15
}
},
"admission-1x1.00-mc20": {
"series": "admission",
"repeats": 1,
"median": {
"capacity_slo": 200,
"capacity_err1": 320,
"peak": 326.3
},
"values": {
"capacity_slo": [
200
],
"capacity_err1": [
320
],
"peak": [
326.3
]
},
"params": {
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": 20,
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 360,
"cpu_percent": {
"api": 91.2,
"lb": 8.6,
"loadgen-run-6ca2812f958b": 18.2,
"postgres": 20.7,
"redis": 3.7
},
"cfs_throttled": 0.0,
"lb_queue_max": 719,
"pg_connections": 14
}
},
"admission-2x0.50-mc10": {
"series": "admission",
"repeats": 1,
"median": {
"capacity_slo": 200,
"capacity_err1": 280,
"peak": 306.3
},
"values": {
"capacity_slo": [
200
],
"capacity_err1": [
280
],
"peak": [
306.3
]
},
"params": {
"API_REPLICAS": 2,
"API_CPUS": "0.5",
"API_MEMORY": "256M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": 10,
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 320,
"cpu_percent": {
"api": 92.3,
"lb": 7.2,
"loadgen-run-65398839d17b": 15.9,
"postgres": 19.1,
"redis": 3.5
},
"cfs_throttled": 0.006,
"lb_queue_max": 640,
"pg_connections": 26
}
},
"fixed-1x1.00": {
"series": "fixed-cpu",
"repeats": 3,
"median": {
"capacity_slo": 240,
"capacity_err1": 280,
"peak": 280
},
"values": {
"capacity_slo": [
280,
240,
240
],
"capacity_err1": [
280,
280,
280
],
"peak": [
280,
316.7,
280
]
},
"params": {
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 280,
"cpu_percent": {
"api": 85.2,
"lb": 6.3,
"loadgen-run-03af3ac2a6d0": 11.9,
"postgres": 18.7,
"redis": 3.7
},
"cfs_throttled": 0.0,
"lb_queue_max": 0,
"pg_connections": 17
}
},
"fixed-2x0.50": {
"series": "fixed-cpu",
"repeats": 3,
"median": {
"capacity_slo": 200,
"capacity_err1": 200,
"peak": 234.7
},
"values": {
"capacity_slo": [
120,
200,
200
],
"capacity_err1": [
200,
240,
200
],
"peak": [
234.7,
240,
234.3
]
},
"params": {
"API_REPLICAS": 2,
"API_CPUS": "0.5",
"API_MEMORY": "256M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 240,
"cpu_percent": {
"api": 78.8,
"lb": 6.6,
"loadgen-run-bc33f10e790a": 12.4,
"postgres": 17.0,
"redis": 3.8
},
"cfs_throttled": 0.223,
"lb_queue_max": 0,
"pg_connections": 28
}
},
"fixed-4x0.25": {
"series": "fixed-cpu",
"repeats": 3,
"median": {
"capacity_slo": 160,
"capacity_err1": 200,
"peak": 200
},
"values": {
"capacity_slo": [
160,
120,
160
],
"capacity_err1": [
200,
200,
160
],
"peak": [
240.6,
200,
194.4
]
},
"params": {
"API_REPLICAS": 4,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 280,
"cpu_percent": {
"api": 87.4,
"lb": 6.6,
"loadgen-run-36ba2530fef5": 12.8,
"postgres": 17.5,
"redis": 3.5
},
"cfs_throttled": 0.525,
"lb_queue_max": 0,
"pg_connections": 49
}
},
"growth-1x0.25": {
"series": "growth",
"repeats": 1,
"median": {
"capacity_slo": 40,
"capacity_err1": 80,
"peak": 83.5
},
"values": {
"capacity_slo": [
40
],
"capacity_err1": [
80
],
"peak": [
83.5
]
},
"params": {
"API_REPLICAS": 1,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 120,
"cpu_percent": {
"api": 25.7,
"lb": 4.1,
"loadgen-run-083085fa5191": 8.4,
"postgres": 7.4,
"redis": 2.6
},
"cfs_throttled": 1.0,
"lb_queue_max": 0,
"pg_connections": 13
}
},
"growth-2x0.25": {
"series": "growth",
"repeats": 1,
"median": {
"capacity_slo": 80,
"capacity_err1": 159.8,
"peak": 159.8
},
"values": {
"capacity_slo": [
80
],
"capacity_err1": [
159.8
],
"peak": [
159.8
]
},
"params": {
"API_REPLICAS": 2,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 160,
"cpu_percent": {
"api": 42.6,
"lb": 5.0,
"loadgen-run-7ce67b9d067d": 9.4,
"postgres": 11.5,
"redis": 2.6
},
"cfs_throttled": 0.918,
"lb_queue_max": 0,
"pg_connections": 24
}
},
"growth-3x0.25": {
"series": "growth",
"repeats": 1,
"median": {
"capacity_slo": 80,
"capacity_err1": 160,
"peak": 177.8
},
"values": {
"capacity_slo": [
80
],
"capacity_err1": [
160
],
"peak": [
177.8
]
},
"params": {
"API_REPLICAS": 3,
"API_CPUS": "0.25",
"API_MEMORY": "128M",
"API_CPUSET": "0",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 240,
"cpu_percent": {
"api": 70.6,
"lb": 6.7,
"loadgen-run-69e377d75d2b": 12.0,
"postgres": 13.5,
"redis": 3.5
},
"cfs_throttled": 0.912,
"lb_queue_max": 0,
"pg_connections": 35
}
},
"machine-2x1.00": {
"series": "machine",
"repeats": 1,
"median": {
"capacity_slo": 320,
"capacity_err1": 320,
"peak": 320
},
"values": {
"capacity_slo": [
320
],
"capacity_err1": [
320
],
"peak": [
320
]
},
"params": {
"API_REPLICAS": 2,
"API_CPUS": "1.0",
"API_MEMORY": "256M",
"API_CPUSET": "0-1",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 320,
"cpu_percent": {
"api": 111.0,
"lb": 7.3,
"loadgen-run-0cfe7ec82689": 17.4,
"postgres": 19.3,
"redis": 3.8
},
"cfs_throttled": 0.0,
"lb_queue_max": 0,
"pg_connections": 20
}
},
"machine-4x1.00": {
"series": "machine",
"repeats": 1,
"median": {
"capacity_slo": 240,
"capacity_err1": 280,
"peak": 311.5
},
"values": {
"capacity_slo": [
240
],
"capacity_err1": [
280
],
"peak": [
311.5
]
},
"params": {
"API_REPLICAS": 4,
"API_CPUS": "1.0",
"API_MEMORY": "256M",
"API_CPUSET": "0-1",
"DB_POOL_MAX": "10",
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 320,
"cpu_percent": {
"api": 119.8,
"lb": 8.8,
"loadgen-run-7cfbba2a71d5": 17.5,
"postgres": 21.3,
"redis": 4.0
},
"cfs_throttled": 0.003,
"lb_queue_max": 0,
"pg_connections": 41
}
},
"pool-1x1.00-p20": {
"series": "pool",
"repeats": 1,
"median": {
"capacity_slo": 200,
"capacity_err1": 280,
"peak": 280
},
"values": {
"capacity_slo": [
200
],
"capacity_err1": [
280
],
"peak": [
280
]
},
"params": {
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"API_CPUSET": "0",
"DB_POOL_MAX": 20,
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 280,
"cpu_percent": {
"api": 89.7,
"lb": 6.8,
"loadgen-run-d4289cdc835a": 14.2,
"postgres": 18.6,
"redis": 3.9
},
"cfs_throttled": 0.0,
"lb_queue_max": 0,
"pg_connections": 21
}
},
"pool-1x1.00-p40": {
"series": "pool",
"repeats": 1,
"median": {
"capacity_slo": 200,
"capacity_err1": 280,
"peak": 280
},
"values": {
"capacity_slo": [
200
],
"capacity_err1": [
280
],
"peak": [
280
]
},
"params": {
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"API_CPUSET": "0",
"DB_POOL_MAX": 40,
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 280,
"cpu_percent": {
"api": 87.5,
"lb": 6.6,
"loadgen-run-dfb98a6b4250": 14.0,
"postgres": 20.0,
"redis": 3.9
},
"cfs_throttled": 0.0,
"lb_queue_max": 0,
"pg_connections": 39.4
}
},
"pool-1x1.00-p5": {
"series": "pool",
"repeats": 1,
"median": {
"capacity_slo": 280,
"capacity_err1": 320,
"peak": 320
},
"values": {
"capacity_slo": [
280
],
"capacity_err1": [
320
],
"peak": [
320
]
},
"params": {
"API_REPLICAS": 1,
"API_CPUS": "1.0",
"API_MEMORY": "512M",
"API_CPUSET": "0",
"DB_POOL_MAX": 5,
"LB_SERVER_MAXCONN": "0",
"LB_QUEUE_TIMEOUT": "2s"
},
"at_peak": {
"rate": 320,
"cpu_percent": {
"api": 90.3,
"lb": 6.5,
"loadgen-run-b334fcc6230c": 13.0,
"postgres": 17.7,
"redis": 3.5
},
"cfs_throttled": 0.0,
"lb_queue_max": 0,
"pg_connections": 5
}
}
}
+17
View File
@@ -0,0 +1,17 @@
| Конфигурация | Серия | Реплики × CPU, RAM | Пул / LB | Ёмкость при SLO, rps | Макс. при ошибках ≤1 %, rps | Пик полезной, rps | Повторов |
|---|---|---|---|---|---|---|---|
| admission-1x1.00-mc10 | admission | 1×1.0 CPU, 512M | pool 10, maxconn 10, queue 2s | 320/280/240 | 360/359.7/320 | 383.7/359.7/339 | 3 |
| admission-1x1.00-mc10-q250 | admission | 1×1.0 CPU, 512M | pool 10, maxconn 10, queue 250ms | 200 | 239.4 | 303.5 | 1 |
| admission-1x1.00-mc20 | admission | 1×1.0 CPU, 512M | pool 10, maxconn 20, queue 2s | 200 | 320 | 326.3 | 1 |
| admission-2x0.50-mc10 | admission | 2×0.5 CPU, 256M | pool 10, maxconn 10, queue 2s | 200 | 280 | 306.3 | 1 |
| fixed-1x1.00 | fixed-cpu | 1×1.0 CPU, 512M | pool 10, maxconn 0, queue 2s | 280/240/240 | 280/280/280 | 280/316.7/280 | 3 |
| fixed-2x0.50 | fixed-cpu | 2×0.5 CPU, 256M | pool 10, maxconn 0, queue 2s | 120/200/200 | 200/240/200 | 234.7/240/234.3 | 3 |
| fixed-4x0.25 | fixed-cpu | 4×0.25 CPU, 128M | pool 10, maxconn 0, queue 2s | 160/120/160 | 200/200/160 | 240.6/200/194.4 | 3 |
| growth-1x0.25 | growth | 1×0.25 CPU, 128M | pool 10, maxconn 0, queue 2s | 40 | 80 | 83.5 | 1 |
| growth-2x0.25 | growth | 2×0.25 CPU, 128M | pool 10, maxconn 0, queue 2s | 80 | 159.8 | 159.8 | 1 |
| growth-3x0.25 | growth | 3×0.25 CPU, 128M | pool 10, maxconn 0, queue 2s | 80 | 160 | 177.8 | 1 |
| machine-2x1.00 | machine | 2×1.0 CPU, 256M | pool 10, maxconn 0, queue 2s | 320 | 320 | 320 | 1 |
| machine-4x1.00 | machine | 4×1.0 CPU, 256M | pool 10, maxconn 0, queue 2s | 240 | 280 | 311.5 | 1 |
| pool-1x1.00-p20 | pool | 1×1.0 CPU, 512M | pool 20, maxconn 0, queue 2s | 200 | 280 | 280 | 1 |
| pool-1x1.00-p40 | pool | 1×1.0 CPU, 512M | pool 40, maxconn 0, queue 2s | 200 | 280 | 280 | 1 |
| pool-1x1.00-p5 | pool | 1×1.0 CPU, 512M | pool 5, maxconn 0, queue 2s | 280 | 320 | 320 | 1 |
+87
View File
@@ -0,0 +1,87 @@
# HAProxy 3.0 — единственная публичная точка входа lab3.
# Бэкенды находятся через DNS Docker (имя сервиса `api` → IP всех реплик), без sticky sessions.
#
# Обнаружение отказа реплики:
# * активная проверка GET /health/ready: inter 1s, после первой ошибки fastinter 500ms, fall 2,
# timeout check 300ms → худший случай обнаружения ≈ 1 + 0.3 + 0.5 + 0.3 ≈ 2.1 с;
# * пассивно: соединение отвергнуто → retry-on conn-failure + redispatch на другую реплику
# (только если запрос ещё не отправлен — безопасно и для неидемпотентных POST).
# * graceful shutdown реплики: SIGTERM → /health/ready = 503 на SHUTDOWN_DRAIN_SECONDS=3 с
# (> fall × fastinter) → LB выводит реплику до того, как она закроет listener.
global
log stdout format raw local0 info
maxconn 4096
defaults
mode http
log global
option httplog
option dontlognull
timeout connect 300ms # сеть Docker: connect < 1 мс; к «мёртвому» IP (kill) нет RST — ждём не дольше 300 мс
timeout check 300ms # проверка не висит 1 с на убитой реплике → быстрее fall
timeout client 30s
timeout server 15s # > REQUEST_TIMEOUT_SECONDS (10 с) приложения
timeout http-request 10s
timeout http-keep-alive 10s
timeout queue "${LB_QUEUE_TIMEOUT}" # сколько запрос может ждать свободный слот maxconn в очереди LB
retries 2
option redispatch 1
retry-on conn-failure # НЕ empty-response/response-timeout: иначе возможен повтор выполненного POST
unique-id-format %[uuid()] # request_id ответов самого LB: случайный, без IP/портов
default-server inter 1s fastinter 500ms downinter 2s fall 2 rise 2 init-addr none resolvers docker
# Ответы, созданные самим LB, — в формате ошибок API (X-Instance-Id у них нет, это допустимо).
http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Нет доступных реплик API","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] hdr Retry-After 1
http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Реплика API недоступна","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id]
http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Таймаут реплики API","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id]
resolvers docker
nameserver dns 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold obsolete 5s # исчезнувший из DNS контейнер убирается из слота через 5 с
accepted_payload_size 8192
frontend api
bind :8080
# Внутренняя диагностика реплик снаружи недоступна.
http-request return status 404 content-type application/json lf-string '{"error":{"code":"not_found","message":"Объект не найден","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] if { path_beg /internal }
default_backend api
frontend api_ttl
bind :8081
http-request return status 404 content-type application/json lf-string '{"error":{"code":"not_found","message":"Объект не найден","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] if { path_beg /internal }
default_backend api_ttl
frontend stats
bind :8404
stats enable
stats uri /stats
stats refresh 2s
stats show-legends
backend api
balance roundrobin # без stickiness: состояние не хранится в реплике
option httpchk
http-check send meth GET uri /health/ready
http-check expect status 200
http-reuse safe
# maxconn: не больше LB_SERVER_MAXCONN одновременных запросов на реплику, излишек ждёт в очереди LB
# (0 = без ограничения). Обоснование значения — трек 5.1 в REPORT.md.
server-template api 8 api:8080 check maxconn "${LB_SERVER_MAXCONN}"
# «Panic mode»: если readiness не проходит НИ У ОДНОЙ реплики (например, упала БД), запросы всё
# равно идут в живые процессы, и клиент получает ответ приложения (401 анониму, 503 unavailable
# в формате API, /health/live = 200), а не безликий 503 LB. Мёртвый процесс → conn-failure → 503 LB.
option allbackups
server-template apibk 8 api:8080 backup resolve-opts allow-dup-ip
backend api_ttl
balance roundrobin
option httpchk
http-check send meth GET uri /health/ready
http-check expect status 200
server-template ttl 2 api-ttl:8080 check
option allbackups
server-template ttlbk 2 api-ttl:8080 backup resolve-opts allow-dup-ip
+63
View File
@@ -0,0 +1,63 @@
-- 0001: базовая схема lab2. Источник истины — PostgreSQL.
-- Миграция выполняется раннером в одной транзакции и регистрируется в schema_migrations.
CREATE TABLE users (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
username text NOT NULL,
display_name text NOT NULL,
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT users_username_key UNIQUE (username),
CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'),
CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100),
CONSTRAINT users_password_hash_kdf CHECK (password_hash LIKE '$scrypt$%')
);
CREATE TABLE chats (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
title text NOT NULL,
created_by uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100)
);
-- Состав участников неизменен в обязательном API; position сохраняет порядок member_ids из запроса.
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users (id),
position smallint NOT NULL,
PRIMARY KEY (chat_id, user_id),
CONSTRAINT chat_members_position_key UNIQUE (chat_id, position),
CONSTRAINT chat_members_position_range CHECK (position BETWEEN 0 AND 99)
);
-- GET /chats: «чаты пользователя» — поиск по user_id.
CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY DEFAULT gen_random_uuid(),
chat_id uuid NOT NULL,
sender_id uuid NOT NULL,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
-- Отправитель обязан быть участником этого чата: инвариант защищён на уровне БД.
CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id)
REFERENCES chat_members (chat_id, user_id),
CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000)
);
-- История: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n.
-- Обоснование и EXPLAIN — REPORT.md и evidence/track2/.
CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id);
-- Серверные сессии. Хранится только SHA-256 от токена (токен — 256 бит случайности),
-- поэтому утечка дампа не даёт готовых Bearer-токенов.
CREATE TABLE sessions (
token_hash bytea PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users (id),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
expires_at timestamptz NOT NULL,
revoked_at timestamptz,
CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32),
CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at)
);
CREATE INDEX sessions_user_idx ON sessions (user_id);
CREATE INDEX sessions_expires_idx ON sessions (expires_at);
@@ -0,0 +1,8 @@
-- 0002: транзакционный outbox инвалидации кэша сессий (уровень 4).
-- Строка пишется в той же транзакции, что и revoked_at. Если Redis недоступен в момент logout,
-- фоновый процесс API повторит инвалидацию после восстановления Redis и только затем снова
-- начнёт читать сессии из кэша.
CREATE TABLE session_cache_invalidations (
token_hash bytea PRIMARY KEY REFERENCES sessions (token_hash) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
+17
View File
@@ -0,0 +1,17 @@
-- 0003 (lab3): общая идемпотентность POST /chats/{chat_id}/messages.
-- Область ключа = пользователь + метод + путь чата + ключ. Единственный идемпотентный endpoint
-- обязательного API — POST сообщения, поэтому метод и шаблон пути фиксированы, а «путь» = chat_id.
-- Миграция аддитивна (expand): код lab2 её не использует и продолжает работать при rolling update.
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users (id),
chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE,
key text NOT NULL,
request_hash bytea NOT NULL,
message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (user_id, chat_id, key),
CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
CONSTRAINT idempotency_request_hash_len CHECK (octet_length(request_hash) = 32)
);
-- Очистка ключей старше IDEMPOTENCY_TTL_SECONDS.
CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at);
+12
View File
@@ -0,0 +1,12 @@
# Runtime-зависимости API (точные версии, проверенные на стенде; см. REPORT.md).
starlette==1.7.0
anyio==4.15.1
idna==3.20
typing_extensions==4.16.0
uvicorn==0.54.0
h11==0.16.0
click==8.5.0
psycopg[binary]==3.2.10
psycopg-binary==3.2.10
psycopg-pool==3.2.6
redis==5.2.1
+33
View File
@@ -0,0 +1,33 @@
# Трек 1 «Цена масштабирования»: override к compose.yaml.
# docker compose --env-file .env -f compose.yaml -f research/compose.track1.yaml ...
# Изоляция по ядрам: реплики API — только на API_CPUSET (их суммарный бюджет CPU фиксирован
# квотой API_CPUS на реплику), всё остальное (LB, PostgreSQL, Redis, генератор) — на AUX_CPUSET,
# чтобы генератор и БД не отнимали CPU у измеряемого API и наоборот. Значения задаёт
# research/track1.py для каждой конфигурации.
services:
api:
cpuset: "${API_CPUSET:-0}"
environment:
DB_POOL_MAX: ${DB_POOL_MAX:-10}
SESSION_CACHE: ${SESSION_CACHE:-redis}
lb:
cpuset: "${AUX_CPUSET:-1}"
deploy:
resources:
limits: {cpus: "1.0", memory: 256M}
postgres:
cpuset: "${AUX_CPUSET:-1}"
redis:
cpuset: "${AUX_CPUSET:-1}"
migrate:
cpuset: "${AUX_CPUSET:-1}"
loadgen:
build: research/loadgen
image: mtusi-lab3-loadgen:local
profiles: [bench]
networks: [edge]
cpuset: "${AUX_CPUSET:-1}"
read_only: true
deploy:
resources:
limits: {cpus: "1.0", memory: 512M}
+41
View File
@@ -0,0 +1,41 @@
#!/bin/sh
# Демонстрация «expand-only» миграций при rolling update (уровень 4). Шаги:
# 1. новая версия образа содержит дополнительную миграцию 0004 (временная, создаётся этим скриптом);
# 2. контейнер новой версии, запущенный ДО миграции, не становится ready (схема отстаёт) —
# LB не отправил бы на него трафик;
# 3. migrate применяет 0004 один раз; старые реплики (знают 0001–0003) остаются ready: более новая
# схема для них допустима;
# 4. rolling update на новую версию под нагрузкой — research/lb_experiments.py rolling-update.
# Вывод — evidence/lb/expand-migration.txt. После шага 4 удалите временный $MIG (БД сохранит таблицу;
# код без файла 0004 остаётся совместимым — проверяются только известные ему миграции).
set -eu
cd "$(dirname "$0")/.."
. scripts/lib.sh
OUT=evidence/lb/expand-migration.txt
mkdir -p evidence/lb
MIG=migrations/0004_demo_expand.sql
VERSION=${APP_VERSION:-expand-demo}
BUILD_CMD=${BUILD_CMD:-"docker compose --env-file $ENV_FILE build --build-arg APP_VERSION=$VERSION api"}
cat > "$MIG" <<'SQL'
-- ДЕМО (expand): новая необязательная таблица; старый код её не использует.
CREATE TABLE IF NOT EXISTS rollout_demo (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, note text);
SQL
{
echo "# $(now) expand-migration demo, new version $VERSION"
APP_VERSION=$VERSION $BUILD_CMD >/dev/null
echo "== 1) new-version container before migrate:"
cid=$(dc run -d --rm --no-deps api serve)
sleep 6
echo "health/ready inside new container: $(docker exec "$cid" python -m messenger healthcheck && echo 200 || echo NOT-READY)"
docker logs "$cid" 2>&1 | grep -m1 'schema is behind' | sed 's/^/log: /' || true
docker stop -t 5 "$cid" >/dev/null
echo "== 2) migrate (one-shot):"
dc run --rm --no-deps migrate 2>/dev/null | tail -1
echo "== 3) old replicas after migrate:"
for c in $(dc ps -q api); do
printf '%s ready=%s version=%s\n' "$(docker inspect -f '{{.Name}}' "$c")" \
"$(docker exec "$c" python -m messenger healthcheck && echo yes || echo no)" \
"$(docker exec "$c" python -m messenger stats | python3 -c 'import json,sys;print(json.load(sys.stdin)["version"])')"
done
} > "$OUT" 2>&1
cat "$OUT"
+249
View File
@@ -0,0 +1,249 @@
#!/usr/bin/env python3
"""Эксперименты уровня 4 (stdlib, Python 3.9+): отказ каждой API-реплики под нагрузкой и rolling update.
python3 research/lb_experiments.py failover-series [--mode stop|kill|both]
python3 research/lb_experiments.py rolling-update
Нагрузка: closed-loop, W потоков через LB; каждый поток — свой пользователь, 1 POST сообщения
(Idempotency-Key; при ошибке/таймауте повтор С ТЕМ ЖЕ ключом и телом) на 3 GET истории.
Каждая попытка журналируется (время, операция, статус, X-Instance-Id, задержка). Итог — JSON в
evidence/lb/*.json + текстовая сводка. Токены не сохраняются.
"""
import argparse
import json
import os
import shlex
import statistics
import subprocess
import sys
import threading
import time
import uuid
from urllib.parse import quote
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "student-tests"))
from common import BASE_URL, Client, Failure, login, new_user # noqa: E402
OUT = ROOT / "evidence" / "lb"
DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env"))
def sh(*args, check=True):
return subprocess.run(list(args), cwd=ROOT, check=check, capture_output=True, text=True).stdout.strip()
def api_containers():
"""[(container_id, name, instance_id)] запущенных реплик сервиса api."""
res = []
for cid in sh(*DC, "ps", "-q", "api").split():
name = sh("docker", "inspect", "-f", "{{.Name}}", cid).lstrip("/")
iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False)
res.append((cid, name, iid))
return sorted(res, key=lambda x: x[1])
def wait_healthy(cid, timeout=60):
t = time.monotonic()
while time.monotonic() - t < timeout:
if sh("docker", "inspect", "-f", "{{.State.Health.Status}}", cid, check=False) == "healthy":
return time.monotonic() - t
time.sleep(0.25)
raise RuntimeError("container not healthy: " + cid)
class Load:
def __init__(self, workers, timeout=2.0):
self.workers, self.timeout = workers, timeout
self.events, self.lock, self.stop = [], threading.Lock(), threading.Event()
boot = Client(BASE_URL)
self.actors = []
for i in range(workers):
a = new_user(boot, "lb%d" % i); login(boot, a)
a["chat"] = boot.call("POST", "/api/v1/chats", 201, actor=a, schema="Chat",
body={"title": "lb", "member_ids": [a["id"]]})["json"]
self.actors.append(a)
self.t0 = None
self.confirmed = {} # message id -> text (подтверждено 201)
def record(self, op, ok, status, instance, latency):
with self.lock:
self.events.append((time.monotonic() - self.t0, op, ok, status, instance, latency))
def worker(self, a):
c = Client(BASE_URL, timeout=self.timeout)
path = "/api/v1/chats/" + a["chat"]["id"] + "/messages"
n, pending = 0, None
while not self.stop.is_set():
n += 1
if n % 4 == 1:
if pending is None:
pending = (uuid.uuid4().hex, "lb-%s-%d" % (a["id"][:8], n))
key, text = pending
op, kw = "post", dict(method="POST", path=path, expected=201, actor=a, body={"text": text},
headers={"Idempotency-Key": key}, schema="Message")
else:
op, kw = "get", dict(method="GET", path=path + "?limit=20", expected=200, actor=a, schema="MessagePage")
t = time.monotonic()
try:
r = c.call(**kw)
self.record(op, True, r["status"], r["headers"].get("X-Instance-Id"), time.monotonic() - t)
if op == "post":
with self.lock:
self.confirmed[r["json"]["id"]] = r["json"]["text"]
pending = None
except Failure as exc:
code = str(exc).rsplit("received ", 1)[-1] if "received" in str(exc) else "conn"
self.record(op, False, code, None, time.monotonic() - t)
time.sleep(0.05) # pending остаётся: повтор с тем же ключом и телом
def __enter__(self):
self.t0 = time.monotonic()
self.threads = [threading.Thread(target=self.worker, args=(a,), daemon=True) for a in self.actors]
[t.start() for t in self.threads]
return self
def __exit__(self, *exc):
self.stop.set()
[t.join(timeout=10) for t in self.threads]
def now(self):
return time.monotonic() - self.t0
def verify(self):
"""Все подтверждённые сообщения на месте, дублей текста (неуверенных повторов) нет."""
c = Client(BASE_URL, timeout=10)
texts, ids = [], set()
for a in self.actors:
path = "/api/v1/chats/" + a["chat"]["id"] + "/messages?limit=100"
cursor = None
while True:
q = path + ("&cursor=" + quote(cursor, safe="") if cursor else "")
page = c.call("GET", q, actor=a, schema="MessagePage")["json"]
for m in page["items"]:
texts.append(m["text"]); ids.add(m["id"])
cursor = page["next_cursor"]
if not cursor:
break
lost = [i for i in self.confirmed if i not in ids]
return {"confirmed": len(self.confirmed), "lost_confirmed": len(lost),
"stored": len(texts), "duplicate_texts": len(texts) - len(set(texts))}
def window(events, a, b):
ev = [e for e in events if a <= e[0] < b]
ok = [e for e in ev if e[2]]
lat = sorted(e[5] for e in ok)
pct = (lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1)) if lat else (lambda p: None)
return {"seconds": round(b - a, 2), "ok": len(ok), "errors": len(ev) - len(ok),
"rps_ok": round(len(ok) / (b - a), 1) if b > a else None,
"p50_ms": pct(0.5), "p99_ms": pct(0.99),
"by_instance": dict(sorted({i: sum(1 for e in ok if e[4] == i) for i in {e[4] for e in ok}}.items())),
"error_codes": dict(sorted({s: sum(1 for e in ev if not e[2] and e[3] == s)
for s in {e[3] for e in ev if not e[2]}}.items()))}
def failover_series(args):
modes = ["stop", "kill"] if args.mode == "both" else [args.mode]
results = {"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers,
"client_timeout_s": 2.0, "phases_s": {"before": args.before, "after_action": args.after},
"runs": []}
for mode in modes:
for idx in range(len(api_containers())):
targets = api_containers()
cid, name, iid = targets[idx]
load = Load(args.workers)
with load:
time.sleep(args.before)
t_action = load.now()
if mode == "stop":
sh("docker", "stop", "-t", "15", cid) # SIGTERM → drain → exit
else:
sh("docker", "kill", "-s", "KILL", cid) # внезапная смерть процесса
t_done = load.now()
time.sleep(args.after)
t_restore = load.now()
sh("docker", "start", cid)
healthy_after = wait_healthy(cid)
new_iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False)
time.sleep(args.settle)
t_end = load.now()
ev = load.events
errs = [e for e in ev if not e[2] and e[0] >= t_action]
target_seen = [e[0] for e in ev if e[2] and e[4] == iid and e[0] >= t_action]
back_seen = [e[0] for e in ev if e[2] and e[4] == new_iid and e[0] >= t_restore]
posts_ok = sorted(e[0] for e in ev if e[2] and e[1] == "post")
gaps = [b - a for a, b in zip(posts_ok, posts_ok[1:]) if a >= t_action - 1 and b <= t_restore]
run = {
"mode": mode, "container": name, "instance_id": iid,
"action_seconds": round(t_done - t_action, 2),
"before": window(ev, 0, t_action), "during": window(ev, t_action, t_restore),
"after_restore": window(ev, t_restore, t_end),
"errors_after_action": len(errs),
"last_error_after_action_s": round(max(e[0] for e in errs) - t_action, 2) if errs else 0,
"target_last_served_after_action_s": round(max(target_seen) - t_action, 2) if target_seen else 0,
"max_gap_between_successful_writes_s": round(max(gaps), 2) if gaps else None,
"replica_healthy_after_start_s": round(healthy_after, 2),
"restarted_replica_first_served_s": round(min(back_seen) - t_restore, 2) if back_seen else None,
"data": load.verify(),
}
results["runs"].append(run)
print("%-4s %-18s errors=%-3d last_err=%5.2fs target_last=%5.2fs max_write_gap=%ss rps(before/during)=%s/%s data=%s" % (
mode, name, run["errors_after_action"], run["last_error_after_action_s"],
run["target_last_served_after_action_s"], run["max_gap_between_successful_writes_s"],
run["before"]["rps_ok"], run["during"]["rps_ok"], run["data"]), flush=True)
time.sleep(2)
OUT.mkdir(parents=True, exist_ok=True)
(OUT / "failover-series.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n")
print("saved evidence/lb/failover-series.json")
def rolling_update(args):
before = api_containers()
images_before = sorted({sh("docker", "inspect", "-f", "{{.Image}}", b[0])[:19] for b in before})
load = Load(args.workers)
with load:
time.sleep(args.before)
t_a = load.now()
env = dict(os.environ)
proc = subprocess.run([str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env,
capture_output=True, text=True)
t_b = load.now()
time.sleep(args.after)
t_end = load.now()
after = api_containers()
ev = load.events
res = {
"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers,
"script_exit": proc.returncode, "script_seconds": round(t_b - t_a, 2),
"before_replicas": [{"container": n, "instance_id": i} for _, n, i in before],
"after_replicas": [{"container": n, "instance_id": i} for _, n, i in after],
"images_before": images_before,
"images_after": sorted({sh("docker", "inspect", "-f", "{{.Image}}", a[0])[:19] for a in after}),
"before": window(ev, 0, t_a), "during": window(ev, t_a, t_b), "after": window(ev, t_b, t_end),
"data": load.verify(),
}
OUT.mkdir(parents=True, exist_ok=True)
(OUT / "rolling-update.json").write_text(json.dumps(res, ensure_ascii=False, indent=2) + "\n")
print(json.dumps({k: res[k] for k in ("script_exit", "script_seconds", "before_replicas", "after_replicas",
"during", "data")}, ensure_ascii=False, indent=1))
if proc.returncode:
print(proc.stderr[-2000:])
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("experiment", choices=["failover-series", "rolling-update"])
p.add_argument("--mode", choices=["stop", "kill", "both"], default="both")
p.add_argument("--workers", type=int, default=8)
p.add_argument("--before", type=float, default=5)
p.add_argument("--after", type=float, default=10)
p.add_argument("--settle", type=float, default=5)
args = p.parse_args()
{"failover-series": failover_series, "rolling-update": rolling_update}[args.experiment](args)
if __name__ == "__main__":
main()
+10
View File
@@ -0,0 +1,10 @@
# Генератор нагрузки трека 1: только stdlib Go, статический бинарник в scratch.
FROM golang:1.24.7-alpine AS build
WORKDIR /src
COPY go.mod main.go ./
RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /loadgen .
FROM scratch
COPY --from=build /loadgen /loadgen
USER 65534:65534
ENTRYPOINT ["/loadgen"]
+3
View File
@@ -0,0 +1,3 @@
module mtusi/loadgen
go 1.22
+274
View File
@@ -0,0 +1,274 @@
// Open-loop HTTP load generator for track 1 (stdlib only, Go 1.22+).
//
// Сценарий: N синтетических пользователей, у каждого свой чат с историей; смесь запросов через LB:
// GET истории (limit=50), POST сообщения (уникальный Idempotency-Key), GET /users/me.
// Модель — открытая: запросы запускаются по расписанию (rate req/s) независимо от ответов;
// задержка считается ОТ ЗАПЛАНИРОВАННОГО момента отправки (без coordinated omission).
// Вывод — JSON-строки в stdout: {"event":"step",...} на каждую ступень rate. Токены не печатаются.
package main
import (
"bytes"
"crypto/rand"
"encoding/base64"
"encoding/hex"
"encoding/json"
"flag"
"fmt"
"io"
"math"
"net/http"
"os"
"sort"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
)
type actor struct {
token, chat string
}
var (
target = flag.String("target", "http://lb:8080", "LB base URL")
nUsers = flag.Int("users", 64, "synthetic users (one chat each)")
seed = flag.Int("seed", 30, "messages per chat before measuring")
rates = flag.String("rates", "100,200,300", "comma-separated request rates (req/s)")
duration = flag.Duration("duration", 15*time.Second, "measured time per step")
warmup = flag.Duration("warmup", 3*time.Second, "unmeasured time per step at the same rate")
mixFlag = flag.String("mix", "history=70,post=20,me=10", "request mix, percent")
timeout = flag.Duration("timeout", 5*time.Second, "client timeout per request")
maxIn = flag.Int("maxinflight", 4000, "in-flight cap; above it a request is counted as dropped")
label = flag.String("label", "", "free-form label copied into output")
stopErr = flag.Float64("stoperr", 0.3, "stop the sweep after 2 consecutive steps with error ratio above this")
)
var client, setupClient *http.Client
func rnd(n int) string {
b := make([]byte, n)
_, _ = rand.Read(b)
return hex.EncodeToString(b)
}
func do(method, path string, body []byte, hdr map[string]string) (int, []byte, error) {
return doWith(client, method, path, body, hdr)
}
func doWith(c *http.Client, method, path string, body []byte, hdr map[string]string) (int, []byte, error) {
req, err := http.NewRequest(method, *target+path, bytes.NewReader(body))
if err != nil {
return 0, nil, err
}
if body != nil {
req.Header.Set("Content-Type", "application/json")
}
for k, v := range hdr {
req.Header.Set(k, v)
}
resp, err := c.Do(req)
if err != nil {
return 0, nil, err
}
defer resp.Body.Close()
data, err := io.ReadAll(resp.Body)
return resp.StatusCode, data, err
}
func must(want int, what, method, path string, body []byte, hdr map[string]string) []byte {
code, data, err := doWith(setupClient, method, path, body, hdr) // KDF медленный: отдельный таймаут
if err != nil || code != want {
fmt.Fprintf(os.Stderr, "setup %s failed: code=%d err=%v\n", what, code, err)
os.Exit(2)
}
return data
}
func setup() []actor {
actors := make([]actor, *nUsers)
var wg sync.WaitGroup
sem := make(chan struct{}, 4)
for i := range actors {
wg.Add(1)
go func(i int) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
user, pw := "lg_"+rnd(6), "Lg-"+rnd(12)
b, _ := json.Marshal(map[string]string{"username": user, "display_name": "load " + strconv.Itoa(i), "password": pw})
var u struct{ ID string }
_ = json.Unmarshal(must(201, "user", "POST", "/api/v1/users", b, nil), &u)
basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+pw))
var s struct{ Token string }
_ = json.Unmarshal(must(201, "login", "POST", "/api/v1/auth/sessions", nil, map[string]string{"Authorization": basic}), &s)
auth := map[string]string{"Authorization": "Bearer " + s.Token}
cb, _ := json.Marshal(map[string]any{"title": "load", "member_ids": []string{u.ID}})
var c struct{ ID string }
_ = json.Unmarshal(must(201, "chat", "POST", "/api/v1/chats", cb, auth), &c)
for k := 0; k < *seed; k++ {
auth["Idempotency-Key"] = rnd(8)
mb, _ := json.Marshal(map[string]string{"text": "seed " + strconv.Itoa(k)})
must(201, "seed", "POST", "/api/v1/chats/"+c.ID+"/messages", mb, auth)
}
actors[i] = actor{token: s.Token, chat: c.ID}
}(i)
}
wg.Wait()
return actors
}
type sample struct {
op int
latency time.Duration // от запланированного момента
service time.Duration // от фактической отправки
status int
}
func pct(v []float64, p float64) float64 {
if len(v) == 0 {
return math.NaN()
}
i := int(math.Ceil(p*float64(len(v)))) - 1
if i < 0 {
i = 0
}
return math.Round(v[i]*100) / 100
}
func main() {
flag.Parse()
client = &http.Client{Timeout: *timeout, Transport: &http.Transport{
MaxIdleConns: 10000, MaxIdleConnsPerHost: 10000, IdleConnTimeout: 30 * time.Second, DisableCompression: true,
}}
setupClient = &http.Client{Timeout: 60 * time.Second}
ops := []string{"history", "post", "me"}
weights := map[string]int{}
for _, part := range strings.Split(*mixFlag, ",") {
kv := strings.SplitN(part, "=", 2)
w, _ := strconv.Atoi(kv[1])
weights[kv[0]] = w
}
var wheel []int
for i, op := range ops {
for k := 0; k < weights[op]; k++ {
wheel = append(wheel, i)
}
}
t0 := time.Now()
actors := setup()
setupSec := time.Since(t0).Seconds()
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"event": "setup", "label": *label, "users": len(actors), "seed": *seed, "seconds": setupSec})
bad := 0
for _, rs := range strings.Split(*rates, ",") {
if bad >= 2 {
break
}
rate, _ := strconv.ParseFloat(strings.TrimSpace(rs), 64)
var mu sync.Mutex
var samples []sample
var inflight, dropped, errs int64
errKinds := map[string]int{}
total := *warmup + *duration
interval := time.Duration(float64(time.Second) / rate)
start := time.Now().Add(50 * time.Millisecond)
measureFrom := start.Add(*warmup)
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{"event": "step_start", "label": *label, "rate": rate,
"warmup_s": warmup.Seconds(), "duration_s": duration.Seconds(), "ts": float64(start.UnixNano()) / 1e9})
var wg sync.WaitGroup
for i := 0; ; i++ {
sched := start.Add(time.Duration(i) * interval)
if sched.Sub(start) >= total {
break
}
if d := time.Until(sched); d > 0 {
time.Sleep(d)
}
measured := !sched.Before(measureFrom)
if atomic.LoadInt64(&inflight) >= int64(*maxIn) {
if measured {
atomic.AddInt64(&dropped, 1)
}
continue
}
atomic.AddInt64(&inflight, 1)
wg.Add(1)
a := actors[i%len(actors)]
op := wheel[i%len(wheel)]
go func(a actor, op int, sched time.Time, measured bool) {
defer wg.Done()
defer atomic.AddInt64(&inflight, -1)
hdr := map[string]string{"Authorization": "Bearer " + a.token}
var code int
var err error
sent := time.Now()
switch op {
case 0:
code, _, err = do("GET", "/api/v1/chats/"+a.chat+"/messages?limit=50", nil, hdr)
case 1:
hdr["Idempotency-Key"] = rnd(8)
code, _, err = do("POST", "/api/v1/chats/"+a.chat+"/messages", []byte(`{"text":"load message"}`), hdr)
case 2:
code, _, err = do("GET", "/api/v1/users/me", nil, hdr)
}
done := time.Now()
if !measured {
return
}
want := 200
if op == 1 {
want = 201
}
mu.Lock()
defer mu.Unlock()
if err != nil || code != want {
errs++
k := strconv.Itoa(code)
if err != nil {
k = "transport"
if strings.Contains(err.Error(), "Timeout") || strings.Contains(err.Error(), "deadline") {
k = "timeout"
}
}
errKinds[k]++
return
}
samples = append(samples, sample{op, done.Sub(sched), done.Sub(sent), code})
}(a, op, sched, measured)
}
wg.Wait()
lat := make([]float64, 0, len(samples))
svc := make([]float64, 0, len(samples))
perOp := map[string][]float64{}
for _, s := range samples {
ms := float64(s.latency.Microseconds()) / 1000
lat = append(lat, ms)
svc = append(svc, float64(s.service.Microseconds())/1000)
perOp[ops[s.op]] = append(perOp[ops[s.op]], ms)
}
sort.Float64s(lat)
sort.Float64s(svc)
opStats := map[string]any{}
for k, v := range perOp {
sort.Float64s(v)
opStats[k] = map[string]any{"ok": len(v), "p50_ms": pct(v, .5), "p99_ms": pct(v, .99)}
}
attempted := float64(len(samples)) + float64(errs) + float64(dropped)
if (float64(errs)+float64(dropped))/math.Max(attempted, 1) > *stopErr {
bad++
} else {
bad = 0
}
_ = json.NewEncoder(os.Stdout).Encode(map[string]any{
"event": "step", "label": *label, "rate": rate, "duration_s": duration.Seconds(),
"attempted": attempted, "ok": len(samples), "errors": errs, "dropped": dropped, "error_kinds": errKinds,
"throughput_ok_rps": math.Round(float64(len(samples))/duration.Seconds()*10) / 10,
"error_ratio": math.Round((float64(errs)+float64(dropped))/math.Max(attempted, 1)*1e4) / 1e4,
"latency_ms": map[string]float64{"p50": pct(lat, .5), "p90": pct(lat, .9), "p99": pct(lat, .99), "p999": pct(lat, .999), "max": pct(lat, 1)},
"service_ms": map[string]float64{"p50": pct(svc, .5), "p99": pct(svc, .99)},
"per_op": opStats,
})
}
}
+319
View File
@@ -0,0 +1,319 @@
#!/usr/bin/env python3
"""Трек 1 «Цена масштабирования» (stdlib, Python 3.9+). Оркестратор серий нагрузки.
python3 research/track1.py run <series>... # fixed-cpu | growth | machine | admission | all
python3 research/track1.py report # сводка + графики из evidence/track1/*.json
Для каждой конфигурации: пересоздать реплики с заданными CPU/RAM/числом (compose.yaml +
research/compose.track1.yaml), дождаться, пока LB видит все реплики UP, запустить генератор
(research/loadgen, open-loop, ступени rate) внутри сети стенда и параллельно собирать:
CPU%/RAM контейнеров (docker stats), очередь/сессии HAProxy (stats CSV), состояния соединений
PostgreSQL (pg_stat_activity), счётчики пула psycopg в каждой реплике (до/после серии).
Результат — evidence/track1/<config>.json. Токенов и паролей в выводе нет.
"""
import argparse
import csv
import io
import json
import os
import re
import shlex
import statistics
import subprocess
import sys
import threading
import time
import urllib.request
from datetime import datetime, timezone
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence" / "track1"
BASE_FILES = os.environ.get("COMPOSE_FILE", "compose.yaml:research/compose.track1.yaml")
DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env"))
STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv")
SLO_P99_MS, SLO_ERR = 250.0, 0.01
# Ступени offered rate (req/s). Генератор останавливает серию после 2 ступеней с >30 % ошибок.
RATES = "40,80,120,160,200,240,280,320,360,400,460,520,600,700,800"
CONFIGS = {
# Серия A: фиксированный суммарный бюджет API = 1 ядро и 512 MiB (KDF-параллелизм 4) делим на 1/2/4.
"fixed-1x1.00": dict(series="fixed-cpu", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4),
"fixed-2x0.50": dict(series="fixed-cpu", API_REPLICAS=2, API_CPUS="0.5", API_MEMORY="256M", KDF_CONCURRENCY=2),
"fixed-4x0.25": dict(series="fixed-cpu", API_REPLICAS=4, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
# Серия B: рост ресурсов — добавляем реплики по 0.25 ядра/128 MiB (4x0.25 = fixed-4x0.25).
"growth-1x0.25": dict(series="growth", API_REPLICAS=1, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
"growth-2x0.25": dict(series="growth", API_REPLICAS=2, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
"growth-3x0.25": dict(series="growth", API_REPLICAS=3, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1),
# Серия C: снять изоляцию ядер — API делит всю машину (2 vCPU) с LB/БД/генератором.
"machine-2x1.00": dict(series="machine", API_REPLICAS=2, API_CPUS="1.0", API_MEMORY="256M", KDF_CONCURRENCY=2,
API_CPUSET="0-1", AUX_CPUSET="0-1"),
"machine-4x1.00": dict(series="machine", API_REPLICAS=4, API_CPUS="1.0", API_MEMORY="256M", KDF_CONCURRENCY=2,
API_CPUSET="0-1", AUX_CPUSET="0-1"),
# Серия P: размер пула соединений при 1x1.00 (p10 = fixed-1x1.00).
"pool-1x1.00-p5": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=5),
"pool-1x1.00-p20": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=20),
"pool-1x1.00-p40": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=40),
# Серия D: улучшение — ограничение конкуренции на LB (maxconn на реплику; mc0 = fixed-1x1.00).
"admission-1x1.00-mc10": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=10),
"admission-1x1.00-mc20": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=20),
"admission-1x1.00-mc10-q250": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M",
KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=10, LB_QUEUE_TIMEOUT="250ms"),
"admission-2x0.50-mc10": dict(series="admission", API_REPLICAS=2, API_CPUS="0.5", API_MEMORY="256M",
KDF_CONCURRENCY=2, LB_SERVER_MAXCONN=10),
}
DEFAULTS = dict(API_CPUSET="0", AUX_CPUSET="1", DB_POOL_MAX="10", SESSION_CACHE="redis",
LB_SERVER_MAXCONN="0", LB_QUEUE_TIMEOUT="2s")
def run(args, env=None, check=True, capture=True, timeout=900):
return subprocess.run(args, cwd=ROOT, env=env, check=check, timeout=timeout,
stdout=subprocess.PIPE if capture else None, stderr=subprocess.PIPE, text=True)
def cfg_env(cfg):
env = dict(os.environ)
env.setdefault("COMPOSE_FILE", BASE_FILES)
for k, v in {**DEFAULTS, **cfg}.items():
if k.isupper():
env[k] = str(v)
return env
def lb_rows():
with urllib.request.urlopen(STATS_URL, timeout=2) as r:
text = r.read().decode().lstrip("# ")
return list(csv.DictReader(io.StringIO(text)))
def wait_lb(n, timeout=90):
t = time.monotonic()
while time.monotonic() - t < timeout:
try:
up = [r for r in lb_rows() if r["pxname"] == "api" and r["svname"].startswith("api")
and not r["svname"].startswith("apibk") and r["status"] == "UP"]
if len(up) == n:
return
except OSError:
pass
time.sleep(1)
raise SystemExit("LB does not see %d UP replicas" % n)
def pool_stats(env):
res = {}
for cid in run(DC + ["ps", "-q", "api"], env).stdout.split():
out = run(["docker", "exec", cid, "python", "-m", "messenger", "stats"], check=False).stdout
try:
d = json.loads(out)
res[d["instance_id"]] = {"pool": d["pool"], "cache": d["cache"], "requests": d["app"]["requests"]}
except (ValueError, KeyError):
pass
return res
class Sampler:
"""docker stats (CPU%, RAM), HAProxy CSV и pg_stat_activity раз в ~1 с с отметками времени."""
ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
def __init__(self, env):
self.env, self.stop = env, threading.Event()
self.docker, self.lb, self.pg, self.throttle = [], [], [], []
def _docker(self):
p = subprocess.Popen(["docker", "stats", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"],
stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, cwd=ROOT)
try:
for line in p.stdout:
if self.stop.is_set():
break
line = self.ANSI.sub("", line).strip()
parts = line.split("\t")
if len(parts) == 3 and parts[1].endswith("%") and parts[0].startswith("mtusi-lab3-"):
mem = parts[2].split("/")[0].strip()
mib = float(re.sub(r"[^0-9.]", "", mem) or 0) * (1024 if "GiB" in mem else 1 / 1024 if "KiB" in mem else 1)
self.docker.append((time.time(), parts[0].replace("mtusi-lab3-", ""), float(parts[1][:-1]), mib))
finally:
p.kill()
@staticmethod
def _throttle(cid):
for path in ("/sys/fs/cgroup/cpu/docker/%s/cpu.stat" % cid, "/sys/fs/cgroup/system.slice/docker-%s.scope/cpu.stat" % cid):
try:
d = dict(line.split() for line in open(path))
return int(d.get("nr_periods", 0)), int(d.get("nr_throttled", 0))
except OSError:
continue
return None
def _poll(self):
ids = [c for c in run(DC + ["ps", "-q", "--no-trunc", "api"], self.env, check=False).stdout.split()]
while not self.stop.is_set():
t = time.time()
thr = [self._throttle(c) for c in ids]
if all(thr) and thr:
self.throttle.append((t, sum(x[0] for x in thr), sum(x[1] for x in thr)))
try:
rows = lb_rows()
api = [r for r in rows if r["pxname"] == "api" and r["svname"] not in ("FRONTEND",)]
be = next(r for r in api if r["svname"] == "BACKEND")
self.lb.append((t, int(be["qcur"] or 0), int(be["scur"] or 0),
sum(int(r["qcur"] or 0) for r in api if r["svname"].startswith("api"))))
except (OSError, StopIteration, ValueError):
pass
self.stop.wait(1.0)
PG_SQL = ("SELECT 'S', extract(epoch FROM now()), coalesce(state, '?'), count(*) FROM pg_stat_activity"
" WHERE datname = 'messenger' AND backend_type = 'client backend' AND pid <> pg_backend_pid() GROUP BY 3"
" UNION ALL SELECT 'W', extract(epoch FROM now()), coalesce(wait_event_type || ':' || wait_event, 'CPU'), count(*)"
" FROM pg_stat_activity WHERE datname = 'messenger' AND state = 'active' AND pid <> pg_backend_pid() GROUP BY 3")
def _pg(self):
"""ОДНА долгоживущая сессия psql с \\watch 1 (не docker exec каждую секунду — он сам грузит хост)."""
p = subprocess.Popen(DC + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True,
cwd=ROOT, env=self.env)
p.stdin.write(self.PG_SQL + " \\watch 1\n"); p.stdin.flush()
snap, cur_ts = {}, None
try:
for line in p.stdout:
if self.stop.is_set():
break
parts = line.strip().split("|")
if len(parts) != 4:
continue
kind, ts, key, n = parts
ts = float(ts)
if cur_ts is not None and ts != cur_ts:
self.pg.append((cur_ts, snap)); snap = {}
cur_ts = ts
snap[("state:" if kind == "S" else "wait:") + key] = int(n)
finally:
p.kill()
def __enter__(self):
self.threads = [threading.Thread(target=self._docker, daemon=True), threading.Thread(target=self._poll, daemon=True),
threading.Thread(target=self._pg, daemon=True)]
[t.start() for t in self.threads]
return self
def __exit__(self, *exc):
self.stop.set()
def window(self, a, b):
cpu, mem = {}, {}
for t, name, c, m in self.docker:
if a <= t <= b:
key = "api" if re.fullmatch(r"api-\d+", name) else re.sub(r"-\d+$", "", name)
cpu.setdefault((key, name), []).append(c)
mem.setdefault(key, []).append(m)
agg = {}
for (key, name), vals in cpu.items():
agg.setdefault(key, 0.0)
agg[key] += statistics.mean(vals)
lb = [x for x in self.lb if a <= x[0] <= b]
th = [x for x in self.throttle if a <= x[0] <= b]
throttled = (round((th[-1][2] - th[0][2]) / max(1, th[-1][1] - th[0][1]), 3) if len(th) >= 2 else None)
pg = [x[1] for x in self.pg if a <= x[0] <= b]
return {
"cpu_percent_mean": {k: round(v, 1) for k, v in sorted(agg.items())},
"mem_mib_max": {k: round(max(v), 1) for k, v in sorted(mem.items())},
"lb_queue_max": max((x[1] + x[3] for x in lb), default=None),
"lb_sessions_mean": round(statistics.mean(x[2] for x in lb), 1) if lb else None,
"pg_active_mean": round(statistics.mean(p.get("state:active", 0) for p in pg), 2) if pg else None,
"pg_wait_events_mean": ({k[5:]: round(sum(p.get(k, 0) for p in pg) / len(pg), 2)
for k in sorted({k for p in pg for k in p if k.startswith("wait:")})} if pg else None),
"api_cfs_throttled_period_ratio": throttled,
"pg_connections_mean": round(statistics.mean(sum(v for k, v in p.items() if k.startswith("state:")) for p in pg), 1) if pg else None,
}
def run_config(name, cfg, args, rep=1):
if rep > 1:
name = "%s-r%d" % (name, rep)
env = cfg_env(cfg)
print("== %s %s" % (name, {k: v for k, v in cfg.items() if k.isupper()}), flush=True)
run(DC + ["up", "-d", "--wait", "--force-recreate", "lb", "api"], env, timeout=300)
wait_lb(int(cfg["API_REPLICAS"]))
time.sleep(3)
pools_before = pool_stats(env)
rates = cfg.get("RATES", args.rates)
cmd = DC + ["--profile", "bench", "run", "--rm", "-T", "loadgen", "-rates", rates,
"-duration", "%ds" % args.duration, "-warmup", "%ds" % args.warmup,
"-users", str(args.users), "-seed", str(args.seed), "-label", name]
steps, starts = [], {}
with Sampler(env) as sampler:
p = subprocess.Popen(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
for line in p.stdout:
try:
ev = json.loads(line)
except ValueError:
continue
if ev["event"] == "step_start":
starts[ev["rate"]] = ev["ts"] + ev["warmup_s"]
elif ev["event"] == "step":
a = starts[ev["rate"]]
ev["resources"] = sampler.window(a, a + ev["duration_s"])
ev["slo_ok"] = ev["latency_ms"]["p99"] <= SLO_P99_MS and ev["error_ratio"] <= SLO_ERR
steps.append(ev)
print(" rate %5.0f ok %6.1f rps err %5.1f%% p50 %7.1f p99 %8.1f ms cpu %s pgActive %s lbQ %s" % (
ev["rate"], ev["throughput_ok_rps"], 100 * ev["error_ratio"], ev["latency_ms"]["p50"],
ev["latency_ms"]["p99"], ev["resources"]["cpu_percent_mean"], ev["resources"]["pg_active_mean"],
ev["resources"]["lb_queue_max"]), flush=True)
elif ev["event"] == "setup":
print(" setup %.1fs" % ev["seconds"], flush=True)
p.wait()
if p.returncode:
print(p.stderr.read()[-1500:])
pools_after = pool_stats(env)
ok = [s for s in steps if s["slo_ok"]]
result = {
"config": name, "repeat": rep, "series": cfg["series"], "params": {**DEFAULTS, **{k: v for k, v in cfg.items() if k.isupper()}},
"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
"slo": {"p99_ms": SLO_P99_MS, "error_ratio": SLO_ERR},
"capacity_at_slo_rps": max((s["throughput_ok_rps"] for s in ok), default=0),
"peak_ok_rps": max((s["throughput_ok_rps"] for s in steps), default=0),
"steps": steps, "pool_before": pools_before, "pool_after": pools_after,
"loadgen": {"users": args.users, "seed": args.seed, "duration_s": args.duration, "warmup_s": args.warmup,
"mix": "history=70,post=20,me=10", "model": "open-loop, latency from scheduled time"},
}
OUT.mkdir(parents=True, exist_ok=True)
(OUT / (name + ".json")).write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n")
print(" => capacity@SLO %.1f rps, peak %.1f rps" % (result["capacity_at_slo_rps"], result["peak_ok_rps"]), flush=True)
def report(args):
from track1_report import build # noqa: E402 (рядом в research/)
build(OUT)
def main():
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
p.add_argument("action", choices=["run", "report", "list"])
p.add_argument("targets", nargs="*", default=["all"])
p.add_argument("--rates", default=RATES)
p.add_argument("--duration", type=int, default=10)
p.add_argument("--warmup", type=int, default=3)
p.add_argument("--users", type=int, default=48)
p.add_argument("--seed", type=int, default=10)
p.add_argument("--rep", type=int, default=1, help="номер повтора (сохраняется как <config>-r<rep>.json)")
args = p.parse_args()
if args.action == "list":
for k, v in CONFIGS.items():
print(k, v)
return
if args.action == "report":
sys.path.insert(0, str(Path(__file__).parent))
return report(args)
names = [n for n, c in CONFIGS.items() if "all" in args.targets or c["series"] in args.targets or n in args.targets]
for n in names:
run_config(n, CONFIGS[n], args, args.rep)
# вернуть штатную конфигурацию стенда
run(DC + ["up", "-d", "--wait", "--force-recreate", "lb", "api"], cfg_env({"API_REPLICAS": 2}), timeout=300)
if __name__ == "__main__":
main()
+208
View File
@@ -0,0 +1,208 @@
"""Сводка и графики трека 1 из evidence/track1/*.json (stdlib; SVG без внешних библиотек).
Графики: одна ось Y на график, линии 2 px, маркеры ≥ 8 px разной формы (вторичное кодирование),
подписи серий у конца линии + легенда, всплывающие подсказки через <title>, таблица в summary.md.
"""
import json
import statistics
from pathlib import Path
# Категориальная палитра (validated: dataviz reference palette, slots 1–4, light surface).
SERIES = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100"]
SURFACE, INK, INK2, GRID = "#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df"
SHAPES = ["circle", "square", "triangle", "diamond"]
def load(out):
runs = {}
for f in sorted(Path(out).glob("*.json")):
if f.name.startswith("summary"):
continue
d = json.loads(f.read_text())
base = d["config"].rsplit("-r", 1)[0] if d.get("repeat", 1) > 1 else d["config"]
runs.setdefault(base, []).append(d)
return runs
def metrics(d):
steps = d["steps"]
err_ok = [s for s in steps if s["error_ratio"] <= 0.01]
return {
"capacity_slo": d["capacity_at_slo_rps"],
"capacity_err1": max((s["throughput_ok_rps"] for s in err_ok), default=0),
"peak": d["peak_ok_rps"],
}
def marker(shape, x, y, color, tip):
t = "<title>%s</title>" % tip
if shape == "circle":
return '<circle cx="%.1f" cy="%.1f" r="4.5" fill="%s" stroke="%s" stroke-width="2">%s</circle>' % (x, y, color, SURFACE, t)
if shape == "square":
return '<rect x="%.1f" y="%.1f" width="9" height="9" rx="1.5" fill="%s" stroke="%s" stroke-width="2">%s</rect>' % (x - 4.5, y - 4.5, color, SURFACE, t)
if shape == "triangle":
return '<path d="M%.1f %.1fL%.1f %.1fL%.1f %.1fZ" fill="%s" stroke="%s" stroke-width="2">%s</path>' % (
x, y - 5.5, x + 5.5, y + 4.5, x - 5.5, y + 4.5, color, SURFACE, t)
return '<path d="M%.1f %.1fL%.1f %.1fL%.1f %.1fL%.1f %.1fZ" fill="%s" stroke="%s" stroke-width="2">%s</path>' % (
x, y - 6, x + 6, y, x, y + 6, x - 6, y, color, SURFACE, t)
def line_chart(path, title, subtitle, series, xlabel, ylabel, ymax=None, diagonal=False, refline=None):
"""series: [(label, [(x, y, tip)])]."""
W, H, L, R, T, B = 760, 400, 64, 190, 64, 52
xs = [p[0] for _, pts in series for p in pts]
ys = [p[1] for _, pts in series for p in pts]
xmax = max(xs) * 1.05
ymax = ymax or max(ys) * 1.12
sx = lambda v: L + (W - L - R) * v / xmax # noqa: E731
sy = lambda v: H - B - (H - T - B) * v / ymax # noqa: E731
o = ['<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 %d %d" font-family="system-ui,-apple-system,Segoe UI,sans-serif">' % (W, H),
'<rect width="100%%" height="100%%" fill="%s"/>' % SURFACE,
'<text x="%d" y="26" font-size="16" font-weight="600" fill="%s">%s</text>' % (L, INK, title),
'<text x="%d" y="46" font-size="12" fill="%s">%s</text>' % (L, INK2, subtitle)]
step = 10 ** len(str(int(ymax / 5))) / 10
step = next(s for s in (step, step * 2, step * 2.5, step * 5, step * 10) if ymax / s <= 6)
v = 0
while v <= ymax:
o.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="%s" stroke-width="1"/>' % (L, W - R, sy(v), sy(v), GRID))
o.append('<text x="%d" y="%.1f" font-size="11" fill="%s" text-anchor="end">%g</text>' % (L - 8, sy(v) + 4, INK2, v))
v += step
xstep = next(s for s in (50, 100, 200, 250, 500) if xmax / s <= 9)
v = 0
while v <= xmax:
o.append('<text x="%.1f" y="%d" font-size="11" fill="%s" text-anchor="middle">%g</text>' % (sx(v), H - B + 18, INK2, v))
v += xstep
o.append('<line x1="%d" x2="%d" y1="%d" y2="%d" stroke="%s" stroke-width="1"/>' % (L, W - R, H - B, H - B, INK2))
o.append('<text x="%d" y="%d" font-size="12" fill="%s" text-anchor="middle">%s</text>' % ((L + W - R) / 2, H - 12, INK2, xlabel))
o.append('<text x="16" y="%d" font-size="12" fill="%s" transform="rotate(-90 16 %d)" text-anchor="middle">%s</text>' % (
(T + H - B) / 2, INK2, (T + H - B) / 2, ylabel))
if refline:
val, text = refline
o.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="%s" stroke-width="1" stroke-dasharray="4 4"/>' % (L, W - R, sy(val), sy(val), INK2))
o.append('<text x="%d" y="%.1f" font-size="11" fill="%s" text-anchor="end">%s</text>' % (W - R - 4, sy(val) - 5, INK2, text))
if diagonal:
m = min(xmax, ymax)
o.append('<line x1="%.1f" y1="%.1f" x2="%.1f" y2="%.1f" stroke="%s" stroke-width="1" stroke-dasharray="4 4"/>' % (
sx(0), sy(0), sx(m), sy(m), INK2))
o.append('<text x="%.1f" y="%.1f" font-size="11" fill="%s" text-anchor="end">пунктир: полезная = поданная</text>' % (sx(m * 0.92) - 8, sy(m * 0.92), INK2))
for i, (label, pts) in enumerate(series):
c = SERIES[i % len(SERIES)]
pts = sorted(pts)
o.append('<polyline fill="none" stroke="%s" stroke-width="2" stroke-linejoin="round" points="%s"/>' % (
c, " ".join("%.1f,%.1f" % (sx(x), sy(y)) for x, y, _ in pts)))
for x, y, tip in pts:
o.append(marker(SHAPES[i % 4], sx(x), sy(y), c, "%s — %s" % (label, tip)))
# легенда сверху справа
lx = W - R + 10
for i, (label, _) in enumerate(series):
y = T + 4 + i * 18
o.append(marker(SHAPES[i % 4], lx + 6, y, SERIES[i % len(SERIES)], label))
o.append('<text x="%d" y="%.1f" font-size="11" fill="%s">%s</text>' % (lx + 16, y + 4, INK2, label))
o.append("</svg>")
Path(path).write_text("\n".join(o) + "\n")
def bar_chart(path, title, subtitle, groups, ylabel):
"""groups: [(label, value, tip)] — одна серия, одна ось."""
W, H, L, R, T, B = 760, 360, 64, 24, 64, 70
ymax = max(v for _, v, _ in groups) * 1.18
n = len(groups)
bw = (W - L - R) / n
sy = lambda v: H - B - (H - T - B) * v / ymax # noqa: E731
o = ['<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 %d %d" font-family="system-ui,-apple-system,Segoe UI,sans-serif">' % (W, H),
'<rect width="100%%" height="100%%" fill="%s"/>' % SURFACE,
'<text x="%d" y="26" font-size="16" font-weight="600" fill="%s">%s</text>' % (L, INK, title),
'<text x="%d" y="46" font-size="12" fill="%s">%s</text>' % (L, INK2, subtitle)]
step = next(s for s in (25, 50, 100, 200) if ymax / s <= 6)
v = 0
while v <= ymax:
o.append('<line x1="%d" x2="%d" y1="%.1f" y2="%.1f" stroke="%s"/>' % (L, W - R, sy(v), sy(v), GRID))
o.append('<text x="%d" y="%.1f" font-size="11" fill="%s" text-anchor="end">%g</text>' % (L - 8, sy(v) + 4, INK2, v))
v += step
for i, (label, val, tip) in enumerate(groups):
x = L + i * bw + bw * 0.22
w = bw * 0.56
y = sy(val)
h = H - B - y
r = min(4, w / 2)
o.append('<path d="M%.1f %.1fV%.1fQ%.1f %.1f %.1f %.1fH%.1fQ%.1f %.1f %.1f %.1fV%.1fZ" fill="%s"><title>%s</title></path>' % (
x, H - B, y + r, x, y, x + r, y, x + w - r, x + w, y, x + w, y + r, H - B, SERIES[0], tip))
o.append('<text x="%.1f" y="%.1f" font-size="12" fill="%s" text-anchor="middle">%g</text>' % (x + w / 2, y - 6, INK, val))
for k, part in enumerate(label.split("\n")):
o.append('<text x="%.1f" y="%.1f" font-size="11" fill="%s" text-anchor="middle">%s</text>' % (
x + w / 2, H - B + 16 + 13 * k, INK2, part))
o.append('<line x1="%d" x2="%d" y1="%d" y2="%d" stroke="%s"/>' % (L, W - R, H - B, H - B, INK2))
o.append('<text x="16" y="%d" font-size="12" fill="%s" transform="rotate(-90 16 %d)" text-anchor="middle">%s</text>' % (
(T + H - B) / 2, INK2, (T + H - B) / 2, ylabel))
o.append("</svg>")
Path(path).write_text("\n".join(o) + "\n")
def build(out):
out = Path(out)
runs = load(out)
table, summary = [], {}
for name, ds in sorted(runs.items()):
ms = [metrics(d) for d in ds]
agg = {k: statistics.median(m[k] for m in ms) for k in ms[0]}
spread = {k: [m[k] for m in ms] for k in ms[0]}
p = ds[0]["params"]
sat = max(ds[0]["steps"], key=lambda s: s["throughput_ok_rps"])
res = sat["resources"]
summary[name] = {"series": ds[0]["series"], "repeats": len(ds), "median": agg, "values": spread,
"params": {k: p[k] for k in ("API_REPLICAS", "API_CPUS", "API_MEMORY", "API_CPUSET", "DB_POOL_MAX",
"LB_SERVER_MAXCONN", "LB_QUEUE_TIMEOUT") if k in p},
"at_peak": {"rate": sat["rate"], "cpu_percent": res["cpu_percent_mean"],
"cfs_throttled": res.get("api_cfs_throttled_period_ratio"),
"lb_queue_max": res["lb_queue_max"], "pg_connections": res["pg_connections_mean"]}}
table.append("| %s | %s | %s×%s CPU, %s | pool %s, maxconn %s, queue %s | %s | %s | %s | %s |" % (
name, ds[0]["series"], p["API_REPLICAS"], p["API_CPUS"], p["API_MEMORY"], p.get("DB_POOL_MAX"),
p.get("LB_SERVER_MAXCONN"), p.get("LB_QUEUE_TIMEOUT"),
"/".join("%g" % v for v in spread["capacity_slo"]), "/".join("%g" % v for v in spread["capacity_err1"]),
"/".join("%g" % v for v in spread["peak"]), len(ds)))
(out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1) + "\n")
head = ("| Конфигурация | Серия | Реплики × CPU, RAM | Пул / LB | Ёмкость при SLO, rps | Макс. при ошибках ≤1 %, rps | "
"Пик полезной, rps | Повторов |\n|---|---|---|---|---|---|---|---|")
(out / "summary.md").write_text(head + "\n" + "\n".join(table) + "\n")
def curve(name, field):
"""Медиана по повторам для каждой ступени rate (повторы могут остановиться на разных ступенях)."""
by_rate = {}
for d in runs[name]:
for s in d["steps"]:
by_rate.setdefault(s["rate"], []).append(s)
pts = []
for rate, ss in sorted(by_rate.items()):
ok = statistics.median(s["throughput_ok_rps"] for s in ss)
p99 = statistics.median(s["latency_ms"]["p99"] for s in ss)
err = statistics.median(s["error_ratio"] for s in ss)
pts.append((rate, ok if field == "ok" else p99, "%g req/s: полезная %.0f rps, ошибок %.1f %%, p99 %.0f мс (медиана %d прогонов)" % (
rate, ok, 100 * err, p99, len(ss))))
return pts
fixed = [("1 × 1.0 CPU", "fixed-1x1.00"), ("2 × 0.5 CPU", "fixed-2x0.50"), ("4 × 0.25 CPU", "fixed-4x0.25")]
fixed = [f for f in fixed if f[1] in runs]
if fixed:
line_chart(out / "fixed-cpu-goodput.svg", "Фиксированный бюджет API (1 CPU, 512 MiB): дробление на реплики",
"Полезная пропускная способность (2xx), медиана 3 прогонов; open-loop, смесь 70/20/10",
[(l, curve(n, "ok")) for l, n in fixed], "Поданная нагрузка, req/s", "Полезная, rps", diagonal=True)
line_chart(out / "fixed-cpu-p99.svg", "Фиксированный бюджет API: хвост задержки",
"p99 успешных ответов от запланированного момента, медиана 3 прогонов",
[(l, [(x, min(y, 2500), t) for x, y, t in curve(n, "p99")]) for l, n in fixed],
"Поданная нагрузка, req/s", "p99, мс (обрезано на 2500)", ymax=2600, refline=(250, "SLO 250 мс"))
adm = [("без лимита (maxconn 0)", "fixed-1x1.00"), ("maxconn 10", "admission-1x1.00-mc10"),
("maxconn 20", "admission-1x1.00-mc20"), ("maxconn 10, queue 250 мс", "admission-1x1.00-mc10-q250")]
adm = [a for a in adm if a[1] in runs]
if len(adm) > 1:
line_chart(out / "admission-goodput.svg", "maxconn на LB против перегрузки (1 × 1.0 CPU)",
"Полезная пропускная способность; пул 10; без лимита и maxconn 10 — медиана 3 прогонов",
[(l, curve(n, "ok")) for l, n in adm], "Поданная нагрузка, req/s", "Полезная, rps", diagonal=True)
bars = [("growth-1x0.25", "1×0.25\n(0.25 CPU)"), ("growth-2x0.25", "2×0.25\n(0.5 CPU)"), ("growth-3x0.25", "3×0.25\n(0.75 CPU)"),
("fixed-4x0.25", "4×0.25\n(1 CPU)"), ("fixed-1x1.00", "1×1.0\n(1 CPU)"), ("machine-2x1.00", "2×1.0\n(2 vCPU хоста)"),
("machine-4x1.00", "4×1.0\n(2 vCPU хоста)")]
bars = [(summary[n]["median"]["peak"], l, n) for n, l in bars if n in summary]
if bars:
bar_chart(out / "growth-peak.svg", "Рост ресурсов: пик полезной пропускной способности",
"Ступени 0.25 CPU на изолированном ядре, затем вся машина (2 vCPU, общие с БД/LB/генератором)",
[(l, round(v), "%s: пик %.0f rps" % (n, v)) for v, l, n in bars], "Пик полезной, rps")
print((out / "summary.md").read_text())
+95
View File
@@ -0,0 +1,95 @@
# Windows PowerShell 5.1+ equivalent of scripts/collect-evidence.sh (lab3).
# Starts the Compose stand (LB + API replicas + migrate + PostgreSQL + Redis), runs every check and writes
# outputs to evidence\ (no tokens/passwords). Stops/restarts containers during failover/restart tests.
# powershell -NoProfile -ExecutionPolicy Bypass -File scripts\collect-evidence.ps1 [-WithTrack1] [-SkipFailures]
param([switch]$WithTrack1, [switch]$SkipFailures)
$ErrorActionPreference = 'Continue'
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
$env:PYTHONUTF8 = '1'
$env:PYTHONIOENCODING = 'utf-8'
# IPv4 explicitly: on Windows 'localhost' resolves to ::1 first; the LB listens on 127.0.0.1 only.
$env:NO_PROXY = '127.0.0.1,localhost'
$B = 'http://127.0.0.1:8080'
$env:BASE_URL = $B
$env:TTL_BASE_URL = 'http://127.0.0.1:8081'
$ev = Join-Path $root 'evidence'
New-Item -ItemType Directory -Force -Path $ev | Out-Null
# Full path of the Python APPLICATION: a bare 'py' would resolve to a PowerShell function first.
$pyApp = Get-Command py -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1
$pythonApp = Get-Command python -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1
if ($pyApp) { $script:PyExe = $pyApp.Source; $script:PyPre = @('-3') }
elseif ($pythonApp) { $script:PyExe = $pythonApp.Source; $script:PyPre = @() }
else { throw 'Python 3.9+ not found (install from python.org, enable "Add to PATH")' }
function Invoke-Python { & $script:PyExe @script:PyPre @args }
function Dc { & docker compose --env-file .env @args }
$hookStop = if ($IsLinux -or $IsMacOS) { 'scripts/stop-one.sh' } else { 'scripts\stop-one.cmd' }
$hookRestart = if ($IsLinux -or $IsMacOS) { 'scripts/restart.sh' } else { 'scripts\restart.cmd' }
$startReplicas = Join-Path $PSScriptRoot 'start-replicas.ps1'
function Get-Stamp {
$commit = 'n/a'
if (Get-Command git -ErrorAction SilentlyContinue) {
$c = (& git rev-parse --short HEAD 2>$null)
if ($LASTEXITCODE -eq 0 -and $c) { $commit = $c }
}
'# ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' commit=' + $commit + ' host=' + [Environment]::OSVersion.VersionString + ' cpus=' + [Environment]::ProcessorCount
}
function Invoke-Step([string]$name, [string]$title, [scriptblock]$block) {
Write-Host ('>>> ' + $name) -ForegroundColor Cyan
$global:LASTEXITCODE = 0
$lines = @(& $block 2>&1 | ForEach-Object { "$_" })
$code = $global:LASTEXITCODE
Set-Content -Path (Join-Path $ev ($name + '.txt')) -Encoding UTF8 -Value (@((Get-Stamp), ('$ ' + $title)) + $lines + @('exit=' + $code))
$lines | Select-Object -Last 2 | ForEach-Object { Write-Host (' ' + $_) }
if ($code -ne 0) { Write-Host (' exit=' + $code + ' (see evidence\' + $name + '.txt)') -ForegroundColor Yellow }
}
& docker version --format '{{.Server.Version}}' *> $null
if ($LASTEXITCODE -ne 0) { throw 'Docker is not running: start Docker Desktop first' }
& (Join-Path $PSScriptRoot 'init-secrets.ps1')
Set-Location $root
Write-Host '>>> docker compose up (build, wait for healthchecks)' -ForegroundColor Cyan
Dc --profile ttl up -d --build --wait
if ($LASTEXITCODE -ne 0) { Dc logs --tail=80; throw 'docker compose up failed' }
Invoke-Step 'stack' 'docker version; compose images; ps' { & docker version --format 'docker {{.Server.Version}}'; Dc version; Dc --profile ttl images; Dc --profile ttl ps -a }
Invoke-Step 'instances' 'X-Instance-Id of 6 requests via LB' {
1..6 | ForEach-Object { (Invoke-WebRequest -UseBasicParsing "$B/health/live").Headers['X-Instance-Id'] }
}
Invoke-Step 'make-check' 'python scripts/check_contract.py' { Invoke-Python scripts/check_contract.py }
Invoke-Step 'smoke' "python tests/smoke.py --base-url $B" { Invoke-Python tests/smoke.py --base-url $B }
Invoke-Step 'test-api' 'python student-tests/test_api.py' { Invoke-Python student-tests/test_api.py }
Invoke-Step 'test-idempotency' 'python student-tests/test_idempotency.py' { Invoke-Python student-tests/test_idempotency.py }
Invoke-Step 'test-expiry' 'python student-tests/test_expiry.py # api-ttl via LB :8081' { Invoke-Python student-tests/test_expiry.py }
Invoke-Step 'test-cache-race' 'docker compose run api: python student-tests/test_cache_race.py' {
Dc run --rm --no-deps -T -v ((Join-Path $root 'student-tests') + ':/app/student-tests:ro') --entrypoint python api student-tests/test_cache_race.py
}
Set-Content -Path (Join-Path $ev 'failover-hook.txt') -Encoding UTF8 -Value @()
foreach ($i in 1, 2) {
Invoke-Step ('failover-' + $i) ('python tests/resilience.py --mode failover --action-script ' + $hookStop) {
Invoke-Python tests/resilience.py --mode failover --action-script $hookStop --base-url $B
}
Invoke-Step ('restore-' + $i) 'scripts\start-replicas.ps1' { & $startReplicas }
}
Set-Content -Path (Join-Path $ev 'restart-hook.txt') -Encoding UTF8 -Value @()
Invoke-Step 'persistence' ('python tests/resilience.py --mode persistence --action-script ' + $hookRestart) {
Invoke-Python tests/resilience.py --mode persistence --action-script $hookRestart --base-url $B
}
if (-not $SkipFailures) {
Invoke-Step 'test-failures' 'python student-tests/test_failures.py # stops/starts redis and postgres' { Invoke-Python student-tests/test_failures.py }
}
Invoke-Step 'migrations-rerun' 'docker compose run --rm --no-deps migrate' { Dc run --rm --no-deps migrate }
Invoke-Step 'lb-failover-series' 'python research/lb_experiments.py failover-series --mode both' { Invoke-Python research/lb_experiments.py failover-series --mode both }
Write-Host 'NOTE: rolling update uses scripts/rolling-update.sh (POSIX sh). On Windows run it from Git Bash/WSL.' -ForegroundColor Yellow
Invoke-Step 'load' 'python tests/load.py --duration 30 --concurrency 4' {
Invoke-Python tests/load.py --base-url $B --duration 30 --concurrency 4 --output evidence/load.json
}
if ($WithTrack1) {
$env:COMPOSE_FILE = 'compose.yaml;research/compose.track1.yaml'
Invoke-Step 'track1' 'python research/track1.py run all' { Dc --profile bench build loadgen; Invoke-Python research/track1.py run all }
Remove-Item Env:COMPOSE_FILE
Invoke-Step 'track1-report' 'python research/track1.py report' { Invoke-Python research/track1.py report }
}
Write-Host 'Done: evidence collected in evidence\' -ForegroundColor Green
+42
View File
@@ -0,0 +1,42 @@
#!/bin/sh
# Полный прогон доказательств lab3 на Compose-стенде; вывод — evidence/ (без токенов и паролей).
# Останавливает/перезапускает контейнеры стенда (failover, restart, отказы БД/Redis, rolling update).
# Трек 5.1 (≈1 ч) запускается отдельно: make track1 (или SKIP_TRACK1=0 ./scripts/collect-evidence.sh).
set -eu
cd "$(dirname "$0")/.."
. scripts/lib.sh
PY=${PYTHON:-python3}
B=${BASE_URL:-http://127.0.0.1:8080}
export BASE_URL=$B TTL_BASE_URL=${TTL_BASE_URL:-http://127.0.0.1:8081}
mkdir -p evidence
stamp() { echo "# $(now) commit=$(git rev-parse --short HEAD 2>/dev/null || echo n/a) host=$(uname -srm) cpus=$(getconf _NPROCESSORS_ONLN)"; }
run() { name=$1; shift; { stamp; echo "\$ $*"; "$@" 2>&1; echo "exit=$?"; } > "evidence/$name.txt" || true; tail -2 "evidence/$name.txt"; }
./scripts/init-secrets.sh
dc --profile ttl up -d --build --wait
{ stamp; docker version --format 'docker {{.Server.Version}}'; dc version; dc --profile ttl images; dc --profile ttl ps; state; } > evidence/stack.txt 2>&1
run instances sh -c "for i in 1 2 3 4 5 6; do curl -s -o /dev/null -D - $B/health/live | grep -i x-instance-id; done"
run make-check make check
run smoke $PY tests/smoke.py --base-url "$B"
run test-api $PY student-tests/test_api.py
run test-idempotency $PY student-tests/test_idempotency.py
run test-expiry $PY student-tests/test_expiry.py
run test-cache-race make test-cache-race
: > evidence/failover-hook.txt
run failover-1 $PY tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$B"
run restore-1 ./scripts/start-replicas.sh
run failover-2 $PY tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$B"
run restore-2 ./scripts/start-replicas.sh
: > evidence/restart-hook.txt
run persistence $PY tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "$B"
run test-failures $PY student-tests/test_failures.py
run migrations-rerun dc run --rm --no-deps migrate
run lb-failover-series $PY research/lb_experiments.py failover-series --mode both
: > evidence/rolling-update.txt
run lb-rolling-update $PY research/lb_experiments.py rolling-update
run load $PY tests/load.py --base-url "$B" --duration 30 --concurrency 4 --output evidence/load.json
if [ "${SKIP_TRACK1:-1}" = 0 ]; then
run track1 env COMPOSE_FILE=compose.yaml:research/compose.track1.yaml $PY research/track1.py run all
run track1-report $PY research/track1.py report
fi
echo "evidence collected in evidence/"
+36
View File
@@ -0,0 +1,36 @@
# Windows PowerShell 5.1+ version of scripts/init-secrets.sh.
# Creates local synthetic secrets in secrets\ and .env (both git-ignored).
# Idempotent: existing files are NOT overwritten. Values are never printed.
$ErrorActionPreference = 'Stop'
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
New-Item -ItemType Directory -Force -Path (Join-Path $root 'secrets'), (Join-Path $root 'backups') | Out-Null
$ascii = New-Object System.Text.ASCIIEncoding
function New-Hex {
$bytes = New-Object byte[] 24
$rng = [System.Security.Cryptography.RandomNumberGenerator]::Create()
$rng.GetBytes($bytes)
($bytes | ForEach-Object { $_.ToString('x2') }) -join ''
}
function Set-SecretFile([string]$name, [string]$value) {
$path = Join-Path (Join-Path $root 'secrets') $name
if (-not (Test-Path $path)) {
# No BOM, no trailing CR/LF: the value is read verbatim inside Linux containers.
[System.IO.File]::WriteAllText($path, $value, $ascii)
}
}
function Get-SecretFile([string]$name) {
([System.IO.File]::ReadAllText((Join-Path (Join-Path $root 'secrets') $name))).Trim()
}
Set-SecretFile 'postgres_password' (New-Hex)
Set-SecretFile 'redis_password' (New-Hex)
$pg = Get-SecretFile 'postgres_password'
$rd = Get-SecretFile 'redis_password'
Set-SecretFile 'database_url' ("postgresql://messenger:" + $pg + "@postgres:5432/messenger")
Set-SecretFile 'cache_url' ("redis://:" + $rd + "@redis:6379/0")
if (-not (Test-Path (Join-Path $root '.env'))) {
Copy-Item (Join-Path $root '.env.example') (Join-Path $root '.env')
}
Write-Output 'secrets\ and .env are ready (values not shown).'
+21
View File
@@ -0,0 +1,21 @@
#!/bin/sh
# Создаёт локальные синтетические secrets для стенда (каталог secrets/ в .gitignore).
# Идемпотентно: существующие файлы НЕ перезаписываются (иначе пароль разойдётся с уже
# инициализированным volume PostgreSQL). Значения не печатаются.
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets backups
chmod 700 secrets
gen() { python3 -c 'import secrets; print(secrets.token_hex(24))'; }
if [ ! -f secrets/postgres_password ]; then gen > secrets/postgres_password; fi
if [ ! -f secrets/redis_password ]; then gen > secrets/redis_password; fi
PG=$(cat secrets/postgres_password)
RD=$(cat secrets/redis_password)
[ -f secrets/database_url ] || printf 'postgresql://messenger:%s@postgres:5432/messenger\n' "$PG" > secrets/database_url
[ -f secrets/cache_url ] || printf 'redis://:%s@redis:6379/0\n' "$RD" > secrets/cache_url
# Файлы монтируются в контейнеры как /run/secrets/*; процесс API работает под uid 10001,
# поэтому файл должен быть читаем внутри контейнера. Каталог secrets/ закрыт (700) на хосте.
chmod 644 secrets/*
[ -f .env ] || cp .env.example .env
echo "secrets/ and .env are ready (values not shown)."
+19
View File
@@ -0,0 +1,19 @@
# Общие функции хуков (POSIX sh). Подключается: . "$(dirname "$0")/lib.sh"
cd "$(dirname "$0")/.."
ENV_FILE=${ENV_FILE:-.env}
dc() { docker compose --env-file "$ENV_FILE" "$@"; }
now() { date -u +%FT%TZ; }
# Состояние всех контейнеров проекта: сервис, имя, статус, StartedAt, PID, alias реплики API.
state() {
for cid in $(dc ps -a -q); do
docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Name}} status={{.State.Status}} started={{.State.StartedAt}} pid={{.State.Pid}}' "$cid" |
sed 's# /# #'
done | sort
}
# Контейнер API-реплики по её X-Instance-Id (alias пишется репликой в /tmp/instance-id).
container_of() {
for cid in $(dc ps -q api); do
if [ "$(docker exec "$cid" cat /tmp/instance-id 2>/dev/null)" = "$1" ]; then echo "$cid"; return 0; fi
done
return 1
}
+4
View File
@@ -0,0 +1,4 @@
@echo off
rem Windows wrapper: tests\resilience.py starts the hook directly; .cmd runs without a shell.
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0restart.ps1"
exit /b %ERRORLEVEL%
+31
View File
@@ -0,0 +1,31 @@
# Windows PowerShell version of scripts/restart.sh (persistence hook): restart LB, ALL API replicas,
# PostgreSQL and Redis keeping volumes. State before/after -> evidence\restart-hook.txt
$ErrorActionPreference = 'Continue'
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
$envFile = if ($env:ENV_FILE) { $env:ENV_FILE } else { '.env' }
$log = Join-Path $root 'evidence\restart-hook.txt'
function Dc { & docker compose --env-file $envFile @args 2>$null | Out-Null; if ($LASTEXITCODE -ne 0) { throw ('docker compose ' + ($args -join ' ') + ' failed') } }
function Get-State {
foreach ($cid in (& docker compose --env-file $envFile ps -a -q)) {
& docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Name}} status={{.State.Status}} started={{.State.StartedAt}} pid={{.State.Pid}}' $cid
}
& docker volume ls --format 'volume {{.Name}}' | Where-Object { $_ -match 'pgdata' }
}
try {
Add-Content -Path $log -Encoding UTF8 -Value ('== ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' before restart')
Add-Content -Path $log -Encoding UTF8 -Value (Get-State)
$ttl = @(& docker compose --env-file $envFile ps -q api-ttl 2>$null).Count -gt 0
Dc stop lb api
if ($ttl) { Dc stop api-ttl }
Dc restart postgres redis
Dc start api
if ($ttl) { Dc start api-ttl }
Dc start lb
Add-Content -Path $log -Encoding UTF8 -Value ('== ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' after restart')
Add-Content -Path $log -Encoding UTF8 -Value (Get-State)
exit 0
} catch {
[Console]::Error.WriteLine($_.Exception.Message)
exit 1
}
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# Persistence hook: перезапускает LB, ВСЕ API-реплики и все хранилища (PostgreSQL, Redis),
# сохраняя volumes. Никаких down -v / FLUSH / повторного seed. Миграции повторно не нужны.
# Состояние контейнеров до/после (StartedAt, PID) — в evidence/restart-hook.txt.
set -eu
. "$(dirname "$0")/lib.sh"
LOG=${RESTART_EVIDENCE:-evidence/restart-hook.txt}
{ echo "== $(now) before restart"; state; docker volume ls --format 'volume {{.Name}}' | grep pgdata; } >> "$LOG"
running_ttl=$(dc ps -q api-ttl 2>/dev/null || true)
dc stop lb api >/dev/null 2>&1
[ -z "$running_ttl" ] || dc stop api-ttl >/dev/null 2>&1
dc restart postgres redis >/dev/null 2>&1
dc start api >/dev/null 2>&1
[ -z "$running_ttl" ] || dc start api-ttl >/dev/null 2>&1
dc start lb >/dev/null 2>&1
{ echo "== $(now) after restart"; state; docker volume ls --format 'volume {{.Name}}' | grep pgdata; } >> "$LOG"
+35
View File
@@ -0,0 +1,35 @@
#!/bin/sh
# Rolling update API-реплик без остановки сервиса (Compose).
# 1. собрать новый образ (старые контейнеры продолжают работать на старом image id);
# 2. применить миграции ОДИН раз (one-shot migrate). Миграции обязаны быть expand-only
# (совместимы с работающей версией), см. REPORT.md;
# 3. для каждой старой реплики: поднять одну новую (--scale N+1 --no-recreate), дождаться
# healthy, затем остановить старую через SIGTERM (drain 3 с, LB выводит её) и удалить.
# Число готовых реплик во время замены не опускается ниже N. Журнал — evidence/rolling-update.txt.
set -eu
. "$(dirname "$0")/lib.sh"
LOG=${ROLLING_EVIDENCE:-evidence/rolling-update.txt}
VERSION=${APP_VERSION:-$(date -u +%Y%m%d%H%M%S)}
BUILD_CMD=${BUILD_CMD:-"docker compose --env-file $ENV_FILE build --build-arg APP_VERSION=$VERSION api"}
wait_healthy() { # $1 = container id
i=0
while [ "$(docker inspect -f '{{.State.Health.Status}}' "$1")" != healthy ]; do
i=$((i + 1)); [ $i -le 120 ] || { echo "replica $1 not healthy" >&2; return 1; }
sleep 0.5
done
}
old=$(dc ps -q api)
n=$(echo "$old" | wc -w)
{ echo "== $(now) rolling update start: $n replicas, new version $VERSION"; for c in $old; do docker inspect -f 'old {{.Name}} image={{.Image}}' "$c"; done; } >> "$LOG"
$BUILD_CMD >/dev/null
dc run --rm --no-deps migrate >> "$LOG" 2>&1
for c in $old; do
before=$(dc ps -q api)
dc up -d --no-deps --no-recreate --scale api=$((n + 1)) api >/dev/null 2>&1
new=$(for x in $(dc ps -q api); do echo "$before" | grep -q "$x" || echo "$x"; done)
wait_healthy "$new"
echo "$(now) new $(docker inspect -f '{{.Name}} image={{.Image}}' "$new") healthy; stopping old $(docker inspect -f '{{.Name}}' "$c")" >> "$LOG"
docker stop -t 15 "$c" >/dev/null
docker rm "$c" >/dev/null
done
{ echo "== $(now) rolling update done"; for c in $(dc ps -q api); do docker inspect -f 'now {{.Name}} image={{.Image}} health={{.State.Health.Status}}' "$c"; done; } >> "$LOG"
+14
View File
@@ -0,0 +1,14 @@
# Windows PowerShell version of scripts/start-replicas.sh: start stopped API replicas, wait for healthy.
$ErrorActionPreference = 'Continue'
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
$envFile = if ($env:ENV_FILE) { $env:ENV_FILE } else { '.env' }
$ids = @(& docker compose --env-file $envFile ps -a -q api)
foreach ($cid in $ids) { & docker start $cid | Out-Null }
for ($i = 0; $i -lt 120; $i++) {
$bad = @($ids | Where-Object { (& docker inspect -f '{{.State.Health.Status}}' $_) -ne 'healthy' }).Count
if ($bad -eq 0) { break }
Start-Sleep -Milliseconds 500
}
foreach ($cid in $ids) { & docker inspect -f '{{.Name}} {{.State.Status}} {{.State.Health.Status}}' $cid }
if ($bad -ne 0) { exit 1 }
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# Восстанавливает остановленные API-реплики (после stop-one) и ждёт их healthcheck (≤60 с).
set -eu
. "$(dirname "$0")/lib.sh"
for cid in $(dc ps -a -q api); do docker start "$cid" >/dev/null; done
i=0
while [ $i -lt 120 ]; do
bad=0
for cid in $(dc ps -a -q api); do
[ "$(docker inspect -f '{{.State.Health.Status}}' "$cid")" = healthy ] || bad=1
done
[ $bad -eq 0 ] && break
i=$((i + 1)); sleep 0.5
done
state | grep '^api '
[ $bad -eq 0 ]
+4
View File
@@ -0,0 +1,4 @@
@echo off
rem Windows wrapper: tests\resilience.py starts the hook directly; .cmd runs without a shell.
powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1"
exit /b %ERRORLEVEL%
+30
View File
@@ -0,0 +1,30 @@
# Windows PowerShell version of scripts/stop-one.sh (failover hook). Stops EXACTLY ONE API replica whose
# X-Instance-Id is in $env:TARGET_INSTANCE_ID and leaves it stopped. DB/Redis/LB/volumes untouched.
# Restore afterwards: scripts\start-replicas.ps1
$ErrorActionPreference = 'Continue'
$root = Split-Path -Parent $PSScriptRoot
Set-Location $root
$envFile = if ($env:ENV_FILE) { $env:ENV_FILE } else { '.env' }
$log = Join-Path $root 'evidence\failover-hook.txt'
if (-not $env:TARGET_INSTANCE_ID) { [Console]::Error.WriteLine('TARGET_INSTANCE_ID is required'); exit 2 }
function Get-State {
foreach ($cid in (& docker compose --env-file $envFile ps -a -q)) {
& docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Name}} status={{.State.Status}} started={{.State.StartedAt}} pid={{.State.Pid}}' $cid
}
}
$target = $null
foreach ($cid in (& docker compose --env-file $envFile ps -q api)) {
$id = (& docker exec $cid cat /tmp/instance-id 2>$null)
if ($id -and $id.Trim() -eq $env:TARGET_INSTANCE_ID) { $target = $cid }
}
if (-not $target) { [Console]::Error.WriteLine('no running api replica with this id'); exit 1 }
$name = (& docker inspect -f '{{.Name}}' $target).TrimStart('/')
$stamp = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
Add-Content -Path $log -Encoding UTF8 -Value ("== $stamp stop-one target=" + $env:TARGET_INSTANCE_ID + " container=$name")
Add-Content -Path $log -Encoding UTF8 -Value (Get-State)
& docker stop -t 15 $target | Out-Null
if ($LASTEXITCODE -ne 0) { exit 1 }
$stamp = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ')
Add-Content -Path $log -Encoding UTF8 -Value ("== $stamp after stop")
Add-Content -Path $log -Encoding UTF8 -Value (Get-State)
exit 0
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
# Failover hook: останавливает РОВНО ОДНУ API-реплику — ту, чей X-Instance-Id передан тестом в
# TARGET_INSTANCE_ID, — и оставляет её остановленной. БД, Redis, LB и volumes не трогает.
# docker stop → SIGTERM → реплика 3 с отвечает 503 на /health/ready (LB выводит её), затем закрывает
# listener и дожидается запросов в полёте. Возврат ≈ 4 с (< 10 с). Восстановление: scripts/start-replicas.sh
set -eu
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
. "$(dirname "$0")/lib.sh"
LOG=${FAILOVER_EVIDENCE:-evidence/failover-hook.txt}
cid=$(container_of "$TARGET_INSTANCE_ID") || { echo "no running api replica with id $TARGET_INSTANCE_ID" >&2; exit 1; }
name=$(docker inspect -f '{{.Name}}' "$cid" | tr -d /)
{ echo "== $(now) stop-one target=$TARGET_INSTANCE_ID container=$name"; state; } >> "$LOG"
docker stop -t 15 "$cid" >/dev/null
{ echo "== $(now) after stop"; state; } >> "$LOG"
+2
View File
@@ -0,0 +1,2 @@
"""MTUSI messenger API, lab3."""
__version__ = "3.0.0"
+125
View File
@@ -0,0 +1,125 @@
"""Точка входа: python -m messenger [serve|migrate|healthcheck|stats]."""
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
import time
import urllib.request
_STD = set(vars(logging.LogRecord("", 0, "", 0, "", (), None))) | {"message", "asctime"}
class JsonFormatter(logging.Formatter):
"""Структурированный лог в stdout. Токены, пароли, Authorization и тела запросов не логируются."""
def format(self, record: logging.LogRecord) -> str:
data = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + "Z",
"level": record.levelname.lower(), "logger": record.name, "msg": record.getMessage()}
data.update({k: v for k, v in vars(record).items() if k not in _STD and not k.startswith("_")})
if record.exc_info:
data["exc"] = self.formatException(record.exc_info).splitlines()[-1]
return json.dumps(data, ensure_ascii=False, default=str)
def setup_logging() -> None:
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), handlers=[handler], force=True)
logging.getLogger("uvicorn.access").disabled = True # свой access-log без query-строк
def serve() -> int:
import uvicorn
from .app import build_app
from .config import load
settings = load()
app = build_app(settings)
state = app.state.messenger
log = logging.getLogger("messenger.shutdown")
class DrainingServer(uvicorn.Server):
"""Graceful shutdown для работы за LB.
1-й SIGTERM/SIGINT: readiness → 503 и `Connection: close`, запросы ещё обслуживаются;
через SHUTDOWN_DRAIN_SECONDS (≥ времени обнаружения LB) uvicorn перестаёт принимать
соединения и дожидается запросов в полёте. Повторный сигнал — немедленный выход."""
drain_started = None
def handle_exit(self, sig, frame):
if self.drain_started is None and settings.shutdown_drain_seconds > 0 and not self.should_exit:
self.drain_started = time.monotonic()
state.draining = True
log.info("drain started", extra={"signal": sig, "instance_id": settings.instance_id,
"drain_seconds": settings.shutdown_drain_seconds})
return
super().handle_exit(sig, frame)
async def on_tick(self, counter):
if (self.drain_started is not None and not self.should_exit
and time.monotonic() - self.drain_started >= settings.shutdown_drain_seconds):
log.info("drain finished; closing listener", extra={"instance_id": settings.instance_id})
self.should_exit = True
return await super().on_tick(counter)
config = uvicorn.Config(app, host=settings.host, port=settings.port, log_config=None,
access_log=False, server_header=False, proxy_headers=False,
# keep-alive дольше, чем idle-пул HAProxy: соединение первым закрывает LB,
# а не реплика (иначе гонка «LB шлёт запрос в закрываемое соединение»).
timeout_keep_alive=int(os.environ.get("KEEPALIVE_SECONDS", "30")),
timeout_graceful_shutdown=10, lifespan="on",
backlog=int(os.environ.get("BACKLOG", "2048")))
server = DrainingServer(config)
server.run()
return 0 if server.started else 1
def migrate() -> int:
from .config import load
from .db import migrate_with_wait
applied = asyncio.run(migrate_with_wait(load()))
print(json.dumps({"applied": applied}))
return 0
def healthcheck() -> int:
port = os.environ.get("APP_PORT", "8080")
path = sys.argv[2] if len(sys.argv) > 2 else "/health/ready"
try:
with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=2) as resp:
return 0 if resp.status == 200 else 1
except Exception: # noqa: BLE001
return 1
def stats() -> int:
port = os.environ.get("APP_PORT", "8080")
with urllib.request.urlopen(f"http://127.0.0.1:{port}/internal/stats", timeout=2) as resp:
print(resp.read().decode())
return 0
def main() -> int:
setup_logging()
command = sys.argv[1] if len(sys.argv) > 1 else "serve"
commands = {"serve": serve, "migrate": migrate, "healthcheck": healthcheck, "stats": stats}
if command not in commands:
print("usage: python -m messenger [serve|migrate|healthcheck|stats]", file=sys.stderr)
return 2
try:
return commands[command]()
except Exception as exc: # noqa: BLE001
from .config import ConfigError
if isinstance(exc, ConfigError):
logging.getLogger("messenger").error("configuration error: %s", exc)
return 2
raise
if __name__ == "__main__":
sys.exit(main())
+620
View File
@@ -0,0 +1,620 @@
"""HTTP API мессенджера (lab3): Starlette + psycopg 3 (async pool) + Redis/Valkey-кэш сессий.
Реплика stateless: всё, от чего зависит корректность (сессии, идемпотентность, данные), лежит
в PostgreSQL; Redis — только кэш. Любую реплику можно остановить или заменить."""
from __future__ import annotations
import asyncio
import base64
import binascii
import contextlib
import hashlib
import json
import os
import re
import logging
import time
import uuid
from pathlib import Path
from psycopg import errors as pg_errors
from starlette.applications import Starlette
from starlette.exceptions import HTTPException
from starlette.requests import Request
from starlette.responses import JSONResponse, Response
from starlette.routing import Route
from . import db
from .config import Settings
from .errors import ApiError, invalid, not_found, unauthorized_basic, unauthorized_bearer, unavailable
from .security import PasswordHasher, looks_like_token, new_token, token_hash
from .sessions import Principal, SessionCache
from .validation import (USERNAME_RE, decode_cursor, encode_cursor, parse_json_object, parse_limit,
parse_uuid, single_param, text_field)
log = logging.getLogger("messenger.http")
# --------------------------------------------------------------------------------------------
# Ответы и обвязка
# --------------------------------------------------------------------------------------------
def request_id(request: Request) -> str:
rid = getattr(request.state, "request_id", None)
if not rid:
rid = uuid.uuid4().hex
request.state.request_id = rid
return rid
def json_response(request: Request, payload, status: int = 200, headers: dict | None = None) -> JSONResponse:
h = {"X-Request-Id": request_id(request), "Cache-Control": "no-store"}
h.update(headers or {})
return JSONResponse(payload, status_code=status, headers=h)
def error_response(request: Request, err: ApiError) -> JSONResponse:
rid = request_id(request)
body = {"error": {"code": err.public_code, "message": err.message, "request_id": rid}}
return json_response(request, body, err.status, err.headers)
class RequestContextMiddleware:
"""Чистый ASGI middleware: X-Request-Id в каждом ответе (включая 404/405 роутера) и access-log
без query-строки, заголовков и тела."""
def __init__(self, app, instance_id: str = "api", state=None):
self.app = app
self.instance_id = instance_id.encode()
self.state = state
async def __call__(self, scope, receive, send):
if scope["type"] != "http":
return await self.app(scope, receive, send)
rid = uuid.uuid4().hex
scope.setdefault("state", {})["request_id"] = rid
started = time.monotonic()
status_holder = {}
async def send_wrapper(message):
if message["type"] == "http.response.start":
status_holder["status"] = message["status"]
headers = list(message.get("headers", []))
if not any(k.lower() == b"x-request-id" for k, _ in headers):
headers.append((b"x-request-id", rid.encode()))
headers.append((b"x-instance-id", self.instance_id))
if self.state is not None and self.state.draining:
# Во время drain просим клиента/LB не переиспользовать соединение с этой репликой.
headers.append((b"connection", b"close"))
message["headers"] = headers
await send(message)
try:
await self.app(scope, receive, send_wrapper)
finally:
if self.state is not None:
self.state.stats["requests"] += 1
log.info("request", extra={"request_id": rid, "method": scope.get("method"),
"path": scope.get("path"), "status": status_holder.get("status"),
"duration_ms": round((time.monotonic() - started) * 1000, 2)})
def endpoint(handler):
"""Единая обработка ошибок и общий таймаут запроса."""
async def wrapper(request: Request):
state: AppState = request.app.state.messenger
try:
async with asyncio.timeout(state.settings.request_timeout_seconds):
return await handler(request, state)
except ApiError as err:
return error_response(request, err)
except Exception as exc: # noqa: BLE001
if db.is_unavailable(exc) or isinstance(exc, TimeoutError):
log.warning("dependency unavailable", extra={"request_id": request_id(request),
"error": type(exc).__name__})
return error_response(request, unavailable())
log.exception("unhandled error", extra={"request_id": request_id(request)})
return error_response(request, ApiError("internal_error"))
wrapper.__name__ = handler.__name__
return wrapper
async def read_body(request: Request, limit: int) -> bytes:
declared = request.headers.get("content-length")
if declared is not None:
if not declared.isdigit():
raise invalid("Некорректный Content-Length")
if int(declared) > limit:
raise ApiError("payload_too_large")
chunks, size = [], 0
async for chunk in request.stream():
size += len(chunk)
if size > limit:
raise ApiError("payload_too_large")
chunks.append(chunk)
return b"".join(chunks)
async def read_json(request: Request, state: "AppState", allowed: set[str], required: set[str]) -> dict:
ctype = request.headers.get("content-type", "").split(";")[0].strip().lower()
body = await read_body(request, state.settings.max_body_bytes)
if ctype != "application/json":
raise invalid("Ожидается Content-Type: application/json")
return parse_json_object(body, allowed, required)
def user_json(row) -> dict:
return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"],
"created_at": db.iso(row["created_at"])}
def chat_json(row) -> dict:
return {"id": str(row["id"]), "title": row["title"],
"member_ids": [str(m) for m in row["member_ids"]], "created_at": db.iso(row["created_at"])}
def message_json(row) -> dict:
return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]),
"text": row["text"], "created_at": db.iso(row["created_at"])}
# --------------------------------------------------------------------------------------------
# Состояние приложения и аутентификация
# --------------------------------------------------------------------------------------------
class AppState:
def __init__(self, settings: Settings):
self.settings = settings
self.pool = db.make_pool(settings)
self.cache = SessionCache(settings)
self.hasher = PasswordHasher(settings.scrypt_log2_n, settings.scrypt_r, settings.scrypt_p,
settings.kdf_concurrency)
self.ready = False # схема проверена, пул открыт
self.draining = False # получен SIGTERM: readiness = 503, LB выводит реплику
self.stop = asyncio.Event()
self.tasks: list[asyncio.Task] = []
self.stats = {"requests": 0, "inflight": 0, "idempotent_replays": 0, "idempotency_conflicts": 0}
async def authenticate(request: Request, state: AppState) -> Principal:
"""Только Authorization: Bearer <opaque>. X-User-Id и Basic игнорируются/отклоняются."""
header = request.headers.get("authorization", "")
scheme, _, token = header.partition(" ")
token = token.strip()
if scheme.lower() != "bearer" or not looks_like_token(token):
raise unauthorized_bearer()
th = token_hash(token)
principal = await state.cache.get(th)
if principal is not None:
return principal
async with state.pool.connection() as conn:
row = await (await conn.execute(
"SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at"
" FROM sessions s JOIN users u ON u.id = s.user_id"
" WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > clock_timestamp()",
(th,))).fetchone()
if row is None:
raise unauthorized_bearer()
principal = Principal(user=user_json(row), expires_at=row["expires_at"].timestamp(), token_hash=th)
await state.cache.put(principal)
return principal
def parse_basic(header: str) -> tuple[str, str]:
scheme, _, value = header.partition(" ")
if scheme.lower() != "basic" or not value.strip():
raise unauthorized_basic()
try:
decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8")
except (binascii.Error, ValueError, UnicodeDecodeError):
raise unauthorized_basic() from None
username, sep, password = decoded.partition(":") # двоеточие допустимо только в пароле
if not sep or not username.isascii() or not USERNAME_RE.fullmatch(username) or not 12 <= len(password) <= 128:
raise unauthorized_basic()
return username, password
# --------------------------------------------------------------------------------------------
# Handlers
# --------------------------------------------------------------------------------------------
@endpoint
async def live(request: Request, state: AppState):
return json_response(request, {"status": "ok"})
@endpoint
async def ready(request: Request, state: AppState):
# Критичная зависимость синхронного пути — только PostgreSQL. Redis — кэш: при его потере
# API продолжает работу через источник истины, поэтому readiness от него не зависит.
if not state.ready or state.draining:
raise unavailable()
try:
async with asyncio.timeout(1.5):
async with state.pool.connection() as conn:
await conn.execute("SELECT 1")
except Exception: # noqa: BLE001
raise unavailable() from None
return json_response(request, {"status": "ok"})
@endpoint
async def internal_stats(request: Request, state: AppState):
"""Только для исследования/диагностики изнутри контейнера; LB отдаёт 404 на /internal/*."""
if request.client is None or request.client.host not in {"127.0.0.1", "::1"}:
raise not_found()
pool = state.pool.get_stats()
return json_response(request, {"instance_id": state.settings.instance_id,
"version": os.environ.get("APP_VERSION", "dev"), "ready": state.ready,
"draining": state.draining, "app": state.stats,
"cache": state.cache.stats, "pool": pool})
@endpoint
async def create_user(request: Request, state: AppState):
data = await read_json(request, state, {"username", "display_name", "password"},
{"username", "display_name", "password"})
username = data["username"]
if not isinstance(username, str) or not USERNAME_RE.fullmatch(username):
raise invalid("username должен соответствовать [a-z0-9_]{3,32}")
display_name = text_field(data, "display_name", 1, 100)
password = text_field(data, "password", 12, 128)
async with state.pool.connection() as conn:
taken = await (await conn.execute("SELECT 1 FROM users WHERE username = %s", (username,))).fetchone()
if taken:
raise ApiError("conflict", "username уже занят")
password_hash = await state.hasher.hash(password)
try:
async with state.pool.connection() as conn:
row = await (await conn.execute(
"INSERT INTO users (username, display_name, password_hash) VALUES (%s, %s, %s)"
" RETURNING id, username, display_name, created_at",
(username, display_name, password_hash))).fetchone()
except pg_errors.UniqueViolation:
# Гонка регистрации: проверку выше прошли оба запроса, уникальный индекс пропустил один.
log.info("registration race resolved by unique constraint", extra={"request_id": request_id(request)})
raise ApiError("conflict", "username уже занят") from None
return json_response(request, user_json(row), 201)
@endpoint
async def me(request: Request, state: AppState):
principal = await authenticate(request, state)
return json_response(request, principal.user)
@endpoint
async def create_session(request: Request, state: AppState):
username, password = parse_basic(request.headers.get("authorization", ""))
body = await read_body(request, 1024)
if body:
raise invalid("POST /auth/sessions не принимает тело")
async with state.pool.connection() as conn:
row = await (await conn.execute(
"SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s",
(username,))).fetchone()
ok = await state.hasher.verify(password, row["password_hash"] if row else None)
if not ok:
raise unauthorized_basic()
token = new_token()
th = token_hash(token)
async with state.pool.connection() as conn:
srow = await (await conn.execute(
"INSERT INTO sessions (token_hash, user_id, expires_at)"
" VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at",
(th, row["id"], state.settings.session_ttl_seconds))).fetchone()
user = user_json(row)
await state.cache.put(Principal(user=user, expires_at=srow["expires_at"].timestamp(), token_hash=th))
return json_response(request, {"token": token, "token_type": "Bearer",
"expires_at": db.iso(srow["expires_at"]), "user": user}, 201)
@endpoint
async def delete_session(request: Request, state: AppState):
principal = await authenticate(request, state)
th = principal.token_hash
async with state.pool.connection() as conn:
async with conn.transaction():
revoked = await (await conn.execute(
"UPDATE sessions SET revoked_at = clock_timestamp()"
" WHERE token_hash = %s AND revoked_at IS NULL RETURNING 1", (th,))).fetchone()
if revoked is None:
raise unauthorized_bearer()
if state.cache.enabled:
await conn.execute(
"INSERT INTO session_cache_invalidations (token_hash) VALUES (%s) ON CONFLICT DO NOTHING",
(th,))
# COMMIT в PostgreSQL уже сделал отзыв окончательным; Redis лишь догоняет источник истины.
if state.cache.enabled and await state.cache.invalidate(th):
with contextlib.suppress(Exception):
async with state.pool.connection() as conn:
await conn.execute("DELETE FROM session_cache_invalidations WHERE token_hash = %s", (th,))
return Response(status_code=204, headers={"X-Request-Id": request_id(request)})
@endpoint
async def create_chat(request: Request, state: AppState):
principal = await authenticate(request, state)
data = await read_json(request, state, {"title", "member_ids"}, {"title", "member_ids"})
title = text_field(data, "title", 1, 100)
raw_members = data["member_ids"]
if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100:
raise invalid("member_ids: массив из 1–100 UUID")
members = [parse_uuid(m, "member_id") for m in raw_members]
if len(set(members)) != len(members):
raise invalid("member_ids содержит дубликаты")
caller = uuid.UUID(principal.user["id"])
if caller not in members:
raise invalid("member_ids должен содержать создателя чата")
async with state.pool.connection() as conn:
try:
async with conn.transaction(): # атомарно: чат и весь состав или ничего
chat = await (await conn.execute(
"INSERT INTO chats (title, created_by) VALUES (%s, %s) RETURNING id, title, created_at",
(title, caller))).fetchone()
await conn.execute(
"INSERT INTO chat_members (chat_id, user_id, position)"
" SELECT %s, m.user_id, m.ord - 1 FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)",
(chat["id"], members))
except pg_errors.ForeignKeyViolation:
raise invalid("member_ids содержит неизвестного пользователя") from None
chat["member_ids"] = members
return json_response(request, chat_json(chat), 201)
_CHAT_LIST_SQL = (
"SELECT c.id, c.title, c.created_at,"
" (SELECT array_agg(x.user_id ORDER BY x.position) FROM chat_members x WHERE x.chat_id = c.id) AS member_ids"
" FROM chat_members m JOIN chats c ON c.id = m.chat_id"
" WHERE m.user_id = %s {after}"
" ORDER BY c.created_at, c.id LIMIT %s")
@endpoint
async def list_chats(request: Request, state: AppState):
principal = await authenticate(request, state)
caller = uuid.UUID(principal.user["id"])
limit = parse_limit(request.query_params)
cursor = single_param(request.query_params, "cursor")
async with state.pool.connection() as conn:
if cursor is not None:
ts, last_id = decode_cursor(cursor, "chats", caller)
rows = await (await conn.execute(_CHAT_LIST_SQL.format(after="AND (c.created_at, c.id) > (%s, %s)"),
(caller, ts, last_id, limit + 1))).fetchall()
else:
rows = await (await conn.execute(_CHAT_LIST_SQL.format(after=""), (caller, limit + 1))).fetchall()
items = rows[:limit]
next_cursor = (encode_cursor("chats", caller, items[-1]["created_at"], items[-1]["id"])
if len(rows) > limit else None)
return json_response(request, {"items": [chat_json(r) for r in items], "next_cursor": next_cursor})
@endpoint
async def get_chat(request: Request, state: AppState):
principal = await authenticate(request, state)
chat_id = parse_uuid(request.path_params["chat_id"], "chat_id")
async with state.pool.connection() as conn:
row = await (await conn.execute(
"SELECT c.id, c.title, c.created_at, array_agg(m.user_id ORDER BY m.position) AS member_ids"
" FROM chats c JOIN chat_members m ON m.chat_id = c.id"
" WHERE c.id = %s AND EXISTS (SELECT 1 FROM chat_members a WHERE a.chat_id = c.id AND a.user_id = %s)"
" GROUP BY c.id", (chat_id, uuid.UUID(principal.user["id"])))).fetchone()
if row is None:
raise not_found() # нет или чужой — одинаковый ответ
return json_response(request, chat_json(row))
async def _is_member(conn, chat_id: uuid.UUID, user_id: uuid.UUID) -> bool:
row = await (await conn.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s",
(chat_id, user_id))).fetchone()
return row is not None
IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}")
def idempotency_key(request: Request) -> str:
values = request.headers.getlist("idempotency-key")
if len(values) != 1 or not IDEMPOTENCY_KEY_RE.fullmatch(values[0]):
raise invalid("Требуется заголовок Idempotency-Key: [A-Za-z0-9._:-]{1,128}")
return values[0]
def request_fingerprint(body: dict) -> bytes:
"""SHA-256 канонического JSON: порядок ключей и пробелы не влияют на равенство запросов."""
canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":"))
return hashlib.sha256(canonical.encode("utf-8")).digest()
class _Replay(Exception):
"""Ключ уже занят: откатить свою вставку и прочитать сохранённый результат."""
@endpoint
async def create_message(request: Request, state: AppState):
principal = await authenticate(request, state)
chat_id = parse_uuid(request.path_params["chat_id"], "chat_id")
sender = uuid.UUID(principal.user["id"])
try:
key = idempotency_key(request)
data = await read_json(request, state, {"text"}, {"text"})
text = text_field(data, "text", 1, 4000)
except ApiError as err:
if err.status == 400:
# Посторонний не должен по 400/404 узнавать о существовании чата.
async with state.pool.connection() as conn:
if not await _is_member(conn, chat_id, sender):
raise not_found() from None
raise
fingerprint = request_fingerprint({"text": text})
ttl = state.settings.idempotency_ttl_seconds
async with state.pool.connection() as conn:
try:
# Одна транзакция: сообщение и запись ключа фиксируются вместе или не фиксируются вовсе.
# Конкурентный INSERT того же ключа (любая реплика) ждёт COMMIT/ROLLBACK первого на
# уникальном индексе: после COMMIT получает конфликт, после ROLLBACK — проходит сам.
async with conn.transaction():
await conn.execute(
"DELETE FROM idempotency_keys WHERE user_id = %s AND chat_id = %s AND key = %s"
" AND created_at < clock_timestamp() - make_interval(secs => %s)",
(sender, chat_id, key, ttl))
row = await (await conn.execute(
"INSERT INTO messages (chat_id, sender_id, text)"
" SELECT %(c)s, %(u)s, %(t)s"
" WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %(c)s AND user_id = %(u)s)"
" RETURNING id, chat_id, sender_id, text, created_at",
{"c": chat_id, "u": sender, "t": text})).fetchone()
if row is None:
raise not_found() # откат: ошибка доступа не фиксируется как результат ключа
claimed = await (await conn.execute(
"INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id)"
" VALUES (%s, %s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING 1",
(sender, chat_id, key, fingerprint, row["id"]))).fetchone()
if claimed is None:
raise _Replay()
except _Replay:
# Повтор: снова проверены сессия (authenticate) и доступ (членство), затем — сохранённый ответ.
stored = await (await conn.execute(
"SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at"
" FROM idempotency_keys k JOIN messages m ON m.id = k.message_id"
" WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s"
" AND EXISTS (SELECT 1 FROM chat_members a WHERE a.chat_id = k.chat_id AND a.user_id = k.user_id)",
(sender, chat_id, key))).fetchone()
if stored is None:
raise not_found() from None
if bytes(stored["request_hash"]) != fingerprint:
state.stats["idempotency_conflicts"] += 1
raise ApiError("conflict", "Idempotency-Key уже использован с другим запросом") from None
state.stats["idempotent_replays"] += 1
return json_response(request, message_json(stored), 201, {"Idempotent-Replayed": "true"})
return json_response(request, message_json(row), 201)
_HISTORY_SQL = (
"SELECT id, chat_id, sender_id, text, created_at FROM messages"
" WHERE chat_id = %s {after} ORDER BY created_at, id LIMIT %s")
@endpoint
async def list_messages(request: Request, state: AppState):
principal = await authenticate(request, state)
chat_id = parse_uuid(request.path_params["chat_id"], "chat_id")
async with state.pool.connection() as conn:
if not await _is_member(conn, chat_id, uuid.UUID(principal.user["id"])):
raise not_found()
limit = parse_limit(request.query_params)
cursor = single_param(request.query_params, "cursor")
if cursor is not None:
ts, last_id = decode_cursor(cursor, "messages", chat_id)
rows = await (await conn.execute(_HISTORY_SQL.format(after="AND (created_at, id) > (%s, %s)"),
(chat_id, ts, last_id, limit + 1))).fetchall()
else:
rows = await (await conn.execute(_HISTORY_SQL.format(after=""), (chat_id, limit + 1))).fetchall()
items = rows[:limit]
next_cursor = (encode_cursor("messages", chat_id, items[-1]["created_at"], items[-1]["id"])
if len(rows) > limit else None)
return json_response(request, {"items": [message_json(r) for r in items], "next_cursor": next_cursor})
# --------------------------------------------------------------------------------------------
# Фоновые задачи и сборка приложения
# --------------------------------------------------------------------------------------------
async def wait_for_schema(state: AppState) -> None:
"""Реплика не мигрирует схему сама (это делает one-shot сервис migrate) и не становится ready,
пока в БД не применены все миграции, которые знает этот код."""
delay = 0.5
while not state.stop.is_set():
try:
ok, missing = await db.schema_status(state.pool)
if ok:
state.ready = True
log.info("schema compatible; replica ready", extra={"instance_id": state.settings.instance_id})
return
log.warning("schema is behind this code; waiting for migrate", extra={"missing": missing})
except Exception as exc: # noqa: BLE001 — БД ещё недоступна или нет schema_migrations
log.warning("schema check failed; retrying", extra={"error": type(exc).__name__})
with contextlib.suppress(asyncio.TimeoutError):
await asyncio.wait_for(state.stop.wait(), timeout=delay)
delay = min(delay * 2, 5)
async def expired_sessions_cleanup(state: AppState) -> None:
"""Удаляет сессии, истёкшие более суток назад, пачками (не держит долгих блокировок)."""
while not state.stop.is_set():
with contextlib.suppress(Exception):
async with state.pool.connection() as conn:
await conn.execute(
"DELETE FROM sessions WHERE token_hash IN (SELECT token_hash FROM sessions"
" WHERE expires_at < clock_timestamp() - interval '1 day' LIMIT 1000)")
await conn.execute(
"DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys"
" WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT 5000)",
(state.settings.idempotency_ttl_seconds,))
with contextlib.suppress(asyncio.TimeoutError):
await asyncio.wait_for(state.stop.wait(), timeout=300)
def build_app(settings: Settings) -> Starlette:
state = AppState(settings)
@contextlib.asynccontextmanager
async def lifespan(app: Starlette):
if settings.migrate_on_start:
applied = await db.migrate_with_wait(settings)
log.info("migrations checked", extra={"applied": applied})
await state.pool.open(wait=False)
with contextlib.suppress(OSError):
Path(settings.instance_id_file).write_text(settings.instance_id + "\n")
state.tasks.append(asyncio.create_task(wait_for_schema(state)))
if state.cache.enabled:
state.tasks.append(asyncio.create_task(state.cache.maintenance_loop(state.pool, state.stop)))
state.tasks.append(asyncio.create_task(expired_sessions_cleanup(state)))
log.info("api started", extra={"instance_id": settings.instance_id, "version": os.environ.get("APP_VERSION", "dev"),
"cache": state.cache.enabled,
"session_ttl_seconds": settings.session_ttl_seconds})
try:
yield
finally:
state.ready = False
state.stop.set()
for task in state.tasks:
task.cancel()
await asyncio.gather(*state.tasks, return_exceptions=True)
await state.cache.close()
await state.pool.close()
async def http_exception(request: Request, exc: HTTPException):
if exc.status_code == 405:
err = ApiError("method_not_allowed", "Метод не поддерживается", headers=dict(exc.headers or {}))
elif exc.status_code == 404:
err = ApiError("not_found")
else:
err = ApiError("invalid_request")
return error_response(request, err)
async def server_error(request: Request, exc: Exception):
return error_response(request, ApiError("internal_error"))
routes = [
Route("/health/live", live, methods=["GET"]),
Route("/health/ready", ready, methods=["GET"]),
Route("/internal/stats", internal_stats, methods=["GET"]),
Route("/api/v1/users", create_user, methods=["POST"]),
Route("/api/v1/users/me", me, methods=["GET"]),
Route("/api/v1/auth/sessions", create_session, methods=["POST"]),
Route("/api/v1/auth/sessions/current", delete_session, methods=["DELETE"]),
Route("/api/v1/chats", create_chat, methods=["POST"]),
Route("/api/v1/chats", list_chats, methods=["GET"]),
Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]),
Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]),
Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]),
]
app = Starlette(routes=routes, lifespan=lifespan,
exception_handlers={HTTPException: http_exception, Exception: server_error})
app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id, state=state)
app.state.messenger = state
return app
__all__ = ["build_app"]
+132
View File
@@ -0,0 +1,132 @@
"""Конфигурация из окружения. Секреты читаются из *_FILE (Docker/Compose secrets)."""
from __future__ import annotations
import os
import secrets
from dataclasses import dataclass
from pathlib import Path
class ConfigError(RuntimeError):
pass
def _secret(name: str, required: bool) -> str | None:
"""Значение NAME или содержимое файла NAME_FILE (приоритет у файла)."""
path = os.environ.get(name + "_FILE")
if path:
try:
value = Path(path).read_text(encoding="utf-8").strip()
except OSError as exc:
raise ConfigError(f"cannot read {name}_FILE") from exc
if not value:
raise ConfigError(f"{name}_FILE is empty")
return value
value = os.environ.get(name, "").strip()
if value:
return value
if required:
raise ConfigError(f"{name} or {name}_FILE is required")
return None
def _int(name: str, default: int, lo: int, hi: int) -> int:
raw = os.environ.get(name)
if raw is None or raw == "":
return default
try:
value = int(raw)
except ValueError as exc:
raise ConfigError(f"{name} must be an integer") from exc
if not lo <= value <= hi:
raise ConfigError(f"{name} must be in [{lo}, {hi}]")
return value
def _float(name: str, default: float, lo: float, hi: float) -> float:
raw = os.environ.get(name)
if raw is None or raw == "":
return default
try:
value = float(raw)
except ValueError as exc:
raise ConfigError(f"{name} must be a number") from exc
if not lo <= value <= hi:
raise ConfigError(f"{name} must be in [{lo}, {hi}]")
return value
@dataclass(frozen=True)
class Settings:
host: str
port: int
env: str
database_url: str
cache_url: str | None
session_ttl_seconds: int
session_cache_ttl_seconds: int
cache_timeout_seconds: float
cache_retry_seconds: float
db_pool_min: int
db_pool_max: int
db_pool_timeout_seconds: float
db_connect_timeout_seconds: int
db_statement_timeout_ms: int
request_timeout_seconds: float
max_body_bytes: int
kdf_concurrency: int
scrypt_log2_n: int
scrypt_r: int
scrypt_p: int
migrate_on_start: bool
migrate_wait_seconds: int
instance_id: str
instance_id_file: str
idempotency_ttl_seconds: int
shutdown_drain_seconds: float
@property
def cache_enabled(self) -> bool:
return self.cache_url is not None
def load() -> Settings:
store = os.environ.get("SESSION_STORE", "postgres").strip()
if store != "postgres":
# Источник истины сессий — всегда PostgreSQL; Redis/Valkey — только кэш (см. REPORT.md).
raise ConfigError("SESSION_STORE must be 'postgres' (Redis/Valkey is a cache via CACHE_URL)")
cache_mode = os.environ.get("SESSION_CACHE", "auto").strip()
if cache_mode not in {"auto", "redis", "off"}:
raise ConfigError("SESSION_CACHE must be auto, redis or off")
cache_url = None if cache_mode == "off" else _secret("CACHE_URL", required=cache_mode == "redis")
pool_max = _int("DB_POOL_MAX", 10, 1, 100)
return Settings(
host=os.environ.get("APP_HOST", "0.0.0.0"),
port=_int("APP_PORT", 8080, 1, 65535),
env=os.environ.get("APP_ENV", "lab"),
database_url=_secret("DATABASE_URL", required=True) or "",
cache_url=cache_url,
session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600),
session_cache_ttl_seconds=_int("SESSION_CACHE_TTL_SECONDS", 60, 1, 3600),
cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.2, 0.01, 5),
cache_retry_seconds=_float("CACHE_RETRY_SECONDS", 2.0, 0.1, 60),
db_pool_min=min(_int("DB_POOL_MIN", 1, 0, 100), pool_max),
db_pool_max=pool_max,
db_pool_timeout_seconds=_float("DB_POOL_TIMEOUT_SECONDS", 2.0, 0.1, 60),
db_connect_timeout_seconds=_int("DB_CONNECT_TIMEOUT_SECONDS", 3, 1, 60),
db_statement_timeout_ms=_int("DB_STATEMENT_TIMEOUT_MS", 5000, 100, 600000),
request_timeout_seconds=_float("REQUEST_TIMEOUT_SECONDS", 10.0, 1, 120),
max_body_bytes=_int("MAX_BODY_BYTES", 65536, 1024, 1048576),
kdf_concurrency=_int("KDF_CONCURRENCY", 4, 1, 64),
scrypt_log2_n=_int("SCRYPT_LOG2_N", 15, 14, 20),
scrypt_r=_int("SCRYPT_R", 8, 1, 32),
scrypt_p=_int("SCRYPT_P", 3, 1, 16),
# lab3: миграции выполняет отдельный one-shot сервис `migrate`; реплики их только проверяют.
migrate_on_start=os.environ.get("MIGRATE_ON_START", "false").lower() in {"1", "true", "yes"},
migrate_wait_seconds=_int("MIGRATE_WAIT_SECONDS", 60, 0, 600),
# Непрозрачный alias реплики: случайный на каждый запуск процесса, без hostname/IP.
instance_id=os.environ.get("INSTANCE_ID") or "api-" + secrets.token_hex(4),
instance_id_file=os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id"),
idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 60, 30 * 24 * 3600),
shutdown_drain_seconds=_float("SHUTDOWN_DRAIN_SECONDS", 3.0, 0, 60),
)
+129
View File
@@ -0,0 +1,129 @@
"""Пул соединений PostgreSQL, версионируемые миграции и классификация ошибок БД."""
from __future__ import annotations
import asyncio
import hashlib
import logging
import time
from datetime import datetime, timezone
from pathlib import Path
import psycopg
from psycopg import errors as pg_errors
from psycopg.rows import dict_row
from psycopg_pool import AsyncConnectionPool, PoolTimeout
from .config import Settings
log = logging.getLogger("messenger.db")
MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations"
MIGRATION_LOCK_ID = 2026_0202 # pg_advisory_lock: параллельные реплики не применяют миграции дважды
def connection_kwargs(settings: Settings) -> dict:
options = (
f"-c statement_timeout={settings.db_statement_timeout_ms} "
"-c lock_timeout=2000 "
"-c idle_in_transaction_session_timeout=10000 "
"-c TimeZone=UTC"
)
return {
"connect_timeout": settings.db_connect_timeout_seconds,
"application_name": "messenger-api",
"options": options,
"row_factory": dict_row,
}
def make_pool(settings: Settings) -> AsyncConnectionPool:
return AsyncConnectionPool(
conninfo=settings.database_url,
kwargs=connection_kwargs(settings),
min_size=settings.db_pool_min,
max_size=settings.db_pool_max,
timeout=settings.db_pool_timeout_seconds, # ожидание свободного соединения
max_waiting=200, # очередь ожидания ограничена
max_idle=300,
reconnect_timeout=30,
check=AsyncConnectionPool.check_connection, # после рестарта PG битые соединения отбрасываются
open=False,
name="messenger",
)
def is_unavailable(exc: BaseException) -> bool:
"""Ошибки, означающие «зависимость недоступна/перегружена», а не баг приложения."""
if isinstance(exc, (PoolTimeout, psycopg.OperationalError, asyncio.TimeoutError, ConnectionError)):
return True
return isinstance(exc, (pg_errors.QueryCanceled, pg_errors.LockNotAvailable,
pg_errors.AdminShutdown, pg_errors.CannotConnectNow))
def iso(ts: datetime) -> str:
"""RFC 3339 UTC с микросекундами и суффиксом Z (точность timestamptz в PostgreSQL)."""
return ts.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
def _migration_files() -> list[tuple[str, str, str]]:
result = []
for path in sorted(MIGRATIONS_DIR.glob("[0-9][0-9][0-9][0-9]_*.sql")):
sql = path.read_text(encoding="utf-8")
result.append((path.stem, sql, hashlib.sha256(sql.encode()).hexdigest()))
return result
async def migrate(settings: Settings) -> list[str]:
"""Применяет недостающие миграции. Повторный запуск — no-op; изменённая применённая миграция —
ошибка (миграции неизменяемы, исправления — новой версией)."""
applied_now: list[str] = []
async with await psycopg.AsyncConnection.connect(
settings.database_url, autocommit=True, **connection_kwargs(settings)) as conn:
await conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_ID,))
try:
await conn.execute(
"CREATE TABLE IF NOT EXISTS schema_migrations ("
" version text PRIMARY KEY, checksum text NOT NULL,"
" applied_at timestamptz NOT NULL DEFAULT clock_timestamp())")
rows = await (await conn.execute("SELECT version, checksum FROM schema_migrations")).fetchall()
done = {r["version"]: r["checksum"] for r in rows}
for version, sql, checksum in _migration_files():
if version in done:
if done[version] != checksum:
raise RuntimeError(f"applied migration {version} was modified")
continue
async with conn.transaction():
await conn.execute(sql)
await conn.execute(
"INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)",
(version, checksum))
applied_now.append(version)
log.info("migration applied", extra={"version": version})
finally:
await conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_ID,))
return applied_now
async def schema_status(pool) -> tuple[bool, list[str]]:
"""Совместима ли схема с этим кодом: применены все известные ему миграции с теми же checksum.
Более новые миграции (expand-фаза следующей версии) допустимы."""
async with pool.connection() as conn:
rows = await (await conn.execute(
"SELECT version, checksum FROM schema_migrations")).fetchall()
done = {r["version"]: r["checksum"] for r in rows}
missing = [v for v, _, c in _migration_files() if done.get(v) != c]
return not missing, missing
async def migrate_with_wait(settings: Settings) -> list[str]:
deadline = time.monotonic() + settings.migrate_wait_seconds
delay = 0.5
while True:
try:
return await migrate(settings)
except psycopg.OperationalError:
if time.monotonic() >= deadline:
raise
log.warning("database not reachable yet; retrying migrations")
await asyncio.sleep(delay)
delay = min(delay * 2, 5)
+67
View File
@@ -0,0 +1,67 @@
"""Единый формат ошибок API: {"error":{"code","message","request_id"}}."""
from __future__ import annotations
BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"'
BEARER_CHALLENGE = 'Bearer realm="messenger"'
_STATUS = {
"invalid_request": 400,
"unauthorized": 401,
"not_found": 404,
"method_not_allowed": 405,
"conflict": 409,
"payload_too_large": 413,
"unsupported_media_type": 415,
"rate_limited": 429,
"internal_error": 500,
"unavailable": 503,
}
_DEFAULT_MESSAGE = {
"invalid_request": "Некорректный запрос",
"unauthorized": "Требуется действительная аутентификация",
"not_found": "Объект не найден",
"conflict": "Конфликт состояния",
"payload_too_large": "Слишком большой запрос",
"unsupported_media_type": "Неподдерживаемый тип содержимого",
"rate_limited": "Слишком много запросов",
"internal_error": "Внутренняя ошибка",
"unavailable": "Сервис временно недоступен",
}
class ApiError(Exception):
def __init__(self, code: str, message: str | None = None, *, challenge: str | None = None,
headers: dict[str, str] | None = None):
super().__init__(code)
self.code = code
self.status = _STATUS[code]
self.message = message or _DEFAULT_MESSAGE.get(code, "Ошибка")
self.headers = dict(headers or {})
if challenge:
self.headers["WWW-Authenticate"] = challenge
@property
def public_code(self) -> str:
# 405 не входит в enum контракта: отдаём invalid_request с корректным HTTP-статусом.
return "invalid_request" if self.code == "method_not_allowed" else self.code
def invalid(message: str) -> ApiError:
return ApiError("invalid_request", message)
def unauthorized_bearer() -> ApiError:
return ApiError("unauthorized", challenge=BEARER_CHALLENGE)
def unauthorized_basic() -> ApiError:
return ApiError("unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE)
def not_found() -> ApiError:
return ApiError("not_found")
def unavailable() -> ApiError:
return ApiError("unavailable", headers={"Retry-After": "1"})
+93
View File
@@ -0,0 +1,93 @@
"""Password KDF (scrypt из OpenSSL через hashlib) и opaque-токены сессий.
Пароль хешируется медленным memory-hard KDF с солью: цель — сделать перебор украденных
hash дорогим. Токен сессии уже содержит 256 бит случайности, перебирать его бессмысленно,
поэтому для индекса в БД/кэше достаточно быстрого SHA-256 — это разные задачи.
"""
from __future__ import annotations
import asyncio
import base64
import hashlib
import hmac
import secrets
_PREFIX = "$scrypt$"
def _b64(data: bytes) -> str:
return base64.b64encode(data).decode("ascii").rstrip("=")
def _unb64(text: str) -> bytes:
return base64.b64decode(text + "=" * (-len(text) % 4), validate=True)
class PasswordHasher:
"""Формат PHC-подобный: $scrypt$ln=15,r=8,p=3$<salt>$<hash>. Параметры хранятся в строке,
поэтому их можно усилить позднее без потери старых hash."""
def __init__(self, log2_n: int, r: int, p: int, concurrency: int):
self.log2_n, self.r, self.p = log2_n, r, p
# Ограничиваем одновременные KDF: каждый вызов занимает 128*r*N байт RAM (32 MiB при ln=15).
self._slots = asyncio.Semaphore(concurrency)
self._dummy: str | None = None
def _maxmem(self, log2_n: int, r: int, p: int) -> int:
return 128 * r * (2 ** log2_n) * 2 + 1024 * 1024 * p
def _derive(self, password: str, salt: bytes, log2_n: int, r: int, p: int) -> bytes:
return hashlib.scrypt(password.encode("utf-8"), salt=salt, n=2 ** log2_n, r=r, p=p,
maxmem=self._maxmem(log2_n, r, p), dklen=32)
def hash_sync(self, password: str) -> str:
salt = secrets.token_bytes(16)
digest = self._derive(password, salt, self.log2_n, self.r, self.p)
return f"{_PREFIX}ln={self.log2_n},r={self.r},p={self.p}${_b64(salt)}${_b64(digest)}"
def verify_sync(self, password: str, encoded: str) -> bool:
try:
if not encoded.startswith(_PREFIX):
return False
params, salt_b64, hash_b64 = encoded[len(_PREFIX):].split("$")
fields = dict(item.split("=", 1) for item in params.split(","))
log2_n, r, p = int(fields["ln"]), int(fields["r"]), int(fields["p"])
if not (10 <= log2_n <= 22 and 1 <= r <= 32 and 1 <= p <= 16):
return False
expected = _unb64(hash_b64)
actual = self._derive(password, _unb64(salt_b64), log2_n, r, p)
except (ValueError, KeyError):
return False
return hmac.compare_digest(actual, expected)
async def hash(self, password: str) -> str:
async with self._slots:
return await asyncio.to_thread(self.hash_sync, password)
async def verify(self, password: str, encoded: str | None) -> bool:
"""Для неизвестного пользователя проверяем фиктивный hash: время ответа не выдаёт,
существует ли username."""
if encoded is None:
if self._dummy is None:
self._dummy = await self.hash("dummy-password-for-timing")
encoded, password = self._dummy, password + "\x00never"
async with self._slots:
await asyncio.to_thread(self.verify_sync, password, encoded)
return False
async with self._slots:
return await asyncio.to_thread(self.verify_sync, password, encoded)
TOKEN_BYTES = 32 # 256 бит энтропии
def new_token() -> str:
return secrets.token_urlsafe(TOKEN_BYTES) # 43 символа base64url
def token_hash(token: str) -> bytes:
return hashlib.sha256(token.encode("ascii")).digest()
def looks_like_token(token: str) -> bool:
return len(token) == 43 and all(c.isascii() and (c.isalnum() or c in "-_") for c in token)
+185
View File
@@ -0,0 +1,185 @@
"""Сессии: PostgreSQL — источник истины, Redis/Valkey — read-through кэш с TTL.
Политика (подробно — REPORT.md):
* ключ sess:v1:<sha256(token)> хранит {user, expires_at}; TTL = min(SESSION_CACHE_TTL_SECONDS,
остаток жизни сессии) — кэш никогда не переживает срок сессии;
* logout: revoked_at + строка outbox в одной транзакции PostgreSQL, затем в Redis атомарно
«надгробие» sess:v1:revoked:<hash> и DEL ключа. Заполнение кэша — Lua SET … только если
надгробия нет, поэтому конкурентный read-through не вернёт отозванную сессию в кэш;
* Redis недоступен → аутентификация идёт в PostgreSQL (fail-open только к источнику истины,
не к анонимному доступу); после ошибки кэш не читается, пока фоновый цикл не проверит Redis
и не дообработает outbox инвалидаций;
* flush/restart Redis = холодный кэш: сессии остаются в PostgreSQL.
"""
from __future__ import annotations
import asyncio
import json
import logging
import time
from dataclasses import dataclass
from psycopg_pool import AsyncConnectionPool
from redis import asyncio as aioredis
from redis.exceptions import RedisError
from .config import Settings
log = logging.getLogger("messenger.sessions")
KEY = "sess:v1:"
TOMBSTONE = "sess:v1:revoked:"
# KEYS[1]=session key, KEYS[2]=tombstone; ARGV[1]=value, ARGV[2]=ttl ms
_SET_IF_NOT_REVOKED = """
if redis.call('EXISTS', KEYS[2]) == 1 then return 0 end
redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
return 1
"""
# KEYS[1]=session key, KEYS[2]=tombstone; ARGV[1]=tombstone ttl ms
_INVALIDATE = """
redis.call('SET', KEYS[2], '1', 'PX', ARGV[1])
redis.call('DEL', KEYS[1])
return 1
"""
@dataclass
class Principal:
user: dict
expires_at: float # epoch seconds
token_hash: bytes
class SessionCache:
def __init__(self, settings: Settings):
self.settings = settings
self.client: aioredis.Redis | None = None
self._set_script = None
self._invalidate_script = None
# degraded=True: кэш не читаем (было исключение Redis или есть недообработанный outbox).
self.degraded = True
self._next_probe = 0.0
self.stats = {"hits": 0, "misses": 0, "errors": 0}
if settings.cache_url:
t = settings.cache_timeout_seconds
self.client = aioredis.from_url(
settings.cache_url, socket_timeout=t, socket_connect_timeout=t,
retry_on_timeout=False, health_check_interval=0, max_connections=50,
decode_responses=False)
self._set_script = self.client.register_script(_SET_IF_NOT_REVOKED)
self._invalidate_script = self.client.register_script(_INVALIDATE)
@property
def enabled(self) -> bool:
return self.client is not None
def _tombstone_ms(self) -> int:
# Надгробие живёт дольше любой записи кэша и любого запроса в полёте.
return int((self.settings.session_cache_ttl_seconds + self.settings.request_timeout_seconds + 5) * 1000)
def _fail(self, op: str, exc: Exception) -> None:
self.stats["errors"] += 1
if not self.degraded:
log.warning("cache unavailable; falling back to PostgreSQL",
extra={"op": op, "error": type(exc).__name__})
self.degraded = True
self._next_probe = time.monotonic() + self.settings.cache_retry_seconds
async def get(self, th: bytes) -> Principal | None:
if not self.client or self.degraded:
return None
try:
raw = await self.client.get(KEY + th.hex())
except (RedisError, OSError, asyncio.TimeoutError) as exc:
self._fail("get", exc)
return None
if raw is None:
self.stats["misses"] += 1
return None
try:
data = json.loads(raw)
principal = Principal(user=data["user"], expires_at=float(data["exp"]), token_hash=th)
except (ValueError, KeyError, TypeError):
return None
if principal.expires_at <= time.time():
return None
self.stats["hits"] += 1
return principal
async def put(self, p: Principal) -> None:
if not self.client or self.degraded:
return
ttl_ms = int(min(self.settings.session_cache_ttl_seconds, p.expires_at - time.time()) * 1000)
if ttl_ms < 1:
return
value = json.dumps({"user": p.user, "exp": p.expires_at}, separators=(",", ":"))
try:
await self._set_script(keys=[KEY + p.token_hash.hex(), TOMBSTONE + p.token_hash.hex()],
args=[value, ttl_ms])
except (RedisError, OSError, asyncio.TimeoutError) as exc:
self._fail("put", exc)
async def invalidate(self, th: bytes) -> bool:
if not self.client:
return True
try:
await self._invalidate_script(keys=[KEY + th.hex(), TOMBSTONE + th.hex()],
args=[self._tombstone_ms()])
return True
except (RedisError, OSError, asyncio.TimeoutError) as exc:
self._fail("invalidate", exc)
return False
async def ping(self) -> bool:
if not self.client:
return False
try:
return bool(await self.client.ping())
except (RedisError, OSError, asyncio.TimeoutError):
return False
async def drain_outbox(self, pool: AsyncConnectionPool) -> int:
"""Повторяет инвалидации из session_cache_invalidations. True-результат: outbox пуст."""
drained = 0
while True:
async with pool.connection() as conn:
rows = await (await conn.execute(
"SELECT token_hash FROM session_cache_invalidations ORDER BY created_at LIMIT 500"
)).fetchall()
if not rows:
return drained
done = []
for row in rows:
th = bytes(row["token_hash"])
if not await self.invalidate(th):
raise ConnectionError("cache still unavailable")
done.append(th)
async with pool.connection() as conn:
await conn.execute("DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)", (done,))
drained += len(done)
async def maintenance_loop(self, pool: AsyncConnectionPool, stop: asyncio.Event) -> None:
"""Возвращает кэш в работу только после успешного PING и пустого outbox."""
while not stop.is_set():
if self.client and time.monotonic() >= self._next_probe:
try:
if await self.ping():
drained = await self.drain_outbox(pool)
if self.degraded:
log.info("cache available again", extra={"replayed_invalidations": drained})
self.degraded = False
else:
self._fail("probe", ConnectionError("ping failed"))
except ConnectionError as exc:
self._fail("drain", exc)
except Exception as exc: # noqa: BLE001 — БД недоступна: состояние кэша не меняем
log.warning("outbox drain skipped", extra={"error": type(exc).__name__})
try:
await asyncio.wait_for(stop.wait(), timeout=1.0)
except asyncio.TimeoutError:
pass
async def close(self) -> None:
if self.client:
await self.client.aclose()

Some files were not shown because too many files have changed in this diff Show More