diff --git a/.env.example b/.env.example index d32b9be0..175e6943 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,23 @@ SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 IDEMPOTENCY_TTL_SECONDS=86400 +# --- реализация lab3 (см. REPORT.md, «Соответствие переменных») --- +# Порты LB на хосте (только 127.0.0.1): API, api-ttl (профиль ttl), статистика HAProxy. +API_PORT=8080 +API_TTL_PORT=8081 +LB_STATS_PORT=8404 +# Ресурсы одной реплики API и PostgreSQL. +API_CPUS=1.0 +API_MEMORY=512M +PG_CPUS=1.0 +PG_MAX_CONNECTIONS=100 +# Redis — read-through кэш сессий; источник истины — PostgreSQL. +SESSION_CACHE=redis +SESSION_CACHE_TTL_SECONDS=60 +DB_POOL_MAX=10 +KDF_CONCURRENCY=4 +# Graceful shutdown: сколько секунд реплика отвечает 503 на readiness перед закрытием listener. +SHUTDOWN_DRAIN_SECONDS=3 +# Ограничение конкуренции на LB (трек 5.1): одновременных запросов на реплику и ожидание в очереди LB. +LB_SERVER_MAXCONN=10 +LB_QUEUE_TIMEOUT=2s diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f67b9b23 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# Shell-скрипты и файлы, читаемые внутри Linux-контейнеров, — всегда LF (иначе ломаются на Windows с autocrlf). +*.sh text eol=lf +*.sql text eol=lf +*.py text eol=lf +*.yaml text eol=lf +*.yml text eol=lf +Dockerfile text eol=lf +.env.example text eol=lf +# Windows-обёртки +*.cmd text eol=crlf +*.ps1 text eol=crlf diff --git a/.github/workflows/app.yml b/.github/workflows/app.yml new file mode 100644 index 00000000..783888f1 --- /dev/null +++ b/.github/workflows/app.yml @@ -0,0 +1,37 @@ +name: Application stack +# Отдельный job решения: сборка образа API, Compose-стенд (LB + 2 реплики + migrate + PostgreSQL + Redis), +# публичные smoke/persistence/failover и собственные проверки. Секреты синтетические, генерируются на лету. +on: [push, pull_request] +permissions: + contents: read +jobs: + stack: + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - name: Secrets and env (synthetic) + run: ./scripts/init-secrets.sh + - name: Compose config + run: docker compose --env-file .env config --quiet + - name: Build and start + run: docker compose --env-file .env --profile ttl up -d --build --wait + - name: Public smoke via LB + run: make test BASE_URL=http://127.0.0.1:8080 + - name: Failover (stop one replica), then restore + run: | + make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 + ./scripts/start-replicas.sh + cat evidence/failover-hook.txt + - name: Persistence (restart LB, replicas, PostgreSQL, Redis; volumes kept) + run: make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 + - name: Student tests + run: make test-student + - name: Rolling update under load + run: python3 research/lb_experiments.py rolling-update + - name: Logs on failure + if: failure() + run: docker compose --env-file .env --profile ttl logs --no-color --tail=200 + - name: Stop + if: always() + run: docker compose --env-file .env --profile ttl down -v diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..dca85456 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +# API мессенджера lab3. Python 3.13, зависимости ставятся в venv в отдельной стадии. +ARG PYTHON_IMAGE=python:3.13.5-slim-bookworm + +FROM ${PYTHON_IMAGE} AS deps +ENV PIP_NO_CACHE_DIR=1 PIP_DISABLE_PIP_VERSION_CHECK=1 +WORKDIR /build +COPY requirements.txt ./ +RUN python -m venv /opt/venv \ + && /opt/venv/bin/pip install --require-virtualenv --no-compile -r requirements.txt + +FROM ${PYTHON_IMAGE} +ARG APP_VERSION=dev +LABEL org.opencontainers.image.version=$APP_VERSION +ENV PATH=/opt/venv/bin:$PATH \ + PYTHONPATH=/app/src \ + PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + APP_VERSION=$APP_VERSION \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app +COPY --from=deps /opt/venv /opt/venv +WORKDIR /app +COPY migrations ./migrations +COPY src ./src +USER 10001:10001 +EXPOSE 8080 +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=5s --timeout=3s --start-period=30s --retries=6 \ + CMD ["python", "-m", "messenger", "healthcheck"] +# PID 1 — сам HTTP-процесс (exec-форма), SIGTERM даёт graceful shutdown uvicorn. +ENTRYPOINT ["python", "-m", "messenger"] +CMD ["serve"] diff --git a/Makefile b/Makefile index 5d86d171..7ba70d8d 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,76 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +# --------------------------------------------------------------------------------------------- +# Цели решения (lab3). Публичные цели выше не изменены. +# --------------------------------------------------------------------------------------------- +COMPOSE ?= docker compose --env-file .env +TRACK1 = COMPOSE_FILE=compose.yaml:research/compose.track1.yaml + +.PHONY: secrets up down ps logs destroy-data restore rolling-update test-student test-api test-idempotency \ + test-expiry test-cache-race test-failures lb-experiments track1 track1-report evidence + +secrets: + ./scripts/init-secrets.sh + +# Единая команда запуска: LB + 2 реплики API + migrate (one-shot) + PostgreSQL + Redis (+ api-ttl для теста TTL). +up: secrets + $(COMPOSE) --profile ttl up -d --build --wait + +ps: + $(COMPOSE) --profile ttl ps -a + +logs: + $(COMPOSE) logs --tail=100 lb api + +# Безопасная остановка: контейнеры удаляются, volume pgdata сохраняется. +down: + $(COMPOSE) --profile ttl --profile bench down + +# НАМЕРЕННОЕ удаление учебных данных (volume pgdata). Необратимо. +destroy-data: + @echo "This deletes ALL lab3 data volumes. Type 'yes' to continue:"; read ans; [ "$$ans" = yes ] + $(COMPOSE) --profile ttl --profile bench down -v + +# Вернуть реплики после make test-failover ACTION_SCRIPT=scripts/stop-one.sh +restore: + ./scripts/start-replicas.sh + +rolling-update: + ./scripts/rolling-update.sh + +test-api: + $(PYTHON) student-tests/test_api.py + +test-idempotency: + $(PYTHON) student-tests/test_idempotency.py + +test-expiry: + $(PYTHON) student-tests/test_expiry.py + +test-cache-race: + $(COMPOSE) run --rm --no-deps -T -v "$(CURDIR)/student-tests:/app/student-tests:ro" --entrypoint python api student-tests/test_cache_race.py + +# Останавливает/запускает redis и postgres (volumes не трогает). +test-failures: + $(PYTHON) student-tests/test_failures.py + +test-student: test-api test-idempotency test-expiry test-cache-race test-failures + +# Уровень 4: отказ каждой реплики (SIGTERM и SIGKILL) под нагрузкой + rolling update под нагрузкой. +lb-experiments: + $(PYTHON) research/lb_experiments.py failover-series --mode both + $(PYTHON) research/lb_experiments.py rolling-update + +# Трек 5.1 (≈1 ч): серии нагрузки; затем сводка и графики в evidence/track1/. +track1: + $(TRACK1) $(COMPOSE) --profile bench build loadgen + $(TRACK1) $(PYTHON) research/track1.py run all + $(PYTHON) research/track1.py report + +track1-report: + $(PYTHON) research/track1.py report + +evidence: + ./scripts/collect-evidence.sh diff --git a/REPORT.md b/REPORT.md index c2679005..c5439603 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,293 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Лебедев Дмитрий, БВТ2302 +- Целевая оценка; выбранный трек 5: **5, трек 1 «Цена масштабирования»** - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: + - библиотеки: Starlette 1.7.0, uvicorn 0.54.0, psycopg 3.2.10 + psycopg_pool 3.2.6, redis-py 5.2.1; HAProxy 3.0; генератор нагрузки — Go, только стандартная библиотека; + - документация: Docker Compose (services, deploy, networks), HAProxy 3.0 Configuration Manual (server-template, resolvers, http-check, retry-on, maxconn/queue, http-error), PostgreSQL (INSERT … ON CONFLICT, ожидание конкурентной вставки), Linux CFS bandwidth control (cpu.cfs_quota_us, nr_throttled); + - **ИИ:** реализация, тесты, эксперименты и этот отчёт подготовлены с помощью ИИ-ассистента Claude (Anthropic). Код других студентов не использовался. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Выбранный вариант: Docker Compose + HAProxy.** Prerequisites: Docker Engine 24+ с Compose v2, Python 3.9+ на хосте (тесты — только stdlib), `make` (Linux/macOS; для Windows — PowerShell-скрипты ниже). Свободны `127.0.0.1:8080` (API через LB), `:8081` (api-ttl через LB), `:8404` (статистика HAProxy). + +**Образы (закреплены тегами):** `haproxy:3.0.5-alpine`, `python:3.13.5-slim-bookworm` (база API), `postgres:16.10-alpine`, `redis:7.4.2-alpine`, `golang:1.24.7-alpine` (только сборка генератора трека 1). API и генератор собираются из исходников репозитория. + +```sh +./scripts/init-secrets.sh # secrets/* и .env (значения не печатаются) +docker compose --env-file .env config --quiet +docker compose --env-file .env --profile ttl up -d --build --wait # = make up +curl -si http://127.0.0.1:8080/health/ready | grep -i x-instance-id # запросы чередуются между репликами + +make test BASE_URL=http://127.0.0.1:8080 # публичный smoke через LB +make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080 +./scripts/start-replicas.sh # = make restore: вернуть остановленную реплику +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 +make test-student # API/auth/ACL, идемпотентность, TTL, гонка кэша, отказы БД/Redis +make lb-experiments # уровень 4: отказ каждой реплики (SIGTERM/SIGKILL) и rolling update под нагрузкой +make track1 # трек 5.1, ≈ 1 ч; затем evidence/track1/summary.md и *.svg +./scripts/collect-evidence.sh # всё, кроме трека, с выводом в evidence/ + +make down # безопасно: контейнеры удаляются, volume pgdata сохраняется +make destroy-data # НАМЕРЕННОЕ удаление данных (down -v), с подтверждением +``` + +**Windows (PowerShell 5.1, без make/WSL).** Docker Desktop запущен, Python с python.org. Используйте `127.0.0.1`, а не `localhost` (Windows сначала пробует `::1`), команды — по одной строке (в PowerShell 5.1 нет `&&`): + +```powershell +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\init-secrets.ps1 +docker compose --env-file .env --profile ttl up -d --build --wait +$env:NO_PROXY = '127.0.0.1,localhost' +py -3 tests\smoke.py --base-url http://127.0.0.1:8080 +py -3 tests\resilience.py --mode failover --action-script scripts\stop-one.cmd --base-url http://127.0.0.1:8080 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\start-replicas.ps1 +py -3 tests\resilience.py --mode persistence --action-script scripts\restart.cmd --base-url http://127.0.0.1:8080 +powershell -NoProfile -ExecutionPolicy Bypass -File scripts\collect-evidence.ps1 # всё сразу; -WithTrack1 — с треком +``` + +`*.cmd` — обёртки над `*.ps1`: публичный тест запускает хук напрямую, а `.ps1` Windows так не исполняет. Rolling update (`scripts/rolling-update.sh`) — POSIX sh: на Windows запускайте из Git Bash/WSL. + +**Действия оператора:** только создание secrets. Миграции применяет one-shot сервис `migrate` до старта реплик; повторно — `docker compose --env-file .env run --rm --no-deps migrate` (no-op). Пароль БД записан в volume при первой инициализации — не удаляйте `secrets/` при живом `pgdata`. + +> **Где собраны доказательства.** В среде разработки Docker daemon работал, но Docker Hub и PyPI были закрыты политикой сети. Поэтому `evidence/` получены на **том же `compose.yaml`** с локально собранными образами-заменителями из бинарников хоста: HAProxy 3.0.0 из исходников, PostgreSQL 16.15, Redis 7.0.15, Python 3.13 с теми же библиотеками. Подмена — только поле `image` / сборка API через override-файл, не входящий в репозиторий; топология, сети, лимиты, healthcheck, хуки и конфиг HAProxy — те же. Перед защитой выполните `make up && ./scripts/collect-evidence.sh` (Windows: `scripts\collect-evidence.ps1`), чтобы `evidence/` содержал вывод именно ваших официальных образов; тот же сценарий гоняет `.github/workflows/app.yml`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` + 127.0.0.1:8080 / :8081 / :8404 (stats) + клиент ──HTTP──► lb: HAProxy 3.0 ──(сеть app, internal)──► api × API_REPLICAS (без ports) + │ DNS Docker: server-template api 8 api:8080 │ uvicorn, 1 процесс, пул ≤ 10 + │ GET /health/ready каждые 1 с, fall 2, rise 2 │ + │ maxconn 10 на реплику, очередь ≤ 2 с ▼ (сеть data, internal) + │ retry-on conn-failure + redispatch postgres:16 [pgdata] ← источник истины: + └ panic-mode: backup = те же реплики пользователи, чаты, сообщения, сессии, + ключи идемпотентности + migrate (one-shot, до реплик) ──────────────────────────────► redis:7 — кэш сессий (без persistence) +``` + +| Роль | Service | Образ | Health/readiness | Volume | Сеть/порт | Credentials | +| --- | --- | --- | --- | --- | --- | --- | +| load_balancer | `lb` | `haproxy:3.0.5-alpine` + `infra/haproxy/haproxy.cfg` | `wget /stats`; бэкенды — HTTP-проверка `/health/ready` | — | edge+app; `127.0.0.1:8080/8081/8404` | — | +| api | `api` (`deploy.replicas: ${API_REPLICAS:-2}`) | сборка `Dockerfile` | `python -m messenger healthcheck` → `/health/ready` | нет (stateless) | app+data, **порты не публикуются** | secrets `database_url`, `cache_url` | +| migrations | `migrate` (one-shot, `restart: "no"`) | тот же образ, `command: migrate` | завершение с кодом 0 — условие старта `api` | — | data | `database_url` | +| postgres | `postgres` | `postgres:16.10-alpine` | `pg_isready` | `pgdata` | data | `POSTGRES_PASSWORD_FILE` | +| redis | `redis` | `redis:7.4.2-alpine` (user redis) | `redis-cli ping` | — (кэш) | data | `requirepass` из secret | +| api-ttl (тест TTL) | `api-ttl`, профиль `ttl` | образ API, `SESSION_TTL_SECONDS=2` | как у api | — | app+data; только через LB `:8081` | как у api | + +Лимиты: API — `API_CPUS`/`API_MEMORY` (по умолчанию 1 CPU / 512 MiB на реплику), LB 0.5 CPU / 128 MiB, PostgreSQL 1 CPU / 1 GiB, Redis 0.25 CPU / 128 MiB, migrate 0.5 / 256 MiB. Нет `container_name` — `api` масштабируется (`--scale`, `deploy.replicas`). Контейнер API: uid 10001, `read_only`, `cap_drop: ALL`, `no-new-privileges`. Сети `app` и `data` — `internal: true`: PostgreSQL, Redis и реплики API снаружи недоступны, единственная точка входа — LB. + +**Что перенесено из lab2.** Весь API, PostgreSQL как источник истины, сессии (scrypt, opaque Bearer, TTL, logout), Redis-кэш сессий с outbox-инвалидацией, keyset-пагинация, тесты lab2. **Что изменено в lab3:** +1. **Stateless-реплики.** Всё, от чего зависит корректность, — в PostgreSQL (сессии, ключи идемпотентности), Redis — общий кэш. Sticky sessions нет: `balance roundrobin`. +2. **`X-Instance-Id`** — случайный alias процесса (`api-<8 hex>`), без hostname/IP; реплика пишет его в `/tmp/instance-id` (tmpfs) — так `stop-one.sh` находит контейнер. +3. **Миграции отдельно и однократно:** сервис `migrate` выполняется до реплик (`depends_on: service_completed_successfully`), под `pg_advisory_lock`. Реплики (`MIGRATE_ON_START=false`) только проверяют, что в `schema_migrations` есть все известные им версии с теми же checksum, и до этого не становятся ready. Более новая схема допустима (expand-фаза следующей версии). +4. **Общая идемпотентность** `POST /chats/{id}/messages` (миграция `0003_idempotency_keys.sql`). +5. **Graceful shutdown** под LB и **panic mode** HAProxy (ниже). + +**Идемпотентность — транзакционная граница.** Одна транзакция: удалить просроченный ключ этой тройки → `INSERT message … WHERE EXISTS(membership)` → `INSERT idempotency_keys (user, chat, key, sha256(канонический JSON), message_id) ON CONFLICT DO NOTHING`. Если ключ уже есть — `ROLLBACK` своей вставки и чтение сохранённого результата: тот же hash → тот же 201 и то же Message (`Idempotent-Replayed: true`), другой → 409. Конкурентная вставка того же ключа с **любой** реплики ждёт на уникальном индексе COMMIT/ROLLBACK первой транзакции: после COMMIT получает конфликт, после ROLLBACK вставляет сама. Поэтому одновременные повторы через разные реплики дают ровно одно сообщение, а транзиентная ошибка (откат) ключ не занимает. Повтор заново проверяет сессию (Bearer) и членство в чате. Хеш — от канонического JSON (`sort_keys`, без пробелов): порядок ключей и форматирование не важны. Срок — `IDEMPOTENCY_TTL_SECONDS` (24 ч); очистка — фоновой задачей реплик пачками по 5000. + +**Health, readiness, timeouts, SIGTERM — согласованно.** + +| Параметр | Значение | Зачем | +| --- | --- | --- | +| Проверка LB | `GET /health/ready`, `inter 1s`, после первой ошибки `fastinter 500ms`, `fall 2`, `rise 2`, `timeout check 300ms` | Максимальное время обнаружения отказа ≈ 1 + 0.3 + 0.5 + 0.3 ≈ **2.1 с**; возврат — 2 успешные проверки (≈ 1–2 с) | +| `timeout connect` | 300 ms | к убитому контейнеру нет RST — ждём не дольше 300 мс, затем `retry-on conn-failure` + `redispatch` на другую реплику (повтор только если запрос ещё не отправлен — безопасно и для неидемпотентных POST) | +| `timeout server` | 15 s | > `REQUEST_TIMEOUT_SECONDS` = 10 с приложения | +| SIGTERM в реплике | 3 с drain (`SHUTDOWN_DRAIN_SECONDS`): `/health/ready` = 503, ответы с `Connection: close`, запросы ещё обслуживаются; затем listener закрывается, запросы в полёте завершаются (≤ 10 с); 2-й сигнал — немедленный выход | drain (3 с) > худшего времени обнаружения LB (≈ 2.1 с) — LB выводит реплику до закрытия listener | +| `stop_grace_period` | 15 s | drain + завершение запросов в полёте | +| uvicorn keep-alive | 30 s (> idle-пула HAProxy) | соединение первым закрывает LB, а не реплика | +| `maxconn` на реплику / `timeout queue` | 10 / 2 s | ограничение конкуренции: излишек ждёт в очереди LB, а не в event loop (обоснование — трек 5.1) | +| DNS | `resolvers docker` (127.0.0.11), `hold obsolete 5s` | новые/удалённые реплики (`--scale`, rolling update) подхватываются без перезапуска LB | + +`depends_on`/healthcheck Compose лишь упорядочивают старт и показывают состояние контейнера, но не распределяют трафик и не исключают больной backend из маршрута. Это делает HAProxy своими проверками. + +**Panic mode.** Если readiness не проходит ни у одной реплики (например, упала БД), LB всё равно отправляет запросы в живые процессы (`backup`-слоты на те же адреса, `option allbackups`, `resolve-opts allow-dup-ip`). Клиент получает ответы приложения: анониму 401, бизнес-запросам 503 `unavailable` в формате API, `/health/live` = 200. Без этого LB отвечал бы безликим 503 на всё. Мёртвый процесс в backup-слоте даёт conn-failure и 503 LB. + +**Ответы, созданные LB** (нет живых реплик, 502/504, `/internal/*` → 404), — в формате ошибок API с `request_id` = `X-Request-Id` (uuid). `X-Instance-Id` у них нет, контракт это допускает. Расширения API описаны в `contracts/EXTENSIONS.md`. + +**Соответствие переменных `.env.example`.** `APP_HOST`/`APP_PORT` — адрес uvicorn (`0.0.0.0:8080`); `SESSION_TTL_SECONDS` — срок сессии; `DATABASE_URL_FILE`/`CACHE_URL_FILE` — `/run/secrets/*`; `SESSION_STORE=postgres` — источник истины сессий; `API_REPLICAS` → `deploy.replicas`; `IDEMPOTENCY_TTL_SECONDS` → срок ключа. Дополнительно: `API_PORT`, `API_TTL_PORT`, `LB_STATS_PORT`, `API_CPUS`, `API_MEMORY`, `PG_CPUS`, `PG_MAX_CONNECTIONS`, `SESSION_CACHE`, `SESSION_CACHE_TTL_SECONDS`, `DB_POOL_MAX`, `KDF_CONCURRENCY`, `SHUTDOWN_DRAIN_SECONDS`, `LB_SERVER_MAXCONN`, `LB_QUEUE_TIMEOUT` (см. комментарии в `.env.example` и `compose.yaml`). + +**Rolling update и миграции.** `scripts/rolling-update.sh`: собрать образ новой версии (`APP_VERSION` → label и env); один раз применить миграции (`run --rm migrate`); для каждой старой реплики поднять одну новую (`--scale N+1 --no-recreate`), дождаться healthy, остановить старую SIGTERM-ом (drain) и удалить. Число готовых реплик не опускается ниже N. **Ограничение:** миграции обязаны быть *expand-only* (добавлять, не ломая работающую версию: новые таблицы/nullable-колонки/индексы). Удаление и переименование делаются отдельным релизом (*contract*) после того, как старых реплик не осталось. Скрипт это не проверяет: это дисциплина разработки, проверенная демонстрацией ниже. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Все файлы — в `evidence/`. Стенд описан в примечании выше; версии и контейнеры — `evidence/stack.txt` (2 vCPU Intel Xeon 2.8 GHz, 8 GiB RAM, Docker 29.9.2, Compose 5.6.1). Прогон с **пустого volume** `pgdata`. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл | +| --- | --- | --- | --- | +| Публичный smoke через LB | `tests/smoke.py --base-url http://127.0.0.1:8080` | `PASS: lab3 public API scenario` | `smoke.txt` | +| 3: стенд одной командой; ≥2 реплики, одна точка входа, общие БД и сессии | `docker compose --profile ttl up -d --build --wait` | `lb` (единственный с портами), `api-1`, `api-2`, `migrate` (exit 0), `postgres`, `redis`, `api-ttl` — healthy | `stack.txt` | +| 3: разные запросы одной сессии — разные реплики, без повторного входа | 6 запросов через LB; smoke/тесты с одним токеном | `X-Instance-Id` чередуется `api-0ac6…`/`api-ea54…`; 10 повторов POST с одним токеном обслужены 2 репликами | `instances.txt`, `test-idempotency.txt` | +| 3: остановка любой реплики; чтение/запись ≤ 10 с; подтверждённые сообщения целы | `tests/resilience.py --mode failover --action-script scripts/stop-one.sh` ×2 + `start-replicas.sh` | PASS дважды: ≥2 реплики до отказа, 0 неудачных попыток из 138–139 записей; `first_recovery_seconds` 4.7–4.8 (это время работы хука `docker stop`, записи не прерывались); остановленный контейнер (`status=exited`) в `failover-hook.txt` | `failover-1.txt`, `failover-2.txt`, `failover-hook.txt`, `restore-*.txt` | +| 3: повтор POST с тем же ключом без дублей, в т. ч. конкурентно и через другую реплику | `student-tests/test_idempotency.py` | 30 одновременных POST одного ключа: 14 + 16 ответов от двух реплик, одно тело, **1 строка**; 2 разных тела на один ключ: 1 сообщение, 10×201 + 10×409; повторы с другой реплики; 400 на неверный ключ; неуспешный запрос ключ не занимает; повтор после logout — 401; ключ старше 24 ч переиспользуется | `test-idempotency.txt` | +| 3: неподтверждённую запись можно повторить с тем же ключом | failover-тест (повтор неопределённого POST тем же ключом), серия отказов (повтор после 502/таймаута) | дублей текста 0, потерянных подтверждённых 0 во всех прогонах | `failover-*.txt`, `lb/failover-series.json` | +| 3: persistence | `tests/resilience.py --mode persistence --action-script scripts/restart.sh` | PASS тем же токеном; новые `StartedAt`/PID у LB, обеих реплик, PostgreSQL, Redis; volume сохранён | `persistence.txt`, `restart-hook.txt` | +| 3: миграции отдельно, однократно, повтор без дублей | пустой volume → сервис `migrate`; `run --rm migrate` | `applied: [0001, 0002, 0003]` один раз; повтор — `applied: []`; реплики логируют `schema compatible; replica ready` | `migrations.txt`, `migrations-rerun.txt` | +| Сохранены возможности lab2 | `test_api.py` (auth, ACL, валидация, пагинация, rollback, гонка регистрации), `test_expiry.py` (через LB :8081), `test_cache_race`, `test_failures.py` | 18/18, 3/3, all passed, 2/2 | `test-api.txt`, `test-expiry.txt`, `test-cache-race.txt`, `test-failures.txt` | +| 4: health/readiness/timeouts/SIGTERM согласованы; LB исключает недоступный backend | таблица выше; серия отказов SIGTERM | после SIGTERM реплика получает последний запрос через 0.95–1.53 с, ошибок 0 | `lb/failover-series.json` | +| 4: последовательно остановить каждую реплику; RPS, ошибки, время восстановления | `research/lb_experiments.py failover-series --mode both` | таблица ниже: SIGTERM — 0 ошибок; SIGKILL — 5–8 ответов 502 за ≤ 0.1 с; восстановление — 5.6–6.0 с до healthy | `lb/failover-series.json`, `lb-failover-series.txt` | +| 4: идемпотентность параллельными запросами с одним ключом | см. уровень 3 | 30 параллельных → 1 сообщение | `test-idempotency.txt` | +| 4: CPU/RAM-лимиты; API без портов; нет `container_name` | `compose.yaml` | у всех сервисов `deploy.resources.limits`; у `api` нет `ports`, сети `app`/`data` internal | `compose.yaml`, `stack.txt` | +| 4: rolling update под нагрузкой; миграции однократны и совместимы | `research/lb_experiments.py rolling-update` (3 обновления подряд), `research/expand_migration_demo.sh` | 0 ошибок за 27–28 с замены, образы сменились, потерь и дублей нет; новая версия без миграции не ready (`missing 0004`), миграция применена один раз, старые реплики остались ready | `lb/rolling-update-*.json`, `rolling-update.txt`, `lb/expand-migration.txt` | +| 5, трек 1 | `research/track1.py run …` (15 конфигураций, ключевые ×3) + `report` | см. раздел трека | `track1/*.json`, `track1/summary.md`, `track1/*.svg` | +| Нагрузка (профиль) | `tests/load.py --duration 30 --concurrency 4` через LB | 236.8 успешных RPS, 0 ошибок, p50/p95/p99 = 15.3/30.8/47.0 мс | `load.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +**Последовательный отказ каждой реплики под нагрузкой** (`evidence/lb/failover-series.json`). 8 потоков closed-loop через LB, у каждого свой пользователь: 1 POST (с ключом; при ошибке повтор тем же ключом и телом) на 3 GET истории; клиентский таймаут 2 с. Фазы: 5 с до отказа → действие → 10 с с одной репликой → `docker start` → 5 с. 2 реплики × 1 CPU / 512 MiB, `maxconn 10`. + +| Воздействие | Реплика | Ошибки после действия | Последняя ошибка | Реплика ещё отвечала | Макс. пауза между записями | RPS до / во время / после | p99 до / во время, мс | До healthy после старта | Потери / дубли | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| SIGTERM (`docker stop`) | api-1 | 0 | — | 0.95 с | 0.12 с | 265 / 252 / 253 | 101 / 68 | 5.6 с | 0 / 0 | +| SIGTERM | api-2 | 0 | — | 1.53 с | 0.18 с | 279 / 220 / 242 | 83 / 89 | 6.0 с | 0 / 0 | +| SIGKILL (`docker kill`) | api-1 | 5 × 502 | 0.08 с | 0.04 с | 0.32 с | 296 / 187 / 237 | 93 / 310 | 5.7 с | 0 / 0 | +| SIGKILL | api-2 | 8 × 502 | 0.10 с | 0.05 с | 0.31 с | 280 / 212 / 218 | 79 / 308 | 5.7 с | 0 / 0 | + +SIGTERM: реплика 3 с отвечает 503 на readiness и дообслуживает запросы, LB выводит её за ~1 с — ни одной ошибки. SIGKILL: ошибки — только запросы «в полёте» на убитом процессе (502), дальше `retry-on conn-failure` + `redispatch` и проверка (`fall 2`) выводят реплику. Неопределённые POST повторены тем же ключом — дублей нет. Падение RPS «во время» — работа на одной реплике (её CPU и пул). «До healthy» — старт контейнера, проверка схемы и healthcheck Docker; в маршрут LB реплика возвращается через 1.3–3.6 с после старта. + +**Настройка, подтверждённая измерением (до/после).** Сначала стояли `timeout connect 1s` и проверка с таймаутом по `inter` (1 с). При SIGKILL к убитому контейнеру нет RST, поэтому каждая попытка соединения и каждая проверка висели по 1 с: максимальная пауза между записями — 0.99–1.06 с, p99 ≈ 1010 мс, RPS во время отказа — 142–165. После `timeout connect 300ms` + `timeout check 300ms`: пауза 0.26–0.32 с, p99 ≈ 310 мс, RPS 187–215 (сохранены `evidence/lb/failover-series-connect1s.json` и текущий ряд). + +**Rolling update под нагрузкой** (`evidence/lb/rolling-update-3.0.1.json`, `-3.0.2-expand.json`, `-3.0.3.json`): три замены подряд — 0 ошибок из 5274–5710 запросов за 27–28 с, p99 135–173 мс, потерянных/дублированных сообщений 0; image id обеих реплик сменился (`rolling-update.txt`). Вторая замена несла новую expand-миграцию 0004 (`lb/expand-migration.txt`): контейнер новой версии до миграции остался NOT-READY с логом `schema is behind … missing 0004`; `migrate` применил её один раз; старые реплики 3.0.1 остались ready. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| SIGTERM одной реплики | 0 ошибок, LB выводит её до закрытия | 0 ошибок, последний запрос на неё через ≤ 1.5 с | новые запросы не прерывались; 0 потерь | +| SIGKILL одной реплики | ошибки только у запросов в полёте | 5–8 ответов 502 за ≤ 0.1 с | ≤ 0.32 с до следующей успешной записи; 0 потерь | +| Рестарт всего стенда (restart hook) | данные и токен живы | PASS | 0 потерь | +| Остановка Redis | работа через PostgreSQL | всё 2xx, `/users/me` за 5 мс | outbox дообработан после старта; отозванный токен — 401 | +| Остановка PostgreSQL | ограниченный 503, аноним — 401 | 503 `unavailable` за 2.0 с, `/health/live` = 200 через panic mode LB | ready через 2.3 с после старта БД | +| Rolling update ×3 | без ошибок | 0 ошибок | 0 потерь | + +### Трек 5.1. Цена масштабирования + +**Вопрос и гипотезы.** Что даёт добавление API-реплик и где предел? +- H1. При фиксированном суммарном CPU/RAM больше реплик не дают больше пропускной способности: Python-API ограничен CPU, а у каждой реплики свои накладные расходы. +- H2. При росте ресурсов пропускная способность растёт до насыщения общего ресурса, после чего новые реплики не помогают. +- H3. При перегрузке без ограничения конкуренции полезная пропускная способность не держит плато, а проваливается (congestion collapse). Ограничение `maxconn` на LB это устраняет. + +**Методика (повторяемая):** `make track1` = `research/track1.py run all` + `report`. +- **Стенд.** Тот же `compose.yaml` + `research/compose.track1.yaml`. Реплики API привязаны к ядру `API_CPUSET=0`, их суммарный бюджет задаётся квотой `API_CPUS`; LB, PostgreSQL, Redis и генератор — на ядре `AUX_CPUSET=1`. Перед каждой конфигурацией реплики и LB пересоздаются (`--force-recreate`), запуск только после того, как LB видит все реплики UP. +- **Нагрузка.** Генератор `research/loadgen` — Go, только stdlib, внутри сети стенда (без docker-proxy). + - Модель **open-loop**: запросы по расписанию, задержка — от запланированного момента, без coordinated omission. + - 48 синтетических пользователей по одному чату с историей; смесь 70% GET истории (limit 50), 20% POST сообщения (уникальный ключ), 10% GET `/users/me`. + - Ступени 40…800 req/s по 3 с прогрева + 10 с измерения; серия останавливается после двух ступеней с > 30% ошибок. +- **Метрики.** + - Ёмкость при SLO: максимальная полезная пропускная способность при p99 ≤ 250 мс и ошибках ≤ 1%. + - Максимум при ошибках ≤ 1%. + - Пик полезной пропускной способности. + - Одновременно: CPU%/RAM контейнеров (`docker stats`), очередь HAProxy (stats CSV), соединения и события ожидания PostgreSQL (одна сессия `psql \watch 1`), доля периодов CFS с троттлингом (`cpu.stat`), счётчики пула psycopg в репликах до/после. +- **Повторы.** Ключевые конфигурации (серия A, `maxconn 10`) — 3 прогона, в выводах медиана. Остальные — по 1 прогону, это отмечено. + +**Исходные данные:** `evidence/track1/[-rN].json` (все ступени и ресурсы), `summary.md`/`summary.json`, графики `*.svg`. + +**Серия A — фиксированный бюджет API 1 CPU / 512 MiB** (scrypt-параллелизм делится так же: 4/2/1), медиана 3 прогонов: + +| Конфигурация | Ёмкость при SLO, rps | Макс. при ошибках ≤ 1 %, rps | Пик, rps | CPU API при 80 rps | CFS-троттлинг при 80 rps | p99 при 80 rps | +| --- | --- | --- | --- | --- | --- | --- | +| 1 × 1.0 CPU | **240** (280/240/240) | **280** | 280 | 28 % | 0 % | 27 мс | +| 2 × 0.5 CPU | 200 (120/200/200) | 200 | 235 | 32 % | 6.7 % | 48 мс | +| 4 × 0.25 CPU | 160 (160/120/160) | 200 | 200 | 45 % | 4.7 % | 66 мс | + +![Фиксированный бюджет: полезная пропускная способность](evidence/track1/fixed-cpu-goodput.svg) +![Фиксированный бюджет: p99](evidence/track1/fixed-cpu-p99.svg) + +H1 подтверждена, и дробление даже вредит. Механизмы: +- **Накладные расходы на реплику.** На одной и той же нагрузке 80 rps суммарный CPU API растёт с 28 до 45%: у каждой реплики свои health-проверки LB (1/с), фоновые циклы (кэш, очистка), event loop и рантайм. Стоимость запроса на пике растёт с 3.2 до 4.4 мс CPU. +- **Квота CFS.** Реплика с квотой 0.25–0.5 ядра, выбравшая квоту за период 100 мс, замораживается до конца периода: 5–7% периодов с троттлингом уже при 80 rps и до 36% на пике. Отсюда хвост задержки при одинаковом суммарном CPU. +- **RAM.** У каждой реплики фиксированные ~30–45 MiB рантайма плюс по 32 MiB на одновременный scrypt. При 128 MiB на реплику и scrypt-параллелизме 4 первые прогоны получили **cgroup OOM kill** при регистрации пользователей (`dmesg: Memory cgroup out of memory`). Дробить RAM без пересчёта параллелизма KDF нельзя. + +**Серия B — рост ресурсов** (реплики по 0.25 ядра на изолированном ядре, 1 прогон; 4×0.25 — медиана серии A) и **серия C — вся машина** (2 vCPU без изоляции, общие с LB/БД/генератором, 1 прогон): + +| Конфигурация | Суммарно CPU API | Пик, rps | Макс. при ошибках ≤ 1 %, rps | +| --- | --- | --- | --- | +| 1 × 0.25 | 0.25 | 84 | 80 | +| 2 × 0.25 | 0.5 | 160 | 160 | +| 3 × 0.25 | 0.75 | 178 | 160 | +| 4 × 0.25 | 1.0 | 200 | 200 | +| 1 × 1.0 (для сравнения) | 1.0 | 280 | 280 | +| 2 × 1.0, вся машина | ≤ 2 (2 vCPU хоста) | 320 | 320 | +| 4 × 1.0, вся машина | ≤ 2 | 312 | 280 | + +![Рост ресурсов](evidence/track1/growth-peak.svg) + +Почему рост реплик перестаёт помогать (H2): +- До 0.5 ядра рост почти линеен (84 → 160 rps), дальше выгода падает: 0.75 → 178, 1.0 → 200. Четыре реплики по 0.25 уступают одной реплике на целом ядре (280) по причинам серии A. +- На всей машине 2 реплики дают +14% к одной (320 против 280), а 4 реплики — уже не лучше двух. На пике суммарная загрузка контейнеров — 159–171% из 200%, остальное съедают dockerd/containerd и сэмплер: **хост насыщен**. +- CPU на запрос по компонентам (machine-2×1.0, на пике): + + | Компонент | мс CPU/запрос | + | --- | --- | + | API | 3.5 | + | PostgreSQL | 0.6 | + | генератор | 0.5 | + | LB | 0.23 | + | Redis | 0.12 | + + API в ~6 раз дороже PostgreSQL на запрос. Значит, на этой машине первым кончается CPU для API, а БД и LB далеки от насыщения: PostgreSQL ~20% CPU; события ожидания — почти только `CPU`, изредка `IO:WALSync`, `pg_test_fsync` 0.25 мс. +- Каждая новая реплика добавляет процессы, которые конкурируют за те же 2 ядра, и пул из 10 соединений к БД: 10 → 20 → 41 соединение. Время удержания соединения на запрос (среднее по всей серии): 16 мс у 1×1.0 → 18 мс у 2×1.0 → 40 мс у 4×1.0. +- **Оценка следующего предела (экстраполяция, не измерено).** При изобилии CPU для API первым насытится PostgreSQL на одном ядре — около 1 / 0.6 мс ≈ 1600 rps на этой смеси. Затем `max_connections=100` при пуле 10 ограничивает число реплик ~9 (с учётом migrate и служебных соединений). Дальше нужен пул на стороне БД (PgBouncer) или меньший пул на реплику. + +**Насыщение пула.** При 1×1.0 под перегрузкой запрос в среднем по серии ждёт свободное соединение 279–308 мс (3 прогона), а держит его 16 мс. При этом сам SQL выполняется ~1 мс: соединение «держит» корутина, которая ждёт своей очереди в перегруженном event loop. `/users/me` без БД (из кэша Redis) в тех же ступенях отвечает за ~25 мс, history/post — около 1 с. Ошибки — `PoolTimeout`/`TooManyRequests` пула, а не LB. + +Серия P (1×1.0, по 1 прогону), пик / ёмкость при SLO: + +| Пул | Пик, rps | Ёмкость при SLO, rps | +| --- | --- | --- | +| 5 | 320 | 280 | +| 10 | 280 | 240 | +| 20 | 280 | 200 | +| 40 | 280 | 200 | + +Больший пул не помогает: ограничитель — CPU event loop, а не число соединений. Меньший пул лишь ограничивает конкуренцию. Это подводит к улучшению. + +**Подтверждённое улучшение: ограничение конкуренции на LB (`maxconn 10` на реплику, `timeout queue 2s`)** — 1×1.0 CPU, пул 10, медиана 3 прогонов каждого варианта: + +| Поданная нагрузка | Без лимита: полезная / ошибки / p50 | `maxconn 10`: полезная / ошибки / p50 | +| --- | --- | --- | +| 280 req/s | 280 / 0 % / 72 мс | 280 / 0 % / **12 мс** | +| 320 req/s | 276 / 13.8 % / 744 мс | **320 / 0 %** / 162 мс | +| 360 req/s | 277 / 22.9 % / 794 мс | **360 / 0.1 %** / 1323 мс | +| 400 req/s | 253 / 36.8 % / 931 мс | **354** / 11.4 % / 1670 мс | +| 460 req/s | 223 / 51.6 % / 1103 мс | **328** / 28.7 % / 2026 мс | +| Ёмкость при SLO | 240 (280/240/240) | **280** (320/280/240) | +| Макс. при ошибках ≤ 1 % | 280 (280/280/280) | **360** (360/360/320) — **+28 %**, каждый из 3 прогонов выше каждого базового | + +![maxconn против перегрузки](evidence/track1/admission-goodput.svg) + +H3 подтверждена. Без лимита LB отдаёт реплике всё сразу: сотни корутин в event loop борются за CPU и 10 соединений, время удержания соединения раздувается, запросы истекают по таймауту пула. CPU тратится на работу, которую потом выбрасывают, и полезная пропускная способность падает с 280 до 223 rps. С `maxconn 10` в реплике не больше 10 запросов; остальные ждут в дешёвой очереди HAProxy и при переполнении получают быстрый 503. Полезная пропускная способность держит плато 330–360 rps вместо провала. + +Проверенные варианты (по 1 прогону): + +| Вариант | Макс. при ошибках ≤ 1 %, rps | +| --- | --- | +| `maxconn 10`, `timeout queue 2s` | 360 (медиана) | +| `maxconn 20` | 320 | +| `maxconn 10`, `timeout queue 250ms` | 239: очередь 250 мс слишком коротка и отбрасывает всплески, которые иначе уложились бы | +| 2×0.5 с `maxconn 10` | 280 (без лимита 200) | + +Значения `maxconn 10` и `timeout queue 2s` **приняты по умолчанию** в `compose.yaml`; уровни 3–4 повторно проверены уже с ними. + +**Ограничения измерений:** +- Одна машина на 2 vCPU. Генератор, LB, БД и рантайм Docker делят её с API. Изоляция ядер снимает конкуренцию только для API. +- Абсолютные числа зависят от CPU хоста. Переносимы соотношения и механизмы, не rps. +- Ключевые конфигурации повторены 3 раза; разброс ёмкости при SLO — одна ступень (±40 rps), из-за этого шага сетки ёмкость определена с точностью до ступени. Остальные конфигурации — 1 прогон. +- Тёплый кэш, данные — тысячи сообщений: стоимость БД здесь минимальна; большие истории — в треке 5.2 lab2. +- Числа получены на локальных образах-заменителях: HAProxy 3.0.0 вместо 3.0.5, PostgreSQL 16.15 вместо 16.10, Redis 7.0 вместо 7.4, psycopg без C-расширения (pure-Python + системная libpq) вместо `psycopg[binary]`. Последнее завышает CPU API на запрос: в официальном образе ожидаемы более высокие абсолютные значения. Повторите `make track1`. +- Сэмплер сам потребляет CPU хоста. Первая версия с `docker exec` раз в секунду заметно занижала результаты и заменена одной сессией `psql`. + +## Ответы на вопросы защиты + +- **Отказ процесса vs отказ узла.** Отказ процесса (SIGKILL реплики) оставляет живыми хост, сеть, Docker и остальные контейнеры: LB видит отказ соединения, перенаправляет запросы, Docker перезапускает контейнер, volume на месте. При отказе узла пропадает всё, что на нём: все реплики, LB, PostgreSQL и его диск. На одном Docker-хосте этот стенд отказ узла не выдерживает — для этого нужны другие узлы, репликация БД и переносимый LB (трек 5.2, не выбран). +- **Почему depends_on/healthcheck — не балансировщик.** `depends_on` задаёт только порядок старта, healthcheck Docker лишь помечает контейнер unhealthy. Ни то, ни другое не распределяет запросы, не исключает больной backend из маршрута и не повторяет запрос на другой реплике. Это делает LB: активные проверки, `fall`/`rise`, `redispatch`. +- **Какие POST можно повторять после timeout и что защищает Idempotency-Key.** Без ключа повторять безопасно только то, что не создаёт новый объект, — в обязательном API таких POST нет: повтор регистрации даст 409, повтор создания чата создаст ещё один чат. `POST /messages` с тем же `Idempotency-Key` и тем же телом можно повторять сколько угодно и через любую реплику: результат первой успешной транзакции сохранён атомарно с сообщением, повтор возвращает его же, другой body — 409. LB сам повторяет только то, что не было отправлено (`retry-on conn-failure`). +- **Что сломается с сессиями в локальной памяти.** Токен, выданный репликой A, будет неизвестен реплике B: при roundrobin каждый второй запрос — 401. Logout на одной реплике не отзовёт токен на другой. Рестарт или отказ реплики разлогинит её пользователей. Sticky sessions лишь маскируют проблему и ломаются при отказе реплики. Здесь сессии в PostgreSQL, Redis — общий кэш. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- **Выдерживается:** остановка/убийство любой одной API-реплики (без ошибок при SIGTERM, ≤ 8 ошибок 502 при SIGKILL), рестарт всего стенда с volumes, rolling update под нагрузкой, отказ Redis, отказ PostgreSQL (ограниченные 503). +- **Не выдерживается:** отказ хоста; отказ единственного LB или PostgreSQL — это SPOF стенда. +- `scripts/stop-one.sh` находит реплику по alias через `docker exec` (файл `/tmp/instance-id`) — это требует работающего процесса. Убитую реплику так не найти, но этот хук останавливает живую реплику. +- Публичный failover-тест сам выбирает реплику, которая обслужила запрос. Поэтому поочерёдная остановка каждой реплики показана отдельно (`lb/failover-series.json`: api-1 и api-2, SIGTERM и SIGKILL). +- Окно устаревания кэша сессий при сетевом разделении реплики и Redis — как в lab2 (≤ `SESSION_CACHE_TTL_SECONDS`). +- Rolling update на Windows — только из Git Bash/WSL (скрипт POSIX sh). +- `evidence/` получены на образах-заменителях (см. примечание в начале). Повторите `./scripts/collect-evidence.sh` на официальных образах. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..87f3c5ac --- /dev/null +++ b/compose.yaml @@ -0,0 +1,172 @@ +# Учебный стенд lab3: HAProxy (единственная точка входа) → API × N → PostgreSQL + Redis. +# Запуск: ./scripts/init-secrets.sh && docker compose --env-file .env up -d --build --wait +name: mtusi-lab3 + +x-api-common: &api-common + build: + context: . + dockerfile: Dockerfile + image: mtusi-lab3-api:local + environment: &api-env + APP_HOST: 0.0.0.0 + APP_PORT: "8080" + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: postgres + SESSION_CACHE: ${SESSION_CACHE:-redis} + SESSION_CACHE_TTL_SECONDS: ${SESSION_CACHE_TTL_SECONDS:-60} + IDEMPOTENCY_TTL_SECONDS: ${IDEMPOTENCY_TTL_SECONDS:-86400} + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + DB_POOL_MIN: "1" + DB_POOL_MAX: ${DB_POOL_MAX:-10} + DB_POOL_TIMEOUT_SECONDS: "2" + KDF_CONCURRENCY: ${KDF_CONCURRENCY:-4} # каждый scrypt ≈ 32 MiB RAM: учитывайте в API_MEMORY + DB_STATEMENT_TIMEOUT_MS: "5000" + CACHE_TIMEOUT_SECONDS: "0.2" + REQUEST_TIMEOUT_SECONDS: "10" + SHUTDOWN_DRAIN_SECONDS: ${SHUTDOWN_DRAIN_SECONDS:-3} + MIGRATE_ON_START: "false" # миграции — только сервис migrate + secrets: [database_url, cache_url] + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_started # Redis — кэш: реплики работают и без него + networks: [app, data] # НЕТ ports: API доступен только через LB + read_only: true + tmpfs: [/tmp] + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + restart: unless-stopped + stop_signal: SIGTERM + stop_grace_period: 15s # drain 3 с + завершение запросов в полёте (≤10 с) + +services: + lb: + image: haproxy:3.0.5-alpine + volumes: + - ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + environment: + LB_SERVER_MAXCONN: ${LB_SERVER_MAXCONN:-10} + LB_QUEUE_TIMEOUT: ${LB_QUEUE_TIMEOUT:-2s} + ports: + - "127.0.0.1:${API_PORT:-8080}:8080" # публичный API + - "127.0.0.1:${API_TTL_PORT:-8081}:8081" # api-ttl (профиль ttl), тоже только через LB + - "127.0.0.1:${LB_STATS_PORT:-8404}:8404" # статистика HAProxy (только чтение) + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8404/stats"] + interval: 5s + timeout: 2s + retries: 3 + networks: [edge, app] + restart: unless-stopped + stop_grace_period: 5s + deploy: + resources: + limits: {cpus: "0.5", memory: 128M} + + # One-shot: миграции применяются ОДИН раз до запуска реплик (не гонкой реплик). + # Повторный запуск (docker compose run --rm migrate) — no-op. Реплики не становятся ready, + # пока схема не содержит все известные им миграции. + migrate: + <<: *api-common + command: ["migrate"] + depends_on: + postgres: + condition: service_healthy + networks: [data] + restart: "no" + secrets: [database_url] + environment: + <<: *api-env + SESSION_CACHE: "off" + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + + api: + <<: *api-common + deploy: + replicas: ${API_REPLICAS:-2} + resources: + limits: {cpus: "${API_CPUS:-1.0}", memory: "${API_MEMORY:-512M}"} + + # Экземпляр с коротким TTL — только для теста истечения; доступен через LB на :8081. + api-ttl: + <<: *api-common + profiles: [ttl] + environment: + <<: *api-env + SESSION_TTL_SECONDS: "2" + deploy: + resources: + limits: {cpus: "0.5", memory: 256M} + + postgres: + image: postgres:16.10-alpine + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: "--auth-host=scram-sha-256 --data-checksums" + command: ["postgres", "-c", "max_connections=${PG_MAX_CONNECTIONS:-100}"] + secrets: [postgres_password] + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger -h 127.0.0.1"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s + networks: [data] + shm_size: 256m + restart: unless-stopped + stop_grace_period: 30s + deploy: + resources: + limits: {cpus: "${PG_CPUS:-1.0}", memory: 1G} + + # Redis — общий кэш сессий без persistence; источник истины сессий — PostgreSQL. + redis: + image: redis:7.4.2-alpine + command: + - sh + - -c + - | + umask 077 + printf 'requirepass %s\n' "$$(cat /run/secrets/redis_password)" > /tmp/redis.conf + exec redis-server /tmp/redis.conf --save '' --appendonly no --maxmemory 64mb --maxmemory-policy allkeys-lru + user: redis + secrets: [redis_password] + healthcheck: + test: ["CMD-SHELL", "REDISCLI_AUTH=\"$$(cat /run/secrets/redis_password)\" redis-cli ping | grep -q PONG"] + interval: 2s + timeout: 2s + retries: 15 + networks: [data] + restart: unless-stopped + deploy: + resources: + limits: {cpus: "0.25", memory: 128M} + +networks: + edge: {} # только LB: публикация портов на 127.0.0.1 + app: + internal: true # LB ↔ API + data: + internal: true # API ↔ PostgreSQL/Redis + +volumes: + pgdata: {} + +secrets: + postgres_password: + file: ./secrets/postgres_password + redis_password: + file: ./secrets/redis_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url diff --git a/contracts/EXTENSIONS.md b/contracts/EXTENSIONS.md new file mode 100644 index 00000000..922ba4e6 --- /dev/null +++ b/contracts/EXTENSIONS.md @@ -0,0 +1,13 @@ +# Расширения решения (обратно совместимые) + +Обязательный контракт (`openapi.json`, `README.md`) не изменён. Дополнительно реализовано: + +| Что | Где | Семантика | +| --- | --- | --- | +| Заголовок ответа `Idempotent-Replayed: true` | `POST /api/v1/chats/{chat_id}/messages` | Ответ 201 — повтор ранее сохранённого результата по тому же `Idempotency-Key` (тело и код совпадают с первым ответом). Клиент может игнорировать. | +| `GET /internal/stats` | только изнутри контейнера реплики (`127.0.0.1`); LB отвечает на `/internal/*` 404 в формате ошибок API | Диагностика для исследования: alias реплики, версия, счётчики запросов, кэша сессий и пула соединений psycopg. `python -m messenger stats` в контейнере. | +| Ответы, созданные самим LB (502/503/504/404 для `/internal/*`) | HAProxy | Тело в формате ошибок API с `request_id` = `X-Request-Id` (случайный uuid). `X-Instance-Id` у них нет — контракт это допускает. | + +Область ключа идемпотентности: пользователь + `POST` + `/api/v1/chats/{chat_id}/messages` + ключ. Других +идемпотентных endpoint в обязательном API нет, поэтому хранится тройка `(user_id, chat_id, key)`. +Срок хранения — `IDEMPOTENCY_TTL_SECONDS` (по умолчанию 86 400 с = 24 ч). diff --git a/evidence/failover-1.txt b/evidence/failover-1.txt new file mode 100644 index 00000000..ceba8313 --- /dev/null +++ b/evidence/failover-1.txt @@ -0,0 +1,6 @@ +# 2026-11-03T09:51:34Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 139, "failed_attempts": 0, "first_recovery_seconds": 4.675} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 diff --git a/evidence/failover-2.txt b/evidence/failover-2.txt new file mode 100644 index 00000000..c285aca0 --- /dev/null +++ b/evidence/failover-2.txt @@ -0,0 +1,6 @@ +# 2026-11-03T09:51:57Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8080 +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 138, "failed_attempts": 0, "first_recovery_seconds": 4.763} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 diff --git a/evidence/failover-hook.txt b/evidence/failover-hook.txt new file mode 100644 index 00000000..a171b4a3 --- /dev/null +++ b/evidence/failover-hook.txt @@ -0,0 +1,32 @@ +== 2026-11-03T09:51:36Z stop-one target=api-5dbb8cd6 container=mtusi-lab3-api-1 +api mtusi-lab3-api-1 status=running started=2026-11-03T09:50:54.761533652Z pid=19437 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359 +api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502 +lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923 +migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0 +postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033 +redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102 +== 2026-11-03T09:51:40Z after stop +api mtusi-lab3-api-1 status=exited started=2026-11-03T09:50:54.761533652Z pid=0 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359 +api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502 +lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923 +migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0 +postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033 +redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102 +== 2026-11-03T09:51:59Z stop-one target=api-5fd104b6 container=mtusi-lab3-api-1 +api mtusi-lab3-api-1 status=running started=2026-11-03T09:51:51.150461585Z pid=22901 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359 +api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502 +lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923 +migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0 +postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033 +redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102 +== 2026-11-03T09:52:03Z after stop +api mtusi-lab3-api-1 status=exited started=2026-11-03T09:51:51.150461585Z pid=0 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359 +api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502 +lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923 +migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0 +postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033 +redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102 diff --git a/evidence/instances.txt b/evidence/instances.txt new file mode 100644 index 00000000..8c9ed88d --- /dev/null +++ b/evidence/instances.txt @@ -0,0 +1,9 @@ +# 2026-11-03T09:51:02Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ sh -c for i in 1 2 3 4 5 6; do curl -s -o /dev/null -D - http://127.0.0.1:8080/health/live | grep -i x-instance-id; done +x-instance-id: api-7e27119d +x-instance-id: api-5dbb8cd6 +x-instance-id: api-7e27119d +x-instance-id: api-5dbb8cd6 +x-instance-id: api-7e27119d +x-instance-id: api-5dbb8cd6 +exit=0 diff --git a/evidence/lb-failover-series.txt b/evidence/lb-failover-series.txt new file mode 100644 index 00000000..8fe97115 --- /dev/null +++ b/evidence/lb-failover-series.txt @@ -0,0 +1,8 @@ +# 2026-11-03T09:53:04Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 research/lb_experiments.py failover-series --mode both +stop mtusi-lab3-api-1 errors=0 last_err= 0.00s target_last= 0.95s max_write_gap=0.12s rps(before/during)=264.6/251.9 data={'confirmed': 1928, 'lost_confirmed': 0, 'stored': 1928, 'duplicate_texts': 0} +stop mtusi-lab3-api-2 errors=0 last_err= 0.00s target_last= 1.53s max_write_gap=0.18s rps(before/during)=278.9/220.4 data={'confirmed': 1865, 'lost_confirmed': 0, 'stored': 1865, 'duplicate_texts': 0} +kill mtusi-lab3-api-1 errors=5 last_err= 0.08s target_last= 0.04s max_write_gap=0.32s rps(before/during)=295.8/186.6 data={'confirmed': 1551, 'lost_confirmed': 0, 'stored': 1551, 'duplicate_texts': 0} +kill mtusi-lab3-api-2 errors=8 last_err= 0.10s target_last= 0.05s max_write_gap=0.31s rps(before/during)=279.9/211.8 data={'confirmed': 1552, 'lost_confirmed': 0, 'stored': 1552, 'duplicate_texts': 0} +saved evidence/lb/failover-series.json +exit=0 diff --git a/evidence/lb-rolling-update.txt b/evidence/lb-rolling-update.txt new file mode 100644 index 00000000..8e19e8ef --- /dev/null +++ b/evidence/lb-rolling-update.txt @@ -0,0 +1,48 @@ +# 2026-11-03T09:55:32Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 research/lb_experiments.py rolling-update +{ + "script_exit": 0, + "script_seconds": 27.88, + "before_replicas": [ + { + "container": "mtusi-lab3-api-1", + "instance_id": "api-471b30bc" + }, + { + "container": "mtusi-lab3-api-2", + "instance_id": "api-50205d38" + } + ], + "after_replicas": [ + { + "container": "mtusi-lab3-api-3", + "instance_id": "api-730666b6" + }, + { + "container": "mtusi-lab3-api-4", + "instance_id": "api-87d10eab" + } + ], + "during": { + "seconds": 27.88, + "ok": 5696, + "errors": 0, + "rps_ok": 204.3, + "p50_ms": 31.4, + "p99_ms": 152.4, + "by_instance": { + "api-471b30bc": 896, + "api-50205d38": 2061, + "api-730666b6": 1928, + "api-87d10eab": 811 + }, + "error_codes": {} + }, + "data": { + "confirmed": 2444, + "lost_confirmed": 0, + "stored": 2444, + "duplicate_texts": 0 + } +} +exit=0 diff --git a/evidence/lb/expand-migration.txt b/evidence/lb/expand-migration.txt new file mode 100644 index 00000000..8cbc004d --- /dev/null +++ b/evidence/lb/expand-migration.txt @@ -0,0 +1,11 @@ +# 2026-11-03T09:57:04Z expand-migration demo, new version 3.0.2-expand +== 1) new-version container before migrate: + Container mtusi-lab3-api-run-b781fee6ea25 Creating + Container mtusi-lab3-api-run-b781fee6ea25 Created +health/ready inside new container: NOT-READY +log: {"ts": "2026-11-03T09:57:05Z", "level": "warning", "logger": "messenger.http", "msg": "schema is behind this code; waiting for migrate", "missing": ["0004_demo_expand"]} +== 2) migrate (one-shot): +{"applied": ["0004_demo_expand"]} +== 3) old replicas after migrate: +/mtusi-lab3-api-3 ready=yes version=3.0.1 +/mtusi-lab3-api-4 ready=yes version=3.0.1 diff --git a/evidence/lb/failover-series.json b/evidence/lb/failover-series.json new file mode 100644 index 00000000..436020eb --- /dev/null +++ b/evidence/lb/failover-series.json @@ -0,0 +1,243 @@ +{ + "measured_at": "2026-11-03T09:53:05+00:00", + "workers": 8, + "client_timeout_s": 2.0, + "phases_s": { + "before": 5, + "after_action": 10 + }, + "runs": [ + { + "mode": "stop", + "container": "mtusi-lab3-api-1", + "instance_id": "api-fd0cdd87", + "action_seconds": 3.91, + "before": { + "seconds": 5.03, + "ok": 1331, + "errors": 0, + "rps_ok": 264.6, + "p50_ms": 27.1, + "p99_ms": 100.7, + "by_instance": { + "api-79e08828": 666, + "api-fd0cdd87": 665 + }, + "error_codes": {} + }, + "during": { + "seconds": 13.91, + "ok": 3504, + "errors": 0, + "rps_ok": 251.9, + "p50_ms": 29.8, + "p99_ms": 68.3, + "by_instance": { + "api-79e08828": 3360, + "api-fd0cdd87": 144 + }, + "error_codes": {} + }, + "after_restore": { + "seconds": 11.32, + "ok": 2858, + "errors": 0, + "rps_ok": 252.5, + "p50_ms": 27.0, + "p99_ms": 107.7, + "by_instance": { + "api-4a34d5cd": 1336, + "api-79e08828": 1522 + }, + "error_codes": {} + }, + "errors_after_action": 0, + "last_error_after_action_s": 0, + "target_last_served_after_action_s": 0.95, + "max_gap_between_successful_writes_s": 0.12, + "replica_healthy_after_start_s": 5.6, + "restarted_replica_first_served_s": 1.28, + "data": { + "confirmed": 1928, + "lost_confirmed": 0, + "stored": 1928, + "duplicate_texts": 0 + } + }, + { + "mode": "stop", + "container": "mtusi-lab3-api-2", + "instance_id": "api-79e08828", + "action_seconds": 4.26, + "before": { + "seconds": 5.04, + "ok": 1407, + "errors": 0, + "rps_ok": 278.9, + "p50_ms": 24.8, + "p99_ms": 82.5, + "by_instance": { + "api-4a34d5cd": 701, + "api-79e08828": 706 + }, + "error_codes": {} + }, + "during": { + "seconds": 14.26, + "ok": 3143, + "errors": 0, + "rps_ok": 220.4, + "p50_ms": 33.8, + "p99_ms": 88.7, + "by_instance": { + "api-4a34d5cd": 2931, + "api-79e08828": 212 + }, + "error_codes": {} + }, + "after_restore": { + "seconds": 11.97, + "ok": 2890, + "errors": 0, + "rps_ok": 241.5, + "p50_ms": 29.0, + "p99_ms": 103.4, + "by_instance": { + "api-4a34d5cd": 1579, + "api-563af4cd": 1311 + }, + "error_codes": {} + }, + "errors_after_action": 0, + "last_error_after_action_s": 0, + "target_last_served_after_action_s": 1.53, + "max_gap_between_successful_writes_s": 0.18, + "replica_healthy_after_start_s": 6.01, + "restarted_replica_first_served_s": 1.48, + "data": { + "confirmed": 1865, + "lost_confirmed": 0, + "stored": 1865, + "duplicate_texts": 0 + } + }, + { + "mode": "kill", + "container": "mtusi-lab3-api-1", + "instance_id": "api-4a34d5cd", + "action_seconds": 0.26, + "before": { + "seconds": 5.04, + "ok": 1492, + "errors": 0, + "rps_ok": 295.8, + "p50_ms": 23.1, + "p99_ms": 93.0, + "by_instance": { + "api-4a34d5cd": 747, + "api-563af4cd": 745 + }, + "error_codes": {} + }, + "during": { + "seconds": 10.26, + "ok": 1914, + "errors": 5, + "rps_ok": 186.6, + "p50_ms": 32.8, + "p99_ms": 309.9, + "by_instance": { + "api-4a34d5cd": 6, + "api-563af4cd": 1908 + }, + "error_codes": { + "502": 5 + } + }, + "after_restore": { + "seconds": 11.75, + "ok": 2780, + "errors": 0, + "rps_ok": 236.5, + "p50_ms": 30.5, + "p99_ms": 92.7, + "by_instance": { + "api-471b30bc": 1048, + "api-563af4cd": 1732 + }, + "error_codes": {} + }, + "errors_after_action": 5, + "last_error_after_action_s": 0.08, + "target_last_served_after_action_s": 0.04, + "max_gap_between_successful_writes_s": 0.32, + "replica_healthy_after_start_s": 5.73, + "restarted_replica_first_served_s": 3.57, + "data": { + "confirmed": 1551, + "lost_confirmed": 0, + "stored": 1551, + "duplicate_texts": 0 + } + }, + { + "mode": "kill", + "container": "mtusi-lab3-api-2", + "instance_id": "api-563af4cd", + "action_seconds": 0.27, + "before": { + "seconds": 5.03, + "ok": 1408, + "errors": 0, + "rps_ok": 279.9, + "p50_ms": 25.5, + "p99_ms": 78.9, + "by_instance": { + "api-471b30bc": 707, + "api-563af4cd": 701 + }, + "error_codes": {} + }, + "during": { + "seconds": 10.27, + "ok": 2175, + "errors": 8, + "rps_ok": 211.8, + "p50_ms": 31.9, + "p99_ms": 307.9, + "by_instance": { + "api-471b30bc": 2171, + "api-563af4cd": 4 + }, + "error_codes": { + "502": 8 + } + }, + "after_restore": { + "seconds": 11.95, + "ok": 2609, + "errors": 0, + "rps_ok": 218.3, + "p50_ms": 33.3, + "p99_ms": 112.1, + "by_instance": { + "api-471b30bc": 1448, + "api-50205d38": 1161 + }, + "error_codes": {} + }, + "errors_after_action": 8, + "last_error_after_action_s": 0.1, + "target_last_served_after_action_s": 0.05, + "max_gap_between_successful_writes_s": 0.31, + "replica_healthy_after_start_s": 5.73, + "restarted_replica_first_served_s": 1.78, + "data": { + "confirmed": 1552, + "lost_confirmed": 0, + "stored": 1552, + "duplicate_texts": 0 + } + } + ] +} diff --git a/evidence/lb/rolling-update-3.0.1.json b/evidence/lb/rolling-update-3.0.1.json new file mode 100644 index 00000000..76f86723 --- /dev/null +++ b/evidence/lb/rolling-update-3.0.1.json @@ -0,0 +1,79 @@ +{ + "measured_at": "2026-11-03T09:56:21+00:00", + "workers": 8, + "script_exit": 0, + "script_seconds": 27.88, + "before_replicas": [ + { + "container": "mtusi-lab3-api-1", + "instance_id": "api-471b30bc" + }, + { + "container": "mtusi-lab3-api-2", + "instance_id": "api-50205d38" + } + ], + "after_replicas": [ + { + "container": "mtusi-lab3-api-3", + "instance_id": "api-730666b6" + }, + { + "container": "mtusi-lab3-api-4", + "instance_id": "api-87d10eab" + } + ], + "images_before": [ + "sha256:d9d8402d14a0" + ], + "images_after": [ + "sha256:fb847cbe9916" + ], + "before": { + "seconds": 5.04, + "ok": 1236, + "errors": 0, + "rps_ok": 245.0, + "p50_ms": 28.1, + "p99_ms": 109.7, + "by_instance": { + "api-471b30bc": 618, + "api-50205d38": 618 + }, + "error_codes": {} + }, + "during": { + "seconds": 27.88, + "ok": 5696, + "errors": 0, + "rps_ok": 204.3, + "p50_ms": 31.4, + "p99_ms": 152.4, + "by_instance": { + "api-471b30bc": 896, + "api-50205d38": 2061, + "api-730666b6": 1928, + "api-87d10eab": 811 + }, + "error_codes": {} + }, + "after": { + "seconds": 10.0, + "ok": 2821, + "errors": 0, + "rps_ok": 282.1, + "p50_ms": 26.0, + "p99_ms": 74.1, + "by_instance": { + "api-730666b6": 1410, + "api-87d10eab": 1411 + }, + "error_codes": {} + }, + "data": { + "confirmed": 2444, + "lost_confirmed": 0, + "stored": 2444, + "duplicate_texts": 0 + } +} diff --git a/evidence/lb/rolling-update-3.0.2-expand.json b/evidence/lb/rolling-update-3.0.2-expand.json new file mode 100644 index 00000000..46eb14ed --- /dev/null +++ b/evidence/lb/rolling-update-3.0.2-expand.json @@ -0,0 +1,79 @@ +{ + "measured_at": "2026-11-03T09:58:07+00:00", + "workers": 8, + "script_exit": 0, + "script_seconds": 28.32, + "before_replicas": [ + { + "container": "mtusi-lab3-api-3", + "instance_id": "api-730666b6" + }, + { + "container": "mtusi-lab3-api-4", + "instance_id": "api-87d10eab" + } + ], + "after_replicas": [ + { + "container": "mtusi-lab3-api-5", + "instance_id": "api-ed7bd071" + }, + { + "container": "mtusi-lab3-api-6", + "instance_id": "api-f4fc385b" + } + ], + "images_before": [ + "sha256:fb847cbe9916" + ], + "images_after": [ + "sha256:5c115f1efe4d" + ], + "before": { + "seconds": 5.04, + "ok": 1465, + "errors": 0, + "rps_ok": 290.7, + "p50_ms": 22.7, + "p99_ms": 77.3, + "by_instance": { + "api-730666b6": 730, + "api-87d10eab": 735 + }, + "error_codes": {} + }, + "during": { + "seconds": 28.32, + "ok": 5710, + "errors": 0, + "rps_ok": 201.6, + "p50_ms": 32.6, + "p99_ms": 134.9, + "by_instance": { + "api-730666b6": 998, + "api-87d10eab": 2125, + "api-ed7bd071": 1910, + "api-f4fc385b": 677 + }, + "error_codes": {} + }, + "after": { + "seconds": 10.0, + "ok": 2450, + "errors": 0, + "rps_ok": 245.0, + "p50_ms": 29.8, + "p99_ms": 87.3, + "by_instance": { + "api-ed7bd071": 1219, + "api-f4fc385b": 1231 + }, + "error_codes": {} + }, + "data": { + "confirmed": 2412, + "lost_confirmed": 0, + "stored": 2412, + "duplicate_texts": 0 + } +} diff --git a/evidence/lb/rolling-update-3.0.3.json b/evidence/lb/rolling-update-3.0.3.json new file mode 100644 index 00000000..97eb80af --- /dev/null +++ b/evidence/lb/rolling-update-3.0.3.json @@ -0,0 +1,79 @@ +{ + "measured_at": "2026-11-03T09:58:56+00:00", + "workers": 8, + "script_exit": 0, + "script_seconds": 27.48, + "before_replicas": [ + { + "container": "mtusi-lab3-api-5", + "instance_id": "api-ed7bd071" + }, + { + "container": "mtusi-lab3-api-6", + "instance_id": "api-f4fc385b" + } + ], + "after_replicas": [ + { + "container": "mtusi-lab3-api-7", + "instance_id": "api-8e374d62" + }, + { + "container": "mtusi-lab3-api-8", + "instance_id": "api-b25f5827" + } + ], + "images_before": [ + "sha256:5c115f1efe4d" + ], + "images_after": [ + "sha256:d5045137ab03" + ], + "before": { + "seconds": 5.03, + "ok": 1381, + "errors": 0, + "rps_ok": 274.4, + "p50_ms": 25.7, + "p99_ms": 80.7, + "by_instance": { + "api-ed7bd071": 693, + "api-f4fc385b": 688 + }, + "error_codes": {} + }, + "during": { + "seconds": 27.48, + "ok": 5274, + "errors": 0, + "rps_ok": 191.9, + "p50_ms": 33.8, + "p99_ms": 172.5, + "by_instance": { + "api-8e374d62": 1644, + "api-b25f5827": 756, + "api-ed7bd071": 972, + "api-f4fc385b": 1902 + }, + "error_codes": {} + }, + "after": { + "seconds": 10.0, + "ok": 2689, + "errors": 0, + "rps_ok": 268.9, + "p50_ms": 27.2, + "p99_ms": 86.0, + "by_instance": { + "api-8e374d62": 1348, + "api-b25f5827": 1341 + }, + "error_codes": {} + }, + "data": { + "confirmed": 2342, + "lost_confirmed": 0, + "stored": 2342, + "duplicate_texts": 0 + } +} diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..f0ef09fd --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.05, + "concurrency": 4, + "successful_requests": 7116, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 236.805, + "successful_latency_ms": { + "p50": 15.312, + "p95": 30.786, + "p99": 47.039 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 00000000..c5e51754 --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,20 @@ +# 2026-11-03T09:56:22Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 tests/load.py --base-url http://127.0.0.1:8080 --duration 30 --concurrency 4 --output evidence/load.json +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.05, + "concurrency": 4, + "successful_requests": 7116, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 236.805, + "successful_latency_ms": { + "p50": 15.312, + "p95": 30.786, + "p99": 47.039 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} +exit=0 diff --git a/evidence/make-check.txt b/evidence/make-check.txt new file mode 100644 index 00000000..01424f6d --- /dev/null +++ b/evidence/make-check.txt @@ -0,0 +1,5 @@ +# 2026-11-03T09:51:02Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ make check +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. +exit=0 diff --git a/evidence/migrations-rerun.txt b/evidence/migrations-rerun.txt new file mode 100644 index 00000000..14e94e1a --- /dev/null +++ b/evidence/migrations-rerun.txt @@ -0,0 +1,6 @@ +# 2026-11-03T09:53:03Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ dc run --rm --no-deps migrate + Container mtusi-lab3-migrate-run-fa71ca88729f Creating + Container mtusi-lab3-migrate-run-fa71ca88729f Created +{"applied": []} +exit=0 diff --git a/evidence/migrations.txt b/evidence/migrations.txt new file mode 100644 index 00000000..f4719efc --- /dev/null +++ b/evidence/migrations.txt @@ -0,0 +1,12 @@ +# 2026-11-03T09:57:03Z empty volume pgdata -> one-shot migrate service (before replicas) +$ docker compose logs migrate +{"ts": "2026-11-03T09:50:54Z", "level": "info", "logger": "messenger.db", "msg": "migration applied", "version": "0001_init"} +{"ts": "2026-11-03T09:50:54Z", "level": "info", "logger": "messenger.db", "msg": "migration applied", "version": "0002_cache_invalidation_outbox"} +{"ts": "2026-11-03T09:50:54Z", "level": "info", "logger": "messenger.db", "msg": "migration applied", "version": "0003_idempotency_keys"} +{"applied": ["0001_init", "0002_cache_invalidation_outbox", "0003_idempotency_keys"]} +{"applied": []} +{"applied": []} + +$ api replicas: schema check +{"ts": "2026-11-03T09:55:49Z", "level": "info", "logger": "messenger.http", "msg": "schema compatible; replica ready", "instance_id": "api-730666b6"} +{"ts": "2026-11-03T09:56:01Z", "level": "info", "logger": "messenger.http", "msg": "schema compatible; replica ready", "instance_id": "api-87d10eab"} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..5ded45e5 --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,5 @@ +# 2026-11-03T09:52:21Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +exit=0 diff --git a/evidence/restart-hook.txt b/evidence/restart-hook.txt new file mode 100644 index 00000000..25ce6bf8 --- /dev/null +++ b/evidence/restart-hook.txt @@ -0,0 +1,18 @@ +== 2026-11-03T09:52:23Z before restart +api mtusi-lab3-api-1 status=running started=2026-11-03T09:52:14.481518513Z pid=24423 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359 +api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502 +lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923 +migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0 +postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033 +redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102 +volume mtusi-lab3_pgdata +== 2026-11-03T09:52:40Z after restart +api mtusi-lab3-api-1 status=running started=2026-11-03T09:52:37.731388393Z pid=26144 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:52:37.507388069Z pid=26087 +api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:52:39.487250115Z pid=26364 +lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:52:39.921663108Z pid=26460 +migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:52:38.793556879Z pid=0 +postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:52:31.371856267Z pid=25812 +redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:52:31.401186275Z pid=25828 +volume mtusi-lab3_pgdata diff --git a/evidence/restore-1.txt b/evidence/restore-1.txt new file mode 100644 index 00000000..10414718 --- /dev/null +++ b/evidence/restore-1.txt @@ -0,0 +1,5 @@ +# 2026-11-03T09:51:50Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ ./scripts/start-replicas.sh +api mtusi-lab3-api-1 status=running started=2026-11-03T09:51:51.150461585Z pid=22901 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359 +exit=0 diff --git a/evidence/restore-2.txt b/evidence/restore-2.txt new file mode 100644 index 00000000..421bf289 --- /dev/null +++ b/evidence/restore-2.txt @@ -0,0 +1,5 @@ +# 2026-11-03T09:52:14Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ ./scripts/start-replicas.sh +api mtusi-lab3-api-1 status=running started=2026-11-03T09:52:14.481518513Z pid=24423 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359 +exit=0 diff --git a/evidence/rolling-update.txt b/evidence/rolling-update.txt new file mode 100644 index 00000000..97491bc3 --- /dev/null +++ b/evidence/rolling-update.txt @@ -0,0 +1,33 @@ +== 2026-11-03T09:55:43Z rolling update start: 2 replicas, new version 3.0.1 +old /mtusi-lab3-api-1 image=sha256:d9d8402d14a0d254f843480ba3d1da45ab5e43b8a9a98c73283cf4205136087b +old /mtusi-lab3-api-2 image=sha256:d9d8402d14a0d254f843480ba3d1da45ab5e43b8a9a98c73283cf4205136087b + Container mtusi-lab3-migrate-run-c15ccd39df8d Creating + Container mtusi-lab3-migrate-run-c15ccd39df8d Created +{"applied": []} +2026-11-03T09:55:54Z new /mtusi-lab3-api-3 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 healthy; stopping old /mtusi-lab3-api-1 +2026-11-03T09:56:06Z new /mtusi-lab3-api-4 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 healthy; stopping old /mtusi-lab3-api-2 +== 2026-11-03T09:56:10Z rolling update done +now /mtusi-lab3-api-3 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 health=healthy +now /mtusi-lab3-api-4 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 health=healthy +== 2026-11-03T09:57:28Z rolling update start: 2 replicas, new version 3.0.2-expand +old /mtusi-lab3-api-3 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 +old /mtusi-lab3-api-4 image=sha256:fb847cbe991672fec484b95bdfe4c640f390268364613e64b216cc5e056d8768 + Container mtusi-lab3-migrate-run-97c8514a64ea Creating + Container mtusi-lab3-migrate-run-97c8514a64ea Created +{"applied": []} +2026-11-03T09:57:38Z new /mtusi-lab3-api-5 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 healthy; stopping old /mtusi-lab3-api-3 +2026-11-03T09:57:51Z new /mtusi-lab3-api-6 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 healthy; stopping old /mtusi-lab3-api-4 +== 2026-11-03T09:57:56Z rolling update done +now /mtusi-lab3-api-5 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 health=healthy +now /mtusi-lab3-api-6 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 health=healthy +== 2026-11-03T09:58:18Z rolling update start: 2 replicas, new version 3.0.3 +old /mtusi-lab3-api-5 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 +old /mtusi-lab3-api-6 image=sha256:5c115f1efe4d3286866d887b3399e8f16f61db3a0462b02a0fada9f38a797a57 + Container mtusi-lab3-migrate-run-728fb26bee21 Creating + Container mtusi-lab3-migrate-run-728fb26bee21 Created +{"applied": []} +2026-11-03T09:58:28Z new /mtusi-lab3-api-7 image=sha256:d5045137ab0313c449760900fee392167eb7b6ef6c6e15cf8d598be6768bbdb3 healthy; stopping old /mtusi-lab3-api-5 +2026-11-03T09:58:40Z new /mtusi-lab3-api-8 image=sha256:d5045137ab0313c449760900fee392167eb7b6ef6c6e15cf8d598be6768bbdb3 healthy; stopping old /mtusi-lab3-api-6 +== 2026-11-03T09:58:44Z rolling update done +now /mtusi-lab3-api-7 image=sha256:d5045137ab0313c449760900fee392167eb7b6ef6c6e15cf8d598be6768bbdb3 health=healthy +now /mtusi-lab3-api-8 image=sha256:d5045137ab0313c449760900fee392167eb7b6ef6c6e15cf8d598be6768bbdb3 health=healthy diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..c4fd1b46 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,4 @@ +# 2026-11-03T09:51:02Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. +exit=0 diff --git a/evidence/stack.txt b/evidence/stack.txt new file mode 100644 index 00000000..998f4671 --- /dev/null +++ b/evidence/stack.txt @@ -0,0 +1,25 @@ +# 2026-11-03T09:51:01Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +docker 29.9.2 +Docker Compose version v5.6.1 +CONTAINER REPOSITORY TAG PLATFORM IMAGE ID SIZE CREATED +mtusi-lab3-api-1 mtusi-lab3-api local linux/amd64 d9d8402d14a0 2.63GB 2 hours ago +mtusi-lab3-api-2 mtusi-lab3-api local linux/amd64 d9d8402d14a0 2.63GB 2 hours ago +mtusi-lab3-api-ttl-1 mtusi-lab3-api local linux/amd64 d9d8402d14a0 2.63GB 2 hours ago +mtusi-lab3-lb-1 mtusi-local/haproxy 3.0 linux/amd64 c93e4aeed30e 42MB 3 hours ago +mtusi-lab3-migrate-1 mtusi-lab3-api local linux/amd64 d9d8402d14a0 2.63GB 2 hours ago +mtusi-lab3-postgres-1 mtusi-local/postgres 16 linux/amd64 2f004dbde9b5 337MB 3 hours ago +mtusi-lab3-redis-1 mtusi-local/redis 7 linux/amd64 01c11b3e9bd0 37.1MB 3 hours ago +NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS +mtusi-lab3-api-1 mtusi-lab3-api:local "python -m messenger…" api 14 seconds ago Up 6 seconds (healthy) 8080/tcp +mtusi-lab3-api-2 mtusi-lab3-api:local "python -m messenger…" api 14 seconds ago Up 7 seconds (healthy) 8080/tcp +mtusi-lab3-api-ttl-1 mtusi-lab3-api:local "python -m messenger…" api-ttl 14 seconds ago Up 6 seconds (healthy) 8080/tcp +mtusi-lab3-lb-1 mtusi-local/haproxy:3.0 "haproxy -W -db -f /…" lb 14 seconds ago Up 14 seconds (healthy) 127.0.0.1:8080-8081->8080-8081/tcp, 127.0.0.1:8404->8404/tcp +mtusi-lab3-postgres-1 mtusi-local/postgres:16 "docker-entrypoint.s…" postgres 14 seconds ago Up 13 seconds (healthy) +mtusi-lab3-redis-1 mtusi-local/redis:7 "sh -c 'umask 077\npr…" redis 14 seconds ago Up 13 seconds (healthy) +api mtusi-lab3-api-1 status=running started=2026-11-03T09:50:54.761533652Z pid=19437 +api mtusi-lab3-api-2 status=running started=2026-11-03T09:50:54.441955467Z pid=19359 +api-ttl mtusi-lab3-api-ttl-1 status=running started=2026-11-03T09:50:55.168689243Z pid=19502 +lb mtusi-lab3-lb-1 status=running started=2026-11-03T09:50:47.557773135Z pid=18923 +migrate mtusi-lab3-migrate-1 status=exited started=2026-11-03T09:50:53.689659096Z pid=0 +postgres mtusi-lab3-postgres-1 status=running started=2026-11-03T09:50:47.95406907Z pid=19033 +redis mtusi-lab3-redis-1 status=running started=2026-11-03T09:50:48.18721571Z pid=19102 diff --git a/evidence/test-api.txt b/evidence/test-api.txt new file mode 100644 index 00000000..eec48f31 --- /dev/null +++ b/evidence/test-api.txt @@ -0,0 +1,23 @@ +# 2026-11-03T09:51:04Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 student-tests/test_api.py +## API, auth, validation, ACL, transactions (http://127.0.0.1:8080) +PASS password stored only as scrypt KDF hash; User has no secret fields 0.66s +PASS wrong password and unknown user: identical 401 + Basic challenge 0.86s +PASS malformed Basic / Bearer on /auth/sessions / body on login 0.01s +PASS password with ':' and non-ASCII UTF-8 works (RFC 7617) 0.53s +PASS opaque 256-bit token, unique per login, DB stores only SHA-256 0.97s +PASS only Bearer accepted; Basic and X-User-Id never authenticate 0.02s +PASS logout revokes only the current session; repeat -> 401 1.14s +PASS strict JSON validation -> 400 0.04s +PASS message text: 4000 code points OK, 4001 / whitespace / sender_id -> 400 0.04s +PASS SQL is parameterized: injection payloads stored verbatim 0.02s +PASS ACL: outsider gets 404 everywhere (even for invalid body) 0.03s +PASS create chat validation: duplicates, caller missing, unknown member 0.01s +PASS compound create-chat rolls back on FK failure (no orphan chat) 0.56s chats before=2, failed insert left 0 rows +PASS registration race: 20 parallel same username -> exactly one 201 3.49s 201x1, 409x19, rows=1 +PASS keyset pagination: 120 msgs, varying limit, no gaps/duplicates 0.94s 4 pages +PASS chat list: only own chats, ordered, cursor bound to user 0.77s +PASS read-after-write: confirmed message visible to next read 0.01s +PASS unknown route 404 / wrong method 405 in error format 0.00s +summary: 18 passed, 0 failed +exit=0 diff --git a/evidence/test-cache-race.txt b/evidence/test-cache-race.txt new file mode 100644 index 00000000..57579551 --- /dev/null +++ b/evidence/test-cache-race.txt @@ -0,0 +1,16 @@ +# 2026-11-03T09:51:31Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ make test-cache-race +docker compose --env-file .env run --rm --no-deps -T -v "/home/claude/work3/mtusi2026lab3/student-tests:/app/student-tests:ro" --entrypoint python api student-tests/test_cache_race.py + Container mtusi-lab3-api-run-bc892bef496f Creating + Container mtusi-lab3-api-run-bc892bef496f Created +PASS read-through put makes entry readable +PASS entry TTL capped by SESSION_CACHE_TTL_SECONDS pttl=60000ms +PASS invalidate removes entry +PASS late put after logout is refused by tombstone +PASS tombstone outlives any cache entry pttl=74998ms +PASS entry never outlives session expiry pttl=1199ms +PASS expired session not served from cache +cache unavailable; falling back to PostgreSQL +PASS degraded mode: cache is not read until outbox drained +summary: all passed +exit=0 diff --git a/evidence/test-expiry.txt b/evidence/test-expiry.txt new file mode 100644 index 00000000..ce875094 --- /dev/null +++ b/evidence/test-expiry.txt @@ -0,0 +1,8 @@ +# 2026-11-03T09:51:21Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 student-tests/test_expiry.py +## session expiry, SESSION_TTL_SECONDS=2 (http://127.0.0.1:8081) +PASS expires_at = now + SESSION_TTL_SECONDS 1.20s remaining 2.00s +PASS token valid before expiry, 401 Bearer after (cache warmed) 4.52s +PASS re-login after expiry issues a new valid session 4.82s +summary: 3 passed, 0 failed +exit=0 diff --git a/evidence/test-failures.txt b/evidence/test-failures.txt new file mode 100644 index 00000000..9d35e608 --- /dev/null +++ b/evidence/test-failures.txt @@ -0,0 +1,7 @@ +# 2026-11-03T09:52:40Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 student-tests/test_failures.py +## dependency failures (http://127.0.0.1:8080) +PASS Redis down: API bypasses cache via PostgreSQL, no anonymous trust 6.95s fallback latency 0.006s; outbox rows during outage=1, drained after recovery +PASS PostgreSQL down: bounded 503 in error format; anonymous still 401 14.78s latency(s)={'ready': 1.5, 'post_message': 2.01, 'history': 2.01, 'login': 2.01, 'register': 2.0}; ready again in 2.6s +summary: 2 passed, 0 failed +exit=0 diff --git a/evidence/test-idempotency.txt b/evidence/test-idempotency.txt new file mode 100644 index 00000000..ded92dd3 --- /dev/null +++ b/evidence/test-idempotency.txt @@ -0,0 +1,14 @@ +# 2026-11-03T09:51:16Z commit=n/a host=Linux 6.18.44-fc-v230 x86_64 cpus=2 +$ python3 student-tests/test_idempotency.py +## idempotency across replicas (http://127.0.0.1:8080) +PASS sequential replays: same 201 + same Message from different replicas 0.29s replicas=2, rows=1 +PASS 30 concurrent POSTs, same key + body, via LB -> exactly one message 0.63s per-replica responses {'api-7e27119d': 15, 'api-5dbb8cd6': 15}, rows=1 +PASS concurrent same key, two different bodies -> one message, others 201-same or 409 0.78s 201x10, 409x10 +PASS after commit: different body -> 409; reordered/whitespace JSON -> replay 0.04s +PASS scope: user + chat + key (same key elsewhere is independent) 0.03s +PASS key format: missing/empty/129 chars/invalid -> 400; 128 chars ok 0.05s +PASS failed attempt does not claim the key (400/404 not recorded) 0.02s +PASS replay re-checks session and access 0.34s +PASS key older than IDEMPOTENCY_TTL_SECONDS is reusable (24 h window) 0.49s +summary: 9 passed, 0 failed +exit=0 diff --git a/evidence/track1/admission-1x1.00-mc10-q250.json b/evidence/track1/admission-1x1.00-mc10-q250.json new file mode 100644 index 00000000..c13c42ee --- /dev/null +++ b/evidence/track1/admission-1x1.00-mc10-q250.json @@ -0,0 +1,887 @@ +{ + "config": "admission-1x1.00-mc10-q250", + "repeat": 1, + "series": "admission", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": 10, + "LB_QUEUE_TIMEOUT": "250ms", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T09:11:19+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 200, + "peak_ok_rps": 303.5, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 28.13, + "p50": 5.56, + "p90": 8.81, + "p99": 16.48, + "p999": 28.13 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.38, + "p99_ms": 18.47 + }, + "me": { + "ok": 40, + "p50_ms": 2.84, + "p99_ms": 13.55 + }, + "post": { + "ok": 80, + "p50_ms": 6.51, + "p99_ms": 21.29 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.9, + "p99": 15.4 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 17.1, + "lb": 2.4, + "loadgen-run-57259902c18b": 2.9, + "postgres": 6.3, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 31.5, + "lb": 4.4, + "loadgen-run-57259902c18b": 5.9, + "postgres": 99.1, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 70.47, + "p50": 5.47, + "p90": 8.7, + "p99": 20.74, + "p999": 70.47 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.34, + "p99_ms": 18.1 + }, + "me": { + "ok": 80, + "p50_ms": 2.56, + "p99_ms": 70.47 + }, + "post": { + "ok": 160, + "p50_ms": 6.46, + "p99_ms": 16.02 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.88, + "p99": 20.42 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 30.7, + "lb": 3.5, + "loadgen-run-57259902c18b": 5.4, + "postgres": 8.5, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 34.5, + "lb": 4.4, + "loadgen-run-57259902c18b": 6.2, + "postgres": 99.8, + "redis": 4.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 114.87, + "p50": 5.52, + "p90": 11.24, + "p99": 35.76, + "p999": 78.88 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.5, + "p99_ms": 43.8 + }, + "me": { + "ok": 120, + "p50_ms": 2.54, + "p99_ms": 7.92 + }, + "post": { + "ok": 240, + "p50_ms": 6.19, + "p99_ms": 22.32 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.92, + "p99": 35.08 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 45.0, + "lb": 4.7, + "loadgen-run-57259902c18b": 7.6, + "postgres": 12.0, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 32.4, + "lb": 4.5, + "loadgen-run-57259902c18b": 6.7, + "postgres": 107.0, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 12.4 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 85, + "p50": 5.71, + "p90": 18.4, + "p99": 68.32, + "p999": 83.44 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.79, + "p99_ms": 53.82 + }, + "me": { + "ok": 160, + "p50_ms": 2.38, + "p99_ms": 67.83 + }, + "post": { + "ok": 320, + "p50_ms": 6.16, + "p99_ms": 78.9 + } + }, + "rate": 160, + "service_ms": { + "p50": 5.12, + "p99": 67.77 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 57.3, + "lb": 5.3, + "loadgen-run-57259902c18b": 8.8, + "postgres": 14.3, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 33.8, + "lb": 4.5, + "loadgen-run-57259902c18b": 6.7, + "postgres": 112.1, + "redis": 5.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.5, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14.9 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 243.98, + "p50": 8.73, + "p90": 82.82, + "p99": 216.96, + "p999": 240.66 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 11.33, + "p99_ms": 216.55 + }, + "me": { + "ok": 200, + "p50_ms": 2.43, + "p99_ms": 163.11 + }, + "post": { + "ok": 400, + "p50_ms": 6.82, + "p99_ms": 227.73 + } + }, + "rate": 200, + "service_ms": { + "p50": 8.1, + "p99": 216.86 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 71.7, + "lb": 6.3, + "loadgen-run-57259902c18b": 10.7, + "postgres": 16.1, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 33.5, + "lb": 5.2, + "loadgen-run-57259902c18b": 7.2, + "postgres": 113.6, + "redis": 3.8 + }, + "lb_queue_max": 4, + "lb_sessions_mean": 5.2, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 6 + }, + "error_ratio": 0.0025, + "errors": 6, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 317.64, + "p50": 84.99, + "p90": 228.57, + "p99": 272.63, + "p999": 309.43 + }, + "ok": 2394, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 85.11, + "p99_ms": 269.69 + }, + "me": { + "ok": 240, + "p50_ms": 58.49, + "p99_ms": 239.04 + }, + "post": { + "ok": 474, + "p50_ms": 98.95, + "p99_ms": 301.35 + } + }, + "rate": 240, + "service_ms": { + "p50": 84.12, + "p99": 271.95 + }, + "throughput_ok_rps": 239.4, + "resources": { + "cpu_percent_mean": { + "api": 86.7, + "lb": 7.2, + "loadgen-run-57259902c18b": 13.2, + "postgres": 18.9, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 44.4, + "lb": 5.8, + "loadgen-run-57259902c18b": 7.8, + "postgres": 114.5, + "redis": 5.1 + }, + "lb_queue_max": 37, + "lb_sessions_mean": 23.2, + "pg_active_mean": 0.67, + "pg_wait_events_mean": { + "CPU": 0.22, + "IO:WALSync": 0.33, + "LWLock:WALWrite": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 329 + }, + "error_ratio": 0.1175, + "errors": 329, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 380.71, + "p50": 256.44, + "p90": 301.41, + "p99": 337.81, + "p999": 377.64 + }, + "ok": 2471, + "per_op": { + "history": { + "ok": 1757, + "p50_ms": 260.64, + "p99_ms": 337.81 + }, + "me": { + "ok": 255, + "p50_ms": 234.22, + "p99_ms": 277.5 + }, + "post": { + "ok": 459, + "p50_ms": 273.47, + "p99_ms": 358.69 + } + }, + "rate": 280, + "service_ms": { + "p50": 256.04, + "p99": 337.24 + }, + "throughput_ok_rps": 247.1, + "resources": { + "cpu_percent_mean": { + "api": 89.0, + "lb": 7.8, + "loadgen-run-57259902c18b": 15.2, + "postgres": 19.1, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 35.0, + "lb": 6.1, + "loadgen-run-57259902c18b": 8.1, + "postgres": 113.3, + "redis": 3.8 + }, + "lb_queue_max": 69, + "lb_sessions_mean": 62.6, + "pg_active_mean": 0.2, + "pg_wait_events_mean": { + "Client:ClientRead": 0.1, + "LWLock:WALWrite": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 320 + }, + "error_ratio": 0.1, + "errors": 320, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 401.65, + "p50": 257.26, + "p90": 291.54, + "p99": 325.68, + "p999": 369.28 + }, + "ok": 2880, + "per_op": { + "history": { + "ok": 1994, + "p50_ms": 260.09, + "p99_ms": 331.06 + }, + "me": { + "ok": 318, + "p50_ms": 232.19, + "p99_ms": 271.83 + }, + "post": { + "ok": 568, + "p50_ms": 262.35, + "p99_ms": 311.67 + } + }, + "rate": 320, + "service_ms": { + "p50": 256.65, + "p99": 323.66 + }, + "throughput_ok_rps": 288, + "resources": { + "cpu_percent_mean": { + "api": 90.4, + "lb": 7.9, + "loadgen-run-57259902c18b": 16.0, + "postgres": 20.5, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 37.5, + "lb": 6.2, + "loadgen-run-57259902c18b": 8.3, + "postgres": 115.0, + "redis": 3.9 + }, + "lb_queue_max": 80, + "lb_sessions_mean": 81.6, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 565 + }, + "error_ratio": 0.1569, + "errors": 565, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 344.98, + "p50": 269.26, + "p90": 293.54, + "p99": 319.02, + "p999": 338.42 + }, + "ok": 3035, + "per_op": { + "history": { + "ok": 2134, + "p50_ms": 272.49, + "p99_ms": 322.22 + }, + "me": { + "ok": 338, + "p50_ms": 242.11, + "p99_ms": 270.86 + }, + "post": { + "ok": 563, + "p50_ms": 272.11, + "p99_ms": 314.63 + } + }, + "rate": 360, + "service_ms": { + "p50": 268.51, + "p99": 319 + }, + "throughput_ok_rps": 303.5, + "resources": { + "cpu_percent_mean": { + "api": 91.5, + "lb": 8.2, + "loadgen-run-57259902c18b": 16.9, + "postgres": 22.9, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 33.9, + "lb": 6.6, + "loadgen-run-57259902c18b": 8.9, + "postgres": 120.7, + "redis": 3.9 + }, + "lb_queue_max": 90, + "lb_sessions_mean": 93.7, + "pg_active_mean": 0.78, + "pg_wait_events_mean": { + "IO:WALSync": 0.11, + "LWLock:WALWrite": 0.67, + "Timeout:VacuumDelay": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1070 + }, + "error_ratio": 0.2675, + "errors": 1070, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 350.75, + "p50": 276.72, + "p90": 299.81, + "p99": 325.41, + "p999": 339.11 + }, + "ok": 2930, + "per_op": { + "history": { + "ok": 2039, + "p50_ms": 279.88, + "p99_ms": 326.46 + }, + "me": { + "ok": 372, + "p50_ms": 246.41, + "p99_ms": 264.6 + }, + "post": { + "ok": 519, + "p50_ms": 276.53, + "p99_ms": 320.36 + } + }, + "rate": 400, + "service_ms": { + "p50": 276, + "p99": 324.62 + }, + "throughput_ok_rps": 293, + "resources": { + "cpu_percent_mean": { + "api": 91.2, + "lb": 8.3, + "loadgen-run-57259902c18b": 17.3, + "postgres": 19.7, + "redis": 4.2 + }, + "mem_mib_max": { + "api": 33.9, + "lb": 6.8, + "loadgen-run-57259902c18b": 10.2, + "postgres": 115.2, + "redis": 3.7 + }, + "lb_queue_max": 101, + "lb_sessions_mean": 108.1, + "pg_active_mean": 1.22, + "pg_wait_events_mean": { + "IO:WALSync": 0.33, + "LWLock:BufferContent": 0.11, + "LWLock:WALWrite": 0.78 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1616 + }, + "error_ratio": 0.3513, + "errors": 1616, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 372.94, + "p50": 279.33, + "p90": 298.59, + "p99": 337.51, + "p999": 371.1 + }, + "ok": 2984, + "per_op": { + "history": { + "ok": 2061, + "p50_ms": 281.81, + "p99_ms": 353.35 + }, + "me": { + "ok": 392, + "p50_ms": 249.67, + "p99_ms": 277.77 + }, + "post": { + "ok": 531, + "p50_ms": 277.99, + "p99_ms": 335.47 + } + }, + "rate": 460, + "service_ms": { + "p50": 278.68, + "p99": 336.83 + }, + "throughput_ok_rps": 298.4, + "resources": { + "cpu_percent_mean": { + "api": 89.1, + "lb": 8.8, + "loadgen-run-57259902c18b": 18.6, + "postgres": 19.8, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 41.2, + "lb": 7.1, + "loadgen-run-57259902c18b": 10.6, + "postgres": 115.5, + "redis": 4.4 + }, + "lb_queue_max": 116, + "lb_sessions_mean": 123.1, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": false + }, + { + "attempted": 5200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2178 + }, + "error_ratio": 0.4188, + "errors": 2178, + "event": "step", + "label": "admission-1x1.00-mc10-q250", + "latency_ms": { + "max": 338.87, + "p50": 279.63, + "p90": 301.71, + "p99": 326.56, + "p999": 336.07 + }, + "ok": 3022, + "per_op": { + "history": { + "ok": 2079, + "p50_ms": 281.76, + "p99_ms": 328.77 + }, + "me": { + "ok": 405, + "p50_ms": 252.07, + "p99_ms": 278.26 + }, + "post": { + "ok": 538, + "p50_ms": 278.28, + "p99_ms": 320.24 + } + }, + "rate": 520, + "service_ms": { + "p50": 278.89, + "p99": 326.13 + }, + "throughput_ok_rps": 302.2, + "resources": { + "cpu_percent_mean": { + "api": 89.5, + "lb": 9.3, + "loadgen-run-57259902c18b": 19.7, + "postgres": 20.1, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 33.9, + "lb": 7.5, + "loadgen-run-57259902c18b": 11.6, + "postgres": 117.0, + "redis": 4.5 + }, + "lb_queue_max": 130, + "lb_sessions_mean": 137.8, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "Client:ClientRead": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-a6b2b034": { + "pool": { + "connections_num": 3, + "requests_num": 19, + "requests_queued": 3, + "connections_ms": 74, + "requests_wait_ms": 74, + "usage_ms": 53, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 8 + } + }, + "pool_after": { + "api-a6b2b034": { + "pool": { + "connections_num": 10, + "requests_num": 31049, + "requests_queued": 195, + "connections_ms": 424, + "requests_wait_ms": 1850, + "usage_ms": 570234, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 33917, + "misses": 108, + "errors": 0 + }, + "requests": 34351 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/admission-1x1.00-mc10-r2.json b/evidence/track1/admission-1x1.00-mc10-r2.json new file mode 100644 index 00000000..2abb8bdd --- /dev/null +++ b/evidence/track1/admission-1x1.00-mc10-r2.json @@ -0,0 +1,929 @@ +{ + "config": "admission-1x1.00-mc10-r2", + "repeat": 2, + "series": "admission", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": 10, + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T09:28:38+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 320, + "peak_ok_rps": 383.7, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 31.44, + "p50": 4.89, + "p90": 7.88, + "p99": 23.86, + "p999": 31.44 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 4.77, + "p99_ms": 12.48 + }, + "me": { + "ok": 40, + "p50_ms": 2.41, + "p99_ms": 17.55 + }, + "post": { + "ok": 80, + "p50_ms": 5.69, + "p99_ms": 28.51 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.27, + "p99": 16.5 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 14.9, + "lb": 2.0, + "loadgen-run-3312d64e3026": 2.5, + "postgres": 6.3, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 39.8, + "lb": 4.3, + "loadgen-run-3312d64e3026": 5.9, + "postgres": 135.4, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 28, + "p50": 4.34, + "p90": 6.93, + "p99": 16.14, + "p999": 28 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 4.26, + "p99_ms": 16.33 + }, + "me": { + "ok": 80, + "p50_ms": 2.17, + "p99_ms": 12.02 + }, + "post": { + "ok": 160, + "p50_ms": 5.3, + "p99_ms": 17.57 + } + }, + "rate": 80, + "service_ms": { + "p50": 3.72, + "p99": 15.43 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 24.6, + "lb": 2.8, + "loadgen-run-3312d64e3026": 4.1, + "postgres": 8.6, + "redis": 2.7 + }, + "mem_mib_max": { + "api": 31.9, + "lb": 4.4, + "loadgen-run-3312d64e3026": 6.2, + "postgres": 135.8, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 172.91, + "p50": 4.28, + "p90": 8.49, + "p99": 106.25, + "p999": 137.4 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 4.17, + "p99_ms": 25.3 + }, + "me": { + "ok": 120, + "p50_ms": 2.22, + "p99_ms": 22.61 + }, + "post": { + "ok": 240, + "p50_ms": 5.15, + "p99_ms": 126.08 + } + }, + "rate": 120, + "service_ms": { + "p50": 3.71, + "p99": 88.98 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 35.8, + "lb": 3.5, + "loadgen-run-3312d64e3026": 5.8, + "postgres": 10.9, + "redis": 3.2 + }, + "mem_mib_max": { + "api": 43.5, + "lb": 4.5, + "loadgen-run-3312d64e3026": 6.7, + "postgres": 145.0, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 361.47, + "p50": 4.41, + "p90": 17.32, + "p99": 297.48, + "p999": 359.57 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 4.53, + "p99_ms": 310.75 + }, + "me": { + "ok": 160, + "p50_ms": 1.77, + "p99_ms": 223.01 + }, + "post": { + "ok": 320, + "p50_ms": 4.63, + "p99_ms": 299.66 + } + }, + "rate": 160, + "service_ms": { + "p50": 3.8, + "p99": 275.17 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 49.6, + "lb": 4.3, + "loadgen-run-3312d64e3026": 7.3, + "postgres": 13.3, + "redis": 3.0 + }, + "mem_mib_max": { + "api": 32.9, + "lb": 5.1, + "loadgen-run-3312d64e3026": 6.9, + "postgres": 147.4, + "redis": 3.9 + }, + "lb_queue_max": 17, + "lb_sessions_mean": 3.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 18.2 + }, + "slo_ok": false + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 85.88, + "p50": 4.59, + "p90": 11.96, + "p99": 49.95, + "p999": 68.67 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 4.8, + "p99_ms": 52.14 + }, + "me": { + "ok": 200, + "p50_ms": 1.79, + "p99_ms": 11.98 + }, + "post": { + "ok": 400, + "p50_ms": 4.52, + "p99_ms": 47.34 + } + }, + "rate": 200, + "service_ms": { + "p50": 4, + "p99": 49.53 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 61.3, + "lb": 4.8, + "loadgen-run-3312d64e3026": 8.6, + "postgres": 14.6, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 41.8, + "lb": 5.2, + "loadgen-run-3312d64e3026": 7.2, + "postgres": 150.1, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 160.53, + "p50": 5.22, + "p90": 26.54, + "p99": 131.48, + "p999": 152.71 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 5.58, + "p99_ms": 134.06 + }, + "me": { + "ok": 240, + "p50_ms": 1.82, + "p99_ms": 74.51 + }, + "post": { + "ok": 480, + "p50_ms": 4.95, + "p99_ms": 125.58 + } + }, + "rate": 240, + "service_ms": { + "p50": 4.69, + "p99": 131.27 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 71.8, + "lb": 5.5, + "loadgen-run-3312d64e3026": 9.7, + "postgres": 16.5, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 33.5, + "lb": 5.1, + "loadgen-run-3312d64e3026": 7.2, + "postgres": 149.8, + "redis": 4.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 158.74, + "p50": 11.25, + "p90": 74.76, + "p99": 147.64, + "p999": 156.21 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 13.02, + "p99_ms": 149.26 + }, + "me": { + "ok": 280, + "p50_ms": 1.84, + "p99_ms": 111.47 + }, + "post": { + "ok": 560, + "p50_ms": 8.86, + "p99_ms": 141.55 + } + }, + "rate": 280, + "service_ms": { + "p50": 10.56, + "p99": 146.93 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 80.9, + "lb": 6.1, + "loadgen-run-3312d64e3026": 11.5, + "postgres": 18.6, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 44.4, + "lb": 5.4, + "loadgen-run-3312d64e3026": 7.1, + "postgres": 151.1, + "redis": 5.2 + }, + "lb_queue_max": 29, + "lb_sessions_mean": 11, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 223.6, + "p50": 52.59, + "p90": 145.14, + "p99": 189.02, + "p999": 207.16 + }, + "ok": 3200, + "per_op": { + "history": { + "ok": 2240, + "p50_ms": 52.17, + "p99_ms": 194.35 + }, + "me": { + "ok": 320, + "p50_ms": 23.12, + "p99_ms": 150.98 + }, + "post": { + "ok": 640, + "p50_ms": 58.37, + "p99_ms": 170.51 + } + }, + "rate": 320, + "service_ms": { + "p50": 52.06, + "p99": 188.24 + }, + "throughput_ok_rps": 320, + "resources": { + "cpu_percent_mean": { + "api": 89.2, + "lb": 6.6, + "loadgen-run-3312d64e3026": 12.9, + "postgres": 19.9, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 46.6, + "lb": 6.3, + "loadgen-run-3312d64e3026": 8.0, + "postgres": 150.3, + "redis": 3.9 + }, + "lb_queue_max": 30, + "lb_sessions_mean": 18.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 443.81, + "p50": 209.96, + "p90": 388.26, + "p99": 418.5, + "p999": 435.16 + }, + "ok": 3600, + "per_op": { + "history": { + "ok": 2520, + "p50_ms": 210.07, + "p99_ms": 418.5 + }, + "me": { + "ok": 360, + "p50_ms": 183.94, + "p99_ms": 393.26 + }, + "post": { + "ok": 720, + "p50_ms": 216.42, + "p99_ms": 421.47 + } + }, + "rate": 360, + "service_ms": { + "p50": 209.57, + "p99": 418.08 + }, + "throughput_ok_rps": 360, + "resources": { + "cpu_percent_mean": { + "api": 92.0, + "lb": 7.1, + "loadgen-run-3312d64e3026": 15.0, + "postgres": 20.5, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 34.2, + "lb": 7.5, + "loadgen-run-3312d64e3026": 11.6, + "postgres": 150.8, + "redis": 4.0 + }, + "lb_queue_max": 134, + "lb_sessions_mean": 94.6, + "pg_active_mean": 0.33, + "pg_wait_events_mean": { + "CPU": 0.11, + "IO:WALSync": 0.11, + "LWLock:WALWrite": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 163 + }, + "error_ratio": 0.0408, + "errors": 163, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 2079.46, + "p50": 1478.7, + "p90": 2024.57, + "p99": 2046.5, + "p999": 2071.4 + }, + "ok": 3837, + "per_op": { + "history": { + "ok": 2685, + "p50_ms": 1475.46, + "p99_ms": 2045.91 + }, + "me": { + "ok": 396, + "p50_ms": 1463.34, + "p99_ms": 2016.25 + }, + "post": { + "ok": 756, + "p50_ms": 1484.41, + "p99_ms": 2056.84 + } + }, + "rate": 400, + "service_ms": { + "p50": 1478.04, + "p99": 2045.97 + }, + "throughput_ok_rps": 383.7, + "resources": { + "cpu_percent_mean": { + "api": 92.8, + "lb": 7.8, + "loadgen-run-3312d64e3026": 17.4, + "postgres": 19.9, + "redis": 3.2 + }, + "mem_mib_max": { + "api": 37.8, + "lb": 18.6, + "loadgen-run-3312d64e3026": 42.2, + "postgres": 151.3, + "redis": 4.1 + }, + "lb_queue_max": 799, + "lb_sessions_mean": 509.8, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1121 + }, + "error_ratio": 0.2437, + "errors": 1121, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 2136.81, + "p50": 2019.07, + "p90": 2038.79, + "p99": 2080.88, + "p999": 2124.48 + }, + "ok": 3479, + "per_op": { + "history": { + "ok": 2395, + "p50_ms": 2024.2, + "p99_ms": 2084.75 + }, + "me": { + "ok": 417, + "p50_ms": 1995.39, + "p99_ms": 2028.96 + }, + "post": { + "ok": 667, + "p50_ms": 2018.88, + "p99_ms": 2078.09 + } + }, + "rate": 460, + "service_ms": { + "p50": 2018.44, + "p99": 2079.6 + }, + "throughput_ok_rps": 347.9, + "resources": { + "cpu_percent_mean": { + "api": 91.3, + "lb": 8.0, + "loadgen-run-3312d64e3026": 17.2, + "postgres": 19.8, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 42.0, + "lb": 23.1, + "loadgen-run-3312d64e3026": 54.8, + "postgres": 152.3, + "redis": 3.9 + }, + "lb_queue_max": 921, + "lb_sessions_mean": 828.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": false + }, + { + "attempted": 5200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1774 + }, + "error_ratio": 0.3412, + "errors": 1774, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 2103.38, + "p50": 2026.19, + "p90": 2041.71, + "p99": 2068.1, + "p999": 2097.32 + }, + "ok": 3426, + "per_op": { + "history": { + "ok": 2351, + "p50_ms": 2029.29, + "p99_ms": 2070.26 + }, + "me": { + "ok": 448, + "p50_ms": 2000.47, + "p99_ms": 2016.55 + }, + "post": { + "ok": 627, + "p50_ms": 2022.82, + "p99_ms": 2067.05 + } + }, + "rate": 520, + "service_ms": { + "p50": 2025.47, + "p99": 2067.67 + }, + "throughput_ok_rps": 342.6, + "resources": { + "cpu_percent_mean": { + "api": 90.7, + "lb": 8.1, + "loadgen-run-3312d64e3026": 18.0, + "postgres": 19.6, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 48.6, + "lb": 25.5, + "loadgen-run-3312d64e3026": 60.2, + "postgres": 153.8, + "redis": 4.0 + }, + "lb_queue_max": 1042, + "lb_sessions_mean": 993, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": false + }, + { + "attempted": 6000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2623 + }, + "error_ratio": 0.4372, + "errors": 2623, + "event": "step", + "label": "admission-1x1.00-mc10-r2", + "latency_ms": { + "max": 2132.2, + "p50": 2027.89, + "p90": 2042.9, + "p99": 2087.73, + "p999": 2109.01 + }, + "ok": 3377, + "per_op": { + "history": { + "ok": 2308, + "p50_ms": 2029.99, + "p99_ms": 2087.06 + }, + "me": { + "ok": 467, + "p50_ms": 2002.35, + "p99_ms": 2036.52 + }, + "post": { + "ok": 602, + "p50_ms": 2024.5, + "p99_ms": 2100.69 + } + }, + "rate": 600, + "service_ms": { + "p50": 2027.28, + "p99": 2086.75 + }, + "throughput_ok_rps": 337.7, + "resources": { + "cpu_percent_mean": { + "api": 90.1, + "lb": 8.9, + "loadgen-run-3312d64e3026": 19.4, + "postgres": 22.7, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 39.8, + "lb": 29.1, + "loadgen-run-3312d64e3026": 70.3, + "postgres": 164.4, + "redis": 5.2 + }, + "lb_queue_max": 1206, + "lb_sessions_mean": 1161.1, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1, + "Client:ClientRead": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-87aae2e3": { + "pool": { + "connections_num": 3, + "requests_num": 19, + "requests_queued": 3, + "connections_ms": 74, + "requests_wait_ms": 86, + "usage_ms": 42, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 8 + } + }, + "pool_after": { + "api-87aae2e3": { + "pool": { + "connections_num": 10, + "requests_num": 40145, + "requests_queued": 196, + "connections_ms": 523, + "requests_wait_ms": 1731, + "usage_ms": 563973, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 43819, + "misses": 138, + "errors": 0 + }, + "requests": 44305 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/admission-1x1.00-mc10-r3.json b/evidence/track1/admission-1x1.00-mc10-r3.json new file mode 100644 index 00000000..5ab735a3 --- /dev/null +++ b/evidence/track1/admission-1x1.00-mc10-r3.json @@ -0,0 +1,931 @@ +{ + "config": "admission-1x1.00-mc10-r3", + "repeat": 3, + "series": "admission", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": 10, + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T09:41:30+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 280, + "peak_ok_rps": 359.7, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 34.98, + "p50": 5.3, + "p90": 8.99, + "p99": 21.81, + "p999": 34.98 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.2, + "p99_ms": 23.17 + }, + "me": { + "ok": 40, + "p50_ms": 2.54, + "p99_ms": 6.17 + }, + "post": { + "ok": 80, + "p50_ms": 6.32, + "p99_ms": 34.98 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.65, + "p99": 21.02 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 16.0, + "lb": 2.2, + "loadgen-run-d7f8aed8393a": 2.7, + "postgres": 8.1, + "redis": 2.5 + }, + "mem_mib_max": { + "api": 31.4, + "lb": 4.4, + "loadgen-run-d7f8aed8393a": 5.9, + "postgres": 163.6, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 19 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 38.36, + "p50": 4.78, + "p90": 8.54, + "p99": 21.18, + "p999": 38.36 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 4.73, + "p99_ms": 21.18 + }, + "me": { + "ok": 80, + "p50_ms": 2.27, + "p99_ms": 4.19 + }, + "post": { + "ok": 160, + "p50_ms": 5.48, + "p99_ms": 31.65 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.25, + "p99": 20.02 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 27.2, + "lb": 3.2, + "loadgen-run-d7f8aed8393a": 4.7, + "postgres": 9.4, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 31.9, + "lb": 4.4, + "loadgen-run-d7f8aed8393a": 6.1, + "postgres": 164.0, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 19 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 35.4, + "p50": 4.88, + "p90": 9.14, + "p99": 20.6, + "p999": 34.41 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 4.93, + "p99_ms": 20.6 + }, + "me": { + "ok": 120, + "p50_ms": 2.3, + "p99_ms": 6.95 + }, + "post": { + "ok": 240, + "p50_ms": 5.24, + "p99_ms": 24.37 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.22, + "p99": 18.43 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 39.0, + "lb": 4.2, + "loadgen-run-d7f8aed8393a": 6.5, + "postgres": 12.7, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 32.9, + "lb": 4.5, + "loadgen-run-d7f8aed8393a": 6.6, + "postgres": 166.5, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 119.37, + "p50": 4.89, + "p90": 11.07, + "p99": 42.87, + "p999": 81.01 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.05, + "p99_ms": 54.86 + }, + "me": { + "ok": 160, + "p50_ms": 2, + "p99_ms": 13.04 + }, + "post": { + "ok": 320, + "p50_ms": 5.09, + "p99_ms": 18.23 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.33, + "p99": 41.88 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 52.2, + "lb": 4.9, + "loadgen-run-d7f8aed8393a": 8.2, + "postgres": 15.1, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 32.7, + "lb": 4.5, + "loadgen-run-d7f8aed8393a": 6.4, + "postgres": 173.3, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20.4 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 136.41, + "p50": 4.66, + "p90": 17.46, + "p99": 81.94, + "p999": 101.49 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 4.81, + "p99_ms": 88.62 + }, + "me": { + "ok": 200, + "p50_ms": 1.95, + "p99_ms": 57.93 + }, + "post": { + "ok": 400, + "p50_ms": 4.66, + "p99_ms": 74.58 + } + }, + "rate": 200, + "service_ms": { + "p50": 4.04, + "p99": 81.36 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 60.9, + "lb": 5.3, + "loadgen-run-d7f8aed8393a": 9.2, + "postgres": 16.1, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 32.7, + "lb": 4.7, + "loadgen-run-d7f8aed8393a": 6.6, + "postgres": 177.4, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 174.67, + "p50": 6.15, + "p90": 57.49, + "p99": 154.11, + "p999": 169.4 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 6.92, + "p99_ms": 161.47 + }, + "me": { + "ok": 240, + "p50_ms": 1.95, + "p99_ms": 91.32 + }, + "post": { + "ok": 480, + "p50_ms": 6.23, + "p99_ms": 139.23 + } + }, + "rate": 240, + "service_ms": { + "p50": 5.59, + "p99": 153.76 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 72.9, + "lb": 5.5, + "loadgen-run-d7f8aed8393a": 9.9, + "postgres": 17.6, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 36.0, + "lb": 5.0, + "loadgen-run-d7f8aed8393a": 7.1, + "postgres": 178.0, + "redis": 3.9 + }, + "lb_queue_max": 17, + "lb_sessions_mean": 6.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 214.78, + "p50": 11.51, + "p90": 94.27, + "p99": 158.26, + "p999": 206.54 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 13.2, + "p99_ms": 154.64 + }, + "me": { + "ok": 280, + "p50_ms": 1.83, + "p99_ms": 160.91 + }, + "post": { + "ok": 560, + "p50_ms": 9.32, + "p99_ms": 194.89 + } + }, + "rate": 280, + "service_ms": { + "p50": 10.94, + "p99": 157.84 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 81.3, + "lb": 6.1, + "loadgen-run-d7f8aed8393a": 11.3, + "postgres": 19.2, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 33.4, + "lb": 5.5, + "loadgen-run-d7f8aed8393a": 7.6, + "postgres": 178.8, + "redis": 5.2 + }, + "lb_queue_max": 33, + "lb_sessions_mean": 10.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": true + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 340.17, + "p50": 161.86, + "p90": 225.6, + "p99": 297.75, + "p999": 334.72 + }, + "ok": 3200, + "per_op": { + "history": { + "ok": 2240, + "p50_ms": 164.01, + "p99_ms": 290.18 + }, + "me": { + "ok": 320, + "p50_ms": 135.46, + "p99_ms": 290.54 + }, + "post": { + "ok": 640, + "p50_ms": 168.49, + "p99_ms": 324.71 + } + }, + "rate": 320, + "service_ms": { + "p50": 161.03, + "p99": 296.81 + }, + "throughput_ok_rps": 320, + "resources": { + "cpu_percent_mean": { + "api": 91.0, + "lb": 7.3, + "loadgen-run-d7f8aed8393a": 14.7, + "postgres": 22.2, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 33.9, + "lb": 6.6, + "loadgen-run-d7f8aed8393a": 8.6, + "postgres": 180.3, + "redis": 5.5 + }, + "lb_queue_max": 62, + "lb_sessions_mean": 49, + "pg_active_mean": 1.11, + "pg_wait_events_mean": { + "IO:WALSync": 0.11, + "LWLock:WALWrite": 1.0 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 3 + }, + "error_ratio": 0.0008, + "errors": 3, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 2043.41, + "p50": 1323.29, + "p90": 1777.62, + "p99": 2019.65, + "p999": 2029.59 + }, + "ok": 3597, + "per_op": { + "history": { + "ok": 2517, + "p50_ms": 1323.29, + "p99_ms": 2021.81 + }, + "me": { + "ok": 360, + "p50_ms": 1103.57, + "p99_ms": 1986.08 + }, + "post": { + "ok": 720, + "p50_ms": 1338.29, + "p99_ms": 2015.65 + } + }, + "rate": 360, + "service_ms": { + "p50": 1322.33, + "p99": 2018.72 + }, + "throughput_ok_rps": 359.7, + "resources": { + "cpu_percent_mean": { + "api": 90.7, + "lb": 7.7, + "loadgen-run-d7f8aed8393a": 16.4, + "postgres": 20.9, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 46.3, + "lb": 15.6, + "loadgen-run-d7f8aed8393a": 33.0, + "postgres": 181.1, + "redis": 4.0 + }, + "lb_queue_max": 609, + "lb_sessions_mean": 381.5, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 457 + }, + "error_ratio": 0.1143, + "errors": 457, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 2103.19, + "p50": 1669.91, + "p90": 2033.99, + "p99": 2067.95, + "p999": 2097.14 + }, + "ok": 3543, + "per_op": { + "history": { + "ok": 2469, + "p50_ms": 1657.97, + "p99_ms": 2076.56 + }, + "me": { + "ok": 381, + "p50_ms": 1922.4, + "p99_ms": 2012.47 + }, + "post": { + "ok": 693, + "p50_ms": 1677.06, + "p99_ms": 2062.11 + } + }, + "rate": 400, + "service_ms": { + "p50": 1668.88, + "p99": 2067.42 + }, + "throughput_ok_rps": 354.3, + "resources": { + "cpu_percent_mean": { + "api": 89.8, + "lb": 7.8, + "loadgen-run-d7f8aed8393a": 16.2, + "postgres": 20.8, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 50.8, + "lb": 20.3, + "loadgen-run-d7f8aed8393a": 44.6, + "postgres": 182.1, + "redis": 3.9 + }, + "lb_queue_max": 794, + "lb_sessions_mean": 576.6, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "Client:ClientRead": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1322 + }, + "error_ratio": 0.2874, + "errors": 1322, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 2134.47, + "p50": 2025.69, + "p90": 2042.13, + "p99": 2079.24, + "p999": 2114.4 + }, + "ok": 3278, + "per_op": { + "history": { + "ok": 2232, + "p50_ms": 2029.09, + "p99_ms": 2084.21 + }, + "me": { + "ok": 408, + "p50_ms": 1999.45, + "p99_ms": 2018.38 + }, + "post": { + "ok": 638, + "p50_ms": 2024.68, + "p99_ms": 2073.64 + } + }, + "rate": 460, + "service_ms": { + "p50": 2025, + "p99": 2079.02 + }, + "throughput_ok_rps": 327.8, + "resources": { + "cpu_percent_mean": { + "api": 92.0, + "lb": 8.6, + "loadgen-run-d7f8aed8393a": 17.7, + "postgres": 22.1, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 43.3, + "lb": 23.2, + "loadgen-run-d7f8aed8393a": 55.2, + "postgres": 182.2, + "redis": 3.9 + }, + "lb_queue_max": 923, + "lb_sessions_mean": 851, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": false + }, + { + "attempted": 5200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2107 + }, + "error_ratio": 0.4052, + "errors": 2107, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 2120.21, + "p50": 2029.66, + "p90": 2046.03, + "p99": 2082.03, + "p999": 2111.54 + }, + "ok": 3093, + "per_op": { + "history": { + "ok": 2101, + "p50_ms": 2031.92, + "p99_ms": 2083.95 + }, + "me": { + "ok": 431, + "p50_ms": 2002.59, + "p99_ms": 2042.16 + }, + "post": { + "ok": 561, + "p50_ms": 2026.16, + "p99_ms": 2069.46 + } + }, + "rate": 520, + "service_ms": { + "p50": 2028.9, + "p99": 2081.41 + }, + "throughput_ok_rps": 309.3, + "resources": { + "cpu_percent_mean": { + "api": 88.5, + "lb": 8.8, + "loadgen-run-d7f8aed8393a": 18.4, + "postgres": 21.5, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 47.5, + "lb": 25.8, + "loadgen-run-d7f8aed8393a": 60.9, + "postgres": 184.2, + "redis": 4.0 + }, + "lb_queue_max": 1040, + "lb_sessions_mean": 1017.4, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": false + }, + { + "attempted": 6000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 3092 + }, + "error_ratio": 0.5153, + "errors": 3092, + "event": "step", + "label": "admission-1x1.00-mc10-r3", + "latency_ms": { + "max": 2138.73, + "p50": 2032.96, + "p90": 2055.36, + "p99": 2094.22, + "p999": 2131.98 + }, + "ok": 2908, + "per_op": { + "history": { + "ok": 2009, + "p50_ms": 2034.57, + "p99_ms": 2100.68 + }, + "me": { + "ok": 375, + "p50_ms": 2004.51, + "p99_ms": 2037.16 + }, + "post": { + "ok": 524, + "p50_ms": 2032.28, + "p99_ms": 2086.16 + } + }, + "rate": 600, + "service_ms": { + "p50": 2032.22, + "p99": 2093.3 + }, + "throughput_ok_rps": 290.8, + "resources": { + "cpu_percent_mean": { + "api": 88.9, + "lb": 9.7, + "loadgen-run-d7f8aed8393a": 20.3, + "postgres": 21.6, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 38.9, + "lb": 29.6, + "loadgen-run-d7f8aed8393a": 68.9, + "postgres": 184.0, + "redis": 5.3 + }, + "lb_queue_max": 1200, + "lb_sessions_mean": 1192.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-7679011c": { + "pool": { + "connections_num": 3, + "requests_num": 19, + "requests_queued": 3, + "connections_ms": 73, + "requests_wait_ms": 83, + "usage_ms": 123, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 8 + } + }, + "pool_after": { + "api-7679011c": { + "pool": { + "connections_num": 10, + "requests_num": 38669, + "requests_queued": 214, + "connections_ms": 390, + "requests_wait_ms": 1639, + "usage_ms": 599558, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 42186, + "misses": 141, + "errors": 0 + }, + "requests": 42677 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/admission-1x1.00-mc10.json b/evidence/track1/admission-1x1.00-mc10.json new file mode 100644 index 00000000..3328d1b7 --- /dev/null +++ b/evidence/track1/admission-1x1.00-mc10.json @@ -0,0 +1,872 @@ +{ + "config": "admission-1x1.00-mc10", + "repeat": 1, + "series": "admission", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": 10, + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T09:03:48+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 240, + "peak_ok_rps": 339, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 26.28, + "p50": 4.92, + "p90": 7.39, + "p99": 15.4, + "p999": 26.28 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 4.8, + "p99_ms": 15.4 + }, + "me": { + "ok": 40, + "p50_ms": 2.35, + "p99_ms": 6.76 + }, + "post": { + "ok": 80, + "p50_ms": 5.71, + "p99_ms": 20.94 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.31, + "p99": 13.37 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 15.4, + "lb": 2.1, + "loadgen-run-c28b896f7dda": 2.5, + "postgres": 5.6, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 31.7, + "lb": 4.4, + "loadgen-run-c28b896f7dda": 5.9, + "postgres": 82.2, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 8 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 23.4, + "p50": 5.05, + "p90": 7.68, + "p99": 15.67, + "p999": 23.4 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 4.89, + "p99_ms": 15.67 + }, + "me": { + "ok": 80, + "p50_ms": 2.48, + "p99_ms": 17.94 + }, + "post": { + "ok": 160, + "p50_ms": 5.96, + "p99_ms": 15.68 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.39, + "p99": 14.68 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 28.4, + "lb": 3.3, + "loadgen-run-c28b896f7dda": 5.2, + "postgres": 8.2, + "redis": 2.9 + }, + "mem_mib_max": { + "api": 32.0, + "lb": 4.3, + "loadgen-run-c28b896f7dda": 6.2, + "postgres": 82.6, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "IO:WALSync": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 8 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 24.11, + "p50": 5, + "p90": 8.16, + "p99": 16.2, + "p999": 22.92 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 4.88, + "p99_ms": 12.84 + }, + "me": { + "ok": 120, + "p50_ms": 2.35, + "p99_ms": 18.12 + }, + "post": { + "ok": 240, + "p50_ms": 5.88, + "p99_ms": 19.11 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.44, + "p99": 15.24 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 39.1, + "lb": 4.4, + "loadgen-run-c28b896f7dda": 6.8, + "postgres": 10.3, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 32.2, + "lb": 4.4, + "loadgen-run-c28b896f7dda": 6.4, + "postgres": 83.6, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 8 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 208.18, + "p50": 5.64, + "p90": 24.22, + "p99": 116.98, + "p999": 188.4 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.67, + "p99_ms": 89.72 + }, + "me": { + "ok": 160, + "p50_ms": 2.3, + "p99_ms": 103.39 + }, + "post": { + "ok": 320, + "p50_ms": 6.16, + "p99_ms": 143.31 + } + }, + "rate": 160, + "service_ms": { + "p50": 5.04, + "p99": 116.66 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 56.5, + "lb": 5.2, + "loadgen-run-c28b896f7dda": 8.9, + "postgres": 14.3, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 44.8, + "lb": 4.7, + "loadgen-run-c28b896f7dda": 6.8, + "postgres": 94.9, + "redis": 5.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.3, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 12.4 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 87.46, + "p50": 5.64, + "p90": 28.46, + "p99": 67.19, + "p999": 81.25 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 5.87, + "p99_ms": 71.04 + }, + "me": { + "ok": 200, + "p50_ms": 2.05, + "p99_ms": 20.35 + }, + "post": { + "ok": 400, + "p50_ms": 5.66, + "p99_ms": 58.55 + } + }, + "rate": 200, + "service_ms": { + "p50": 4.97, + "p99": 66.73 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 66.9, + "lb": 5.6, + "loadgen-run-c28b896f7dda": 9.6, + "postgres": 14.3, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 33.1, + "lb": 4.6, + "loadgen-run-c28b896f7dda": 6.8, + "postgres": 95.4, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 232.48, + "p50": 11.54, + "p90": 111.14, + "p99": 223.22, + "p999": 231.73 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 14.64, + "p99_ms": 225.68 + }, + "me": { + "ok": 240, + "p50_ms": 2.22, + "p99_ms": 174.16 + }, + "post": { + "ok": 480, + "p50_ms": 7.44, + "p99_ms": 223.22 + } + }, + "rate": 240, + "service_ms": { + "p50": 10.89, + "p99": 222.87 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 78.0, + "lb": 6.1, + "loadgen-run-c28b896f7dda": 11.3, + "postgres": 16.6, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 42.3, + "lb": 5.4, + "loadgen-run-c28b896f7dda": 7.2, + "postgres": 95.9, + "redis": 3.9 + }, + "lb_queue_max": 17, + "lb_sessions_mean": 7.2, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 449.54, + "p50": 63.09, + "p90": 274.85, + "p99": 432.81, + "p999": 445.7 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 63.04, + "p99_ms": 432.76 + }, + "me": { + "ok": 280, + "p50_ms": 39.06, + "p99_ms": 393.59 + }, + "post": { + "ok": 560, + "p50_ms": 83.47, + "p99_ms": 438.73 + } + }, + "rate": 280, + "service_ms": { + "p50": 62.55, + "p99": 431.42 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 88.6, + "lb": 6.9, + "loadgen-run-c28b896f7dda": 14.1, + "postgres": 18.7, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 39.0, + "lb": 6.8, + "loadgen-run-c28b896f7dda": 9.9, + "postgres": 97.3, + "redis": 3.8 + }, + "lb_queue_max": 80, + "lb_sessions_mean": 24.1, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 426.08, + "p50": 233.15, + "p90": 345.38, + "p99": 400.44, + "p999": 421.35 + }, + "ok": 3200, + "per_op": { + "history": { + "ok": 2240, + "p50_ms": 236.18, + "p99_ms": 392.66 + }, + "me": { + "ok": 320, + "p50_ms": 205.69, + "p99_ms": 379.24 + }, + "post": { + "ok": 640, + "p50_ms": 237.5, + "p99_ms": 407.5 + } + }, + "rate": 320, + "service_ms": { + "p50": 232.65, + "p99": 399.91 + }, + "throughput_ok_rps": 320, + "resources": { + "cpu_percent_mean": { + "api": 90.6, + "lb": 7.1, + "loadgen-run-c28b896f7dda": 14.8, + "postgres": 19.7, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 41.8, + "lb": 7.0, + "loadgen-run-c28b896f7dda": 9.6, + "postgres": 96.9, + "redis": 3.8 + }, + "lb_queue_max": 96, + "lb_sessions_mean": 73.4, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 210 + }, + "error_ratio": 0.0583, + "errors": 210, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 2093.05, + "p50": 1512.71, + "p90": 2029.09, + "p99": 2060.89, + "p999": 2089.69 + }, + "ok": 3390, + "per_op": { + "history": { + "ok": 2352, + "p50_ms": 1502.11, + "p99_ms": 2068.9 + }, + "me": { + "ok": 353, + "p50_ms": 1499.05, + "p99_ms": 2010.4 + }, + "post": { + "ok": 685, + "p50_ms": 1524.93, + "p99_ms": 2052.03 + } + }, + "rate": 360, + "service_ms": { + "p50": 1512.04, + "p99": 2060.26 + }, + "throughput_ok_rps": 339, + "resources": { + "cpu_percent_mean": { + "api": 90.3, + "lb": 8.1, + "loadgen-run-c28b896f7dda": 18.4, + "postgres": 19.3, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 37.4, + "lb": 16.9, + "loadgen-run-c28b896f7dda": 36.5, + "postgres": 98.1, + "redis": 3.9 + }, + "lb_queue_max": 716, + "lb_sessions_mean": 404.8, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 740 + }, + "error_ratio": 0.185, + "errors": 740, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 2122.25, + "p50": 1990.94, + "p90": 2042.5, + "p99": 2083.5, + "p999": 2107.02 + }, + "ok": 3260, + "per_op": { + "history": { + "ok": 2291, + "p50_ms": 1997.39, + "p99_ms": 2084.35 + }, + "me": { + "ok": 363, + "p50_ms": 1989.82, + "p99_ms": 2030.34 + }, + "post": { + "ok": 606, + "p50_ms": 1976.43, + "p99_ms": 2078.11 + } + }, + "rate": 400, + "service_ms": { + "p50": 1989.83, + "p99": 2082.86 + }, + "throughput_ok_rps": 326, + "resources": { + "cpu_percent_mean": { + "api": 91.9, + "lb": 8.7, + "loadgen-run-c28b896f7dda": 18.3, + "postgres": 20.4, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 38.5, + "lb": 20.8, + "loadgen-run-c28b896f7dda": 46.3, + "postgres": 98.6, + "redis": 3.8 + }, + "lb_queue_max": 799, + "lb_sessions_mean": 658.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1500 + }, + "error_ratio": 0.3261, + "errors": 1500, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 2141.66, + "p50": 2027.38, + "p90": 2048.46, + "p99": 2080.13, + "p999": 2125.89 + }, + "ok": 3100, + "per_op": { + "history": { + "ok": 2120, + "p50_ms": 2030.97, + "p99_ms": 2086.38 + }, + "me": { + "ok": 405, + "p50_ms": 2000.17, + "p99_ms": 2034.48 + }, + "post": { + "ok": 575, + "p50_ms": 2024.36, + "p99_ms": 2072.63 + } + }, + "rate": 460, + "service_ms": { + "p50": 2026.65, + "p99": 2079.89 + }, + "throughput_ok_rps": 310, + "resources": { + "cpu_percent_mean": { + "api": 89.6, + "lb": 8.6, + "loadgen-run-c28b896f7dda": 17.9, + "postgres": 18.9, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 38.9, + "lb": 23.5, + "loadgen-run-c28b896f7dda": 53.2, + "postgres": 99.6, + "redis": 3.8 + }, + "lb_queue_max": 920, + "lb_sessions_mean": 865.2, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": false + }, + { + "attempted": 5200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2368 + }, + "error_ratio": 0.4554, + "errors": 2368, + "event": "step", + "label": "admission-1x1.00-mc10", + "latency_ms": { + "max": 2154.91, + "p50": 2032, + "p90": 2053.04, + "p99": 2111.5, + "p999": 2145.08 + }, + "ok": 2832, + "per_op": { + "history": { + "ok": 1930, + "p50_ms": 2033.94, + "p99_ms": 2112.49 + }, + "me": { + "ok": 385, + "p50_ms": 2002.98, + "p99_ms": 2028.39 + }, + "post": { + "ok": 517, + "p50_ms": 2030.67, + "p99_ms": 2125.35 + } + }, + "rate": 520, + "service_ms": { + "p50": 2031.37, + "p99": 2111.05 + }, + "throughput_ok_rps": 283.2, + "resources": { + "cpu_percent_mean": { + "api": 90.6, + "lb": 9.5, + "loadgen-run-c28b896f7dda": 19.9, + "postgres": 19.7, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 39.1, + "lb": 26.1, + "loadgen-run-c28b896f7dda": 61.8, + "postgres": 99.2, + "redis": 5.3 + }, + "lb_queue_max": 1040, + "lb_sessions_mean": 1014.6, + "pg_active_mean": 0.2, + "pg_wait_events_mean": { + "Client:ClientRead": 0.2 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-2e5d1b71": { + "pool": { + "connections_num": 3, + "requests_num": 19, + "requests_queued": 3, + "connections_ms": 64, + "requests_wait_ms": 68, + "usage_ms": 37, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 8 + } + }, + "pool_after": { + "api-2e5d1b71": { + "pool": { + "connections_num": 10, + "requests_num": 33873, + "requests_queued": 201, + "connections_ms": 435, + "requests_wait_ms": 1656, + "usage_ms": 551219, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 36857, + "misses": 116, + "errors": 0 + }, + "requests": 37305 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/admission-1x1.00-mc20.json b/evidence/track1/admission-1x1.00-mc20.json new file mode 100644 index 00000000..069aafeb --- /dev/null +++ b/evidence/track1/admission-1x1.00-mc20.json @@ -0,0 +1,869 @@ +{ + "config": "admission-1x1.00-mc20", + "repeat": 1, + "series": "admission", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": 20, + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T09:07:55+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 200, + "peak_ok_rps": 326.3, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 72.89, + "p50": 5.74, + "p90": 10.45, + "p99": 34.26, + "p999": 72.89 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.56, + "p99_ms": 34.26 + }, + "me": { + "ok": 40, + "p50_ms": 2.86, + "p99_ms": 68.16 + }, + "post": { + "ok": 80, + "p50_ms": 6.81, + "p99_ms": 22.77 + } + }, + "rate": 40, + "service_ms": { + "p50": 5.09, + "p99": 33.75 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 17.4, + "lb": 2.4, + "loadgen-run-6ca2812f958b": 3.1, + "postgres": 6.0, + "redis": 2.7 + }, + "mem_mib_max": { + "api": 33.4, + "lb": 4.4, + "loadgen-run-6ca2812f958b": 5.9, + "postgres": 88.5, + "redis": 4.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 8 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 23.76, + "p50": 5.26, + "p90": 8.36, + "p99": 16.86, + "p999": 23.76 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.2, + "p99_ms": 14.05 + }, + "me": { + "ok": 80, + "p50_ms": 2.65, + "p99_ms": 8.51 + }, + "post": { + "ok": 160, + "p50_ms": 6.33, + "p99_ms": 23.41 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.68, + "p99": 16.17 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 30.1, + "lb": 3.6, + "loadgen-run-6ca2812f958b": 5.3, + "postgres": 8.8, + "redis": 3.1 + }, + "mem_mib_max": { + "api": 31.8, + "lb": 4.5, + "loadgen-run-6ca2812f958b": 6.2, + "postgres": 90.8, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 8 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 104.11, + "p50": 5.47, + "p90": 15.4, + "p99": 59, + "p999": 97.17 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.48, + "p99_ms": 63.84 + }, + "me": { + "ok": 120, + "p50_ms": 2.46, + "p99_ms": 7.53 + }, + "post": { + "ok": 240, + "p50_ms": 6.02, + "p99_ms": 24.99 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.92, + "p99": 58.96 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 44.3, + "lb": 4.5, + "loadgen-run-6ca2812f958b": 7.4, + "postgres": 12.3, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 36.7, + "lb": 4.6, + "loadgen-run-6ca2812f958b": 6.7, + "postgres": 98.4, + "redis": 5.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10.4 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 117.45, + "p50": 4.86, + "p90": 12.77, + "p99": 55.34, + "p999": 94.65 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 4.93, + "p99_ms": 53.02 + }, + "me": { + "ok": 160, + "p50_ms": 2.05, + "p99_ms": 20.05 + }, + "post": { + "ok": 320, + "p50_ms": 5.22, + "p99_ms": 57.33 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.25, + "p99": 53.68 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 51.2, + "lb": 4.9, + "loadgen-run-6ca2812f958b": 8.3, + "postgres": 12.9, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 45.5, + "lb": 4.6, + "loadgen-run-6ca2812f958b": 6.9, + "postgres": 101.4, + "redis": 5.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.8, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 12.1 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 210.78, + "p50": 5.74, + "p90": 66.14, + "p99": 188.58, + "p999": 207.28 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 6.04, + "p99_ms": 192.47 + }, + "me": { + "ok": 200, + "p50_ms": 2.09, + "p99_ms": 95.94 + }, + "post": { + "ok": 400, + "p50_ms": 5.72, + "p99_ms": 186.99 + } + }, + "rate": 200, + "service_ms": { + "p50": 5.09, + "p99": 188.36 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 65.8, + "lb": 5.7, + "loadgen-run-6ca2812f958b": 9.9, + "postgres": 15.0, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 33.4, + "lb": 5.0, + "loadgen-run-6ca2812f958b": 7.1, + "postgres": 104.4, + "redis": 3.8 + }, + "lb_queue_max": 17, + "lb_sessions_mean": 6.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 495.13, + "p50": 74.78, + "p90": 394.07, + "p99": 469.64, + "p999": 489.39 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 75.26, + "p99_ms": 461.44 + }, + "me": { + "ok": 240, + "p50_ms": 34.09, + "p99_ms": 401.2 + }, + "post": { + "ok": 480, + "p50_ms": 90.18, + "p99_ms": 477.79 + } + }, + "rate": 240, + "service_ms": { + "p50": 74.15, + "p99": 468.97 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 85.4, + "lb": 6.9, + "loadgen-run-6ca2812f958b": 13.1, + "postgres": 19.1, + "redis": 4.3 + }, + "mem_mib_max": { + "api": 34.2, + "lb": 6.5, + "loadgen-run-6ca2812f958b": 9.2, + "postgres": 104.5, + "redis": 3.8 + }, + "lb_queue_max": 87, + "lb_sessions_mean": 30.7, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 601.55, + "p50": 169.96, + "p90": 463.64, + "p99": 573.13, + "p999": 588.34 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 171.21, + "p99_ms": 566.91 + }, + "me": { + "ok": 280, + "p50_ms": 119.76, + "p99_ms": 522.34 + }, + "post": { + "ok": 560, + "p50_ms": 186.06, + "p99_ms": 585.45 + } + }, + "rate": 280, + "service_ms": { + "p50": 169.11, + "p99": 572.41 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 89.8, + "lb": 7.2, + "loadgen-run-6ca2812f958b": 14.4, + "postgres": 19.8, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 41.8, + "lb": 7.6, + "loadgen-run-6ca2812f958b": 11.3, + "postgres": 105.3, + "redis": 3.9 + }, + "lb_queue_max": 135, + "lb_sessions_mean": 60.6, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 592.61, + "p50": 282.65, + "p90": 490.74, + "p99": 577.06, + "p999": 585.74 + }, + "ok": 3200, + "per_op": { + "history": { + "ok": 2240, + "p50_ms": 287.08, + "p99_ms": 576.17 + }, + "me": { + "ok": 320, + "p50_ms": 241.97, + "p99_ms": 521.63 + }, + "post": { + "ok": 640, + "p50_ms": 294.77, + "p99_ms": 580.88 + } + }, + "rate": 320, + "service_ms": { + "p50": 282.07, + "p99": 576.32 + }, + "throughput_ok_rps": 320, + "resources": { + "cpu_percent_mean": { + "api": 92.2, + "lb": 6.9, + "loadgen-run-6ca2812f958b": 14.7, + "postgres": 19.9, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 36.1, + "lb": 7.7, + "loadgen-run-6ca2812f958b": 11.5, + "postgres": 106.7, + "redis": 5.5 + }, + "lb_queue_max": 145, + "lb_sessions_mean": 91.7, + "pg_active_mean": 0.2, + "pg_wait_events_mean": { + "CPU": 0.2 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 337 + }, + "error_ratio": 0.0936, + "errors": 337, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 2223.99, + "p50": 1971.49, + "p90": 2074.67, + "p99": 2130.15, + "p999": 2211.47 + }, + "ok": 3263, + "per_op": { + "history": { + "ok": 2315, + "p50_ms": 1984.72, + "p99_ms": 2152.29 + }, + "me": { + "ok": 334, + "p50_ms": 1915.43, + "p99_ms": 2018.04 + }, + "post": { + "ok": 614, + "p50_ms": 1966.23, + "p99_ms": 2106.86 + } + }, + "rate": 360, + "service_ms": { + "p50": 1970.5, + "p99": 2129.39 + }, + "throughput_ok_rps": 326.3, + "resources": { + "cpu_percent_mean": { + "api": 91.2, + "lb": 8.6, + "loadgen-run-6ca2812f958b": 18.2, + "postgres": 20.7, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 41.7, + "lb": 18.1, + "loadgen-run-6ca2812f958b": 42.6, + "postgres": 106.5, + "redis": 4.5 + }, + "lb_queue_max": 719, + "lb_sessions_mean": 516.9, + "pg_active_mean": 0.4, + "pg_wait_events_mean": { + "IO:WALSync": 0.2, + "LWLock:WALWrite": 0.2 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 970 + }, + "error_ratio": 0.2425, + "errors": 970, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 2256.48, + "p50": 2056.27, + "p90": 2092.45, + "p99": 2166.54, + "p999": 2234.81 + }, + "ok": 3030, + "per_op": { + "history": { + "ok": 2188, + "p50_ms": 2059.33, + "p99_ms": 2166.54 + }, + "me": { + "ok": 303, + "p50_ms": 2004.44, + "p99_ms": 2041.52 + }, + "post": { + "ok": 539, + "p50_ms": 2062.03, + "p99_ms": 2178.87 + } + }, + "rate": 400, + "service_ms": { + "p50": 2054.84, + "p99": 2166.11 + }, + "throughput_ok_rps": 303, + "resources": { + "cpu_percent_mean": { + "api": 91.8, + "lb": 7.8, + "loadgen-run-6ca2812f958b": 16.7, + "postgres": 19.2, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 40.1, + "lb": 21.0, + "loadgen-run-6ca2812f958b": 48.7, + "postgres": 106.7, + "redis": 4.0 + }, + "lb_queue_max": 800, + "lb_sessions_mean": 752.9, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1603 + }, + "error_ratio": 0.3485, + "errors": 1603, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 2220.71, + "p50": 2061.29, + "p90": 2099.38, + "p99": 2150.8, + "p999": 2216.83 + }, + "ok": 2997, + "per_op": { + "history": { + "ok": 2168, + "p50_ms": 2063.37, + "p99_ms": 2152.38 + }, + "me": { + "ok": 331, + "p50_ms": 2007.81, + "p99_ms": 2033.45 + }, + "post": { + "ok": 498, + "p50_ms": 2065.91, + "p99_ms": 2168.52 + } + }, + "rate": 460, + "service_ms": { + "p50": 2060.51, + "p99": 2150.43 + }, + "throughput_ok_rps": 299.7, + "resources": { + "cpu_percent_mean": { + "api": 91.4, + "lb": 8.6, + "loadgen-run-6ca2812f958b": 18.4, + "postgres": 19.6, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 50.7, + "lb": 23.6, + "loadgen-run-6ca2812f958b": 53.4, + "postgres": 107.6, + "redis": 4.5 + }, + "lb_queue_max": 920, + "lb_sessions_mean": 893.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14 + }, + "slo_ok": false + }, + { + "attempted": 5200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2322 + }, + "error_ratio": 0.4465, + "errors": 2322, + "event": "step", + "label": "admission-1x1.00-mc20", + "latency_ms": { + "max": 2194.73, + "p50": 2067.03, + "p90": 2097.92, + "p99": 2157.48, + "p999": 2187.12 + }, + "ok": 2878, + "per_op": { + "history": { + "ok": 2069, + "p50_ms": 2069.51, + "p99_ms": 2161.06 + }, + "me": { + "ok": 313, + "p50_ms": 2011.02, + "p99_ms": 2045.56 + }, + "post": { + "ok": 496, + "p50_ms": 2071.98, + "p99_ms": 2174.02 + } + }, + "rate": 520, + "service_ms": { + "p50": 2066.54, + "p99": 2157.21 + }, + "throughput_ok_rps": 287.8, + "resources": { + "cpu_percent_mean": { + "api": 90.4, + "lb": 8.9, + "loadgen-run-6ca2812f958b": 19.0, + "postgres": 19.3, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 45.7, + "lb": 27.3, + "loadgen-run-6ca2812f958b": 61.5, + "postgres": 108.6, + "redis": 3.8 + }, + "lb_queue_max": 1041, + "lb_sessions_mean": 1040.6, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 14 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-6c123de5": { + "pool": { + "connections_num": 3, + "requests_num": 19, + "requests_queued": 3, + "connections_ms": 93, + "requests_wait_ms": 101, + "usage_ms": 53, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 8 + } + }, + "pool_after": { + "api-6c123de5": { + "pool": { + "connections_num": 10, + "requests_num": 33711, + "requests_queued": 23037, + "connections_ms": 438, + "requests_wait_ms": 614175, + "usage_ms": 711040, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 36469, + "misses": 116, + "errors": 0 + }, + "requests": 36915 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/admission-2x0.50-mc10.json b/evidence/track1/admission-2x0.50-mc10.json new file mode 100644 index 00000000..6ba4df6d --- /dev/null +++ b/evidence/track1/admission-2x0.50-mc10.json @@ -0,0 +1,919 @@ +{ + "config": "admission-2x0.50-mc10", + "repeat": 1, + "series": "admission", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": 10, + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 2, + "API_CPUS": "0.5", + "API_MEMORY": "256M", + "KDF_CONCURRENCY": 2 + }, + "measured_at": "2026-11-03T09:14:58+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 200, + "peak_ok_rps": 306.3, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 55.67, + "p50": 5.57, + "p90": 10.78, + "p99": 27.62, + "p999": 55.67 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.35, + "p99_ms": 22.32 + }, + "me": { + "ok": 40, + "p50_ms": 2.8, + "p99_ms": 19.2 + }, + "post": { + "ok": 80, + "p50_ms": 7.19, + "p99_ms": 28.45 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.94, + "p99": 26.81 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 21.1, + "lb": 2.3, + "loadgen-run-65398839d17b": 3.0, + "postgres": 6.4, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 31.1, + "lb": 4.4, + "loadgen-run-65398839d17b": 6.0, + "postgres": 110.1, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.039, + "pg_connections_mean": 12 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 73.17, + "p50": 5.26, + "p90": 12.21, + "p99": 34.4, + "p999": 73.17 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.2, + "p99_ms": 36.6 + }, + "me": { + "ok": 80, + "p50_ms": 2.6, + "p99_ms": 26.77 + }, + "post": { + "ok": 160, + "p50_ms": 6.04, + "p99_ms": 37.27 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.67, + "p99": 31.51 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 33.6, + "lb": 3.1, + "loadgen-run-65398839d17b": 5.0, + "postgres": 8.8, + "redis": 3.0 + }, + "mem_mib_max": { + "api": 43.9, + "lb": 4.4, + "loadgen-run-65398839d17b": 6.3, + "postgres": 110.3, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.039, + "pg_connections_mean": 12 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 226.71, + "p50": 5.34, + "p90": 17.43, + "p99": 141.16, + "p999": 190.12 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.26, + "p99_ms": 126.11 + }, + "me": { + "ok": 120, + "p50_ms": 2.4, + "p99_ms": 28.24 + }, + "post": { + "ok": 240, + "p50_ms": 6.41, + "p99_ms": 146.28 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.71, + "p99": 140.5 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 49.8, + "lb": 4.2, + "loadgen-run-65398839d17b": 6.7, + "postgres": 12.5, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 44.5, + "lb": 4.6, + "loadgen-run-65398839d17b": 6.9, + "postgres": 131.8, + "redis": 5.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 18.3 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 331.57, + "p50": 6.06, + "p90": 103.28, + "p99": 260.66, + "p999": 328.14 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 6.2, + "p99_ms": 259.9 + }, + "me": { + "ok": 160, + "p50_ms": 2.64, + "p99_ms": 194.62 + }, + "post": { + "ok": 320, + "p50_ms": 6.56, + "p99_ms": 276.11 + } + }, + "rate": 160, + "service_ms": { + "p50": 5.6, + "p99": 260.05 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 62.8, + "lb": 5.2, + "loadgen-run-65398839d17b": 8.5, + "postgres": 14.6, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 34.6, + "lb": 5.0, + "loadgen-run-65398839d17b": 7.5, + "postgres": 144.4, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 24.2 + }, + "slo_ok": false + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 228.33, + "p50": 10.68, + "p90": 119.22, + "p99": 201.85, + "p999": 215.33 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 11.69, + "p99_ms": 202.53 + }, + "me": { + "ok": 200, + "p50_ms": 2.29, + "p99_ms": 119.21 + }, + "post": { + "ok": 400, + "p50_ms": 12.03, + "p99_ms": 203.06 + } + }, + "rate": 200, + "service_ms": { + "p50": 9.88, + "p99": 201.58 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 77.0, + "lb": 5.7, + "loadgen-run-65398839d17b": 9.6, + "postgres": 15.8, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 44.4, + "lb": 5.1, + "loadgen-run-65398839d17b": 7.3, + "postgres": 146.4, + "redis": 5.0 + }, + "lb_queue_max": 9, + "lb_sessions_mean": 6.1, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "Client:ClientRead": 0.11, + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.006, + "pg_connections_mean": 26 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 770.1, + "p50": 143.33, + "p90": 631.73, + "p99": 749.7, + "p999": 765.09 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 147.35, + "p99_ms": 751.89 + }, + "me": { + "ok": 240, + "p50_ms": 68.25, + "p99_ms": 685.74 + }, + "post": { + "ok": 480, + "p50_ms": 147.53, + "p99_ms": 749.64 + } + }, + "rate": 240, + "service_ms": { + "p50": 142.88, + "p99": 748.55 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 91.2, + "lb": 6.7, + "loadgen-run-65398839d17b": 13.0, + "postgres": 18.6, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 35.0, + "lb": 7.7, + "loadgen-run-65398839d17b": 12.8, + "postgres": 145.8, + "redis": 4.3 + }, + "lb_queue_max": 149, + "lb_sessions_mean": 57.4, + "pg_active_mean": 0.33, + "pg_wait_events_mean": { + "CPU": 0.22, + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 26 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 1985.12, + "p50": 556.1, + "p90": 735.21, + "p99": 1921.66, + "p999": 1970.12 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 556.1, + "p99_ms": 1937.39 + }, + "me": { + "ok": 280, + "p50_ms": 348.18, + "p99_ms": 1869.36 + }, + "post": { + "ok": 560, + "p50_ms": 417.29, + "p99_ms": 1904.67 + } + }, + "rate": 280, + "service_ms": { + "p50": 555.55, + "p99": 1907.7 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 93.1, + "lb": 6.9, + "loadgen-run-65398839d17b": 14.2, + "postgres": 20.3, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 46.2, + "lb": 8.4, + "loadgen-run-65398839d17b": 14.3, + "postgres": 147.5, + "redis": 3.9 + }, + "lb_queue_max": 176, + "lb_sessions_mean": 130.2, + "pg_active_mean": 0.2, + "pg_wait_events_mean": { + "CPU": 0.2 + }, + "api_cfs_throttled_period_ratio": 0.006, + "pg_connections_mean": 26 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 137 + }, + "error_ratio": 0.0428, + "errors": 137, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 2168.21, + "p50": 1571.72, + "p90": 2063.3, + "p99": 2115.18, + "p999": 2160.74 + }, + "ok": 3063, + "per_op": { + "history": { + "ok": 2165, + "p50_ms": 1583.46, + "p99_ms": 2124.16 + }, + "me": { + "ok": 304, + "p50_ms": 1529.66, + "p99_ms": 2022.38 + }, + "post": { + "ok": 594, + "p50_ms": 1529.38, + "p99_ms": 2098.44 + } + }, + "rate": 320, + "service_ms": { + "p50": 1571.09, + "p99": 2113.38 + }, + "throughput_ok_rps": 306.3, + "resources": { + "cpu_percent_mean": { + "api": 92.3, + "lb": 7.2, + "loadgen-run-65398839d17b": 15.9, + "postgres": 19.1, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 47.8, + "lb": 16.2, + "loadgen-run-65398839d17b": 37.6, + "postgres": 147.7, + "redis": 5.5 + }, + "lb_queue_max": 640, + "lb_sessions_mean": 457.2, + "pg_active_mean": 0.2, + "pg_wait_events_mean": { + "CPU": 0.2 + }, + "api_cfs_throttled_period_ratio": 0.006, + "pg_connections_mean": 26 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 600 + }, + "error_ratio": 0.1667, + "errors": 600, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 2220.53, + "p50": 2031.26, + "p90": 2089.23, + "p99": 2155.93, + "p999": 2201.05 + }, + "ok": 3000, + "per_op": { + "history": { + "ok": 2138, + "p50_ms": 2047.34, + "p99_ms": 2163.26 + }, + "me": { + "ok": 317, + "p50_ms": 2000.62, + "p99_ms": 2061.42 + }, + "post": { + "ok": 545, + "p50_ms": 2050.82, + "p99_ms": 2150.96 + } + }, + "rate": 360, + "service_ms": { + "p50": 2030.2, + "p99": 2155.49 + }, + "throughput_ok_rps": 300, + "resources": { + "cpu_percent_mean": { + "api": 94.1, + "lb": 7.4, + "loadgen-run-65398839d17b": 15.5, + "postgres": 19.9, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 42.1, + "lb": 19.3, + "loadgen-run-65398839d17b": 44.4, + "postgres": 147.4, + "redis": 5.3 + }, + "lb_queue_max": 720, + "lb_sessions_mean": 552.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.016, + "pg_connections_mean": 26 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1098 + }, + "error_ratio": 0.2745, + "errors": 1098, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 2296.82, + "p50": 2059.75, + "p90": 2096.99, + "p99": 2153.66, + "p999": 2268.02 + }, + "ok": 2902, + "per_op": { + "history": { + "ok": 2102, + "p50_ms": 2064.07, + "p99_ms": 2155.6 + }, + "me": { + "ok": 300, + "p50_ms": 2010.3, + "p99_ms": 2060.99 + }, + "post": { + "ok": 500, + "p50_ms": 2060.46, + "p99_ms": 2144.71 + } + }, + "rate": 400, + "service_ms": { + "p50": 2059.13, + "p99": 2153.28 + }, + "throughput_ok_rps": 290.2, + "resources": { + "cpu_percent_mean": { + "api": 93.9, + "lb": 8.0, + "loadgen-run-65398839d17b": 16.6, + "postgres": 19.8, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 38.7, + "lb": 21.3, + "loadgen-run-65398839d17b": 47.0, + "postgres": 148.5, + "redis": 4.0 + }, + "lb_queue_max": 797, + "lb_sessions_mean": 737.6, + "pg_active_mean": 0.4, + "pg_wait_events_mean": { + "CPU": 0.2, + "IO:WALSync": 0.2 + }, + "api_cfs_throttled_period_ratio": 0.039, + "pg_connections_mean": 26 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1827 + }, + "error_ratio": 0.3972, + "errors": 1827, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 2271.42, + "p50": 2062.52, + "p90": 2114.85, + "p99": 2182.04, + "p999": 2265.54 + }, + "ok": 2773, + "per_op": { + "history": { + "ok": 2017, + "p50_ms": 2064.67, + "p99_ms": 2178.03 + }, + "me": { + "ok": 294, + "p50_ms": 2012.08, + "p99_ms": 2066.11 + }, + "post": { + "ok": 462, + "p50_ms": 2068.16, + "p99_ms": 2263.76 + } + }, + "rate": 460, + "service_ms": { + "p50": 2061.77, + "p99": 2181.92 + }, + "throughput_ok_rps": 277.3, + "resources": { + "cpu_percent_mean": { + "api": 91.8, + "lb": 8.5, + "loadgen-run-65398839d17b": 18.0, + "postgres": 20.3, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 38.6, + "lb": 23.8, + "loadgen-run-65398839d17b": 54.5, + "postgres": 148.3, + "redis": 3.9 + }, + "lb_queue_max": 921, + "lb_sessions_mean": 929.2, + "pg_active_mean": 0.3, + "pg_wait_events_mean": { + "CPU": 0.1, + "IO:WALSync": 0.1, + "LWLock:WALWrite": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.033, + "pg_connections_mean": 26 + }, + "slo_ok": false + }, + { + "attempted": 5200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2594 + }, + "error_ratio": 0.4988, + "errors": 2594, + "event": "step", + "label": "admission-2x0.50-mc10", + "latency_ms": { + "max": 2300.66, + "p50": 2070.01, + "p90": 2115.36, + "p99": 2187.79, + "p999": 2233.89 + }, + "ok": 2606, + "per_op": { + "history": { + "ok": 1899, + "p50_ms": 2072.18, + "p99_ms": 2184.89 + }, + "me": { + "ok": 248, + "p50_ms": 2019.21, + "p99_ms": 2086.36 + }, + "post": { + "ok": 459, + "p50_ms": 2072.02, + "p99_ms": 2211.08 + } + }, + "rate": 520, + "service_ms": { + "p50": 2069.24, + "p99": 2187.39 + }, + "throughput_ok_rps": 260.6, + "resources": { + "cpu_percent_mean": { + "api": 92.6, + "lb": 8.5, + "loadgen-run-65398839d17b": 18.3, + "postgres": 19.5, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 48.8, + "lb": 26.2, + "loadgen-run-65398839d17b": 61.5, + "postgres": 149.7, + "redis": 4.2 + }, + "lb_queue_max": 1040, + "lb_sessions_mean": 1031.1, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 26 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-a6f26611": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 121, + "requests_wait_ms": 141, + "usage_ms": 50, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-37d6ee24": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 164, + "requests_wait_ms": 184, + "usage_ms": 94, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + } + }, + "pool_after": { + "api-a6f26611": { + "pool": { + "connections_num": 10, + "requests_num": 16737, + "requests_queued": 199, + "connections_ms": 827, + "requests_wait_ms": 3416, + "usage_ms": 621313, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 17286, + "misses": 55, + "errors": 2 + }, + "requests": 17908 + }, + "api-37d6ee24": { + "pool": { + "connections_num": 10, + "requests_num": 16544, + "requests_queued": 208, + "connections_ms": 842, + "requests_wait_ms": 4643, + "usage_ms": 618848, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 17540, + "misses": 73, + "errors": 0 + }, + "requests": 17903 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/admission-goodput.svg b/evidence/track1/admission-goodput.svg new file mode 100644 index 00000000..54005466 --- /dev/null +++ b/evidence/track1/admission-goodput.svg @@ -0,0 +1,87 @@ + + +maxconn на LB против перегрузки (1 × 1.0 CPU) +Полезная пропускная способность; пул 10; без лимита и maxconn 10 — медиана 3 прогонов + +0 + +100 + +200 + +300 + +400 +0 +100 +200 +300 +400 +500 +600 + +Поданная нагрузка, req/s +Полезная, rps + +пунктир: полезная = поданная + +без лимита (maxconn 0) — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 28 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 27 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 56 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 106 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 218 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 322 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 320 req/s: полезная 276 rps, ошибок 13.8 %, p99 1518 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 360 req/s: полезная 277 rps, ошибок 22.9 %, p99 1671 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 400 req/s: полезная 253 rps, ошибок 36.8 %, p99 1340 мс (медиана 3 прогонов) +без лимита (maxconn 0) — 460 req/s: полезная 223 rps, ошибок 51.6 %, p99 1655 мс (медиана 3 прогонов) + +maxconn 10 — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 22 мс (медиана 3 прогонов) +maxconn 10 — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 16 мс (медиана 3 прогонов) +maxconn 10 — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 21 мс (медиана 3 прогонов) +maxconn 10 — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 117 мс (медиана 3 прогонов) +maxconn 10 — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 67 мс (медиана 3 прогонов) +maxconn 10 — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 154 мс (медиана 3 прогонов) +maxconn 10 — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 158 мс (медиана 3 прогонов) +maxconn 10 — 320 req/s: полезная 320 rps, ошибок 0.0 %, p99 298 мс (медиана 3 прогонов) +maxconn 10 — 360 req/s: полезная 360 rps, ошибок 0.1 %, p99 2020 мс (медиана 3 прогонов) +maxconn 10 — 400 req/s: полезная 354 rps, ошибок 11.4 %, p99 2068 мс (медиана 3 прогонов) +maxconn 10 — 460 req/s: полезная 328 rps, ошибок 28.7 %, p99 2080 мс (медиана 3 прогонов) +maxconn 10 — 520 req/s: полезная 309 rps, ошибок 40.5 %, p99 2082 мс (медиана 3 прогонов) +maxconn 10 — 600 req/s: полезная 314 rps, ошибок 47.6 %, p99 2091 мс (медиана 2 прогонов) + +maxconn 20 — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 34 мс (медиана 1 прогонов) +maxconn 20 — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 17 мс (медиана 1 прогонов) +maxconn 20 — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 59 мс (медиана 1 прогонов) +maxconn 20 — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 55 мс (медиана 1 прогонов) +maxconn 20 — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 189 мс (медиана 1 прогонов) +maxconn 20 — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 470 мс (медиана 1 прогонов) +maxconn 20 — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 573 мс (медиана 1 прогонов) +maxconn 20 — 320 req/s: полезная 320 rps, ошибок 0.0 %, p99 577 мс (медиана 1 прогонов) +maxconn 20 — 360 req/s: полезная 326 rps, ошибок 9.4 %, p99 2130 мс (медиана 1 прогонов) +maxconn 20 — 400 req/s: полезная 303 rps, ошибок 24.2 %, p99 2167 мс (медиана 1 прогонов) +maxconn 20 — 460 req/s: полезная 300 rps, ошибок 34.8 %, p99 2151 мс (медиана 1 прогонов) +maxconn 20 — 520 req/s: полезная 288 rps, ошибок 44.6 %, p99 2157 мс (медиана 1 прогонов) + +maxconn 10, queue 250 мс — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 16 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 21 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 36 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 68 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 217 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 240 req/s: полезная 239 rps, ошибок 0.2 %, p99 273 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 280 req/s: полезная 247 rps, ошибок 11.8 %, p99 338 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 320 req/s: полезная 288 rps, ошибок 10.0 %, p99 326 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 360 req/s: полезная 304 rps, ошибок 15.7 %, p99 319 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 400 req/s: полезная 293 rps, ошибок 26.8 %, p99 325 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 460 req/s: полезная 298 rps, ошибок 35.1 %, p99 338 мс (медиана 1 прогонов) +maxconn 10, queue 250 мс — 520 req/s: полезная 302 rps, ошибок 41.9 %, p99 327 мс (медиана 1 прогонов) +без лимита (maxconn 0) +без лимита (maxconn 0) +maxconn 10 +maxconn 10 +maxconn 20 +maxconn 20 +maxconn 10, queue 250 мс +maxconn 10, queue 250 мс + diff --git a/evidence/track1/fixed-1x1.00-r2.json b/evidence/track1/fixed-1x1.00-r2.json new file mode 100644 index 00000000..5f7f15f0 --- /dev/null +++ b/evidence/track1/fixed-1x1.00-r2.json @@ -0,0 +1,800 @@ +{ + "config": "fixed-1x1.00-r2", + "repeat": 2, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T09:18:58+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 280, + "peak_ok_rps": 280, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 248.53, + "p50": 5.38, + "p90": 22.13, + "p99": 213.96, + "p999": 248.53 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.21, + "p99_ms": 22.13 + }, + "me": { + "ok": 40, + "p50_ms": 2.67, + "p99_ms": 7.65 + }, + "post": { + "ok": 80, + "p50_ms": 14.1, + "p99_ms": 248.53 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.8, + "p99": 213.63 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 17.2, + "lb": 2.2, + "loadgen-run-03af3ac2a6d0": 2.7, + "postgres": 7.3, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 31.8, + "lb": 4.4, + "loadgen-run-03af3ac2a6d0": 6.0, + "postgres": 126.6, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.9, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "IO:WALSync": 0.11, + "LWLock:WALWrite": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 46.94, + "p50": 5.28, + "p90": 9.76, + "p99": 29.18, + "p999": 46.94 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.16, + "p99_ms": 32.25 + }, + "me": { + "ok": 80, + "p50_ms": 2.46, + "p99_ms": 11.2 + }, + "post": { + "ok": 160, + "p50_ms": 6.08, + "p99_ms": 26.84 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.65, + "p99": 24.56 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 29.3, + "lb": 3.3, + "loadgen-run-03af3ac2a6d0": 4.9, + "postgres": 9.3, + "redis": 2.9 + }, + "mem_mib_max": { + "api": 32.0, + "lb": 4.4, + "loadgen-run-03af3ac2a6d0": 6.2, + "postgres": 126.9, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 85.76, + "p50": 5.25, + "p90": 9.92, + "p99": 38.83, + "p999": 79.18 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.17, + "p99_ms": 41.01 + }, + "me": { + "ok": 120, + "p50_ms": 2.52, + "p99_ms": 23.77 + }, + "post": { + "ok": 240, + "p50_ms": 6.04, + "p99_ms": 16.59 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.63, + "p99": 37.56 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 41.0, + "lb": 4.5, + "loadgen-run-03af3ac2a6d0": 6.9, + "postgres": 11.1, + "redis": 3.1 + }, + "mem_mib_max": { + "api": 32.3, + "lb": 4.4, + "loadgen-run-03af3ac2a6d0": 6.2, + "postgres": 128.1, + "redis": 4.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.9, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 92.92, + "p50": 4.84, + "p90": 15.94, + "p99": 60.51, + "p999": 83.13 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 4.93, + "p99_ms": 55.7 + }, + "me": { + "ok": 160, + "p50_ms": 2.18, + "p99_ms": 45.8 + }, + "post": { + "ok": 320, + "p50_ms": 5.02, + "p99_ms": 67.91 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.24, + "p99": 59.46 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 51.2, + "lb": 4.8, + "loadgen-run-03af3ac2a6d0": 8.2, + "postgres": 13.5, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 42.8, + "lb": 4.6, + "loadgen-run-03af3ac2a6d0": 6.9, + "postgres": 127.7, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.9, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 137.85, + "p50": 5.46, + "p90": 21.53, + "p99": 101.42, + "p999": 131.71 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 6.04, + "p99_ms": 81.92 + }, + "me": { + "ok": 200, + "p50_ms": 1.94, + "p99_ms": 14.7 + }, + "post": { + "ok": 400, + "p50_ms": 4.75, + "p99_ms": 114.56 + } + }, + "rate": 200, + "service_ms": { + "p50": 4.79, + "p99": 100.58 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 63.3, + "lb": 5.3, + "loadgen-run-03af3ac2a6d0": 8.8, + "postgres": 14.3, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 33.2, + "lb": 4.7, + "loadgen-run-03af3ac2a6d0": 6.9, + "postgres": 129.2, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 3.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 257.5, + "p50": 9.35, + "p90": 113.47, + "p99": 237.9, + "p999": 256.55 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 12.07, + "p99_ms": 217.22 + }, + "me": { + "ok": 240, + "p50_ms": 2.06, + "p99_ms": 73.57 + }, + "post": { + "ok": 480, + "p50_ms": 6.25, + "p99_ms": 248.81 + } + }, + "rate": 240, + "service_ms": { + "p50": 8.74, + "p99": 237.86 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 78.3, + "lb": 5.7, + "loadgen-run-03af3ac2a6d0": 10.9, + "postgres": 17.6, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 39.4, + "lb": 5.0, + "loadgen-run-03af3ac2a6d0": 7.4, + "postgres": 128.4, + "redis": 5.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 6.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 199.51, + "p50": 19.2, + "p90": 111.53, + "p99": 190.39, + "p999": 198.45 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 22.23, + "p99_ms": 191.69 + }, + "me": { + "ok": 280, + "p50_ms": 2.78, + "p99_ms": 33.67 + }, + "post": { + "ok": 560, + "p50_ms": 23.44, + "p99_ms": 183.02 + } + }, + "rate": 280, + "service_ms": { + "p50": 18.61, + "p99": 189.82 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 85.2, + "lb": 6.3, + "loadgen-run-03af3ac2a6d0": 11.9, + "postgres": 18.7, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 47.0, + "lb": 5.1, + "loadgen-run-03af3ac2a6d0": 7.7, + "postgres": 130.8, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 11.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": true + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 529 + }, + "error_ratio": 0.1653, + "errors": 529, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 2327.82, + "p50": 807.39, + "p90": 1125.44, + "p99": 1517.58, + "p999": 2286.06 + }, + "ok": 2671, + "per_op": { + "history": { + "ok": 1864, + "p50_ms": 824.16, + "p99_ms": 1522.62 + }, + "me": { + "ok": 318, + "p50_ms": 21.32, + "p99_ms": 780.29 + }, + "post": { + "ok": 489, + "p50_ms": 833.42, + "p99_ms": 2266.99 + } + }, + "rate": 320, + "service_ms": { + "p50": 806.93, + "p99": 1516.57 + }, + "throughput_ok_rps": 267.1, + "resources": { + "cpu_percent_mean": { + "api": 91.1, + "lb": 7.4, + "loadgen-run-03af3ac2a6d0": 15.0, + "postgres": 19.4, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 40.7, + "lb": 8.1, + "loadgen-run-03af3ac2a6d0": 15.7, + "postgres": 129.4, + "redis": 5.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 207.2, + "pg_active_mean": 0.4, + "pg_wait_events_mean": { + "Client:ClientRead": 0.4 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 826 + }, + "error_ratio": 0.2294, + "errors": 826, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 1991.53, + "p50": 794.31, + "p90": 1047.54, + "p99": 1602.98, + "p999": 1976.35 + }, + "ok": 2774, + "per_op": { + "history": { + "ok": 1897, + "p50_ms": 812.89, + "p99_ms": 1653.82 + }, + "me": { + "ok": 360, + "p50_ms": 18.42, + "p99_ms": 788.44 + }, + "post": { + "ok": 517, + "p50_ms": 806.01, + "p99_ms": 1626.1 + } + }, + "rate": 360, + "service_ms": { + "p50": 793.71, + "p99": 1602.6 + }, + "throughput_ok_rps": 277.4, + "resources": { + "cpu_percent_mean": { + "api": 92.0, + "lb": 7.2, + "loadgen-run-03af3ac2a6d0": 14.6, + "postgres": 17.9, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 52.8, + "lb": 8.1, + "loadgen-run-03af3ac2a6d0": 16.6, + "postgres": 129.9, + "redis": 5.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 214.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1732 + }, + "error_ratio": 0.433, + "errors": 1732, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 1590.66, + "p50": 1014.62, + "p90": 1400.84, + "p99": 1559.84, + "p999": 1584.98 + }, + "ok": 2268, + "per_op": { + "history": { + "ok": 1502, + "p50_ms": 1118.51, + "p99_ms": 1557.17 + }, + "me": { + "ok": 400, + "p50_ms": 26.08, + "p99_ms": 99.93 + }, + "post": { + "ok": 366, + "p50_ms": 1144.47, + "p99_ms": 1584.3 + } + }, + "rate": 400, + "service_ms": { + "p50": 1014.28, + "p99": 1559.56 + }, + "throughput_ok_rps": 226.8, + "resources": { + "cpu_percent_mean": { + "api": 88.6, + "lb": 8.7, + "loadgen-run-03af3ac2a6d0": 16.0, + "postgres": 16.7, + "redis": 4.6 + }, + "mem_mib_max": { + "api": 54.4, + "lb": 7.9, + "loadgen-run-03af3ac2a6d0": 16.8, + "postgres": 131.2, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 214.7, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "Client:ClientRead": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2093 + }, + "error_ratio": 0.455, + "errors": 2093, + "event": "step", + "label": "fixed-1x1.00-r2", + "latency_ms": { + "max": 1463.9, + "p50": 930.75, + "p90": 1322.55, + "p99": 1421.05, + "p999": 1463.03 + }, + "ok": 2507, + "per_op": { + "history": { + "ok": 1632, + "p50_ms": 979.78, + "p99_ms": 1424.25 + }, + "me": { + "ok": 460, + "p50_ms": 18.49, + "p99_ms": 107.52 + }, + "post": { + "ok": 415, + "p50_ms": 1011.43, + "p99_ms": 1458.17 + } + }, + "rate": 460, + "service_ms": { + "p50": 930.2, + "p99": 1420.76 + }, + "throughput_ok_rps": 250.7, + "resources": { + "cpu_percent_mean": { + "api": 89.9, + "lb": 9.1, + "loadgen-run-03af3ac2a6d0": 16.7, + "postgres": 16.5, + "redis": 4.7 + }, + "mem_mib_max": { + "api": 52.9, + "lb": 8.5, + "loadgen-run-03af3ac2a6d0": 17.4, + "postgres": 131.1, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 219.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 17 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-4ff42b6f": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 85, + "requests_wait_ms": 92, + "usage_ms": 90, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + } + }, + "pool_after": { + "api-4ff42b6f": { + "pool": { + "connections_num": 10, + "requests_num": 32520, + "requests_queued": 13942, + "connections_ms": 233, + "requests_wait_ms": 9616407, + "usage_ms": 526852, + "requests_errors": 5825, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 34796, + "misses": 317, + "errors": 0 + }, + "requests": 35401 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-1x1.00-r3.json b/evidence/track1/fixed-1x1.00-r3.json new file mode 100644 index 00000000..278cd29a --- /dev/null +++ b/evidence/track1/fixed-1x1.00-r3.json @@ -0,0 +1,806 @@ +{ + "config": "fixed-1x1.00-r3", + "repeat": 3, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T09:31:59+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 240, + "peak_ok_rps": 316.7, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 59.69, + "p50": 4.89, + "p90": 8.33, + "p99": 28.02, + "p999": 59.69 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 4.87, + "p99_ms": 14.99 + }, + "me": { + "ok": 40, + "p50_ms": 2.42, + "p99_ms": 4.72 + }, + "post": { + "ok": 80, + "p50_ms": 5.9, + "p99_ms": 59.69 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.32, + "p99": 27.01 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 14.0, + "lb": 1.9, + "loadgen-run-b4cc6f85f08e": 2.5, + "postgres": 7.2, + "redis": 2.2 + }, + "mem_mib_max": { + "api": 33.8, + "lb": 4.5, + "loadgen-run-b4cc6f85f08e": 6.0, + "postgres": 144.4, + "redis": 5.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 16 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 19.2, + "p50": 4.77, + "p90": 7.58, + "p99": 14.08, + "p999": 19.2 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 4.72, + "p99_ms": 14.64 + }, + "me": { + "ok": 80, + "p50_ms": 2.27, + "p99_ms": 11.67 + }, + "post": { + "ok": 160, + "p50_ms": 5.76, + "p99_ms": 13.21 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.2, + "p99": 13.5 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 27.2, + "lb": 3.1, + "loadgen-run-b4cc6f85f08e": 4.8, + "postgres": 9.4, + "redis": 2.9 + }, + "mem_mib_max": { + "api": 31.9, + "lb": 4.5, + "loadgen-run-b4cc6f85f08e": 6.2, + "postgres": 146.1, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 16 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 82.75, + "p50": 4.82, + "p90": 8.82, + "p99": 18.5, + "p999": 67.67 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 4.87, + "p99_ms": 14.5 + }, + "me": { + "ok": 120, + "p50_ms": 2.22, + "p99_ms": 58.75 + }, + "post": { + "ok": 240, + "p50_ms": 5.14, + "p99_ms": 21.24 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.21, + "p99": 17.05 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 38.8, + "lb": 3.9, + "loadgen-run-b4cc6f85f08e": 6.5, + "postgres": 11.4, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 35.3, + "lb": 4.5, + "loadgen-run-b4cc6f85f08e": 6.2, + "postgres": 149.8, + "redis": 4.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 18 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 60.48, + "p50": 4.96, + "p90": 13.17, + "p99": 49.06, + "p999": 59.71 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.08, + "p99_ms": 50.46 + }, + "me": { + "ok": 160, + "p50_ms": 2.12, + "p99_ms": 8.97 + }, + "post": { + "ok": 320, + "p50_ms": 5.15, + "p99_ms": 48.86 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.31, + "p99": 48.66 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 51.7, + "lb": 4.9, + "loadgen-run-b4cc6f85f08e": 8.0, + "postgres": 14.0, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 32.7, + "lb": 4.5, + "loadgen-run-b4cc6f85f08e": 6.2, + "postgres": 155.8, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 180.98, + "p50": 5.25, + "p90": 24.89, + "p99": 130.76, + "p999": 147.99 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 5.78, + "p99_ms": 123.83 + }, + "me": { + "ok": 200, + "p50_ms": 2.06, + "p99_ms": 26.29 + }, + "post": { + "ok": 400, + "p50_ms": 4.98, + "p99_ms": 138.22 + } + }, + "rate": 200, + "service_ms": { + "p50": 4.65, + "p99": 130.34 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 63.7, + "lb": 5.3, + "loadgen-run-b4cc6f85f08e": 9.2, + "postgres": 16.0, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 39.1, + "lb": 4.7, + "loadgen-run-b4cc6f85f08e": 7.0, + "postgres": 158.9, + "redis": 5.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 3.8, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 299.19, + "p50": 8.6, + "p90": 94.17, + "p99": 218.04, + "p999": 287.43 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 9.44, + "p99_ms": 218.12 + }, + "me": { + "ok": 240, + "p50_ms": 2.13, + "p99_ms": 45.38 + }, + "post": { + "ok": 480, + "p50_ms": 11.16, + "p99_ms": 220.45 + } + }, + "rate": 240, + "service_ms": { + "p50": 8.01, + "p99": 214.19 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 78.9, + "lb": 6.0, + "loadgen-run-b4cc6f85f08e": 11.0, + "postgres": 18.0, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 46.1, + "lb": 5.0, + "loadgen-run-b4cc6f85f08e": 7.2, + "postgres": 159.0, + "redis": 4.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 9.4, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.11, + "Client:ClientRead": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 342.76, + "p50": 104.12, + "p90": 241.32, + "p99": 322.06, + "p999": 341.03 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 113.98, + "p99_ms": 319.32 + }, + "me": { + "ok": 280, + "p50_ms": 13.4, + "p99_ms": 45.72 + }, + "post": { + "ok": 560, + "p50_ms": 142.28, + "p99_ms": 337.01 + } + }, + "rate": 280, + "service_ms": { + "p50": 103.28, + "p99": 321.51 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 88.9, + "lb": 6.5, + "loadgen-run-b4cc6f85f08e": 12.8, + "postgres": 19.8, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 44.3, + "lb": 5.3, + "loadgen-run-b4cc6f85f08e": 7.9, + "postgres": 158.6, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 28.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 33 + }, + "error_ratio": 0.0103, + "errors": 33, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 1467.42, + "p50": 477.92, + "p90": 700.51, + "p99": 1105.55, + "p999": 1354.96 + }, + "ok": 3167, + "per_op": { + "history": { + "ok": 2217, + "p50_ms": 494.08, + "p99_ms": 1109.71 + }, + "me": { + "ok": 320, + "p50_ms": 16.73, + "p99_ms": 638.61 + }, + "post": { + "ok": 630, + "p50_ms": 501.4, + "p99_ms": 1110.53 + } + }, + "rate": 320, + "service_ms": { + "p50": 477.56, + "p99": 1104.52 + }, + "throughput_ok_rps": 316.7, + "resources": { + "cpu_percent_mean": { + "api": 93.8, + "lb": 7.0, + "loadgen-run-b4cc6f85f08e": 14.4, + "postgres": 20.7, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 52.8, + "lb": 7.6, + "loadgen-run-b4cc6f85f08e": 13.5, + "postgres": 160.8, + "redis": 4.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 142.7, + "pg_active_mean": 0.6, + "pg_wait_events_mean": { + "CPU": 0.1, + "IO:WALSync": 0.1, + "LWLock:WALWrite": 0.4 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 799 + }, + "error_ratio": 0.2219, + "errors": 799, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 1986.54, + "p50": 775.59, + "p90": 1169.18, + "p99": 1670.77, + "p999": 1869.14 + }, + "ok": 2801, + "per_op": { + "history": { + "ok": 1921, + "p50_ms": 795.13, + "p99_ms": 1636.24 + }, + "me": { + "ok": 360, + "p50_ms": 18.67, + "p99_ms": 800.91 + }, + "post": { + "ok": 520, + "p50_ms": 797.11, + "p99_ms": 1814.63 + } + }, + "rate": 360, + "service_ms": { + "p50": 774.85, + "p99": 1670.75 + }, + "throughput_ok_rps": 280.1, + "resources": { + "cpu_percent_mean": { + "api": 92.3, + "lb": 7.4, + "loadgen-run-b4cc6f85f08e": 15.2, + "postgres": 19.7, + "redis": 4.2 + }, + "mem_mib_max": { + "api": 49.8, + "lb": 7.9, + "loadgen-run-b4cc6f85f08e": 17.3, + "postgres": 161.2, + "redis": 4.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 212.3, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "IO:WALSync": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1473 + }, + "error_ratio": 0.3683, + "errors": 1473, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 1359.43, + "p50": 931.49, + "p90": 1160.69, + "p99": 1339.97, + "p999": 1358.42 + }, + "ok": 2527, + "per_op": { + "history": { + "ok": 1711, + "p50_ms": 975.71, + "p99_ms": 1342.7 + }, + "me": { + "ok": 400, + "p50_ms": 20.86, + "p99_ms": 89.1 + }, + "post": { + "ok": 416, + "p50_ms": 975.52, + "p99_ms": 1317.79 + } + }, + "rate": 400, + "service_ms": { + "p50": 931.07, + "p99": 1337.71 + }, + "throughput_ok_rps": 252.7, + "resources": { + "cpu_percent_mean": { + "api": 89.9, + "lb": 8.3, + "loadgen-run-b4cc6f85f08e": 16.2, + "postgres": 18.0, + "redis": 4.5 + }, + "mem_mib_max": { + "api": 53.8, + "lb": 7.6, + "loadgen-run-b4cc6f85f08e": 16.8, + "postgres": 160.1, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 215.2, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "Client:ClientRead": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2472 + }, + "error_ratio": 0.5374, + "errors": 2472, + "event": "step", + "label": "fixed-1x1.00-r3", + "latency_ms": { + "max": 1703.62, + "p50": 1187.41, + "p90": 1429.09, + "p99": 1676.2, + "p999": 1702.67 + }, + "ok": 2128, + "per_op": { + "history": { + "ok": 1366, + "p50_ms": 1293.19, + "p99_ms": 1676.2 + }, + "me": { + "ok": 460, + "p50_ms": 26.44, + "p99_ms": 171.21 + }, + "post": { + "ok": 302, + "p50_ms": 1307.55, + "p99_ms": 1684.72 + } + }, + "rate": 460, + "service_ms": { + "p50": 1186.63, + "p99": 1676.13 + }, + "throughput_ok_rps": 212.8, + "resources": { + "cpu_percent_mean": { + "api": 88.8, + "lb": 9.0, + "loadgen-run-b4cc6f85f08e": 17.0, + "postgres": 16.6, + "redis": 4.6 + }, + "mem_mib_max": { + "api": 53.8, + "lb": 7.7, + "loadgen-run-b4cc6f85f08e": 17.9, + "postgres": 161.3, + "redis": 4.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 220.4, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-f87de045": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 80, + "requests_wait_ms": 76, + "usage_ms": 75, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + } + }, + "pool_after": { + "api-f87de045": { + "pool": { + "connections_num": 10, + "requests_num": 32487, + "requests_queued": 15163, + "connections_ms": 410, + "requests_wait_ms": 9045144, + "usage_ms": 531592, + "requests_errors": 5350, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 34841, + "misses": 272, + "errors": 0 + }, + "requests": 35397 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-1x1.00.json b/evidence/track1/fixed-1x1.00.json new file mode 100644 index 00000000..848bd6a2 --- /dev/null +++ b/evidence/track1/fixed-1x1.00.json @@ -0,0 +1,804 @@ +{ + "config": "fixed-1x1.00", + "repeat": 1, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T08:28:34+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 240, + "peak_ok_rps": 280, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 26.77, + "p50": 5.05, + "p90": 8.12, + "p99": 17.13, + "p999": 26.77 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 4.9, + "p99_ms": 20.07 + }, + "me": { + "ok": 40, + "p50_ms": 2.62, + "p99_ms": 5.13 + }, + "post": { + "ok": 80, + "p50_ms": 6.27, + "p99_ms": 13.07 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.39, + "p99": 16.53 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 15.6, + "lb": 1.9, + "loadgen-run-14b3c75cfc70": 2.6, + "postgres": 4.5, + "redis": 2.2 + }, + "mem_mib_max": { + "api": 31.6, + "lb": 4.4, + "loadgen-run-14b3c75cfc70": 5.9, + "postgres": 51.6, + "redis": 3.5 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 57.5, + "p50": 5.04, + "p90": 8.07, + "p99": 26.76, + "p999": 57.5 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 4.95, + "p99_ms": 15.38 + }, + "me": { + "ok": 80, + "p50_ms": 2.55, + "p99_ms": 6.48 + }, + "post": { + "ok": 160, + "p50_ms": 6.41, + "p99_ms": 56.12 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.45, + "p99": 26.32 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 28.0, + "lb": 3.4, + "loadgen-run-14b3c75cfc70": 5.1, + "postgres": 7.3, + "redis": 2.7 + }, + "mem_mib_max": { + "api": 31.9, + "lb": 4.4, + "loadgen-run-14b3c75cfc70": 6.2, + "postgres": 51.7, + "redis": 3.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 195.02, + "p50": 5.27, + "p90": 10.48, + "p99": 117.23, + "p999": 143.67 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.22, + "p99_ms": 60.1 + }, + "me": { + "ok": 120, + "p50_ms": 2.41, + "p99_ms": 59.21 + }, + "post": { + "ok": 240, + "p50_ms": 5.82, + "p99_ms": 139.6 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.67, + "p99": 116.35 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 41.8, + "lb": 4.4, + "loadgen-run-14b3c75cfc70": 7.2, + "postgres": 10.0, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 42.8, + "lb": 4.4, + "loadgen-run-14b3c75cfc70": 6.4, + "postgres": 61.0, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 6.3 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 76.33, + "p50": 4.77, + "p90": 10.67, + "p99": 56.46, + "p999": 74.06 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 4.78, + "p99_ms": 56.67 + }, + "me": { + "ok": 160, + "p50_ms": 2.19, + "p99_ms": 11.42 + }, + "post": { + "ok": 320, + "p50_ms": 5.4, + "p99_ms": 61.52 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.17, + "p99": 55.54 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 50.6, + "lb": 5.0, + "loadgen-run-14b3c75cfc70": 8.7, + "postgres": 12.1, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 32.8, + "lb": 4.6, + "loadgen-run-14b3c75cfc70": 6.8, + "postgres": 64.4, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 132.03, + "p50": 6.12, + "p90": 22.47, + "p99": 106.21, + "p999": 124.66 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 6.79, + "p99_ms": 91.02 + }, + "me": { + "ok": 200, + "p50_ms": 2.07, + "p99_ms": 51.74 + }, + "post": { + "ok": 400, + "p50_ms": 5.81, + "p99_ms": 118.12 + } + }, + "rate": 200, + "service_ms": { + "p50": 5.54, + "p99": 106 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 67.2, + "lb": 5.8, + "loadgen-run-14b3c75cfc70": 10.1, + "postgres": 14.1, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 33.4, + "lb": 4.7, + "loadgen-run-14b3c75cfc70": 6.9, + "postgres": 65.5, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 247.31, + "p50": 7.82, + "p90": 117.69, + "p99": 213.3, + "p999": 241.35 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 8.94, + "p99_ms": 213.68 + }, + "me": { + "ok": 240, + "p50_ms": 2.59, + "p99_ms": 37.25 + }, + "post": { + "ok": 480, + "p50_ms": 7.72, + "p99_ms": 223.22 + } + }, + "rate": 240, + "service_ms": { + "p50": 7.22, + "p99": 211.96 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 78.8, + "lb": 6.0, + "loadgen-run-14b3c75cfc70": 11.4, + "postgres": 15.8, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 46.5, + "lb": 4.9, + "loadgen-run-14b3c75cfc70": 7.3, + "postgres": 65.2, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 9.5, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.11, + "Client:ClientRead": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 498.37, + "p50": 72.46, + "p90": 297.04, + "p99": 408.37, + "p999": 488.08 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 80, + "p99_ms": 403.35 + }, + "me": { + "ok": 280, + "p50_ms": 9.14, + "p99_ms": 82.06 + }, + "post": { + "ok": 560, + "p50_ms": 98.9, + "p99_ms": 417.27 + } + }, + "rate": 280, + "service_ms": { + "p50": 71.87, + "p99": 407.71 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 89.1, + "lb": 7.0, + "loadgen-run-14b3c75cfc70": 13.8, + "postgres": 17.9, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 45.8, + "lb": 5.7, + "loadgen-run-14b3c75cfc70": 9.1, + "postgres": 65.7, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 27.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 441 + }, + "error_ratio": 0.1378, + "errors": 441, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 1917.11, + "p50": 743.52, + "p90": 1047.11, + "p99": 1615.1, + "p999": 1888.72 + }, + "ok": 2759, + "per_op": { + "history": { + "ok": 1938, + "p50_ms": 755.54, + "p99_ms": 1614.04 + }, + "me": { + "ok": 319, + "p50_ms": 17.88, + "p99_ms": 542.08 + }, + "post": { + "ok": 502, + "p50_ms": 777.13, + "p99_ms": 1866.85 + } + }, + "rate": 320, + "service_ms": { + "p50": 742.61, + "p99": 1614.32 + }, + "throughput_ok_rps": 275.9, + "resources": { + "cpu_percent_mean": { + "api": 92.8, + "lb": 7.6, + "loadgen-run-14b3c75cfc70": 15.7, + "postgres": 18.0, + "redis": 4.2 + }, + "mem_mib_max": { + "api": 41.1, + "lb": 8.1, + "loadgen-run-14b3c75cfc70": 16.1, + "postgres": 66.5, + "redis": 4.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 204.1, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 965 + }, + "error_ratio": 0.2681, + "errors": 965, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 2078.9, + "p50": 806.35, + "p90": 1076.62, + "p99": 1707.09, + "p999": 2066.4 + }, + "ok": 2635, + "per_op": { + "history": { + "ok": 1795, + "p50_ms": 830.08, + "p99_ms": 1844.42 + }, + "me": { + "ok": 348, + "p50_ms": 18.81, + "p99_ms": 843.52 + }, + "post": { + "ok": 492, + "p50_ms": 833.46, + "p99_ms": 1634.82 + } + }, + "rate": 360, + "service_ms": { + "p50": 805.56, + "p99": 1706.51 + }, + "throughput_ok_rps": 263.5, + "resources": { + "cpu_percent_mean": { + "api": 93.0, + "lb": 8.2, + "loadgen-run-14b3c75cfc70": 16.6, + "postgres": 17.7, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 53.5, + "lb": 8.0, + "loadgen-run-14b3c75cfc70": 17.8, + "postgres": 67.0, + "redis": 4.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 212.9, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "Client:ClientRead": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1334 + }, + "error_ratio": 0.3335, + "errors": 1334, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 1155.97, + "p50": 894.03, + "p90": 1041.24, + "p99": 1136.1, + "p999": 1151.19 + }, + "ok": 2666, + "per_op": { + "history": { + "ok": 1827, + "p50_ms": 909.57, + "p99_ms": 1132.15 + }, + "me": { + "ok": 400, + "p50_ms": 20.84, + "p99_ms": 70.83 + }, + "post": { + "ok": 439, + "p50_ms": 904.37, + "p99_ms": 1144.54 + } + }, + "rate": 400, + "service_ms": { + "p50": 893.59, + "p99": 1135.1 + }, + "throughput_ok_rps": 266.6, + "resources": { + "cpu_percent_mean": { + "api": 90.6, + "lb": 8.3, + "loadgen-run-14b3c75cfc70": 16.2, + "postgres": 16.2, + "redis": 4.5 + }, + "mem_mib_max": { + "api": 41.6, + "lb": 7.6, + "loadgen-run-14b3c75cfc70": 17.0, + "postgres": 68.5, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 211.4, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2372 + }, + "error_ratio": 0.5157, + "errors": 2372, + "event": "step", + "label": "fixed-1x1.00", + "latency_ms": { + "max": 1742.42, + "p50": 1102.86, + "p90": 1466.56, + "p99": 1655.37, + "p999": 1696.79 + }, + "ok": 2228, + "per_op": { + "history": { + "ok": 1411, + "p50_ms": 1154.7, + "p99_ms": 1652 + }, + "me": { + "ok": 460, + "p50_ms": 25.32, + "p99_ms": 99.88 + }, + "post": { + "ok": 357, + "p50_ms": 1165.95, + "p99_ms": 1696.22 + } + }, + "rate": 460, + "service_ms": { + "p50": 1102.31, + "p99": 1654.72 + }, + "throughput_ok_rps": 222.8, + "resources": { + "cpu_percent_mean": { + "api": 88.0, + "lb": 9.4, + "loadgen-run-14b3c75cfc70": 18.1, + "postgres": 15.3, + "redis": 5.0 + }, + "mem_mib_max": { + "api": 54.2, + "lb": 8.8, + "loadgen-run-14b3c75cfc70": 17.1, + "postgres": 67.9, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 222.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-a0810b44": { + "pool": { + "connections_num": 3, + "requests_num": 19, + "requests_queued": 3, + "connections_ms": 64, + "requests_wait_ms": 68, + "usage_ms": 47, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 8 + } + }, + "pool_after": { + "api-a0810b44": { + "pool": { + "connections_num": 10, + "requests_num": 32550, + "requests_queued": 14872, + "connections_ms": 413, + "requests_wait_ms": 10013292, + "usage_ms": 536852, + "requests_errors": 5843, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 34713, + "misses": 297, + "errors": 1 + }, + "requests": 35398 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-2x0.50-r2.json b/evidence/track1/fixed-2x0.50-r2.json new file mode 100644 index 00000000..d682f7b9 --- /dev/null +++ b/evidence/track1/fixed-2x0.50-r2.json @@ -0,0 +1,718 @@ +{ + "config": "fixed-2x0.50-r2", + "repeat": 2, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 2, + "API_CPUS": "0.5", + "API_MEMORY": "256M", + "KDF_CONCURRENCY": 2 + }, + "measured_at": "2026-11-03T09:21:48+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 120, + "peak_ok_rps": 234.7, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 61.93, + "p50": 5.16, + "p90": 9.93, + "p99": 38.56, + "p999": 61.93 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.12, + "p99_ms": 53.03 + }, + "me": { + "ok": 40, + "p50_ms": 2.42, + "p99_ms": 10.92 + }, + "post": { + "ok": 80, + "p50_ms": 5.73, + "p99_ms": 34.4 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.57, + "p99": 37.85 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 19.3, + "lb": 2.2, + "loadgen-run-bc33f10e790a": 2.5, + "postgres": 6.5, + "redis": 2.3 + }, + "mem_mib_max": { + "api": 43.7, + "lb": 4.4, + "loadgen-run-bc33f10e790a": 6.1, + "postgres": 130.0, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.039, + "pg_connections_mean": 16 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 44.49, + "p50": 4.91, + "p90": 9.89, + "p99": 22.47, + "p999": 44.49 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 4.76, + "p99_ms": 16.94 + }, + "me": { + "ok": 80, + "p50_ms": 2.26, + "p99_ms": 16.42 + }, + "post": { + "ok": 160, + "p50_ms": 5.97, + "p99_ms": 24.78 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.28, + "p99": 22.1 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 30.4, + "lb": 3.2, + "loadgen-run-bc33f10e790a": 4.6, + "postgres": 8.5, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 42.0, + "lb": 4.4, + "loadgen-run-bc33f10e790a": 6.3, + "postgres": 130.1, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.8, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.017, + "pg_connections_mean": 16 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 206.61, + "p50": 5.1, + "p90": 18.43, + "p99": 112.38, + "p999": 146.51 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.12, + "p99_ms": 107.46 + }, + "me": { + "ok": 120, + "p50_ms": 2.43, + "p99_ms": 53.96 + }, + "post": { + "ok": 240, + "p50_ms": 5.75, + "p99_ms": 130.13 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.49, + "p99": 111.5 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 45.5, + "lb": 4.0, + "loadgen-run-bc33f10e790a": 6.8, + "postgres": 11.5, + "redis": 3.0 + }, + "mem_mib_max": { + "api": 44.6, + "lb": 4.5, + "loadgen-run-bc33f10e790a": 6.8, + "postgres": 137.8, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.011, + "pg_connections_mean": 18.3 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 321.07, + "p50": 5.98, + "p90": 56.53, + "p99": 254.22, + "p999": 315.27 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.95, + "p99_ms": 202.11 + }, + "me": { + "ok": 160, + "p50_ms": 2.53, + "p99_ms": 147.86 + }, + "post": { + "ok": 320, + "p50_ms": 7.06, + "p99_ms": 307.91 + } + }, + "rate": 160, + "service_ms": { + "p50": 5.39, + "p99": 253.54 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 63.6, + "lb": 5.4, + "loadgen-run-bc33f10e790a": 8.7, + "postgres": 16.3, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 42.7, + "lb": 4.8, + "loadgen-run-bc33f10e790a": 7.1, + "postgres": 159.8, + "redis": 5.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 5.1, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.011, + "pg_connections_mean": 25.3 + }, + "slo_ok": false + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 392.21, + "p50": 7.31, + "p90": 63.62, + "p99": 309.74, + "p999": 383.94 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 8.35, + "p99_ms": 276.99 + }, + "me": { + "ok": 200, + "p50_ms": 2.47, + "p99_ms": 100.7 + }, + "post": { + "ok": 400, + "p50_ms": 7.19, + "p99_ms": 342.89 + } + }, + "rate": 200, + "service_ms": { + "p50": 6.81, + "p99": 309.04 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 75.5, + "lb": 6.0, + "loadgen-run-bc33f10e790a": 10.1, + "postgres": 16.8, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 45.9, + "lb": 5.1, + "loadgen-run-bc33f10e790a": 7.5, + "postgres": 160.6, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 5.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.011, + "pg_connections_mean": 28 + }, + "slo_ok": false + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 53 + }, + "error_ratio": 0.0221, + "errors": 53, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 1600.56, + "p50": 338.47, + "p90": 1284.39, + "p99": 1392.57, + "p999": 1590.9 + }, + "ok": 2347, + "per_op": { + "history": { + "ok": 1634, + "p50_ms": 385.93, + "p99_ms": 1401.71 + }, + "me": { + "ok": 240, + "p50_ms": 34.57, + "p99_ms": 172.6 + }, + "post": { + "ok": 473, + "p50_ms": 388.76, + "p99_ms": 1379.86 + } + }, + "rate": 240, + "service_ms": { + "p50": 338.01, + "p99": 1391.75 + }, + "throughput_ok_rps": 234.7, + "resources": { + "cpu_percent_mean": { + "api": 78.8, + "lb": 6.6, + "loadgen-run-bc33f10e790a": 12.4, + "postgres": 17.0, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 49.2, + "lb": 8.4, + "loadgen-run-bc33f10e790a": 14.4, + "postgres": 163.3, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 99.7, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.223, + "pg_connections_mean": 28 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 571 + }, + "error_ratio": 0.2039, + "errors": 571, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 1999.62, + "p50": 1064.78, + "p90": 1647.92, + "p99": 1918.92, + "p999": 1987.41 + }, + "ok": 2229, + "per_op": { + "history": { + "ok": 1510, + "p50_ms": 1222.65, + "p99_ms": 1921.34 + }, + "me": { + "ok": 280, + "p50_ms": 44.87, + "p99_ms": 167.45 + }, + "post": { + "ok": 439, + "p50_ms": 1144.96, + "p99_ms": 1925.78 + } + }, + "rate": 280, + "service_ms": { + "p50": 1063.95, + "p99": 1918.05 + }, + "throughput_ok_rps": 222.9, + "resources": { + "cpu_percent_mean": { + "api": 82.5, + "lb": 7.1, + "loadgen-run-bc33f10e790a": 13.7, + "postgres": 17.8, + "redis": 4.2 + }, + "mem_mib_max": { + "api": 52.9, + "lb": 8.9, + "loadgen-run-bc33f10e790a": 17.8, + "postgres": 161.4, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 221.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.225, + "pg_connections_mean": 28 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1189 + }, + "error_ratio": 0.3716, + "errors": 1189, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 2963.48, + "p50": 1283.78, + "p90": 2024.23, + "p99": 2463.8, + "p999": 2778.97 + }, + "ok": 2011, + "per_op": { + "history": { + "ok": 1318, + "p50_ms": 1359.7, + "p99_ms": 2442.14 + }, + "me": { + "ok": 275, + "p50_ms": 48.66, + "p99_ms": 1645.93 + }, + "post": { + "ok": 418, + "p50_ms": 1366.84, + "p99_ms": 2481.06 + } + }, + "rate": 320, + "service_ms": { + "p50": 1282.87, + "p99": 2463.16 + }, + "throughput_ok_rps": 201.1, + "resources": { + "cpu_percent_mean": { + "api": 84.2, + "lb": 7.6, + "loadgen-run-bc33f10e790a": 13.9, + "postgres": 17.9, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 55.8, + "lb": 9.8, + "loadgen-run-bc33f10e790a": 21.2, + "postgres": 163.2, + "redis": 5.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 247, + "pg_active_mean": 0.3, + "pg_wait_events_mean": { + "CPU": 0.3 + }, + "api_cfs_throttled_period_ratio": 0.172, + "pg_connections_mean": 28 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1770 + }, + "error_ratio": 0.4917, + "errors": 1770, + "event": "step", + "label": "fixed-2x0.50-r2", + "latency_ms": { + "max": 2899.83, + "p50": 1276.49, + "p90": 1994.35, + "p99": 2642.19, + "p999": 2865.02 + }, + "ok": 1830, + "per_op": { + "history": { + "ok": 1252, + "p50_ms": 1392.33, + "p99_ms": 2674.85 + }, + "me": { + "ok": 251, + "p50_ms": 51.72, + "p99_ms": 1485.91 + }, + "post": { + "ok": 327, + "p50_ms": 1421, + "p99_ms": 2151.95 + } + }, + "rate": 360, + "service_ms": { + "p50": 1275.94, + "p99": 2642.17 + }, + "throughput_ok_rps": 183, + "resources": { + "cpu_percent_mean": { + "api": 80.7, + "lb": 8.4, + "loadgen-run-bc33f10e790a": 14.9, + "postgres": 14.8, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 58.5, + "lb": 11.3, + "loadgen-run-bc33f10e790a": 22.9, + "postgres": 162.7, + "redis": 4.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 255.7, + "pg_active_mean": 0.2, + "pg_wait_events_mean": { + "CPU": 0.2 + }, + "api_cfs_throttled_period_ratio": 0.207, + "pg_connections_mean": 28 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-f359a812": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 124, + "requests_wait_ms": 155, + "usage_ms": 47, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-be216ffb": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 153, + "requests_wait_ms": 206, + "usage_ms": 50, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + } + }, + "pool_after": { + "api-f359a812": { + "pool": { + "connections_num": 10, + "requests_num": 12923, + "requests_queued": 5458, + "connections_ms": 837, + "requests_wait_ms": 4909083, + "usage_ms": 375953, + "requests_errors": 2881, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 11210, + "misses": 136, + "errors": 9 + }, + "requests": 13330 + }, + "api-be216ffb": { + "pool": { + "connections_num": 10, + "requests_num": 10324, + "requests_queued": 4305, + "connections_ms": 1050, + "requests_wait_ms": 3791102, + "usage_ms": 317058, + "requests_errors": 1273, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 10680, + "misses": 140, + "errors": 0 + }, + "requests": 11053 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-2x0.50-r3.json b/evidence/track1/fixed-2x0.50-r3.json new file mode 100644 index 00000000..78bd0c15 --- /dev/null +++ b/evidence/track1/fixed-2x0.50-r3.json @@ -0,0 +1,711 @@ +{ + "config": "fixed-2x0.50-r3", + "repeat": 3, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 2, + "API_CPUS": "0.5", + "API_MEMORY": "256M", + "KDF_CONCURRENCY": 2 + }, + "measured_at": "2026-11-03T09:34:50+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 200, + "peak_ok_rps": 240, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 58.6, + "p50": 5.1, + "p90": 9.91, + "p99": 32.8, + "p999": 58.6 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.01, + "p99_ms": 31.39 + }, + "me": { + "ok": 40, + "p50_ms": 2.82, + "p99_ms": 46.05 + }, + "post": { + "ok": 80, + "p50_ms": 6.29, + "p99_ms": 53.02 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.56, + "p99": 32.16 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 18.9, + "lb": 2.1, + "loadgen-run-30ea8f468626": 2.6, + "postgres": 7.4, + "redis": 2.5 + }, + "mem_mib_max": { + "api": 42.3, + "lb": 4.4, + "loadgen-run-30ea8f468626": 6.1, + "postgres": 154.4, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.016, + "pg_connections_mean": 18 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 74.87, + "p50": 5.35, + "p90": 10.64, + "p99": 40.26, + "p999": 74.87 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.26, + "p99_ms": 28.64 + }, + "me": { + "ok": 80, + "p50_ms": 2.57, + "p99_ms": 13.94 + }, + "post": { + "ok": 160, + "p50_ms": 5.99, + "p99_ms": 66.31 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.7, + "p99": 30.89 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 33.3, + "lb": 3.3, + "loadgen-run-30ea8f468626": 4.9, + "postgres": 9.8, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 31.4, + "lb": 4.4, + "loadgen-run-30ea8f468626": 6.2, + "postgres": 155.2, + "redis": 5.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.028, + "pg_connections_mean": 18 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 134.6, + "p50": 4.92, + "p90": 9.82, + "p99": 34.96, + "p999": 127.43 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 4.81, + "p99_ms": 26.58 + }, + "me": { + "ok": 120, + "p50_ms": 2.27, + "p99_ms": 27.75 + }, + "post": { + "ok": 240, + "p50_ms": 5.78, + "p99_ms": 124.94 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.27, + "p99": 34.87 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 45.4, + "lb": 4.1, + "loadgen-run-30ea8f468626": 6.6, + "postgres": 12.4, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 41.6, + "lb": 4.5, + "loadgen-run-30ea8f468626": 6.6, + "postgres": 156.9, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.022, + "pg_connections_mean": 18.3 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 410.6, + "p50": 5.95, + "p90": 73.41, + "p99": 252.29, + "p999": 379.29 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 6.15, + "p99_ms": 248.75 + }, + "me": { + "ok": 160, + "p50_ms": 2.47, + "p99_ms": 79.63 + }, + "post": { + "ok": 320, + "p50_ms": 6.45, + "p99_ms": 269.86 + } + }, + "rate": 160, + "service_ms": { + "p50": 5.39, + "p99": 251.33 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 59.6, + "lb": 5.2, + "loadgen-run-30ea8f468626": 8.3, + "postgres": 15.7, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 33.8, + "lb": 4.7, + "loadgen-run-30ea8f468626": 7.0, + "postgres": 183.9, + "redis": 4.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 3.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.006, + "pg_connections_mean": 30 + }, + "slo_ok": false + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 207.88, + "p50": 7.28, + "p90": 75.29, + "p99": 168.96, + "p999": 192.93 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 8.51, + "p99_ms": 172.54 + }, + "me": { + "ok": 200, + "p50_ms": 2.23, + "p99_ms": 35.27 + }, + "post": { + "ok": 400, + "p50_ms": 6.14, + "p99_ms": 158.59 + } + }, + "rate": 200, + "service_ms": { + "p50": 6.7, + "p99": 168.22 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 72.5, + "lb": 5.5, + "loadgen-run-30ea8f468626": 9.4, + "postgres": 16.6, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 44.0, + "lb": 4.9, + "loadgen-run-30ea8f468626": 7.3, + "postgres": 192.3, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 4.6, + "pg_active_mean": 0.25, + "pg_wait_events_mean": { + "IO:WALSync": 0.12, + "LWLock:WALWrite": 0.12 + }, + "api_cfs_throttled_period_ratio": 0.028, + "pg_connections_mean": 32 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 310.05, + "p50": 10.63, + "p90": 179.93, + "p99": 264.29, + "p999": 301.29 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 11.6, + "p99_ms": 256.9 + }, + "me": { + "ok": 240, + "p50_ms": 1.97, + "p99_ms": 102.26 + }, + "post": { + "ok": 480, + "p50_ms": 11.9, + "p99_ms": 282.57 + } + }, + "rate": 240, + "service_ms": { + "p50": 10.11, + "p99": 263.44 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 81.8, + "lb": 5.5, + "loadgen-run-30ea8f468626": 10.1, + "postgres": 17.9, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 46.0, + "lb": 5.1, + "loadgen-run-30ea8f468626": 7.6, + "postgres": 192.5, + "redis": 5.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 9.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.044, + "pg_connections_mean": 32 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 553 + }, + "error_ratio": 0.1975, + "errors": 553, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 2256.81, + "p50": 973.46, + "p90": 1724.08, + "p99": 1925.48, + "p999": 2225.79 + }, + "ok": 2247, + "per_op": { + "history": { + "ok": 1560, + "p50_ms": 1190.4, + "p99_ms": 1924.57 + }, + "me": { + "ok": 280, + "p50_ms": 38.5, + "p99_ms": 201.59 + }, + "post": { + "ok": 407, + "p50_ms": 1128.39, + "p99_ms": 1935.59 + } + }, + "rate": 280, + "service_ms": { + "p50": 972.79, + "p99": 1924.33 + }, + "throughput_ok_rps": 224.7, + "resources": { + "cpu_percent_mean": { + "api": 77.9, + "lb": 7.3, + "loadgen-run-30ea8f468626": 13.9, + "postgres": 17.1, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 53.0, + "lb": 9.7, + "loadgen-run-30ea8f468626": 18.6, + "postgres": 194.3, + "redis": 5.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 207.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.31, + "pg_connections_mean": 32 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1447 + }, + "error_ratio": 0.4522, + "errors": 1447, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 2648.68, + "p50": 1462.09, + "p90": 2084.07, + "p99": 2498.29, + "p999": 2635.17 + }, + "ok": 1753, + "per_op": { + "history": { + "ok": 1151, + "p50_ms": 1550.53, + "p99_ms": 2530.79 + }, + "me": { + "ok": 272, + "p50_ms": 64, + "p99_ms": 1368.3 + }, + "post": { + "ok": 330, + "p50_ms": 1502.13, + "p99_ms": 2498.29 + } + }, + "rate": 320, + "service_ms": { + "p50": 1461.29, + "p99": 2497.51 + }, + "throughput_ok_rps": 175.3, + "resources": { + "cpu_percent_mean": { + "api": 80.8, + "lb": 7.7, + "loadgen-run-30ea8f468626": 14.0, + "postgres": 18.0, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 56.6, + "lb": 10.5, + "loadgen-run-30ea8f468626": 21.2, + "postgres": 194.5, + "redis": 4.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 259.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.196, + "pg_connections_mean": 32 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1776 + }, + "error_ratio": 0.4933, + "errors": 1776, + "event": "step", + "label": "fixed-2x0.50-r3", + "latency_ms": { + "max": 2349.03, + "p50": 1325.25, + "p90": 2067.91, + "p99": 2292.96, + "p999": 2348.53 + }, + "ok": 1824, + "per_op": { + "history": { + "ok": 1186, + "p50_ms": 1513.93, + "p99_ms": 2300.52 + }, + "me": { + "ok": 331, + "p50_ms": 62.48, + "p99_ms": 271.8 + }, + "post": { + "ok": 307, + "p50_ms": 1439.83, + "p99_ms": 2208.61 + } + }, + "rate": 360, + "service_ms": { + "p50": 1324.88, + "p99": 2292.84 + }, + "throughput_ok_rps": 182.4, + "resources": { + "cpu_percent_mean": { + "api": 82.5, + "lb": 8.5, + "loadgen-run-30ea8f468626": 15.1, + "postgres": 16.3, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 49.3, + "lb": 10.9, + "loadgen-run-30ea8f468626": 21.2, + "postgres": 194.8, + "redis": 4.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 253.8, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "Client:ClientRead": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.214, + "pg_connections_mean": 32 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-474b696e": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 166, + "requests_wait_ms": 186, + "usage_ms": 130, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-c88fcc5c": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 188, + "requests_wait_ms": 180, + "usage_ms": 57, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + } + }, + "pool_after": { + "api-474b696e": { + "pool": { + "connections_num": 10, + "requests_num": 12681, + "requests_queued": 4249, + "connections_ms": 952, + "requests_wait_ms": 4348360, + "usage_ms": 321100, + "requests_errors": 2849, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 12578, + "misses": 58, + "errors": 1 + }, + "requests": 13363 + }, + "api-c88fcc5c": { + "pool": { + "connections_num": 10, + "requests_num": 10515, + "requests_queued": 3617, + "connections_ms": 888, + "requests_wait_ms": 3055712, + "usage_ms": 267700, + "requests_errors": 1231, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 9978, + "misses": 68, + "errors": 19 + }, + "requests": 11029 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-2x0.50.json b/evidence/track1/fixed-2x0.50.json new file mode 100644 index 00000000..2437d875 --- /dev/null +++ b/evidence/track1/fixed-2x0.50.json @@ -0,0 +1,644 @@ +{ + "config": "fixed-2x0.50", + "repeat": 1, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 2, + "API_CPUS": "0.5", + "API_MEMORY": "256M", + "KDF_CONCURRENCY": 2 + }, + "measured_at": "2026-11-03T08:31:11+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 200, + "peak_ok_rps": 234.3, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50", + "latency_ms": { + "max": 74.46, + "p50": 5.6, + "p90": 11.01, + "p99": 54.84, + "p999": 74.46 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.52, + "p99_ms": 58.87 + }, + "me": { + "ok": 40, + "p50_ms": 3.04, + "p99_ms": 74.46 + }, + "post": { + "ok": 80, + "p50_ms": 6.14, + "p99_ms": 35.31 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.96, + "p99": 54.76 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 21.4, + "lb": 2.3, + "loadgen-run-c0edec5e3a16": 2.9, + "postgres": 5.2, + "redis": 2.4 + }, + "mem_mib_max": { + "api": 43.7, + "lb": 4.4, + "loadgen-run-c0edec5e3a16": 6.1, + "postgres": 64.8, + "redis": 5.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.056, + "pg_connections_mean": 8 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50", + "latency_ms": { + "max": 66.59, + "p50": 5.32, + "p90": 10.33, + "p99": 48.05, + "p999": 66.59 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.17, + "p99_ms": 48.45 + }, + "me": { + "ok": 80, + "p50_ms": 2.58, + "p99_ms": 19.69 + }, + "post": { + "ok": 160, + "p50_ms": 6.03, + "p99_ms": 50.48 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.66, + "p99": 46.2 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 31.6, + "lb": 3.4, + "loadgen-run-c0edec5e3a16": 5.2, + "postgres": 7.5, + "redis": 2.7 + }, + "mem_mib_max": { + "api": 37.6, + "lb": 4.4, + "loadgen-run-c0edec5e3a16": 6.6, + "postgres": 65.1, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.067, + "pg_connections_mean": 8 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50", + "latency_ms": { + "max": 482.53, + "p50": 5.63, + "p90": 30.45, + "p99": 234.28, + "p999": 422.39 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.62, + "p99_ms": 226.5 + }, + "me": { + "ok": 120, + "p50_ms": 2.5, + "p99_ms": 59.5 + }, + "post": { + "ok": 240, + "p50_ms": 6.36, + "p99_ms": 237.83 + } + }, + "rate": 120, + "service_ms": { + "p50": 5, + "p99": 233.75 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 50.7, + "lb": 4.5, + "loadgen-run-c0edec5e3a16": 7.1, + "postgres": 11.2, + "redis": 3.2 + }, + "mem_mib_max": { + "api": 44.6, + "lb": 4.6, + "loadgen-run-c0edec5e3a16": 6.9, + "postgres": 81.9, + "redis": 5.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.9, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.033, + "pg_connections_mean": 13.4 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50", + "latency_ms": { + "max": 207.81, + "p50": 5.44, + "p90": 22.5, + "p99": 102.12, + "p999": 186.8 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.46, + "p99_ms": 101.19 + }, + "me": { + "ok": 160, + "p50_ms": 2.43, + "p99_ms": 55.34 + }, + "post": { + "ok": 320, + "p50_ms": 6.11, + "p99_ms": 180.2 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.82, + "p99": 101.34 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 62.1, + "lb": 5.2, + "loadgen-run-c0edec5e3a16": 8.4, + "postgres": 14.4, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 37.0, + "lb": 4.7, + "loadgen-run-c0edec5e3a16": 7.0, + "postgres": 94.9, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.9, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-2x0.50", + "latency_ms": { + "max": 334.19, + "p50": 7.53, + "p90": 99.13, + "p99": 231.85, + "p999": 261.78 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 9.15, + "p99_ms": 227.68 + }, + "me": { + "ok": 200, + "p50_ms": 2.38, + "p99_ms": 88.53 + }, + "post": { + "ok": 400, + "p50_ms": 7.55, + "p99_ms": 233.77 + } + }, + "rate": 200, + "service_ms": { + "p50": 6.9, + "p99": 231.05 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 74.5, + "lb": 5.8, + "loadgen-run-c0edec5e3a16": 10.1, + "postgres": 14.8, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 35.7, + "lb": 5.0, + "loadgen-run-c0edec5e3a16": 7.6, + "postgres": 98.1, + "redis": 5.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 3.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.005, + "pg_connections_mean": 21 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 57 + }, + "error_ratio": 0.0238, + "errors": 57, + "event": "step", + "label": "fixed-2x0.50", + "latency_ms": { + "max": 2013.87, + "p50": 433.83, + "p90": 1311.32, + "p99": 1788.86, + "p999": 1846.65 + }, + "ok": 2343, + "per_op": { + "history": { + "ok": 1632, + "p50_ms": 476.98, + "p99_ms": 1809.07 + }, + "me": { + "ok": 240, + "p50_ms": 42.22, + "p99_ms": 1699.54 + }, + "post": { + "ok": 471, + "p50_ms": 470.16, + "p99_ms": 1658.32 + } + }, + "rate": 240, + "service_ms": { + "p50": 433.33, + "p99": 1787.71 + }, + "throughput_ok_rps": 234.3, + "resources": { + "cpu_percent_mean": { + "api": 80.0, + "lb": 7.1, + "loadgen-run-c0edec5e3a16": 13.5, + "postgres": 15.6, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 50.6, + "lb": 8.5, + "loadgen-run-c0edec5e3a16": 15.3, + "postgres": 98.4, + "redis": 4.5 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 115.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.189, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1168 + }, + "error_ratio": 0.4171, + "errors": 1168, + "event": "step", + "label": "fixed-2x0.50", + "latency_ms": { + "max": 2716.46, + "p50": 1237.32, + "p90": 2215.78, + "p99": 2521.89, + "p999": 2713.48 + }, + "ok": 1632, + "per_op": { + "history": { + "ok": 1136, + "p50_ms": 1383.52, + "p99_ms": 2538.44 + }, + "me": { + "ok": 222, + "p50_ms": 51.78, + "p99_ms": 1233.59 + }, + "post": { + "ok": 274, + "p50_ms": 1390.59, + "p99_ms": 2490.89 + } + }, + "rate": 280, + "service_ms": { + "p50": 1235.83, + "p99": 2521.65 + }, + "throughput_ok_rps": 163.2, + "resources": { + "cpu_percent_mean": { + "api": 82.9, + "lb": 8.1, + "loadgen-run-c0edec5e3a16": 14.7, + "postgres": 15.5, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 55.8, + "lb": 10.8, + "loadgen-run-c0edec5e3a16": 23.5, + "postgres": 98.9, + "redis": 5.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 258.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.175, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1286 + }, + "error_ratio": 0.4019, + "errors": 1286, + "event": "step", + "label": "fixed-2x0.50", + "latency_ms": { + "max": 2542.69, + "p50": 1228.48, + "p90": 1925.6, + "p99": 2445.3, + "p999": 2534.12 + }, + "ok": 1914, + "per_op": { + "history": { + "ok": 1263, + "p50_ms": 1315.98, + "p99_ms": 2268.86 + }, + "me": { + "ok": 276, + "p50_ms": 50.38, + "p99_ms": 2227.43 + }, + "post": { + "ok": 375, + "p50_ms": 1328.73, + "p99_ms": 2531.62 + } + }, + "rate": 320, + "service_ms": { + "p50": 1227.62, + "p99": 2444.99 + }, + "throughput_ok_rps": 191.4, + "resources": { + "cpu_percent_mean": { + "api": 81.9, + "lb": 7.7, + "loadgen-run-c0edec5e3a16": 14.0, + "postgres": 14.2, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 51.5, + "lb": 10.8, + "loadgen-run-c0edec5e3a16": 22.1, + "postgres": 100.5, + "redis": 4.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 229.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.228, + "pg_connections_mean": 21 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-2c6df209": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 137, + "requests_wait_ms": 133, + "usage_ms": 76, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-d9869430": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 204, + "requests_wait_ms": 204, + "usage_ms": 113, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + } + }, + "pool_after": { + "api-2c6df209": { + "pool": { + "connections_num": 10, + "requests_num": 10526, + "requests_queued": 3822, + "connections_ms": 991, + "requests_wait_ms": 3557844, + "usage_ms": 282106, + "requests_errors": 2082, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 8994, + "misses": 23, + "errors": 46 + }, + "requests": 10702 + }, + "api-d9869430": { + "pool": { + "connections_num": 10, + "requests_num": 8580, + "requests_queued": 3170, + "connections_ms": 1170, + "requests_wait_ms": 2813537, + "usage_ms": 252813, + "requests_errors": 644, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 8349, + "misses": 26, + "errors": 10 + }, + "requests": 8982 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-4x0.25-r2.json b/evidence/track1/fixed-4x0.25-r2.json new file mode 100644 index 00000000..853ca535 --- /dev/null +++ b/evidence/track1/fixed-4x0.25-r2.json @@ -0,0 +1,816 @@ +{ + "config": "fixed-4x0.25-r2", + "repeat": 2, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 4, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "KDF_CONCURRENCY": 1 + }, + "measured_at": "2026-11-03T09:24:56+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 160, + "peak_ok_rps": 240.6, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 72.83, + "p50": 5.6, + "p90": 18.79, + "p99": 60.61, + "p999": 72.83 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.51, + "p99_ms": 60.38 + }, + "me": { + "ok": 40, + "p50_ms": 2.27, + "p99_ms": 54.64 + }, + "post": { + "ok": 80, + "p50_ms": 6.29, + "p99_ms": 72.83 + } + }, + "rate": 40, + "service_ms": { + "p50": 5.02, + "p99": 60.19 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 27.1, + "lb": 2.3, + "loadgen-run-36ba2530fef5": 2.6, + "postgres": 7.3, + "redis": 2.1 + }, + "mem_mib_max": { + "api": 43.6, + "lb": 4.4, + "loadgen-run-36ba2530fef5": 6.1, + "postgres": 146.4, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0.44, + "pg_wait_events_mean": { + "CPU": 0.44 + }, + "api_cfs_throttled_period_ratio": 0.133, + "pg_connections_mean": 21 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 91.42, + "p50": 5.15, + "p90": 15.24, + "p99": 81.45, + "p999": 91.42 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.1, + "p99_ms": 81.93 + }, + "me": { + "ok": 80, + "p50_ms": 2.48, + "p99_ms": 34.89 + }, + "post": { + "ok": 160, + "p50_ms": 6, + "p99_ms": 81.45 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.53, + "p99": 80.46 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 37.3, + "lb": 3.2, + "loadgen-run-36ba2530fef5": 4.6, + "postgres": 9.1, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 43.6, + "lb": 4.4, + "loadgen-run-36ba2530fef5": 6.3, + "postgres": 147.1, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.1, + "pg_connections_mean": 21 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 794.96, + "p50": 6.02, + "p90": 72, + "p99": 482.97, + "p999": 559.54 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.86, + "p99_ms": 482.97 + }, + "me": { + "ok": 120, + "p50_ms": 2.58, + "p99_ms": 71.7 + }, + "post": { + "ok": 240, + "p50_ms": 8.2, + "p99_ms": 511.91 + } + }, + "rate": 120, + "service_ms": { + "p50": 5.37, + "p99": 459.72 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 54.9, + "lb": 4.1, + "loadgen-run-36ba2530fef5": 6.4, + "postgres": 13.1, + "redis": 3.2 + }, + "mem_mib_max": { + "api": 44.5, + "lb": 4.9, + "loadgen-run-36ba2530fef5": 7.3, + "postgres": 184.2, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 2.8, + "pg_active_mean": 0.33, + "pg_wait_events_mean": { + "CPU": 0.33 + }, + "api_cfs_throttled_period_ratio": 0.137, + "pg_connections_mean": 27.8 + }, + "slo_ok": false + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 176.91, + "p50": 6, + "p90": 48.44, + "p99": 135.32, + "p999": 173.31 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 6.07, + "p99_ms": 129.04 + }, + "me": { + "ok": 160, + "p50_ms": 2.42, + "p99_ms": 72.19 + }, + "post": { + "ok": 320, + "p50_ms": 7.61, + "p99_ms": 168.93 + } + }, + "rate": 160, + "service_ms": { + "p50": 5.45, + "p99": 134.42 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 67.0, + "lb": 4.8, + "loadgen-run-36ba2530fef5": 7.7, + "postgres": 14.0, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 44.9, + "lb": 4.9, + "loadgen-run-36ba2530fef5": 7.3, + "postgres": 186.5, + "redis": 5.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 3, + "pg_active_mean": 0.44, + "pg_wait_events_mean": { + "CPU": 0.44 + }, + "api_cfs_throttled_period_ratio": 0.177, + "pg_connections_mean": 37 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 1864.69, + "p50": 129.18, + "p90": 1046.29, + "p99": 1634.55, + "p999": 1752.71 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 152.99, + "p99_ms": 1634.55 + }, + "me": { + "ok": 200, + "p50_ms": 37.67, + "p99_ms": 1071 + }, + "post": { + "ok": 400, + "p50_ms": 212.9, + "p99_ms": 1637.95 + } + }, + "rate": 200, + "service_ms": { + "p50": 128.83, + "p99": 1633.77 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 78.8, + "lb": 5.7, + "loadgen-run-36ba2530fef5": 10.2, + "postgres": 17.8, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 47.5, + "lb": 7.0, + "loadgen-run-36ba2530fef5": 14.0, + "postgres": 218.9, + "redis": 4.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 64.7, + "pg_active_mean": 0.56, + "pg_wait_events_mean": { + "CPU": 0.44, + "Client:ClientRead": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.336, + "pg_connections_mean": 47.8 + }, + "slo_ok": false + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 93 + }, + "error_ratio": 0.0388, + "errors": 93, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 2432.27, + "p50": 665.72, + "p90": 1642.63, + "p99": 2389.58, + "p999": 2418.04 + }, + "ok": 2307, + "per_op": { + "history": { + "ok": 1606, + "p50_ms": 735.45, + "p99_ms": 2386.92 + }, + "me": { + "ok": 240, + "p50_ms": 85.07, + "p99_ms": 978.1 + }, + "post": { + "ok": 461, + "p50_ms": 788.2, + "p99_ms": 2403.19 + } + }, + "rate": 240, + "service_ms": { + "p50": 665.45, + "p99": 2389.27 + }, + "throughput_ok_rps": 230.7, + "resources": { + "cpu_percent_mean": { + "api": 84.0, + "lb": 6.2, + "loadgen-run-36ba2530fef5": 11.6, + "postgres": 18.2, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 50.7, + "lb": 9.3, + "loadgen-run-36ba2530fef5": 19.0, + "postgres": 220.5, + "redis": 5.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 176.5, + "pg_active_mean": 0.7, + "pg_wait_events_mean": { + "CPU": 0.5, + "IO:WALSync": 0.1, + "LWLock:WALWrite": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.609, + "pg_connections_mean": 49 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 394 + }, + "error_ratio": 0.1407, + "errors": 394, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 3147.58, + "p50": 1144.24, + "p90": 2259.16, + "p99": 3033.35, + "p999": 3142.23 + }, + "ok": 2406, + "per_op": { + "history": { + "ok": 1691, + "p50_ms": 1299.66, + "p99_ms": 2969.38 + }, + "me": { + "ok": 258, + "p50_ms": 98.8, + "p99_ms": 1279.44 + }, + "post": { + "ok": 457, + "p50_ms": 1247.21, + "p99_ms": 3114.75 + } + }, + "rate": 280, + "service_ms": { + "p50": 1143.61, + "p99": 3032.74 + }, + "throughput_ok_rps": 240.6, + "resources": { + "cpu_percent_mean": { + "api": 87.4, + "lb": 6.6, + "loadgen-run-36ba2530fef5": 12.8, + "postgres": 17.5, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 52.6, + "lb": 11.9, + "loadgen-run-36ba2530fef5": 26.6, + "postgres": 221.1, + "redis": 4.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 302, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.525, + "pg_connections_mean": 49 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1333 + }, + "error_ratio": 0.4166, + "errors": 1333, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 3445.8, + "p50": 1646.59, + "p90": 2594.25, + "p99": 3379.85, + "p999": 3411.29 + }, + "ok": 1867, + "per_op": { + "history": { + "ok": 1284, + "p50_ms": 1767.93, + "p99_ms": 3346.81 + }, + "me": { + "ok": 230, + "p50_ms": 158.03, + "p99_ms": 2152.93 + }, + "post": { + "ok": 353, + "p50_ms": 1858.8, + "p99_ms": 3405.89 + } + }, + "rate": 320, + "service_ms": { + "p50": 1646.43, + "p99": 3379.49 + }, + "throughput_ok_rps": 186.7, + "resources": { + "cpu_percent_mean": { + "api": 88.0, + "lb": 7.6, + "loadgen-run-36ba2530fef5": 13.5, + "postgres": 16.4, + "redis": 3.2 + }, + "mem_mib_max": { + "api": 53.8, + "lb": 13.2, + "loadgen-run-36ba2530fef5": 31.9, + "postgres": 222.0, + "redis": 5.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 441.3, + "pg_active_mean": 0.7, + "pg_wait_events_mean": { + "CPU": 0.3, + "Client:ClientRead": 0.4 + }, + "api_cfs_throttled_period_ratio": 0.424, + "pg_connections_mean": 49 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1580 + }, + "error_ratio": 0.4389, + "errors": 1580, + "event": "step", + "label": "fixed-4x0.25-r2", + "latency_ms": { + "max": 2770.47, + "p50": 1496, + "p90": 2242.21, + "p99": 2507.19, + "p999": 2700.51 + }, + "ok": 2020, + "per_op": { + "history": { + "ok": 1369, + "p50_ms": 1699.72, + "p99_ms": 2460.58 + }, + "me": { + "ok": 299, + "p50_ms": 121.51, + "p99_ms": 343.24 + }, + "post": { + "ok": 352, + "p50_ms": 1567.6, + "p99_ms": 2692.62 + } + }, + "rate": 360, + "service_ms": { + "p50": 1495.47, + "p99": 2506.19 + }, + "throughput_ok_rps": 202, + "resources": { + "cpu_percent_mean": { + "api": 85.3, + "lb": 8.3, + "loadgen-run-36ba2530fef5": 13.8, + "postgres": 15.7, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 56.5, + "lb": 13.4, + "loadgen-run-36ba2530fef5": 36.4, + "postgres": 222.2, + "redis": 5.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 415.3, + "pg_active_mean": 0.5, + "pg_wait_events_mean": { + "CPU": 0.3, + "Client:ClientRead": 0.1, + "IO:WALSync": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.585, + "pg_connections_mean": 48.1 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-1d61817b": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 162, + "requests_wait_ms": 193, + "usage_ms": 67, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 2, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 8 + }, + "api-a25272a7": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 240, + "requests_wait_ms": 263, + "usage_ms": 183, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-d4ce142e": { + "pool": { + "connections_num": 3, + "requests_num": 22, + "requests_queued": 3, + "connections_ms": 213, + "requests_wait_ms": 147, + "usage_ms": 193, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + }, + "api-fba3a55f": { + "pool": { + "connections_num": 3, + "requests_num": 26, + "requests_queued": 4, + "connections_ms": 294, + "requests_wait_ms": 225, + "usage_ms": 185, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 13 + } + }, + "pool_after": { + "api-1d61817b": { + "pool": { + "connections_num": 10, + "requests_num": 6436, + "requests_queued": 2845, + "connections_ms": 1381, + "requests_wait_ms": 3326559, + "usage_ms": 343623, + "requests_errors": 1715, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 5268, + "misses": 12, + "errors": 23 + }, + "requests": 6630 + }, + "api-a25272a7": { + "pool": { + "connections_num": 10, + "requests_num": 6380, + "requests_queued": 3327, + "connections_ms": 1579, + "requests_wait_ms": 3887774, + "usage_ms": 376655, + "requests_errors": 943, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 5519, + "misses": 13, + "errors": 21 + }, + "requests": 6337 + }, + "api-d4ce142e": { + "pool": { + "connections_num": 10, + "requests_num": 5885, + "requests_queued": 3013, + "connections_ms": 1348, + "requests_wait_ms": 3443239, + "usage_ms": 363291, + "requests_errors": 481, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 9, + "requests_waiting": 0 + }, + "cache": { + "hits": 5572, + "misses": 14, + "errors": 9 + }, + "requests": 6030 + }, + "api-fba3a55f": { + "pool": { + "connections_num": 10, + "requests_num": 5848, + "requests_queued": 3089, + "connections_ms": 1764, + "requests_wait_ms": 3174240, + "usage_ms": 357547, + "requests_errors": 454, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 9, + "requests_waiting": 0 + }, + "cache": { + "hits": 4883, + "misses": 13, + "errors": 30 + }, + "requests": 5812 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-4x0.25-r3.json b/evidence/track1/fixed-4x0.25-r3.json new file mode 100644 index 00000000..d6b9704f --- /dev/null +++ b/evidence/track1/fixed-4x0.25-r3.json @@ -0,0 +1,734 @@ +{ + "config": "fixed-4x0.25-r3", + "repeat": 3, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 4, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "KDF_CONCURRENCY": 1 + }, + "measured_at": "2026-11-03T09:37:46+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 120, + "peak_ok_rps": 200, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r3", + "latency_ms": { + "max": 88.29, + "p50": 5.38, + "p90": 13.59, + "p99": 82.74, + "p999": 88.29 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.33, + "p99_ms": 82.74 + }, + "me": { + "ok": 40, + "p50_ms": 2.42, + "p99_ms": 11.79 + }, + "post": { + "ok": 80, + "p50_ms": 5.93, + "p99_ms": 85.85 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.73, + "p99": 82.16 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 25.6, + "lb": 2.1, + "loadgen-run-89815b9f6838": 2.5, + "postgres": 7.7, + "redis": 2.4 + }, + "mem_mib_max": { + "api": 40.4, + "lb": 4.4, + "loadgen-run-89815b9f6838": 6.1, + "postgres": 175.2, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.072, + "pg_connections_mean": 25 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r3", + "latency_ms": { + "max": 92.03, + "p50": 5.28, + "p90": 22.78, + "p99": 61.16, + "p999": 92.03 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.18, + "p99_ms": 61.94 + }, + "me": { + "ok": 80, + "p50_ms": 2.45, + "p99_ms": 54.32 + }, + "post": { + "ok": 160, + "p50_ms": 6.25, + "p99_ms": 71.44 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.65, + "p99": 60.42 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 34.9, + "lb": 3.2, + "loadgen-run-89815b9f6838": 4.7, + "postgres": 9.9, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 43.7, + "lb": 4.4, + "loadgen-run-89815b9f6838": 6.3, + "postgres": 175.7, + "redis": 5.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.9, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.094, + "pg_connections_mean": 25 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r3", + "latency_ms": { + "max": 134.47, + "p50": 5.47, + "p90": 37.31, + "p99": 86.52, + "p999": 119.12 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.33, + "p99_ms": 87.77 + }, + "me": { + "ok": 120, + "p50_ms": 2.73, + "p99_ms": 52.33 + }, + "post": { + "ok": 240, + "p50_ms": 6.41, + "p99_ms": 95.71 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.86, + "p99": 85.73 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 49.3, + "lb": 4.0, + "loadgen-run-89815b9f6838": 6.2, + "postgres": 12.2, + "redis": 3.1 + }, + "mem_mib_max": { + "api": 44.1, + "lb": 4.5, + "loadgen-run-89815b9f6838": 6.6, + "postgres": 178.8, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.8, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.086, + "pg_connections_mean": 25.8 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r3", + "latency_ms": { + "max": 664.67, + "p50": 7.67, + "p90": 234.63, + "p99": 471.46, + "p999": 654.99 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 8.22, + "p99_ms": 481.87 + }, + "me": { + "ok": 160, + "p50_ms": 2.73, + "p99_ms": 154.45 + }, + "post": { + "ok": 320, + "p50_ms": 8.26, + "p99_ms": 470.13 + } + }, + "rate": 160, + "service_ms": { + "p50": 7.1, + "p99": 471.35 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 67.1, + "lb": 4.7, + "loadgen-run-89815b9f6838": 7.8, + "postgres": 17.6, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 44.9, + "lb": 5.1, + "loadgen-run-89815b9f6838": 7.7, + "postgres": 239.6, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 11.5, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.044, + "pg_connections_mean": 42.7 + }, + "slo_ok": false + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25-r3", + "latency_ms": { + "max": 2946.08, + "p50": 426.45, + "p90": 1381.86, + "p99": 1855.88, + "p999": 2676.84 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 469.38, + "p99_ms": 1789.43 + }, + "me": { + "ok": 200, + "p50_ms": 72.89, + "p99_ms": 448.11 + }, + "post": { + "ok": 400, + "p50_ms": 531.44, + "p99_ms": 2000.46 + } + }, + "rate": 200, + "service_ms": { + "p50": 425.95, + "p99": 1855.08 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 74.6, + "lb": 5.6, + "loadgen-run-89815b9f6838": 10.3, + "postgres": 17.5, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 50.4, + "lb": 8.2, + "loadgen-run-89815b9f6838": 15.2, + "postgres": 250.8, + "redis": 4.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 103.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.404, + "pg_connections_mean": 52.2 + }, + "slo_ok": false + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 627 + }, + "error_ratio": 0.2613, + "errors": 627, + "event": "step", + "label": "fixed-4x0.25-r3", + "latency_ms": { + "max": 3109.98, + "p50": 614.79, + "p90": 2033.95, + "p99": 2760.99, + "p999": 3106.01 + }, + "ok": 1773, + "per_op": { + "history": { + "ok": 1223, + "p50_ms": 670.52, + "p99_ms": 2652.06 + }, + "me": { + "ok": 195, + "p50_ms": 88.16, + "p99_ms": 2495.23 + }, + "post": { + "ok": 355, + "p50_ms": 870.35, + "p99_ms": 2933.06 + } + }, + "rate": 240, + "service_ms": { + "p50": 610.97, + "p99": 2760.1 + }, + "throughput_ok_rps": 177.3, + "resources": { + "cpu_percent_mean": { + "api": 78.9, + "lb": 6.4, + "loadgen-run-89815b9f6838": 11.9, + "postgres": 16.7, + "redis": 3.2 + }, + "mem_mib_max": { + "api": 53.9, + "lb": 9.9, + "loadgen-run-89815b9f6838": 21.0, + "postgres": 250.4, + "redis": 5.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 209.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.384, + "pg_connections_mean": 53 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1102 + }, + "error_ratio": 0.3936, + "errors": 1102, + "event": "step", + "label": "fixed-4x0.25-r3", + "latency_ms": { + "max": 4334.56, + "p50": 1383.24, + "p90": 3095.47, + "p99": 3639.35, + "p999": 4302.63 + }, + "ok": 1698, + "per_op": { + "history": { + "ok": 1112, + "p50_ms": 1465.17, + "p99_ms": 3611.04 + }, + "me": { + "ok": 250, + "p50_ms": 120.89, + "p99_ms": 1839.54 + }, + "post": { + "ok": 336, + "p50_ms": 1480.37, + "p99_ms": 3671.15 + } + }, + "rate": 280, + "service_ms": { + "p50": 1382.39, + "p99": 3623.89 + }, + "throughput_ok_rps": 169.8, + "resources": { + "cpu_percent_mean": { + "api": 83.5, + "lb": 7.0, + "loadgen-run-89815b9f6838": 13.0, + "postgres": 17.5, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 53.7, + "lb": 12.2, + "loadgen-run-89815b9f6838": 27.2, + "postgres": 251.7, + "redis": 4.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 349.4, + "pg_active_mean": 0.5, + "pg_wait_events_mean": { + "Client:ClientRead": 0.5 + }, + "api_cfs_throttled_period_ratio": 0.334, + "pg_connections_mean": 53 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1469 + }, + "error_ratio": 0.4591, + "errors": 1469, + "event": "step", + "label": "fixed-4x0.25-r3", + "latency_ms": { + "max": 4102.38, + "p50": 1496.75, + "p90": 3242.68, + "p99": 3974.51, + "p999": 4099.38 + }, + "ok": 1731, + "per_op": { + "history": { + "ok": 1145, + "p50_ms": 1645.44, + "p99_ms": 3926.88 + }, + "me": { + "ok": 255, + "p50_ms": 115.33, + "p99_ms": 2338.79 + }, + "post": { + "ok": 331, + "p50_ms": 1609.47, + "p99_ms": 3998.78 + } + }, + "rate": 320, + "service_ms": { + "p50": 1496.47, + "p99": 3973.21 + }, + "throughput_ok_rps": 173.1, + "resources": { + "cpu_percent_mean": { + "api": 78.7, + "lb": 8.0, + "loadgen-run-89815b9f6838": 13.2, + "postgres": 16.7, + "redis": 3.0 + }, + "mem_mib_max": { + "api": 58.6, + "lb": 13.2, + "loadgen-run-89815b9f6838": 30.5, + "postgres": 252.1, + "redis": 6.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 413.7, + "pg_active_mean": 0.3, + "pg_wait_events_mean": { + "Client:ClientRead": 0.3 + }, + "api_cfs_throttled_period_ratio": 0.375, + "pg_connections_mean": 53 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-cbf60d4e": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 212, + "requests_wait_ms": 203, + "usage_ms": 114, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 2, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-aa2feb60": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 263, + "requests_wait_ms": 285, + "usage_ms": 122, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 2, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-75fe1536": { + "pool": { + "connections_num": 3, + "requests_num": 22, + "requests_queued": 3, + "connections_ms": 218, + "requests_wait_ms": 290, + "usage_ms": 59, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + }, + "api-95ddae14": { + "pool": { + "connections_num": 3, + "requests_num": 26, + "requests_queued": 3, + "connections_ms": 221, + "requests_wait_ms": 280, + "usage_ms": 90, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 13 + } + }, + "pool_after": { + "api-cbf60d4e": { + "pool": { + "connections_num": 10, + "requests_num": 5685, + "requests_queued": 2640, + "connections_ms": 1849, + "requests_wait_ms": 3373255, + "usage_ms": 293816, + "requests_errors": 1303, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 3738, + "misses": 16, + "errors": 17 + }, + "requests": 5343 + }, + "api-aa2feb60": { + "pool": { + "connections_num": 10, + "requests_num": 4088, + "requests_queued": 1775, + "connections_ms": 1616, + "requests_wait_ms": 1525236, + "usage_ms": 262582, + "requests_errors": 98, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 3933, + "misses": 22, + "errors": 2 + }, + "requests": 4204 + }, + "api-75fe1536": { + "pool": { + "connections_num": 10, + "requests_num": 4030, + "requests_queued": 1772, + "connections_ms": 1588, + "requests_wait_ms": 1549683, + "usage_ms": 263095, + "requests_errors": 128, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 3969, + "misses": 23, + "errors": 0 + }, + "requests": 4203 + }, + "api-95ddae14": { + "pool": { + "connections_num": 10, + "requests_num": 6393, + "requests_queued": 3048, + "connections_ms": 1829, + "requests_wait_ms": 3953309, + "usage_ms": 339101, + "requests_errors": 2079, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 4746, + "misses": 22, + "errors": 47 + }, + "requests": 6327 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-4x0.25.json b/evidence/track1/fixed-4x0.25.json new file mode 100644 index 00000000..a84af3cd --- /dev/null +++ b/evidence/track1/fixed-4x0.25.json @@ -0,0 +1,666 @@ +{ + "config": "fixed-4x0.25", + "repeat": 1, + "series": "fixed-cpu", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 4, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "KDF_CONCURRENCY": 1 + }, + "measured_at": "2026-11-03T08:33:59+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 160, + "peak_ok_rps": 194.4, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25", + "latency_ms": { + "max": 81.32, + "p50": 5.54, + "p90": 19.21, + "p99": 46.5, + "p999": 81.32 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.42, + "p99_ms": 46.5 + }, + "me": { + "ok": 40, + "p50_ms": 2.74, + "p99_ms": 37.64 + }, + "post": { + "ok": 80, + "p50_ms": 6.31, + "p99_ms": 81.32 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.91, + "p99": 46.2 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 27.9, + "lb": 2.2, + "loadgen-run-12d799a1eccf": 2.7, + "postgres": 5.4, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 41.9, + "lb": 4.4, + "loadgen-run-12d799a1eccf": 6.1, + "postgres": 86.1, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.147, + "pg_connections_mean": 15 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25", + "latency_ms": { + "max": 96.74, + "p50": 5.83, + "p90": 34.28, + "p99": 66.27, + "p999": 96.74 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.68, + "p99_ms": 72 + }, + "me": { + "ok": 80, + "p50_ms": 2.76, + "p99_ms": 33.16 + }, + "post": { + "ok": 160, + "p50_ms": 7.01, + "p99_ms": 73.64 + } + }, + "rate": 80, + "service_ms": { + "p50": 5.2, + "p99": 66.15 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 45.0, + "lb": 3.4, + "loadgen-run-12d799a1eccf": 5.2, + "postgres": 8.2, + "redis": 2.9 + }, + "mem_mib_max": { + "api": 43.9, + "lb": 4.4, + "loadgen-run-12d799a1eccf": 6.4, + "postgres": 86.5, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.9, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.047, + "pg_connections_mean": 15 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25", + "latency_ms": { + "max": 94.79, + "p50": 5.44, + "p90": 19.88, + "p99": 61.91, + "p999": 89.89 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.37, + "p99_ms": 53.57 + }, + "me": { + "ok": 120, + "p50_ms": 2.47, + "p99_ms": 29.03 + }, + "post": { + "ok": 240, + "p50_ms": 6.46, + "p99_ms": 88.89 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.83, + "p99": 60.94 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 52.8, + "lb": 4.5, + "loadgen-run-12d799a1eccf": 6.9, + "postgres": 10.1, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 44.2, + "lb": 4.5, + "loadgen-run-12d799a1eccf": 6.9, + "postgres": 87.2, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.6, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.08, + "pg_connections_mean": 15 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "fixed-4x0.25", + "latency_ms": { + "max": 230.21, + "p50": 7.32, + "p90": 66.78, + "p99": 183.46, + "p999": 226.59 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 7.43, + "p99_ms": 182.27 + }, + "me": { + "ok": 160, + "p50_ms": 2.71, + "p99_ms": 61.09 + }, + "post": { + "ok": 320, + "p50_ms": 7.79, + "p99_ms": 191.99 + } + }, + "rate": 160, + "service_ms": { + "p50": 6.7, + "p99": 181.31 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 69.2, + "lb": 5.4, + "loadgen-run-12d799a1eccf": 8.6, + "postgres": 17.0, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 40.7, + "lb": 5.0, + "loadgen-run-12d799a1eccf": 7.3, + "postgres": 140.7, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 2.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.014, + "pg_connections_mean": 38 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 222 + }, + "error_ratio": 0.111, + "errors": 222, + "event": "step", + "label": "fixed-4x0.25", + "latency_ms": { + "max": 3740.56, + "p50": 77.46, + "p90": 2713.32, + "p99": 3646.61, + "p999": 3739.67 + }, + "ok": 1778, + "per_op": { + "history": { + "ok": 1255, + "p50_ms": 81.51, + "p99_ms": 3673.79 + }, + "me": { + "ok": 171, + "p50_ms": 33.14, + "p99_ms": 1846.93 + }, + "post": { + "ok": 352, + "p50_ms": 104.81, + "p99_ms": 3504.62 + } + }, + "rate": 200, + "service_ms": { + "p50": 76.72, + "p99": 3646 + }, + "throughput_ok_rps": 177.8, + "resources": { + "cpu_percent_mean": { + "api": 74.2, + "lb": 6.6, + "loadgen-run-12d799a1eccf": 11.4, + "postgres": 16.2, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 46.6, + "lb": 8.0, + "loadgen-run-12d799a1eccf": 16.0, + "postgres": 152.5, + "redis": 4.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 100.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.244, + "pg_connections_mean": 42 + }, + "slo_ok": false + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 797 + }, + "error_ratio": 0.3321, + "errors": 797, + "event": "step", + "label": "fixed-4x0.25", + "latency_ms": { + "max": 3706.47, + "p50": 574.82, + "p90": 2602.1, + "p99": 3632.31, + "p999": 3691.57 + }, + "ok": 1603, + "per_op": { + "history": { + "ok": 1093, + "p50_ms": 622.32, + "p99_ms": 3645.45 + }, + "me": { + "ok": 177, + "p50_ms": 89.5, + "p99_ms": 2342.5 + }, + "post": { + "ok": 333, + "p50_ms": 634.35, + "p99_ms": 3584.81 + } + }, + "rate": 240, + "service_ms": { + "p50": 573.6, + "p99": 3631.91 + }, + "throughput_ok_rps": 160.3, + "resources": { + "cpu_percent_mean": { + "api": 79.7, + "lb": 7.6, + "loadgen-run-12d799a1eccf": 14.0, + "postgres": 16.9, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 53.3, + "lb": 10.4, + "loadgen-run-12d799a1eccf": 19.0, + "postgres": 153.7, + "redis": 4.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 201.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.264, + "pg_connections_mean": 42 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 856 + }, + "error_ratio": 0.3057, + "errors": 856, + "event": "step", + "label": "fixed-4x0.25", + "latency_ms": { + "max": 4265.81, + "p50": 1659.46, + "p90": 3147.39, + "p99": 3983.01, + "p999": 4258.79 + }, + "ok": 1944, + "per_op": { + "history": { + "ok": 1347, + "p50_ms": 1801.24, + "p99_ms": 4114.07 + }, + "me": { + "ok": 226, + "p50_ms": 173.87, + "p99_ms": 2076.44 + }, + "post": { + "ok": 371, + "p50_ms": 1726.89, + "p99_ms": 3875.73 + } + }, + "rate": 280, + "service_ms": { + "p50": 1658.8, + "p99": 3982.24 + }, + "throughput_ok_rps": 194.4, + "resources": { + "cpu_percent_mean": { + "api": 86.2, + "lb": 8.6, + "loadgen-run-12d799a1eccf": 15.2, + "postgres": 18.2, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 54.4, + "lb": 13.7, + "loadgen-run-12d799a1eccf": 29.6, + "postgres": 154.7, + "redis": 5.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 367.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.358, + "pg_connections_mean": 42 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-9fb88c7c": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 267, + "requests_wait_ms": 264, + "usage_ms": 112, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-ed246c7a": { + "pool": { + "connections_num": 3, + "requests_num": 23, + "requests_queued": 3, + "connections_ms": 230, + "requests_wait_ms": 278, + "usage_ms": 87, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 11 + }, + "api-d4560f02": { + "pool": { + "connections_num": 3, + "requests_num": 25, + "requests_queued": 3, + "connections_ms": 215, + "requests_wait_ms": 144, + "usage_ms": 196, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 12 + }, + "api-06619766": { + "pool": { + "connections_num": 3, + "requests_num": 28, + "requests_queued": 3, + "connections_ms": 301, + "requests_wait_ms": 313, + "usage_ms": 229, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 14 + } + }, + "pool_after": { + "api-9fb88c7c": { + "pool": { + "connections_num": 10, + "requests_num": 3407, + "requests_queued": 1188, + "connections_ms": 1921, + "requests_wait_ms": 844090, + "usage_ms": 188476, + "requests_errors": 139, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 2807, + "misses": 10, + "errors": 17 + }, + "requests": 3298 + }, + "api-ed246c7a": { + "pool": { + "connections_num": 10, + "requests_num": 3754, + "requests_queued": 1329, + "connections_ms": 1879, + "requests_wait_ms": 1059850, + "usage_ms": 208745, + "requests_errors": 232, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 3128, + "misses": 12, + "errors": 34 + }, + "requests": 3700 + }, + "api-d4560f02": { + "pool": { + "connections_num": 10, + "requests_num": 4050, + "requests_queued": 1639, + "connections_ms": 1709, + "requests_wait_ms": 1645005, + "usage_ms": 253579, + "requests_errors": 327, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 3828, + "misses": 12, + "errors": 5 + }, + "requests": 4179 + }, + "api-06619766": { + "pool": { + "connections_num": 10, + "requests_num": 5175, + "requests_queued": 2168, + "connections_ms": 1886, + "requests_wait_ms": 2863319, + "usage_ms": 250661, + "requests_errors": 1187, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 2883, + "misses": 15, + "errors": 20 + }, + "requests": 4713 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/fixed-cpu-goodput.svg b/evidence/track1/fixed-cpu-goodput.svg new file mode 100644 index 00000000..483b5970 --- /dev/null +++ b/evidence/track1/fixed-cpu-goodput.svg @@ -0,0 +1,61 @@ + + +Фиксированный бюджет API (1 CPU, 512 MiB): дробление на реплики +Полезная пропускная способность (2xx), медиана 3 прогонов; open-loop, смесь 70/20/10 + +0 + +100 + +200 + +300 +0 +100 +200 +300 +400 + +Поданная нагрузка, req/s +Полезная, rps + +пунктир: полезная = поданная + +1 × 1.0 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 28 мс (медиана 3 прогонов) +1 × 1.0 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 27 мс (медиана 3 прогонов) +1 × 1.0 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов) +1 × 1.0 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 56 мс (медиана 3 прогонов) +1 × 1.0 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 106 мс (медиана 3 прогонов) +1 × 1.0 CPU — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 218 мс (медиана 3 прогонов) +1 × 1.0 CPU — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 322 мс (медиана 3 прогонов) +1 × 1.0 CPU — 320 req/s: полезная 276 rps, ошибок 13.8 %, p99 1518 мс (медиана 3 прогонов) +1 × 1.0 CPU — 360 req/s: полезная 277 rps, ошибок 22.9 %, p99 1671 мс (медиана 3 прогонов) +1 × 1.0 CPU — 400 req/s: полезная 253 rps, ошибок 36.8 %, p99 1340 мс (медиана 3 прогонов) +1 × 1.0 CPU — 460 req/s: полезная 223 rps, ошибок 51.6 %, p99 1655 мс (медиана 3 прогонов) + +2 × 0.5 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов) +2 × 0.5 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 40 мс (медиана 3 прогонов) +2 × 0.5 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 112 мс (медиана 3 прогонов) +2 × 0.5 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 252 мс (медиана 3 прогонов) +2 × 0.5 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 232 мс (медиана 3 прогонов) +2 × 0.5 CPU — 240 req/s: полезная 235 rps, ошибок 2.2 %, p99 1393 мс (медиана 3 прогонов) +2 × 0.5 CPU — 280 req/s: полезная 223 rps, ошибок 20.4 %, p99 1925 мс (медиана 3 прогонов) +2 × 0.5 CPU — 320 req/s: полезная 191 rps, ошибок 40.2 %, p99 2464 мс (медиана 3 прогонов) +2 × 0.5 CPU — 360 req/s: полезная 183 rps, ошибок 49.3 %, p99 2468 мс (медиана 2 прогонов) + +4 × 0.25 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 61 мс (медиана 3 прогонов) +4 × 0.25 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 66 мс (медиана 3 прогонов) +4 × 0.25 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 87 мс (медиана 3 прогонов) +4 × 0.25 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 183 мс (медиана 3 прогонов) +4 × 0.25 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 1856 мс (медиана 3 прогонов) +4 × 0.25 CPU — 240 req/s: полезная 177 rps, ошибок 26.1 %, p99 2761 мс (медиана 3 прогонов) +4 × 0.25 CPU — 280 req/s: полезная 194 rps, ошибок 30.6 %, p99 3639 мс (медиана 3 прогонов) +4 × 0.25 CPU — 320 req/s: полезная 180 rps, ошибок 43.8 %, p99 3677 мс (медиана 2 прогонов) +4 × 0.25 CPU — 360 req/s: полезная 202 rps, ошибок 43.9 %, p99 2507 мс (медиана 1 прогонов) +1 × 1.0 CPU +1 × 1.0 CPU +2 × 0.5 CPU +2 × 0.5 CPU +4 × 0.25 CPU +4 × 0.25 CPU + diff --git a/evidence/track1/fixed-cpu-p99.svg b/evidence/track1/fixed-cpu-p99.svg new file mode 100644 index 00000000..4a7e825e --- /dev/null +++ b/evidence/track1/fixed-cpu-p99.svg @@ -0,0 +1,65 @@ + + +Фиксированный бюджет API: хвост задержки +p99 успешных ответов от запланированного момента, медиана 3 прогонов + +0 + +500 + +1000 + +1500 + +2000 + +2500 +0 +100 +200 +300 +400 + +Поданная нагрузка, req/s +p99, мс (обрезано на 2500) + +SLO 250 мс + +1 × 1.0 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 28 мс (медиана 3 прогонов) +1 × 1.0 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 27 мс (медиана 3 прогонов) +1 × 1.0 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов) +1 × 1.0 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 56 мс (медиана 3 прогонов) +1 × 1.0 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 106 мс (медиана 3 прогонов) +1 × 1.0 CPU — 240 req/s: полезная 240 rps, ошибок 0.0 %, p99 218 мс (медиана 3 прогонов) +1 × 1.0 CPU — 280 req/s: полезная 280 rps, ошибок 0.0 %, p99 322 мс (медиана 3 прогонов) +1 × 1.0 CPU — 320 req/s: полезная 276 rps, ошибок 13.8 %, p99 1518 мс (медиана 3 прогонов) +1 × 1.0 CPU — 360 req/s: полезная 277 rps, ошибок 22.9 %, p99 1671 мс (медиана 3 прогонов) +1 × 1.0 CPU — 400 req/s: полезная 253 rps, ошибок 36.8 %, p99 1340 мс (медиана 3 прогонов) +1 × 1.0 CPU — 460 req/s: полезная 223 rps, ошибок 51.6 %, p99 1655 мс (медиана 3 прогонов) + +2 × 0.5 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 39 мс (медиана 3 прогонов) +2 × 0.5 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 40 мс (медиана 3 прогонов) +2 × 0.5 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 112 мс (медиана 3 прогонов) +2 × 0.5 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 252 мс (медиана 3 прогонов) +2 × 0.5 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 232 мс (медиана 3 прогонов) +2 × 0.5 CPU — 240 req/s: полезная 235 rps, ошибок 2.2 %, p99 1393 мс (медиана 3 прогонов) +2 × 0.5 CPU — 280 req/s: полезная 223 rps, ошибок 20.4 %, p99 1925 мс (медиана 3 прогонов) +2 × 0.5 CPU — 320 req/s: полезная 191 rps, ошибок 40.2 %, p99 2464 мс (медиана 3 прогонов) +2 × 0.5 CPU — 360 req/s: полезная 183 rps, ошибок 49.3 %, p99 2468 мс (медиана 2 прогонов) + +4 × 0.25 CPU — 40 req/s: полезная 40 rps, ошибок 0.0 %, p99 61 мс (медиана 3 прогонов) +4 × 0.25 CPU — 80 req/s: полезная 80 rps, ошибок 0.0 %, p99 66 мс (медиана 3 прогонов) +4 × 0.25 CPU — 120 req/s: полезная 120 rps, ошибок 0.0 %, p99 87 мс (медиана 3 прогонов) +4 × 0.25 CPU — 160 req/s: полезная 160 rps, ошибок 0.0 %, p99 183 мс (медиана 3 прогонов) +4 × 0.25 CPU — 200 req/s: полезная 200 rps, ошибок 0.0 %, p99 1856 мс (медиана 3 прогонов) +4 × 0.25 CPU — 240 req/s: полезная 177 rps, ошибок 26.1 %, p99 2761 мс (медиана 3 прогонов) +4 × 0.25 CPU — 280 req/s: полезная 194 rps, ошибок 30.6 %, p99 3639 мс (медиана 3 прогонов) +4 × 0.25 CPU — 320 req/s: полезная 180 rps, ошибок 43.8 %, p99 3677 мс (медиана 2 прогонов) +4 × 0.25 CPU — 360 req/s: полезная 202 rps, ошибок 43.9 %, p99 2507 мс (медиана 1 прогонов) +1 × 1.0 CPU +1 × 1.0 CPU +2 × 0.5 CPU +2 × 0.5 CPU +4 × 0.25 CPU +4 × 0.25 CPU + diff --git a/evidence/track1/growth-1x0.25.json b/evidence/track1/growth-1x0.25.json new file mode 100644 index 00000000..7b9a5a3c --- /dev/null +++ b/evidence/track1/growth-1x0.25.json @@ -0,0 +1,341 @@ +{ + "config": "growth-1x0.25", + "repeat": 1, + "series": "growth", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "KDF_CONCURRENCY": 1 + }, + "measured_at": "2026-11-03T08:37:56+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 40, + "peak_ok_rps": 83.5, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-1x0.25", + "latency_ms": { + "max": 163.97, + "p50": 5.87, + "p90": 51.55, + "p99": 133.55, + "p999": 163.97 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.69, + "p99_ms": 86.2 + }, + "me": { + "ok": 40, + "p50_ms": 2.74, + "p99_ms": 78.28 + }, + "post": { + "ok": 80, + "p50_ms": 6.87, + "p99_ms": 163.97 + } + }, + "rate": 40, + "service_ms": { + "p50": 5.26, + "p99": 132.57 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 16.0, + "lb": 2.7, + "loadgen-run-083085fa5191": 3.1, + "postgres": 6.4, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 41.7, + "lb": 4.5, + "loadgen-run-083085fa5191": 6.4, + "postgres": 83.0, + "redis": 4.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.222, + "pg_connections_mean": 12 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-1x0.25", + "latency_ms": { + "max": 1106.74, + "p50": 25.44, + "p90": 488.84, + "p99": 684.07, + "p999": 1106.74 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 27.24, + "p99_ms": 642.11 + }, + "me": { + "ok": 80, + "p50_ms": 9.52, + "p99_ms": 95.12 + }, + "post": { + "ok": 160, + "p50_ms": 22.17, + "p99_ms": 719.12 + } + }, + "rate": 80, + "service_ms": { + "p50": 24.84, + "p99": 683.5 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 24.8, + "lb": 3.2, + "loadgen-run-083085fa5191": 5.3, + "postgres": 7.4, + "redis": 2.7 + }, + "mem_mib_max": { + "api": 45.9, + "lb": 4.9, + "loadgen-run-083085fa5191": 7.2, + "postgres": 84.8, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 11.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.912, + "pg_connections_mean": 12.7 + }, + "slo_ok": false + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 365 + }, + "error_ratio": 0.3042, + "errors": 365, + "event": "step", + "label": "growth-1x0.25", + "latency_ms": { + "max": 4351.01, + "p50": 2040.56, + "p90": 2269.49, + "p99": 3927.25, + "p999": 4351.01 + }, + "ok": 835, + "per_op": { + "history": { + "ok": 551, + "p50_ms": 2075.27, + "p99_ms": 3934.79 + }, + "me": { + "ok": 120, + "p50_ms": 71.28, + "p99_ms": 1794.18 + }, + "post": { + "ok": 164, + "p50_ms": 2066.38, + "p99_ms": 3876.14 + } + }, + "rate": 120, + "service_ms": { + "p50": 2040.24, + "p99": 3926.97 + }, + "throughput_ok_rps": 83.5, + "resources": { + "cpu_percent_mean": { + "api": 25.7, + "lb": 4.1, + "loadgen-run-083085fa5191": 8.4, + "postgres": 7.4, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 51.2, + "lb": 7.7, + "loadgen-run-083085fa5191": 14.9, + "postgres": 85.1, + "redis": 5.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 161.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 1.0, + "pg_connections_mean": 13 + }, + "slo_ok": false + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1162 + }, + "error_ratio": 0.7262, + "errors": 1162, + "event": "step", + "label": "growth-1x0.25", + "latency_ms": { + "max": 4794.08, + "p50": 3150.27, + "p90": 3933.96, + "p99": 4099.07, + "p999": 4794.08 + }, + "ok": 438, + "per_op": { + "history": { + "ok": 269, + "p50_ms": 3503.56, + "p99_ms": 4344.37 + }, + "me": { + "ok": 88, + "p50_ms": 992.99, + "p99_ms": 2236.79 + }, + "post": { + "ok": 81, + "p50_ms": 2758.74, + "p99_ms": 4099.07 + } + }, + "rate": 160, + "service_ms": { + "p50": 3149.5, + "p99": 4098.99 + }, + "throughput_ok_rps": 43.8, + "resources": { + "cpu_percent_mean": { + "api": 25.5, + "lb": 5.3, + "loadgen-run-083085fa5191": 9.8, + "postgres": 6.8, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 54.2, + "lb": 9.1, + "loadgen-run-083085fa5191": 17.7, + "postgres": 85.1, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 220.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 1.0, + "pg_connections_mean": 13 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-3d5fcadf": { + "pool": { + "connections_num": 3, + "requests_num": 19, + "requests_queued": 3, + "connections_ms": 215, + "requests_wait_ms": 296, + "usage_ms": 42, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + } + }, + "pool_after": { + "api-3d5fcadf": { + "pool": { + "connections_num": 10, + "requests_num": 6470, + "requests_queued": 2801, + "connections_ms": 1357, + "requests_wait_ms": 4124254, + "usage_ms": 270202, + "requests_errors": 1682, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 4407, + "misses": 40, + "errors": 36 + }, + "requests": 6050 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/growth-2x0.25.json b/evidence/track1/growth-2x0.25.json new file mode 100644 index 00000000..979d0a29 --- /dev/null +++ b/evidence/track1/growth-2x0.25.json @@ -0,0 +1,522 @@ +{ + "config": "growth-2x0.25", + "repeat": 1, + "series": "growth", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 2, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "KDF_CONCURRENCY": 1 + }, + "measured_at": "2026-11-03T08:40:54+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 80, + "peak_ok_rps": 159.8, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-2x0.25", + "latency_ms": { + "max": 81.86, + "p50": 5.5, + "p90": 20.81, + "p99": 72.09, + "p999": 81.86 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.36, + "p99_ms": 72.09 + }, + "me": { + "ok": 40, + "p50_ms": 2.94, + "p99_ms": 67.97 + }, + "post": { + "ok": 80, + "p50_ms": 7.25, + "p99_ms": 77.37 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.92, + "p99": 71.44 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 17.4, + "lb": 2.6, + "loadgen-run-7ce67b9d067d": 2.9, + "postgres": 6.1, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 37.3, + "lb": 4.6, + "loadgen-run-7ce67b9d067d": 6.1, + "postgres": 80.3, + "redis": 4.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.083, + "pg_connections_mean": 10 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-2x0.25", + "latency_ms": { + "max": 119.96, + "p50": 5.94, + "p90": 48.55, + "p99": 90.77, + "p999": 119.96 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.64, + "p99_ms": 94.32 + }, + "me": { + "ok": 80, + "p50_ms": 2.87, + "p99_ms": 71.43 + }, + "post": { + "ok": 160, + "p50_ms": 7.32, + "p99_ms": 87.38 + } + }, + "rate": 80, + "service_ms": { + "p50": 5.27, + "p99": 89.94 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 34.3, + "lb": 3.5, + "loadgen-run-7ce67b9d067d": 5.2, + "postgres": 10.2, + "redis": 3.1 + }, + "mem_mib_max": { + "api": 44.6, + "lb": 4.9, + "loadgen-run-7ce67b9d067d": 7.1, + "postgres": 101.6, + "redis": 4.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.4, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.324, + "pg_connections_mean": 19 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-2x0.25", + "latency_ms": { + "max": 1962.59, + "p50": 287.31, + "p90": 793.16, + "p99": 1436.06, + "p999": 1509.54 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 304.65, + "p99_ms": 1432.21 + }, + "me": { + "ok": 120, + "p50_ms": 49.86, + "p99_ms": 188.42 + }, + "post": { + "ok": 240, + "p50_ms": 438.94, + "p99_ms": 1470.95 + } + }, + "rate": 120, + "service_ms": { + "p50": 286.93, + "p99": 1435.02 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 41.4, + "lb": 4.4, + "loadgen-run-7ce67b9d067d": 7.5, + "postgres": 11.0, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 46.9, + "lb": 5.7, + "loadgen-run-7ce67b9d067d": 8.1, + "postgres": 114.1, + "redis": 4.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 40.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.765, + "pg_connections_mean": 23 + }, + "slo_ok": false + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2 + }, + "error_ratio": 0.0013, + "errors": 2, + "event": "step", + "label": "growth-2x0.25", + "latency_ms": { + "max": 3292.73, + "p50": 979.17, + "p90": 2008.11, + "p99": 2192, + "p999": 2922.43 + }, + "ok": 1598, + "per_op": { + "history": { + "ok": 1119, + "p50_ms": 1070.65, + "p99_ms": 2203.93 + }, + "me": { + "ok": 160, + "p50_ms": 61, + "p99_ms": 507.81 + }, + "post": { + "ok": 319, + "p50_ms": 1205.83, + "p99_ms": 2192 + } + }, + "rate": 160, + "service_ms": { + "p50": 975.73, + "p99": 2191.43 + }, + "throughput_ok_rps": 159.8, + "resources": { + "cpu_percent_mean": { + "api": 42.6, + "lb": 5.0, + "loadgen-run-7ce67b9d067d": 9.4, + "postgres": 11.5, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 49.6, + "lb": 7.4, + "loadgen-run-7ce67b9d067d": 13.4, + "postgres": 115.2, + "redis": 5.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 149.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.918, + "pg_connections_mean": 24 + }, + "slo_ok": false + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 708 + }, + "error_ratio": 0.354, + "errors": 708, + "event": "step", + "label": "growth-2x0.25", + "latency_ms": { + "max": 3749.16, + "p50": 1473.88, + "p90": 2265.03, + "p99": 3631.02, + "p999": 3728.39 + }, + "ok": 1292, + "per_op": { + "history": { + "ok": 883, + "p50_ms": 1650.32, + "p99_ms": 3631.02 + }, + "me": { + "ok": 179, + "p50_ms": 86.56, + "p99_ms": 2081.3 + }, + "post": { + "ok": 230, + "p50_ms": 1675.91, + "p99_ms": 3694.18 + } + }, + "rate": 200, + "service_ms": { + "p50": 1473.65, + "p99": 3630.79 + }, + "throughput_ok_rps": 129.2, + "resources": { + "cpu_percent_mean": { + "api": 47.2, + "lb": 5.7, + "loadgen-run-7ce67b9d067d": 10.5, + "postgres": 11.0, + "redis": 3.1 + }, + "mem_mib_max": { + "api": 54.0, + "lb": 9.9, + "loadgen-run-7ce67b9d067d": 19.6, + "postgres": 115.8, + "redis": 4.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 241.9, + "pg_active_mean": 0.9, + "pg_wait_events_mean": { + "CPU": 0.5, + "IO:WALSync": 0.1, + "LWLock:WALWrite": 0.3 + }, + "api_cfs_throttled_period_ratio": 0.944, + "pg_connections_mean": 24 + }, + "slo_ok": false + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1212 + }, + "error_ratio": 0.505, + "errors": 1212, + "event": "step", + "label": "growth-2x0.25", + "latency_ms": { + "max": 3296.52, + "p50": 1697.67, + "p90": 2757.62, + "p99": 3260.39, + "p999": 3294.14 + }, + "ok": 1188, + "per_op": { + "history": { + "ok": 804, + "p50_ms": 1835.07, + "p99_ms": 3262.87 + }, + "me": { + "ok": 181, + "p50_ms": 96.98, + "p99_ms": 2195.43 + }, + "post": { + "ok": 203, + "p50_ms": 1750.8, + "p99_ms": 2960.61 + } + }, + "rate": 240, + "service_ms": { + "p50": 1694.36, + "p99": 3259.65 + }, + "throughput_ok_rps": 118.8, + "resources": { + "cpu_percent_mean": { + "api": 45.9, + "lb": 6.1, + "loadgen-run-7ce67b9d067d": 10.7, + "postgres": 9.2, + "redis": 2.4 + }, + "mem_mib_max": { + "api": 54.4, + "lb": 9.9, + "loadgen-run-7ce67b9d067d": 20.7, + "postgres": 116.1, + "redis": 5.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 249.7, + "pg_active_mean": 0.3, + "pg_wait_events_mean": { + "CPU": 0.3 + }, + "api_cfs_throttled_period_ratio": 0.942, + "pg_connections_mean": 24 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-2307bb79": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 229, + "requests_wait_ms": 297, + "usage_ms": 55, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + }, + "api-9f69e8f9": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 212, + "requests_wait_ms": 290, + "usage_ms": 44, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + } + }, + "pool_after": { + "api-2307bb79": { + "pool": { + "connections_num": 10, + "requests_num": 6230, + "requests_queued": 3483, + "connections_ms": 1132, + "requests_wait_ms": 3430506, + "usage_ms": 321376, + "requests_errors": 1354, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 4948, + "misses": 47, + "errors": 16 + }, + "requests": 6140 + }, + "api-9f69e8f9": { + "pool": { + "connections_num": 10, + "requests_num": 5583, + "requests_queued": 2977, + "connections_ms": 1605, + "requests_wait_ms": 3531264, + "usage_ms": 316196, + "requests_errors": 855, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 5142, + "misses": 47, + "errors": 8 + }, + "requests": 5767 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/growth-3x0.25.json b/evidence/track1/growth-3x0.25.json new file mode 100644 index 00000000..12c75e39 --- /dev/null +++ b/evidence/track1/growth-3x0.25.json @@ -0,0 +1,704 @@ +{ + "config": "growth-3x0.25", + "repeat": 1, + "series": "growth", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 3, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "KDF_CONCURRENCY": 1 + }, + "measured_at": "2026-11-03T08:44:02+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 80, + "peak_ok_rps": 177.8, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-3x0.25", + "latency_ms": { + "max": 113.11, + "p50": 5.99, + "p90": 22.54, + "p99": 82.06, + "p999": 113.11 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.79, + "p99_ms": 82.06 + }, + "me": { + "ok": 40, + "p50_ms": 2.92, + "p99_ms": 66.08 + }, + "post": { + "ok": 80, + "p50_ms": 6.98, + "p99_ms": 113.11 + } + }, + "rate": 40, + "service_ms": { + "p50": 5.36, + "p99": 81.31 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 22.2, + "lb": 2.3, + "loadgen-run-69e377d75d2b": 2.8, + "postgres": 6.4, + "redis": 2.7 + }, + "mem_mib_max": { + "api": 38.1, + "lb": 4.4, + "loadgen-run-69e377d75d2b": 6.1, + "postgres": 97.1, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.096, + "pg_connections_mean": 15 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-3x0.25", + "latency_ms": { + "max": 140.63, + "p50": 5.78, + "p90": 41.54, + "p99": 89.95, + "p999": 140.63 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.68, + "p99_ms": 93.77 + }, + "me": { + "ok": 80, + "p50_ms": 2.73, + "p99_ms": 58.74 + }, + "post": { + "ok": 160, + "p50_ms": 6.73, + "p99_ms": 78.67 + } + }, + "rate": 80, + "service_ms": { + "p50": 5.15, + "p99": 87.88 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 36.9, + "lb": 3.6, + "loadgen-run-69e377d75d2b": 5.3, + "postgres": 8.8, + "redis": 2.9 + }, + "mem_mib_max": { + "api": 35.9, + "lb": 4.4, + "loadgen-run-69e377d75d2b": 6.3, + "postgres": 98.4, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.188, + "pg_connections_mean": 15 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-3x0.25", + "latency_ms": { + "max": 526.34, + "p50": 6.86, + "p90": 161.34, + "p99": 300.82, + "p999": 456.11 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 6.74, + "p99_ms": 300.82 + }, + "me": { + "ok": 120, + "p50_ms": 2.92, + "p99_ms": 125.9 + }, + "post": { + "ok": 240, + "p50_ms": 9.24, + "p99_ms": 327.57 + } + }, + "rate": 120, + "service_ms": { + "p50": 6.13, + "p99": 298.48 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 55.4, + "lb": 4.4, + "loadgen-run-69e377d75d2b": 7.2, + "postgres": 13.4, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 44.5, + "lb": 4.7, + "loadgen-run-69e377d75d2b": 6.9, + "postgres": 125.7, + "redis": 4.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 5.1, + "pg_active_mean": 0.33, + "pg_wait_events_mean": { + "CPU": 0.11, + "IO:WALSync": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.319, + "pg_connections_mean": 24.7 + }, + "slo_ok": false + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "growth-3x0.25", + "latency_ms": { + "max": 736.64, + "p50": 55.32, + "p90": 395.09, + "p99": 564.5, + "p999": 720.14 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 61.49, + "p99_ms": 567.01 + }, + "me": { + "ok": 160, + "p50_ms": 19.48, + "p99_ms": 324.03 + }, + "post": { + "ok": 320, + "p50_ms": 79.86, + "p99_ms": 564.5 + } + }, + "rate": 160, + "service_ms": { + "p50": 54.68, + "p99": 563.45 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 66.8, + "lb": 5.7, + "loadgen-run-69e377d75d2b": 9.5, + "postgres": 15.5, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 45.3, + "lb": 5.3, + "loadgen-run-69e377d75d2b": 7.8, + "postgres": 145.4, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 24.4, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.11, + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.626, + "pg_connections_mean": 33.9 + }, + "slo_ok": false + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 411 + }, + "error_ratio": 0.2055, + "errors": 411, + "event": "step", + "label": "growth-3x0.25", + "latency_ms": { + "max": 3347.53, + "p50": 750.43, + "p90": 2257.96, + "p99": 2992.67, + "p999": 3341.77 + }, + "ok": 1589, + "per_op": { + "history": { + "ok": 1083, + "p50_ms": 784.79, + "p99_ms": 2972.94 + }, + "me": { + "ok": 186, + "p50_ms": 122.24, + "p99_ms": 2428.17 + }, + "post": { + "ok": 320, + "p50_ms": 871.16, + "p99_ms": 3259.04 + } + }, + "rate": 200, + "service_ms": { + "p50": 750.39, + "p99": 2991.38 + }, + "throughput_ok_rps": 158.9, + "resources": { + "cpu_percent_mean": { + "api": 66.3, + "lb": 6.7, + "loadgen-run-69e377d75d2b": 12.9, + "postgres": 15.0, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 53.9, + "lb": 9.7, + "loadgen-run-69e377d75d2b": 17.2, + "postgres": 145.4, + "redis": 4.5 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 158.8, + "pg_active_mean": 0.3, + "pg_wait_events_mean": { + "Client:ClientRead": 0.2, + "IO:WALSync": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.884, + "pg_connections_mean": 35 + }, + "slo_ok": false + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 622 + }, + "error_ratio": 0.2592, + "errors": 622, + "event": "step", + "label": "growth-3x0.25", + "latency_ms": { + "max": 3100.61, + "p50": 1149.65, + "p90": 2244.54, + "p99": 2549.02, + "p999": 2631.99 + }, + "ok": 1778, + "per_op": { + "history": { + "ok": 1230, + "p50_ms": 1339.55, + "p99_ms": 2565.64 + }, + "me": { + "ok": 220, + "p50_ms": 100.21, + "p99_ms": 693.97 + }, + "post": { + "ok": 328, + "p50_ms": 1548.52, + "p99_ms": 2452.56 + } + }, + "rate": 240, + "service_ms": { + "p50": 1149.31, + "p99": 2548.93 + }, + "throughput_ok_rps": 177.8, + "resources": { + "cpu_percent_mean": { + "api": 70.6, + "lb": 6.7, + "loadgen-run-69e377d75d2b": 12.0, + "postgres": 13.5, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 54.0, + "lb": 11.3, + "loadgen-run-69e377d75d2b": 22.6, + "postgres": 147.3, + "redis": 4.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 275, + "pg_active_mean": 0.4, + "pg_wait_events_mean": { + "CPU": 0.3, + "IO:WALSync": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.912, + "pg_connections_mean": 35 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1612 + }, + "error_ratio": 0.5757, + "errors": 1612, + "event": "step", + "label": "growth-3x0.25", + "latency_ms": { + "max": 4774.18, + "p50": 1901.67, + "p90": 3072.91, + "p99": 4240.71, + "p999": 4773.95 + }, + "ok": 1188, + "per_op": { + "history": { + "ok": 780, + "p50_ms": 2061.96, + "p99_ms": 4241.06 + }, + "me": { + "ok": 182, + "p50_ms": 208.11, + "p99_ms": 2614.16 + }, + "post": { + "ok": 226, + "p50_ms": 1970.44, + "p99_ms": 4772.46 + } + }, + "rate": 280, + "service_ms": { + "p50": 1900.44, + "p99": 4239.84 + }, + "throughput_ok_rps": 118.8, + "resources": { + "cpu_percent_mean": { + "api": 66.8, + "lb": 8.1, + "loadgen-run-69e377d75d2b": 14.5, + "postgres": 11.9, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 57.4, + "lb": 13.3, + "loadgen-run-69e377d75d2b": 27.1, + "postgres": 146.4, + "redis": 4.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 356.8, + "pg_active_mean": 0.4, + "pg_wait_events_mean": { + "IO:WALSync": 0.1, + "LWLock:WALWrite": 0.3 + }, + "api_cfs_throttled_period_ratio": 0.862, + "pg_connections_mean": 35 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2458 + }, + "error_ratio": 0.7681, + "errors": 2458, + "event": "step", + "label": "growth-3x0.25", + "latency_ms": { + "max": 4971.63, + "p50": 2239.43, + "p90": 4064.12, + "p99": 4614.18, + "p999": 4971.63 + }, + "ok": 742, + "per_op": { + "history": { + "ok": 511, + "p50_ms": 2359.66, + "p99_ms": 4614.18 + }, + "me": { + "ok": 83, + "p50_ms": 224.99, + "p99_ms": 3368.95 + }, + "post": { + "ok": 148, + "p50_ms": 2246.82, + "p99_ms": 4922.68 + } + }, + "rate": 320, + "service_ms": { + "p50": 2239.16, + "p99": 4613.09 + }, + "throughput_ok_rps": 74.2, + "resources": { + "cpu_percent_mean": { + "api": 65.2, + "lb": 8.7, + "loadgen-run-69e377d75d2b": 15.7, + "postgres": 12.4, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 57.6, + "lb": 12.8, + "loadgen-run-69e377d75d2b": 30.6, + "postgres": 146.9, + "redis": 5.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 359.3, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "CPU": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.891, + "pg_connections_mean": 35 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-883b7718": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 328, + "requests_wait_ms": 404, + "usage_ms": 197, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-6519d22e": { + "pool": { + "connections_num": 3, + "requests_num": 22, + "requests_queued": 3, + "connections_ms": 236, + "requests_wait_ms": 201, + "usage_ms": 135, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + }, + "api-f113faa6": { + "pool": { + "connections_num": 3, + "requests_num": 22, + "requests_queued": 3, + "connections_ms": 328, + "requests_wait_ms": 491, + "usage_ms": 143, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + } + }, + "pool_after": { + "api-883b7718": { + "pool": { + "connections_num": 10, + "requests_num": 5415, + "requests_queued": 2370, + "connections_ms": 1932, + "requests_wait_ms": 2757267, + "usage_ms": 326768, + "requests_errors": 1021, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 4476, + "misses": 20, + "errors": 25 + }, + "requests": 5489 + }, + "api-6519d22e": { + "pool": { + "connections_num": 10, + "requests_num": 5503, + "requests_queued": 2599, + "connections_ms": 1826, + "requests_wait_ms": 2907009, + "usage_ms": 340635, + "requests_errors": 744, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 4857, + "misses": 26, + "errors": 8 + }, + "requests": 5613 + }, + "api-f113faa6": { + "pool": { + "connections_num": 10, + "requests_num": 9185, + "requests_queued": 3748, + "connections_ms": 1941, + "requests_wait_ms": 5470804, + "usage_ms": 411062, + "requests_errors": 4012, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 4782, + "misses": 38, + "errors": 50 + }, + "requests": 8820 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/growth-peak.svg b/evidence/track1/growth-peak.svg new file mode 100644 index 00000000..397dd387 --- /dev/null +++ b/evidence/track1/growth-peak.svg @@ -0,0 +1,43 @@ + + +Рост ресурсов: пик полезной пропускной способности +Ступени 0.25 CPU на изолированном ядре, затем вся машина (2 vCPU, общие с БД/LB/генератором) + +0 + +100 + +200 + +300 +growth-1x0.25: пик 84 rps +84 +1×0.25 +(0.25 CPU) +growth-2x0.25: пик 160 rps +160 +2×0.25 +(0.5 CPU) +growth-3x0.25: пик 178 rps +178 +3×0.25 +(0.75 CPU) +fixed-4x0.25: пик 200 rps +200 +4×0.25 +(1 CPU) +fixed-1x1.00: пик 280 rps +280 +1×1.0 +(1 CPU) +machine-2x1.00: пик 320 rps +320 +2×1.0 +(2 vCPU хоста) +machine-4x1.00: пик 312 rps +312 +4×1.0 +(2 vCPU хоста) + +Пик полезной, rps + diff --git a/evidence/track1/machine-2x1.00.json b/evidence/track1/machine-2x1.00.json new file mode 100644 index 00000000..09d4e7dc --- /dev/null +++ b/evidence/track1/machine-2x1.00.json @@ -0,0 +1,849 @@ +{ + "config": "machine-2x1.00", + "repeat": 1, + "series": "machine", + "params": { + "API_CPUSET": "0-1", + "AUX_CPUSET": "0-1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 2, + "API_CPUS": "1.0", + "API_MEMORY": "256M", + "KDF_CONCURRENCY": 2 + }, + "measured_at": "2026-11-03T08:47:28+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 320, + "peak_ok_rps": 320, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 21.87, + "p50": 5.44, + "p90": 7.77, + "p99": 13.42, + "p999": 21.87 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.38, + "p99_ms": 13.42 + }, + "me": { + "ok": 40, + "p50_ms": 2.76, + "p99_ms": 7.88 + }, + "post": { + "ok": 80, + "p50_ms": 6.44, + "p99_ms": 16.93 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.86, + "p99": 12.3 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 20.2, + "lb": 2.2, + "loadgen-run-0cfe7ec82689": 3.7, + "postgres": 4.9, + "redis": 2.5 + }, + "mem_mib_max": { + "api": 43.9, + "lb": 4.9, + "loadgen-run-0cfe7ec82689": 6.7, + "postgres": 57.0, + "redis": 3.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "Client:ClientRead": 0.11, + "IO:DataFileRead": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 6 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 55.69, + "p50": 5.56, + "p90": 8.19, + "p99": 23.02, + "p999": 55.69 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.53, + "p99_ms": 24.68 + }, + "me": { + "ok": 80, + "p50_ms": 2.74, + "p99_ms": 7.51 + }, + "post": { + "ok": 160, + "p50_ms": 6.25, + "p99_ms": 22.85 + } + }, + "rate": 80, + "service_ms": { + "p50": 5.01, + "p99": 22.72 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 34.1, + "lb": 3.7, + "loadgen-run-0cfe7ec82689": 7.1, + "postgres": 7.8, + "redis": 3.1 + }, + "mem_mib_max": { + "api": 31.6, + "lb": 4.9, + "loadgen-run-0cfe7ec82689": 7.1, + "postgres": 57.2, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 6 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 58.19, + "p50": 5.8, + "p90": 10.61, + "p99": 36.92, + "p999": 55.22 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.81, + "p99_ms": 36.82 + }, + "me": { + "ok": 120, + "p50_ms": 2.68, + "p99_ms": 26.45 + }, + "post": { + "ok": 240, + "p50_ms": 6.3, + "p99_ms": 50.78 + } + }, + "rate": 120, + "service_ms": { + "p50": 5.17, + "p99": 32.4 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 51.4, + "lb": 4.9, + "loadgen-run-0cfe7ec82689": 10.3, + "postgres": 10.0, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 31.8, + "lb": 4.9, + "loadgen-run-0cfe7ec82689": 7.1, + "postgres": 59.0, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.005, + "pg_connections_mean": 6 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 167.26, + "p50": 5.44, + "p90": 13.3, + "p99": 108.05, + "p999": 145.02 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.49, + "p99_ms": 110.96 + }, + "me": { + "ok": 160, + "p50_ms": 2.4, + "p99_ms": 49.53 + }, + "post": { + "ok": 320, + "p50_ms": 6.14, + "p99_ms": 95.27 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.88, + "p99": 107.62 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 63.4, + "lb": 5.5, + "loadgen-run-0cfe7ec82689": 11.1, + "postgres": 12.7, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 40.6, + "lb": 5.0, + "loadgen-run-0cfe7ec82689": 7.3, + "postgres": 77.1, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.017, + "pg_connections_mean": 10.1 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 312.38, + "p50": 6.54, + "p90": 20.02, + "p99": 130.19, + "p999": 269.03 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 6.86, + "p99_ms": 120.22 + }, + "me": { + "ok": 200, + "p50_ms": 2.31, + "p99_ms": 43.91 + }, + "post": { + "ok": 400, + "p50_ms": 7.28, + "p99_ms": 214.54 + } + }, + "rate": 200, + "service_ms": { + "p50": 6, + "p99": 130.09 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 78.9, + "lb": 5.9, + "loadgen-run-0cfe7ec82689": 12.9, + "postgres": 14.2, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 32.8, + "lb": 5.1, + "loadgen-run-0cfe7ec82689": 7.6, + "postgres": 85.0, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 2.9, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.011, + "pg_connections_mean": 16.7 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 233.7, + "p50": 7.95, + "p90": 47.79, + "p99": 167.31, + "p999": 211.5 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 8.88, + "p99_ms": 170.69 + }, + "me": { + "ok": 240, + "p50_ms": 2.26, + "p99_ms": 41.58 + }, + "post": { + "ok": 480, + "p50_ms": 7.29, + "p99_ms": 168.55 + } + }, + "rate": 240, + "service_ms": { + "p50": 7.35, + "p99": 166.1 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 94.1, + "lb": 6.6, + "loadgen-run-0cfe7ec82689": 14.9, + "postgres": 16.5, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 44.2, + "lb": 5.3, + "loadgen-run-0cfe7ec82689": 7.6, + "postgres": 92.4, + "redis": 4.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.1, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "IO:WALSync": 0.11, + "LWLock:WALWrite": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.006, + "pg_connections_mean": 18.8 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 240.79, + "p50": 9.58, + "p90": 59.16, + "p99": 157.6, + "p999": 200.58 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 11.79, + "p99_ms": 157.59 + }, + "me": { + "ok": 280, + "p50_ms": 2.11, + "p99_ms": 51.53 + }, + "post": { + "ok": 560, + "p50_ms": 8.33, + "p99_ms": 191.05 + } + }, + "rate": 280, + "service_ms": { + "p50": 8.88, + "p99": 156.09 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 103.9, + "lb": 7.0, + "loadgen-run-0cfe7ec82689": 15.6, + "postgres": 17.5, + "redis": 3.6 + }, + "mem_mib_max": { + "api": 44.7, + "lb": 5.4, + "loadgen-run-0cfe7ec82689": 7.8, + "postgres": 93.4, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 4.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 204.39, + "p50": 14.01, + "p90": 82.12, + "p99": 161.5, + "p999": 192.47 + }, + "ok": 3200, + "per_op": { + "history": { + "ok": 2240, + "p50_ms": 15.95, + "p99_ms": 156.65 + }, + "me": { + "ok": 320, + "p50_ms": 2.55, + "p99_ms": 56.48 + }, + "post": { + "ok": 640, + "p50_ms": 17.66, + "p99_ms": 174.15 + } + }, + "rate": 320, + "service_ms": { + "p50": 13.39, + "p99": 158.47 + }, + "throughput_ok_rps": 320, + "resources": { + "cpu_percent_mean": { + "api": 111.0, + "lb": 7.3, + "loadgen-run-0cfe7ec82689": 17.4, + "postgres": 19.3, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 45.9, + "lb": 7.0, + "loadgen-run-0cfe7ec82689": 7.8, + "postgres": 93.9, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 9, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20 + }, + "slo_ok": true + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 720 + }, + "error_ratio": 0.2, + "errors": 720, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 2461.41, + "p50": 715.82, + "p90": 1287.85, + "p99": 1921.22, + "p999": 2429.42 + }, + "ok": 2880, + "per_op": { + "history": { + "ok": 1992, + "p50_ms": 811.39, + "p99_ms": 2041.89 + }, + "me": { + "ok": 353, + "p50_ms": 43.79, + "p99_ms": 1489.58 + }, + "post": { + "ok": 535, + "p50_ms": 805.48, + "p99_ms": 1721.12 + } + }, + "rate": 360, + "service_ms": { + "p50": 714.71, + "p99": 1921.18 + }, + "throughput_ok_rps": 288, + "resources": { + "cpu_percent_mean": { + "api": 108.5, + "lb": 9.4, + "loadgen-run-0cfe7ec82689": 20.0, + "postgres": 18.8, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 52.7, + "lb": 9.8, + "loadgen-run-0cfe7ec82689": 21.9, + "postgres": 96.1, + "redis": 4.5 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 192, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.012, + "pg_connections_mean": 20 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1868 + }, + "error_ratio": 0.467, + "errors": 1868, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 2561.02, + "p50": 1274.55, + "p90": 1694.59, + "p99": 2389.72, + "p999": 2555.42 + }, + "ok": 2132, + "per_op": { + "history": { + "ok": 1425, + "p50_ms": 1358.48, + "p99_ms": 2392.22 + }, + "me": { + "ok": 273, + "p50_ms": 82.93, + "p99_ms": 1469.12 + }, + "post": { + "ok": 434, + "p50_ms": 1368, + "p99_ms": 2392.65 + } + }, + "rate": 400, + "service_ms": { + "p50": 1270.48, + "p99": 2389.29 + }, + "throughput_ok_rps": 213.2, + "resources": { + "cpu_percent_mean": { + "api": 106.8, + "lb": 10.6, + "loadgen-run-0cfe7ec82689": 20.8, + "postgres": 21.4, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 53.8, + "lb": 11.7, + "loadgen-run-0cfe7ec82689": 24.6, + "postgres": 96.4, + "redis": 5.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 273, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.018, + "pg_connections_mean": 20 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2050 + }, + "error_ratio": 0.4457, + "errors": 2050, + "event": "step", + "label": "machine-2x1.00", + "latency_ms": { + "max": 1864.86, + "p50": 1068.02, + "p90": 1507.62, + "p99": 1828.84, + "p999": 1855.65 + }, + "ok": 2550, + "per_op": { + "history": { + "ok": 1670, + "p50_ms": 1153.33, + "p99_ms": 1833.5 + }, + "me": { + "ok": 430, + "p50_ms": 37.12, + "p99_ms": 222.89 + }, + "post": { + "ok": 450, + "p50_ms": 1161.06, + "p99_ms": 1831.85 + } + }, + "rate": 460, + "service_ms": { + "p50": 1067.2, + "p99": 1828.63 + }, + "throughput_ok_rps": 255, + "resources": { + "cpu_percent_mean": { + "api": 104.4, + "lb": 10.4, + "loadgen-run-0cfe7ec82689": 21.7, + "postgres": 16.0, + "redis": 4.4 + }, + "mem_mib_max": { + "api": 56.3, + "lb": 11.5, + "loadgen-run-0cfe7ec82689": 22.4, + "postgres": 96.6, + "redis": 6.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 247.1, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "Client:ClientRead": 0.11, + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.006, + "pg_connections_mean": 20 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-7e1b920a": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 117, + "requests_wait_ms": 95, + "usage_ms": 64, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + }, + "api-f503cd6e": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 80, + "requests_wait_ms": 85, + "usage_ms": 32, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + } + }, + "pool_after": { + "api-7e1b920a": { + "pool": { + "connections_num": 10, + "requests_num": 18225, + "requests_queued": 5467, + "connections_ms": 674, + "requests_wait_ms": 4249639, + "usage_ms": 326862, + "requests_errors": 3391, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 16568, + "misses": 119, + "errors": 3 + }, + "requests": 18926 + }, + "api-f503cd6e": { + "pool": { + "connections_num": 10, + "requests_num": 15491, + "requests_queued": 4334, + "connections_ms": 641, + "requests_wait_ms": 3038536, + "usage_ms": 285901, + "requests_errors": 1914, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 15862, + "misses": 128, + "errors": 2 + }, + "requests": 16676 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/machine-4x1.00.json b/evidence/track1/machine-4x1.00.json new file mode 100644 index 00000000..bf769573 --- /dev/null +++ b/evidence/track1/machine-4x1.00.json @@ -0,0 +1,1010 @@ +{ + "config": "machine-4x1.00", + "repeat": 1, + "series": "machine", + "params": { + "API_CPUSET": "0-1", + "AUX_CPUSET": "0-1", + "DB_POOL_MAX": "10", + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 4, + "API_CPUS": "1.0", + "API_MEMORY": "256M", + "KDF_CONCURRENCY": 2 + }, + "measured_at": "2026-11-03T08:50:45+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 240, + "peak_ok_rps": 311.5, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 31.28, + "p50": 5.12, + "p90": 7.47, + "p99": 16.23, + "p999": 31.28 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.02, + "p99_ms": 21.56 + }, + "me": { + "ok": 40, + "p50_ms": 2.74, + "p99_ms": 8.38 + }, + "post": { + "ok": 80, + "p50_ms": 5.98, + "p99_ms": 15.15 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.47, + "p99": 14.21 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 24.9, + "lb": 2.3, + "loadgen-run-7cfbba2a71d5": 3.5, + "postgres": 5.1, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 40.0, + "lb": 4.9, + "loadgen-run-7cfbba2a71d5": 6.8, + "postgres": 79.2, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 12 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 36.02, + "p50": 5.31, + "p90": 7.83, + "p99": 19.25, + "p999": 36.02 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.25, + "p99_ms": 19.25 + }, + "me": { + "ok": 80, + "p50_ms": 2.63, + "p99_ms": 7.33 + }, + "post": { + "ok": 160, + "p50_ms": 6.15, + "p99_ms": 20.24 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.66, + "p99": 18.45 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 39.5, + "lb": 3.7, + "loadgen-run-7cfbba2a71d5": 6.9, + "postgres": 7.1, + "redis": 3.0 + }, + "mem_mib_max": { + "api": 37.2, + "lb": 4.9, + "loadgen-run-7cfbba2a71d5": 7.1, + "postgres": 80.9, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 12 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 58.9, + "p50": 5.59, + "p90": 10.37, + "p99": 28.59, + "p999": 51.41 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.59, + "p99_ms": 27.05 + }, + "me": { + "ok": 120, + "p50_ms": 2.6, + "p99_ms": 16.06 + }, + "post": { + "ok": 240, + "p50_ms": 6.21, + "p99_ms": 33.76 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.98, + "p99": 25.74 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 59.7, + "lb": 4.9, + "loadgen-run-7cfbba2a71d5": 9.8, + "postgres": 10.5, + "redis": 3.1 + }, + "mem_mib_max": { + "api": 31.6, + "lb": 4.9, + "loadgen-run-7cfbba2a71d5": 7.5, + "postgres": 79.7, + "redis": 4.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 12 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 111.15, + "p50": 6.2, + "p90": 20.88, + "p99": 61.31, + "p999": 106.8 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 6.25, + "p99_ms": 61.96 + }, + "me": { + "ok": 160, + "p50_ms": 2.68, + "p99_ms": 60.23 + }, + "post": { + "ok": 320, + "p50_ms": 7.44, + "p99_ms": 55.7 + } + }, + "rate": 160, + "service_ms": { + "p50": 5.6, + "p99": 59.23 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 72.8, + "lb": 5.7, + "loadgen-run-7cfbba2a71d5": 11.4, + "postgres": 13.4, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 41.0, + "lb": 5.0, + "loadgen-run-7cfbba2a71d5": 7.6, + "postgres": 84.2, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.5, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 13.2 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 938.22, + "p50": 10.89, + "p90": 167.43, + "p99": 540.29, + "p999": 741.99 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 12.21, + "p99_ms": 540.29 + }, + "me": { + "ok": 200, + "p50_ms": 2.94, + "p99_ms": 289.49 + }, + "post": { + "ok": 400, + "p50_ms": 13.09, + "p99_ms": 603.95 + } + }, + "rate": 200, + "service_ms": { + "p50": 10.16, + "p99": 539.8 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 101.1, + "lb": 7.0, + "loadgen-run-7cfbba2a71d5": 13.8, + "postgres": 19.7, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 44.4, + "lb": 6.4, + "loadgen-run-7cfbba2a71d5": 7.9, + "postgres": 140.9, + "redis": 5.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 9.8, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 34.9 + }, + "slo_ok": false + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 276.88, + "p50": 9.75, + "p90": 55.48, + "p99": 171.93, + "p999": 262.88 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 11.16, + "p99_ms": 175.27 + }, + "me": { + "ok": 240, + "p50_ms": 2.57, + "p99_ms": 36.59 + }, + "post": { + "ok": 480, + "p50_ms": 9.44, + "p99_ms": 247.83 + } + }, + "rate": 240, + "service_ms": { + "p50": 9.05, + "p99": 171.04 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 104.5, + "lb": 6.5, + "loadgen-run-7cfbba2a71d5": 14.7, + "postgres": 16.6, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 44.9, + "lb": 6.2, + "loadgen-run-7cfbba2a71d5": 7.8, + "postgres": 142.7, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 4.4, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 38 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 644.95, + "p50": 29.31, + "p90": 218.92, + "p99": 448.95, + "p999": 629.82 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 34.29, + "p99_ms": 453.5 + }, + "me": { + "ok": 280, + "p50_ms": 6.38, + "p99_ms": 235.1 + }, + "post": { + "ok": 560, + "p50_ms": 32.96, + "p99_ms": 468.6 + } + }, + "rate": 280, + "service_ms": { + "p50": 28.19, + "p99": 448.3 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 117.5, + "lb": 7.7, + "loadgen-run-7cfbba2a71d5": 16.0, + "postgres": 19.2, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 45.5, + "lb": 5.7, + "loadgen-run-7cfbba2a71d5": 8.3, + "postgres": 150.1, + "redis": 5.5 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 17.5, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "IO:WALSync": 0.11, + "LWLock:WALWrite": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 40.8 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 85 + }, + "error_ratio": 0.0266, + "errors": 85, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 1907.72, + "p50": 522.5, + "p90": 1096.98, + "p99": 1757.8, + "p999": 1852.77 + }, + "ok": 3115, + "per_op": { + "history": { + "ok": 2181, + "p50_ms": 555.05, + "p99_ms": 1758.2 + }, + "me": { + "ok": 318, + "p50_ms": 52.43, + "p99_ms": 1183 + }, + "post": { + "ok": 616, + "p50_ms": 570.51, + "p99_ms": 1811.04 + } + }, + "rate": 320, + "service_ms": { + "p50": 521.9, + "p99": 1757.51 + }, + "throughput_ok_rps": 311.5, + "resources": { + "cpu_percent_mean": { + "api": 119.8, + "lb": 8.8, + "loadgen-run-7cfbba2a71d5": 17.5, + "postgres": 21.3, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 47.8, + "lb": 9.1, + "loadgen-run-7cfbba2a71d5": 17.4, + "postgres": 151.3, + "redis": 4.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 175.3, + "pg_active_mean": 0.3, + "pg_wait_events_mean": { + "CPU": 0.2, + "IO:WALSync": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.003, + "pg_connections_mean": 41 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 740 + }, + "error_ratio": 0.2056, + "errors": 740, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 3436.18, + "p50": 936.77, + "p90": 2077.02, + "p99": 2610.03, + "p999": 3089.34 + }, + "ok": 2860, + "per_op": { + "history": { + "ok": 1981, + "p50_ms": 1003.42, + "p99_ms": 2667.77 + }, + "me": { + "ok": 302, + "p50_ms": 70.63, + "p99_ms": 1854.56 + }, + "post": { + "ok": 577, + "p50_ms": 996.55, + "p99_ms": 2610.03 + } + }, + "rate": 360, + "service_ms": { + "p50": 936.13, + "p99": 2609.41 + }, + "throughput_ok_rps": 286, + "resources": { + "cpu_percent_mean": { + "api": 119.3, + "lb": 9.6, + "loadgen-run-7cfbba2a71d5": 18.8, + "postgres": 21.0, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 52.5, + "lb": 12.8, + "loadgen-run-7cfbba2a71d5": 28.6, + "postgres": 153.8, + "redis": 6.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 324.3, + "pg_active_mean": 0.4, + "pg_wait_events_mean": { + "Client:ClientRead": 0.1, + "IO:WALSync": 0.2, + "LWLock:WALWrite": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 41 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1165 + }, + "error_ratio": 0.2913, + "errors": 1165, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 2829.88, + "p50": 1094.2, + "p90": 2000.34, + "p99": 2461.73, + "p999": 2728.13 + }, + "ok": 2835, + "per_op": { + "history": { + "ok": 1947, + "p50_ms": 1159.21, + "p99_ms": 2465.3 + }, + "me": { + "ok": 318, + "p50_ms": 74.96, + "p99_ms": 1468.34 + }, + "post": { + "ok": 570, + "p50_ms": 1271.63, + "p99_ms": 2467.72 + } + }, + "rate": 400, + "service_ms": { + "p50": 1093.29, + "p99": 2461.55 + }, + "throughput_ok_rps": 283.5, + "resources": { + "cpu_percent_mean": { + "api": 120.7, + "lb": 9.6, + "loadgen-run-7cfbba2a71d5": 19.3, + "postgres": 18.6, + "redis": 4.2 + }, + "mem_mib_max": { + "api": 55.0, + "lb": 14.7, + "loadgen-run-7cfbba2a71d5": 31.7, + "postgres": 154.3, + "redis": 5.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 382.5, + "pg_active_mean": 0.2, + "pg_wait_events_mean": { + "IO:WALSync": 0.1, + "LWLock:WALWrite": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 41 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2048 + }, + "error_ratio": 0.4452, + "errors": 2048, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 2932.22, + "p50": 1390.1, + "p90": 2301.91, + "p99": 2780.83, + "p999": 2867.14 + }, + "ok": 2552, + "per_op": { + "history": { + "ok": 1766, + "p50_ms": 1558.2, + "p99_ms": 2775.67 + }, + "me": { + "ok": 304, + "p50_ms": 118.22, + "p99_ms": 1969.13 + }, + "post": { + "ok": 482, + "p50_ms": 1503.48, + "p99_ms": 2793.62 + } + }, + "rate": 460, + "service_ms": { + "p50": 1388.56, + "p99": 2748.78 + }, + "throughput_ok_rps": 255.2, + "resources": { + "cpu_percent_mean": { + "api": 118.1, + "lb": 11.1, + "loadgen-run-7cfbba2a71d5": 20.1, + "postgres": 18.0, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 58.9, + "lb": 15.1, + "loadgen-run-7cfbba2a71d5": 37.0, + "postgres": 154.8, + "redis": 5.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 425.7, + "pg_active_mean": 0.4, + "pg_wait_events_mean": { + "CPU": 0.1, + "Client:ClientRead": 0.1, + "IO:WALSync": 0.1, + "LWLock:WALWrite": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 41 + }, + "slo_ok": false + }, + { + "attempted": 5200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 3200 + }, + "error_ratio": 0.6154, + "errors": 3200, + "event": "step", + "label": "machine-4x1.00", + "latency_ms": { + "max": 2975.05, + "p50": 1297.6, + "p90": 2457.19, + "p99": 2839.45, + "p999": 2910.4 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1373, + "p50_ms": 1380.26, + "p99_ms": 2785.66 + }, + "me": { + "ok": 278, + "p50_ms": 107.93, + "p99_ms": 2286.37 + }, + "post": { + "ok": 349, + "p50_ms": 1451.07, + "p99_ms": 2867.01 + } + }, + "rate": 520, + "service_ms": { + "p50": 1297.29, + "p99": 2838.65 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 114.8, + "lb": 12.8, + "loadgen-run-7cfbba2a71d5": 22.3, + "postgres": 15.2, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 61.2, + "lb": 16.8, + "loadgen-run-7cfbba2a71d5": 39.8, + "postgres": 155.2, + "redis": 5.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 500.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.003, + "pg_connections_mean": 40.6 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-d92c6716": { + "pool": { + "connections_num": 3, + "requests_num": 21, + "requests_queued": 3, + "connections_ms": 85, + "requests_wait_ms": 87, + "usage_ms": 50, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + }, + "api-c5e93f47": { + "pool": { + "connections_num": 2, + "requests_num": 22, + "requests_queued": 3, + "connections_ms": 273, + "requests_wait_ms": 387, + "usage_ms": 118, + "pool_min": 1, + "pool_max": 10, + "pool_size": 2, + "pool_available": 2, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + }, + "api-d823bc24": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 75, + "requests_wait_ms": 85, + "usage_ms": 39, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + }, + "api-6757cfbe": { + "pool": { + "connections_num": 3, + "requests_num": 22, + "requests_queued": 3, + "connections_ms": 76, + "requests_wait_ms": 63, + "usage_ms": 49, + "pool_min": 1, + "pool_max": 10, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 10 + } + }, + "pool_after": { + "api-d92c6716": { + "pool": { + "connections_num": 10, + "requests_num": 9872, + "requests_queued": 4247, + "connections_ms": 518, + "requests_wait_ms": 3821650, + "usage_ms": 398890, + "requests_errors": 1846, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 8511, + "misses": 37, + "errors": 13 + }, + "requests": 10169 + }, + "api-c5e93f47": { + "pool": { + "connections_num": 10, + "requests_num": 10175, + "requests_queued": 4529, + "connections_ms": 1565, + "requests_wait_ms": 4073477, + "usage_ms": 426405, + "requests_errors": 1594, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 9, + "requests_waiting": 0 + }, + "cache": { + "hits": 9431, + "misses": 70, + "errors": 5 + }, + "requests": 10744 + }, + "api-d823bc24": { + "pool": { + "connections_num": 10, + "requests_num": 10348, + "requests_queued": 5078, + "connections_ms": 1673, + "requests_wait_ms": 4340407, + "usage_ms": 439976, + "requests_errors": 1666, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 8407, + "misses": 63, + "errors": 10 + }, + "requests": 10428 + }, + "api-6757cfbe": { + "pool": { + "connections_num": 10, + "requests_num": 11341, + "requests_queued": 4591, + "connections_ms": 1170, + "requests_wait_ms": 4084031, + "usage_ms": 414974, + "requests_errors": 2647, + "pool_min": 1, + "pool_max": 10, + "pool_size": 10, + "pool_available": 10, + "requests_waiting": 0 + }, + "cache": { + "hits": 8617, + "misses": 35, + "errors": 50 + }, + "requests": 11469 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/pool-1x1.00-p20.json b/evidence/track1/pool-1x1.00-p20.json new file mode 100644 index 00000000..2bbfa8e4 --- /dev/null +++ b/evidence/track1/pool-1x1.00-p20.json @@ -0,0 +1,809 @@ +{ + "config": "pool-1x1.00-p20", + "repeat": 1, + "series": "pool", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": 20, + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T08:57:33+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 200, + "peak_ok_rps": 280, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 25.35, + "p50": 5.17, + "p90": 8.29, + "p99": 15.35, + "p999": 25.35 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 5.01, + "p99_ms": 15.35 + }, + "me": { + "ok": 40, + "p50_ms": 2.5, + "p99_ms": 7.06 + }, + "post": { + "ok": 80, + "p50_ms": 6.48, + "p99_ms": 25.35 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.61, + "p99": 14.31 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 15.1, + "lb": 2.1, + "loadgen-run-d4289cdc835a": 2.7, + "postgres": 4.7, + "redis": 2.3 + }, + "mem_mib_max": { + "api": 31.5, + "lb": 4.3, + "loadgen-run-d4289cdc835a": 5.9, + "postgres": 66.8, + "redis": 5.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.11, + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 6 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 45.39, + "p50": 5.26, + "p90": 7.78, + "p99": 14.71, + "p999": 45.39 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.24, + "p99_ms": 13.2 + }, + "me": { + "ok": 80, + "p50_ms": 2.43, + "p99_ms": 4.55 + }, + "post": { + "ok": 160, + "p50_ms": 5.84, + "p99_ms": 35.4 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.67, + "p99": 13.63 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 29.2, + "lb": 3.3, + "loadgen-run-d4289cdc835a": 5.1, + "postgres": 7.3, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 31.8, + "lb": 4.4, + "loadgen-run-d4289cdc835a": 6.2, + "postgres": 67.0, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 6 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 27.65, + "p50": 4.82, + "p90": 8.01, + "p99": 19.27, + "p999": 26.18 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 4.76, + "p99_ms": 21.96 + }, + "me": { + "ok": 120, + "p50_ms": 2.21, + "p99_ms": 10.94 + }, + "post": { + "ok": 240, + "p50_ms": 5.52, + "p99_ms": 14.23 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.21, + "p99": 18.71 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 39.5, + "lb": 4.2, + "loadgen-run-d4289cdc835a": 6.7, + "postgres": 9.8, + "redis": 3.2 + }, + "mem_mib_max": { + "api": 37.7, + "lb": 4.5, + "loadgen-run-d4289cdc835a": 6.2, + "postgres": 70.7, + "redis": 5.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 7 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 99.68, + "p50": 5.09, + "p90": 13.32, + "p99": 77.24, + "p999": 96.76 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.1, + "p99_ms": 78.51 + }, + "me": { + "ok": 160, + "p50_ms": 2.06, + "p99_ms": 7.9 + }, + "post": { + "ok": 320, + "p50_ms": 5.79, + "p99_ms": 71.27 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.53, + "p99": 77.04 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 52.9, + "lb": 4.9, + "loadgen-run-d4289cdc835a": 8.2, + "postgres": 12.8, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 32.7, + "lb": 4.6, + "loadgen-run-d4289cdc835a": 6.5, + "postgres": 79.6, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.9, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 10.1 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 134.92, + "p50": 6.69, + "p90": 54.8, + "p99": 116.76, + "p999": 131.97 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 7.37, + "p99_ms": 113.52 + }, + "me": { + "ok": 200, + "p50_ms": 2.28, + "p99_ms": 44.94 + }, + "post": { + "ok": 400, + "p50_ms": 6.85, + "p99_ms": 128.33 + } + }, + "rate": 200, + "service_ms": { + "p50": 6.24, + "p99": 115.72 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 69.8, + "lb": 5.8, + "loadgen-run-d4289cdc835a": 10.1, + "postgres": 16.5, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 36.0, + "lb": 4.9, + "loadgen-run-d4289cdc835a": 7.0, + "postgres": 106.2, + "redis": 4.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 3.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 20.1 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 532.26, + "p50": 56.78, + "p90": 343.8, + "p99": 412.4, + "p999": 523.07 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 64.99, + "p99_ms": 417.7 + }, + "me": { + "ok": 240, + "p50_ms": 15.39, + "p99_ms": 159.45 + }, + "post": { + "ok": 480, + "p50_ms": 85.9, + "p99_ms": 412.4 + } + }, + "rate": 240, + "service_ms": { + "p50": 56.11, + "p99": 412.06 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 85.2, + "lb": 6.3, + "loadgen-run-d4289cdc835a": 12.6, + "postgres": 18.1, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 48.5, + "lb": 5.5, + "loadgen-run-d4289cdc835a": 8.4, + "postgres": 104.4, + "redis": 5.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 26.2, + "pg_active_mean": 1.22, + "pg_wait_events_mean": { + "CPU": 0.11, + "IO:WALSync": 0.11, + "LWLock:WALWrite": 1.0 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 460.29, + "p50": 105.06, + "p90": 387.13, + "p99": 431.08, + "p999": 448.04 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 128.94, + "p99_ms": 432.79 + }, + "me": { + "ok": 280, + "p50_ms": 21.32, + "p99_ms": 112.9 + }, + "post": { + "ok": 560, + "p50_ms": 135.12, + "p99_ms": 419.06 + } + }, + "rate": 280, + "service_ms": { + "p50": 104.62, + "p99": 430.5 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 89.7, + "lb": 6.8, + "loadgen-run-d4289cdc835a": 14.2, + "postgres": 18.6, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 49.5, + "lb": 6.2, + "loadgen-run-d4289cdc835a": 9.4, + "postgres": 105.0, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 38.7, + "pg_active_mean": 0.44, + "pg_wait_events_mean": { + "CPU": 0.33, + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 494 + }, + "error_ratio": 0.1544, + "errors": 494, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 2208.7, + "p50": 857.91, + "p90": 1154.5, + "p99": 1829.56, + "p999": 2202.57 + }, + "ok": 2706, + "per_op": { + "history": { + "ok": 1856, + "p50_ms": 899.64, + "p99_ms": 1829.56 + }, + "me": { + "ok": 319, + "p50_ms": 40.14, + "p99_ms": 904.79 + }, + "post": { + "ok": 531, + "p50_ms": 889.44, + "p99_ms": 2061.05 + } + }, + "rate": 320, + "service_ms": { + "p50": 857.14, + "p99": 1829.14 + }, + "throughput_ok_rps": 270.6, + "resources": { + "cpu_percent_mean": { + "api": 92.7, + "lb": 7.8, + "loadgen-run-d4289cdc835a": 16.8, + "postgres": 19.8, + "redis": 4.2 + }, + "mem_mib_max": { + "api": 54.5, + "lb": 8.1, + "loadgen-run-d4289cdc835a": 17.2, + "postgres": 106.1, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 204.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 978 + }, + "error_ratio": 0.2717, + "errors": 978, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 2431.92, + "p50": 851.53, + "p90": 1453.99, + "p99": 1853.51, + "p999": 2409.37 + }, + "ok": 2622, + "per_op": { + "history": { + "ok": 1769, + "p50_ms": 873.33, + "p99_ms": 1867.17 + }, + "me": { + "ok": 354, + "p50_ms": 40.19, + "p99_ms": 846.62 + }, + "post": { + "ok": 499, + "p50_ms": 879.56, + "p99_ms": 1983.31 + } + }, + "rate": 360, + "service_ms": { + "p50": 851.02, + "p99": 1853.48 + }, + "throughput_ok_rps": 262.2, + "resources": { + "cpu_percent_mean": { + "api": 92.5, + "lb": 7.9, + "loadgen-run-d4289cdc835a": 15.7, + "postgres": 18.0, + "redis": 4.5 + }, + "mem_mib_max": { + "api": 55.4, + "lb": 8.3, + "loadgen-run-d4289cdc835a": 18.1, + "postgres": 106.8, + "redis": 4.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 231.8, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1821 + }, + "error_ratio": 0.4553, + "errors": 1821, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 1704.67, + "p50": 1106.16, + "p90": 1558.94, + "p99": 1681.95, + "p999": 1694.46 + }, + "ok": 2179, + "per_op": { + "history": { + "ok": 1410, + "p50_ms": 1197.31, + "p99_ms": 1683.7 + }, + "me": { + "ok": 400, + "p50_ms": 47.32, + "p99_ms": 116.31 + }, + "post": { + "ok": 369, + "p50_ms": 1199.01, + "p99_ms": 1682.46 + } + }, + "rate": 400, + "service_ms": { + "p50": 1105.9, + "p99": 1681.27 + }, + "throughput_ok_rps": 217.9, + "resources": { + "cpu_percent_mean": { + "api": 88.6, + "lb": 8.6, + "loadgen-run-d4289cdc835a": 16.5, + "postgres": 15.0, + "redis": 4.3 + }, + "mem_mib_max": { + "api": 55.5, + "lb": 8.1, + "loadgen-run-d4289cdc835a": 17.9, + "postgres": 107.4, + "redis": 5.3 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 232.7, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2817 + }, + "error_ratio": 0.6124, + "errors": 2817, + "event": "step", + "label": "pool-1x1.00-p20", + "latency_ms": { + "max": 2241.17, + "p50": 1298.81, + "p90": 1741.3, + "p99": 2197.71, + "p999": 2238.3 + }, + "ok": 1783, + "per_op": { + "history": { + "ok": 1156, + "p50_ms": 1392.51, + "p99_ms": 2197.79 + }, + "me": { + "ok": 295, + "p50_ms": 66.89, + "p99_ms": 1300.77 + }, + "post": { + "ok": 332, + "p50_ms": 1427.86, + "p99_ms": 2226.95 + } + }, + "rate": 460, + "service_ms": { + "p50": 1298.29, + "p99": 2197.03 + }, + "throughput_ok_rps": 178.3, + "resources": { + "cpu_percent_mean": { + "api": 88.7, + "lb": 10.3, + "loadgen-run-d4289cdc835a": 18.8, + "postgres": 18.7, + "redis": 3.3 + }, + "mem_mib_max": { + "api": 55.5, + "lb": 8.0, + "loadgen-run-d4289cdc835a": 19.9, + "postgres": 114.3, + "redis": 4.2 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 234.3, + "pg_active_mean": 0.2, + "pg_wait_events_mean": { + "CPU": 0.1, + "Client:ClientRead": 0.1, + "IO:WALWrite": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 21 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-7df7c520": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 76, + "requests_wait_ms": 81, + "usage_ms": 53, + "pool_min": 1, + "pool_max": 20, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + } + }, + "pool_after": { + "api-7df7c520": { + "pool": { + "connections_num": 20, + "requests_num": 33619, + "requests_queued": 14834, + "connections_ms": 1282, + "requests_wait_ms": 9575838, + "usage_ms": 1121363, + "requests_errors": 7049, + "pool_min": 1, + "pool_max": 20, + "pool_size": 20, + "pool_available": 20, + "requests_waiting": 0 + }, + "cache": { + "hits": 32179, + "misses": 316, + "errors": 1 + }, + "requests": 35397 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/pool-1x1.00-p40.json b/evidence/track1/pool-1x1.00-p40.json new file mode 100644 index 00000000..e279d9fb --- /dev/null +++ b/evidence/track1/pool-1x1.00-p40.json @@ -0,0 +1,670 @@ +{ + "config": "pool-1x1.00-p40", + "repeat": 1, + "series": "pool", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": 40, + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T09:00:19+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 200, + "peak_ok_rps": 280, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 24.88, + "p50": 5.11, + "p90": 7.57, + "p99": 18.06, + "p999": 24.88 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 4.96, + "p99_ms": 18.79 + }, + "me": { + "ok": 40, + "p50_ms": 2.38, + "p99_ms": 17.98 + }, + "post": { + "ok": 80, + "p50_ms": 6.21, + "p99_ms": 13.85 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.5, + "p99": 17.45 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 16.2, + "lb": 2.2, + "loadgen-run-dfb98a6b4250": 2.7, + "postgres": 5.3, + "redis": 2.0 + }, + "mem_mib_max": { + "api": 31.5, + "lb": 4.4, + "loadgen-run-dfb98a6b4250": 5.9, + "postgres": 75.5, + "redis": 3.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 7 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 78.53, + "p50": 5.2, + "p90": 8.48, + "p99": 34.62, + "p999": 78.53 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 5.08, + "p99_ms": 41.45 + }, + "me": { + "ok": 80, + "p50_ms": 2.47, + "p99_ms": 76.29 + }, + "post": { + "ok": 160, + "p50_ms": 6.37, + "p99_ms": 21.92 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.65, + "p99": 34.2 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 29.3, + "lb": 3.5, + "loadgen-run-dfb98a6b4250": 5.3, + "postgres": 8.1, + "redis": 3.1 + }, + "mem_mib_max": { + "api": 32.0, + "lb": 4.4, + "loadgen-run-dfb98a6b4250": 6.2, + "postgres": 79.3, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 7.2 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 46.75, + "p50": 5.16, + "p90": 10.09, + "p99": 26.59, + "p999": 45.19 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.13, + "p99_ms": 26.39 + }, + "me": { + "ok": 120, + "p50_ms": 2.35, + "p99_ms": 6.55 + }, + "post": { + "ok": 240, + "p50_ms": 5.95, + "p99_ms": 35.64 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.56, + "p99": 26.25 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 42.7, + "lb": 4.3, + "loadgen-run-dfb98a6b4250": 7.1, + "postgres": 10.3, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 32.5, + "lb": 4.4, + "loadgen-run-dfb98a6b4250": 6.7, + "postgres": 80.7, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.6, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "IO:WALSync": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 9 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 85.67, + "p50": 5.44, + "p90": 17.43, + "p99": 55.11, + "p999": 85.46 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.55, + "p99_ms": 58.98 + }, + "me": { + "ok": 160, + "p50_ms": 2.35, + "p99_ms": 12.67 + }, + "post": { + "ok": 320, + "p50_ms": 5.62, + "p99_ms": 51.19 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.85, + "p99": 52.82 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 57.6, + "lb": 5.3, + "loadgen-run-dfb98a6b4250": 8.8, + "postgres": 12.7, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 33.8, + "lb": 4.5, + "loadgen-run-dfb98a6b4250": 6.9, + "postgres": 88.2, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 1.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 11.2 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 252.61, + "p50": 6.5, + "p90": 51.08, + "p99": 190.85, + "p999": 243.4 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 7.91, + "p99_ms": 183.04 + }, + "me": { + "ok": 200, + "p50_ms": 2.07, + "p99_ms": 102.47 + }, + "post": { + "ok": 400, + "p50_ms": 6.08, + "p99_ms": 230.48 + } + }, + "rate": 200, + "service_ms": { + "p50": 5.99, + "p99": 190.26 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 70.2, + "lb": 5.7, + "loadgen-run-dfb98a6b4250": 10.0, + "postgres": 15.8, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 45.9, + "lb": 5.9, + "loadgen-run-dfb98a6b4250": 7.5, + "postgres": 109.4, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 3.1, + "pg_active_mean": 0.33, + "pg_wait_events_mean": { + "CPU": 0.33 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 15.7 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 711.16, + "p50": 18.48, + "p90": 363.61, + "p99": 537.32, + "p999": 692.24 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 22.52, + "p99_ms": 667.34 + }, + "me": { + "ok": 240, + "p50_ms": 2.7, + "p99_ms": 393.19 + }, + "post": { + "ok": 480, + "p50_ms": 22.17, + "p99_ms": 520.32 + } + }, + "rate": 240, + "service_ms": { + "p50": 17.84, + "p99": 536.87 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 80.9, + "lb": 6.6, + "loadgen-run-dfb98a6b4250": 12.4, + "postgres": 20.1, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 43.9, + "lb": 6.0, + "loadgen-run-dfb98a6b4250": 9.4, + "postgres": 144.8, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 17.7, + "pg_active_mean": 0.11, + "pg_wait_events_mean": { + "CPU": 0.11, + "Timeout:VacuumDelay": 0.11 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 25.8 + }, + "slo_ok": false + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 518.91, + "p50": 219.27, + "p90": 416.78, + "p99": 473.03, + "p999": 495.86 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 253.72, + "p99_ms": 458.22 + }, + "me": { + "ok": 280, + "p50_ms": 45.34, + "p99_ms": 164.64 + }, + "post": { + "ok": 560, + "p50_ms": 277.69, + "p99_ms": 488.27 + } + }, + "rate": 280, + "service_ms": { + "p50": 219.03, + "p99": 469.76 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 87.5, + "lb": 6.6, + "loadgen-run-dfb98a6b4250": 14.0, + "postgres": 20.0, + "redis": 3.9 + }, + "mem_mib_max": { + "api": 49.9, + "lb": 6.3, + "loadgen-run-dfb98a6b4250": 9.8, + "postgres": 160.8, + "redis": 4.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 57.2, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 39.4 + }, + "slo_ok": false + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 986 + }, + "error_ratio": 0.3081, + "errors": 986, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 3113.46, + "p50": 1173.18, + "p90": 1641.36, + "p99": 2305.47, + "p999": 3110.62 + }, + "ok": 2214, + "per_op": { + "history": { + "ok": 1528, + "p50_ms": 1274.1, + "p99_ms": 2854.56 + }, + "me": { + "ok": 320, + "p50_ms": 85.55, + "p99_ms": 1344.67 + }, + "post": { + "ok": 366, + "p50_ms": 1298.79, + "p99_ms": 1894.6 + } + }, + "rate": 320, + "service_ms": { + "p50": 1172.44, + "p99": 2304.66 + }, + "throughput_ok_rps": 221.4, + "resources": { + "cpu_percent_mean": { + "api": 92.3, + "lb": 8.1, + "loadgen-run-dfb98a6b4250": 15.8, + "postgres": 16.2, + "redis": 4.4 + }, + "mem_mib_max": { + "api": 45.6, + "lb": 8.4, + "loadgen-run-dfb98a6b4250": 19.6, + "postgres": 163.0, + "redis": 5.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 261, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 42 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1255 + }, + "error_ratio": 0.3486, + "errors": 1255, + "event": "step", + "label": "pool-1x1.00-p40", + "latency_ms": { + "max": 2909.46, + "p50": 1044.08, + "p90": 1533.05, + "p99": 1924.77, + "p999": 2721.03 + }, + "ok": 2345, + "per_op": { + "history": { + "ok": 1623, + "p50_ms": 1079.4, + "p99_ms": 1913.46 + }, + "me": { + "ok": 338, + "p50_ms": 74.83, + "p99_ms": 1384.79 + }, + "post": { + "ok": 384, + "p50_ms": 1097.97, + "p99_ms": 1953.4 + } + }, + "rate": 360, + "service_ms": { + "p50": 1043.49, + "p99": 1924.11 + }, + "throughput_ok_rps": 234.5, + "resources": { + "cpu_percent_mean": { + "api": 91.8, + "lb": 8.5, + "loadgen-run-dfb98a6b4250": 16.3, + "postgres": 18.4, + "redis": 4.5 + }, + "mem_mib_max": { + "api": 57.0, + "lb": 8.8, + "loadgen-run-dfb98a6b4250": 19.6, + "postgres": 163.8, + "redis": 4.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 264.7, + "pg_active_mean": 0.1, + "pg_wait_events_mean": { + "Client:ClientRead": 0.1 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 42 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-86705189": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 95, + "requests_wait_ms": 94, + "usage_ms": 64, + "pool_min": 1, + "pool_max": 40, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + } + }, + "pool_after": { + "api-86705189": { + "pool": { + "connections_num": 40, + "requests_num": 22545, + "requests_queued": 7424, + "connections_ms": 4509, + "requests_wait_ms": 4677487, + "usage_ms": 1169861, + "requests_errors": 2332, + "pool_min": 1, + "pool_max": 40, + "pool_size": 40, + "pool_available": 40, + "requests_waiting": 0 + }, + "cache": { + "hits": 23399, + "misses": 280, + "errors": 1 + }, + "requests": 24198 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/pool-1x1.00-p5.json b/evidence/track1/pool-1x1.00-p5.json new file mode 100644 index 00000000..fb61ef6e --- /dev/null +++ b/evidence/track1/pool-1x1.00-p5.json @@ -0,0 +1,793 @@ +{ + "config": "pool-1x1.00-p5", + "repeat": 1, + "series": "pool", + "params": { + "API_CPUSET": "0", + "AUX_CPUSET": "1", + "DB_POOL_MAX": 5, + "SESSION_CACHE": "redis", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s", + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "KDF_CONCURRENCY": 4 + }, + "measured_at": "2026-11-03T08:54:01+00:00", + "slo": { + "p99_ms": 250.0, + "error_ratio": 0.01 + }, + "capacity_at_slo_rps": 280, + "peak_ok_rps": 320, + "steps": [ + { + "attempted": 400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 33.44, + "p50": 5.1, + "p90": 8.04, + "p99": 13.8, + "p999": 33.44 + }, + "ok": 400, + "per_op": { + "history": { + "ok": 280, + "p50_ms": 4.98, + "p99_ms": 11.95 + }, + "me": { + "ok": 40, + "p50_ms": 2.64, + "p99_ms": 13.8 + }, + "post": { + "ok": 80, + "p50_ms": 6.25, + "p99_ms": 33.44 + } + }, + "rate": 40, + "service_ms": { + "p50": 4.56, + "p99": 13.34 + }, + "throughput_ok_rps": 40, + "resources": { + "cpu_percent_mean": { + "api": 16.1, + "lb": 2.2, + "loadgen-run-b334fcc6230c": 2.6, + "postgres": 4.5, + "redis": 2.6 + }, + "mem_mib_max": { + "api": 31.6, + "lb": 4.4, + "loadgen-run-b334fcc6230c": 6.1, + "postgres": 53.4, + "redis": 3.5 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 4 + }, + "slo_ok": true + }, + { + "attempted": 800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 35.89, + "p50": 4.73, + "p90": 6.84, + "p99": 16.1, + "p999": 35.89 + }, + "ok": 800, + "per_op": { + "history": { + "ok": 560, + "p50_ms": 4.61, + "p99_ms": 16.1 + }, + "me": { + "ok": 80, + "p50_ms": 2.26, + "p99_ms": 15.26 + }, + "post": { + "ok": 160, + "p50_ms": 5.84, + "p99_ms": 23.37 + } + }, + "rate": 80, + "service_ms": { + "p50": 4.11, + "p99": 15.45 + }, + "throughput_ok_rps": 80, + "resources": { + "cpu_percent_mean": { + "api": 26.8, + "lb": 3.3, + "loadgen-run-b334fcc6230c": 4.7, + "postgres": 6.9, + "redis": 2.8 + }, + "mem_mib_max": { + "api": 31.8, + "lb": 4.4, + "loadgen-run-b334fcc6230c": 6.3, + "postgres": 54.5, + "redis": 3.6 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 4 + }, + "slo_ok": true + }, + { + "attempted": 1200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 112.91, + "p50": 5, + "p90": 8.32, + "p99": 55.97, + "p999": 99.69 + }, + "ok": 1200, + "per_op": { + "history": { + "ok": 840, + "p50_ms": 5.05, + "p99_ms": 66.08 + }, + "me": { + "ok": 120, + "p50_ms": 2.21, + "p99_ms": 24.33 + }, + "post": { + "ok": 240, + "p50_ms": 5.31, + "p99_ms": 43.99 + } + }, + "rate": 120, + "service_ms": { + "p50": 4.41, + "p99": 55.57 + }, + "throughput_ok_rps": 120, + "resources": { + "cpu_percent_mean": { + "api": 41.2, + "lb": 4.1, + "loadgen-run-b334fcc6230c": 6.8, + "postgres": 9.8, + "redis": 3.4 + }, + "mem_mib_max": { + "api": 39.5, + "lb": 4.5, + "loadgen-run-b334fcc6230c": 6.3, + "postgres": 57.6, + "redis": 4.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.3, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": true + }, + { + "attempted": 1600, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 127.66, + "p50": 5.19, + "p90": 12.29, + "p99": 81.13, + "p999": 115.15 + }, + "ok": 1600, + "per_op": { + "history": { + "ok": 1120, + "p50_ms": 5.23, + "p99_ms": 86.17 + }, + "me": { + "ok": 160, + "p50_ms": 2.17, + "p99_ms": 7.97 + }, + "post": { + "ok": 320, + "p50_ms": 5.52, + "p99_ms": 26.79 + } + }, + "rate": 160, + "service_ms": { + "p50": 4.64, + "p99": 80.4 + }, + "throughput_ok_rps": 160, + "resources": { + "cpu_percent_mean": { + "api": 54.7, + "lb": 5.2, + "loadgen-run-b334fcc6230c": 8.7, + "postgres": 12.0, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 32.6, + "lb": 4.5, + "loadgen-run-b334fcc6230c": 6.8, + "postgres": 58.0, + "redis": 3.8 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 0.8, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": true + }, + { + "attempted": 2000, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 169, + "p50": 8.92, + "p90": 45.38, + "p99": 148.97, + "p999": 162.73 + }, + "ok": 2000, + "per_op": { + "history": { + "ok": 1400, + "p50_ms": 11.07, + "p99_ms": 152.32 + }, + "me": { + "ok": 200, + "p50_ms": 2.52, + "p99_ms": 32.35 + }, + "post": { + "ok": 400, + "p50_ms": 6.59, + "p99_ms": 141.8 + } + }, + "rate": 200, + "service_ms": { + "p50": 8.36, + "p99": 148.57 + }, + "throughput_ok_rps": 200, + "resources": { + "cpu_percent_mean": { + "api": 74.3, + "lb": 6.2, + "loadgen-run-b334fcc6230c": 10.5, + "postgres": 18.1, + "redis": 3.7 + }, + "mem_mib_max": { + "api": 33.1, + "lb": 4.8, + "loadgen-run-b334fcc6230c": 7.0, + "postgres": 59.1, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 2.7, + "pg_active_mean": 0.22, + "pg_wait_events_mean": { + "CPU": 0.22 + }, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": true + }, + { + "attempted": 2400, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 266.6, + "p50": 43.3, + "p90": 136.78, + "p99": 238.74, + "p999": 259.53 + }, + "ok": 2400, + "per_op": { + "history": { + "ok": 1680, + "p50_ms": 52.51, + "p99_ms": 239.03 + }, + "me": { + "ok": 240, + "p50_ms": 3.63, + "p99_ms": 64.38 + }, + "post": { + "ok": 480, + "p50_ms": 75.42, + "p99_ms": 243.05 + } + }, + "rate": 240, + "service_ms": { + "p50": 42.7, + "p99": 237.65 + }, + "throughput_ok_rps": 240, + "resources": { + "cpu_percent_mean": { + "api": 85.0, + "lb": 6.6, + "loadgen-run-b334fcc6230c": 12.0, + "postgres": 16.2, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 46.0, + "lb": 4.8, + "loadgen-run-b334fcc6230c": 7.5, + "postgres": 59.7, + "redis": 3.9 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 16.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": true + }, + { + "attempted": 2800, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 260.24, + "p50": 50.89, + "p90": 173.45, + "p99": 240.32, + "p999": 254.25 + }, + "ok": 2800, + "per_op": { + "history": { + "ok": 1960, + "p50_ms": 58.88, + "p99_ms": 244.24 + }, + "me": { + "ok": 280, + "p50_ms": 4.89, + "p99_ms": 76.83 + }, + "post": { + "ok": 560, + "p50_ms": 72.52, + "p99_ms": 229.92 + } + }, + "rate": 280, + "service_ms": { + "p50": 49.97, + "p99": 239.93 + }, + "throughput_ok_rps": 280, + "resources": { + "cpu_percent_mean": { + "api": 89.0, + "lb": 6.8, + "loadgen-run-b334fcc6230c": 13.4, + "postgres": 17.8, + "redis": 4.0 + }, + "mem_mib_max": { + "api": 45.8, + "lb": 5.1, + "loadgen-run-b334fcc6230c": 7.7, + "postgres": 60.2, + "redis": 3.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 16.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": true + }, + { + "attempted": 3200, + "dropped": 0, + "duration_s": 10, + "error_kinds": {}, + "error_ratio": 0, + "errors": 0, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 417.82, + "p50": 53.71, + "p90": 209.84, + "p99": 268.52, + "p999": 395.31 + }, + "ok": 3200, + "per_op": { + "history": { + "ok": 2240, + "p50_ms": 59.81, + "p99_ms": 257.43 + }, + "me": { + "ok": 320, + "p50_ms": 4.73, + "p99_ms": 49.46 + }, + "post": { + "ok": 640, + "p50_ms": 74.29, + "p99_ms": 280.87 + } + }, + "rate": 320, + "service_ms": { + "p50": 53.03, + "p99": 267.56 + }, + "throughput_ok_rps": 320, + "resources": { + "cpu_percent_mean": { + "api": 90.3, + "lb": 6.5, + "loadgen-run-b334fcc6230c": 13.0, + "postgres": 17.7, + "redis": 3.5 + }, + "mem_mib_max": { + "api": 38.8, + "lb": 5.3, + "loadgen-run-b334fcc6230c": 7.9, + "postgres": 61.0, + "redis": 5.4 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 22.4, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": false + }, + { + "attempted": 3600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 663 + }, + "error_ratio": 0.1842, + "errors": 663, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 2043.4, + "p50": 685.54, + "p90": 940.61, + "p99": 1494.15, + "p999": 2032.18 + }, + "ok": 2937, + "per_op": { + "history": { + "ok": 2047, + "p50_ms": 699.36, + "p99_ms": 1513.61 + }, + "me": { + "ok": 359, + "p50_ms": 10.2, + "p99_ms": 1176.8 + }, + "post": { + "ok": 531, + "p50_ms": 710.38, + "p99_ms": 1384.34 + } + }, + "rate": 360, + "service_ms": { + "p50": 684.96, + "p99": 1493.45 + }, + "throughput_ok_rps": 293.7, + "resources": { + "cpu_percent_mean": { + "api": 90.9, + "lb": 7.6, + "loadgen-run-b334fcc6230c": 15.8, + "postgres": 15.7, + "redis": 4.1 + }, + "mem_mib_max": { + "api": 51.2, + "lb": 8.5, + "loadgen-run-b334fcc6230c": 16.8, + "postgres": 62.4, + "redis": 4.0 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 205.1, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": false + }, + { + "attempted": 4000, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 1441 + }, + "error_ratio": 0.3603, + "errors": 1441, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 1320.84, + "p50": 894.91, + "p90": 1148.13, + "p99": 1290.83, + "p999": 1317.19 + }, + "ok": 2559, + "per_op": { + "history": { + "ok": 1763, + "p50_ms": 911.45, + "p99_ms": 1290.57 + }, + "me": { + "ok": 400, + "p50_ms": 11.26, + "p99_ms": 68.81 + }, + "post": { + "ok": 396, + "p50_ms": 927.42, + "p99_ms": 1306.13 + } + }, + "rate": 400, + "service_ms": { + "p50": 894.37, + "p99": 1290.08 + }, + "throughput_ok_rps": 255.9, + "resources": { + "cpu_percent_mean": { + "api": 89.4, + "lb": 7.8, + "loadgen-run-b334fcc6230c": 15.6, + "postgres": 14.1, + "redis": 4.2 + }, + "mem_mib_max": { + "api": 43.8, + "lb": 8.1, + "loadgen-run-b334fcc6230c": 16.6, + "postgres": 61.7, + "redis": 4.1 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 208.6, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": false + }, + { + "attempted": 4600, + "dropped": 0, + "duration_s": 10, + "error_kinds": { + "503": 2702 + }, + "error_ratio": 0.5874, + "errors": 2702, + "event": "step", + "label": "pool-1x1.00-p5", + "latency_ms": { + "max": 2217.32, + "p50": 999.11, + "p90": 1518.48, + "p99": 2081.03, + "p999": 2215.87 + }, + "ok": 1898, + "per_op": { + "history": { + "ok": 1193, + "p50_ms": 1119.84, + "p99_ms": 2105.92 + }, + "me": { + "ok": 391, + "p50_ms": 13.79, + "p99_ms": 762.31 + }, + "post": { + "ok": 314, + "p50_ms": 1244.02, + "p99_ms": 2024.97 + } + }, + "rate": 460, + "service_ms": { + "p50": 998.63, + "p99": 2080.44 + }, + "throughput_ok_rps": 189.8, + "resources": { + "cpu_percent_mean": { + "api": 86.8, + "lb": 10.1, + "loadgen-run-b334fcc6230c": 18.0, + "postgres": 14.3, + "redis": 3.8 + }, + "mem_mib_max": { + "api": 51.9, + "lb": 8.3, + "loadgen-run-b334fcc6230c": 23.6, + "postgres": 62.2, + "redis": 4.7 + }, + "lb_queue_max": 0, + "lb_sessions_mean": 210.5, + "pg_active_mean": 0, + "pg_wait_events_mean": {}, + "api_cfs_throttled_period_ratio": 0.0, + "pg_connections_mean": 5 + }, + "slo_ok": false + } + ], + "pool_before": { + "api-b0d1c3a3": { + "pool": { + "connections_num": 3, + "requests_num": 20, + "requests_queued": 3, + "connections_ms": 63, + "requests_wait_ms": 72, + "usage_ms": 56, + "pool_min": 1, + "pool_max": 5, + "pool_size": 3, + "pool_available": 3, + "requests_waiting": 0 + }, + "cache": { + "hits": 0, + "misses": 0, + "errors": 0 + }, + "requests": 9 + } + }, + "pool_after": { + "api-b0d1c3a3": { + "pool": { + "connections_num": 5, + "requests_num": 32970, + "requests_queued": 16715, + "connections_ms": 199, + "requests_wait_ms": 7636714, + "usage_ms": 289126, + "requests_errors": 5343, + "pool_min": 1, + "pool_max": 5, + "pool_size": 5, + "pool_available": 5, + "requests_waiting": 0 + }, + "cache": { + "hits": 33637, + "misses": 218, + "errors": 1 + }, + "requests": 35404 + } + }, + "loadgen": { + "users": 48, + "seed": 10, + "duration_s": 10, + "warmup_s": 3, + "mix": "history=70,post=20,me=10", + "model": "open-loop, latency from scheduled time" + } +} diff --git a/evidence/track1/summary.json b/evidence/track1/summary.json new file mode 100644 index 00000000..aa5abcda --- /dev/null +++ b/evidence/track1/summary.json @@ -0,0 +1,656 @@ +{ + "admission-1x1.00-mc10": { + "series": "admission", + "repeats": 3, + "median": { + "capacity_slo": 280, + "capacity_err1": 359.7, + "peak": 359.7 + }, + "values": { + "capacity_slo": [ + 320, + 280, + 240 + ], + "capacity_err1": [ + 360, + 359.7, + 320 + ], + "peak": [ + 383.7, + 359.7, + 339 + ] + }, + "params": { + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": 10, + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 400, + "cpu_percent": { + "api": 92.8, + "lb": 7.8, + "loadgen-run-3312d64e3026": 17.4, + "postgres": 19.9, + "redis": 3.2 + }, + "cfs_throttled": 0.0, + "lb_queue_max": 799, + "pg_connections": 20 + } + }, + "admission-1x1.00-mc10-q250": { + "series": "admission", + "repeats": 1, + "median": { + "capacity_slo": 200, + "capacity_err1": 239.4, + "peak": 303.5 + }, + "values": { + "capacity_slo": [ + 200 + ], + "capacity_err1": [ + 239.4 + ], + "peak": [ + 303.5 + ] + }, + "params": { + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": 10, + "LB_QUEUE_TIMEOUT": "250ms" + }, + "at_peak": { + "rate": 360, + "cpu_percent": { + "api": 91.5, + "lb": 8.2, + "loadgen-run-57259902c18b": 16.9, + "postgres": 22.9, + "redis": 4.1 + }, + "cfs_throttled": 0.0, + "lb_queue_max": 90, + "pg_connections": 15 + } + }, + "admission-1x1.00-mc20": { + "series": "admission", + "repeats": 1, + "median": { + "capacity_slo": 200, + "capacity_err1": 320, + "peak": 326.3 + }, + "values": { + "capacity_slo": [ + 200 + ], + "capacity_err1": [ + 320 + ], + "peak": [ + 326.3 + ] + }, + "params": { + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": 20, + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 360, + "cpu_percent": { + "api": 91.2, + "lb": 8.6, + "loadgen-run-6ca2812f958b": 18.2, + "postgres": 20.7, + "redis": 3.7 + }, + "cfs_throttled": 0.0, + "lb_queue_max": 719, + "pg_connections": 14 + } + }, + "admission-2x0.50-mc10": { + "series": "admission", + "repeats": 1, + "median": { + "capacity_slo": 200, + "capacity_err1": 280, + "peak": 306.3 + }, + "values": { + "capacity_slo": [ + 200 + ], + "capacity_err1": [ + 280 + ], + "peak": [ + 306.3 + ] + }, + "params": { + "API_REPLICAS": 2, + "API_CPUS": "0.5", + "API_MEMORY": "256M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": 10, + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 320, + "cpu_percent": { + "api": 92.3, + "lb": 7.2, + "loadgen-run-65398839d17b": 15.9, + "postgres": 19.1, + "redis": 3.5 + }, + "cfs_throttled": 0.006, + "lb_queue_max": 640, + "pg_connections": 26 + } + }, + "fixed-1x1.00": { + "series": "fixed-cpu", + "repeats": 3, + "median": { + "capacity_slo": 240, + "capacity_err1": 280, + "peak": 280 + }, + "values": { + "capacity_slo": [ + 280, + 240, + 240 + ], + "capacity_err1": [ + 280, + 280, + 280 + ], + "peak": [ + 280, + 316.7, + 280 + ] + }, + "params": { + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 280, + "cpu_percent": { + "api": 85.2, + "lb": 6.3, + "loadgen-run-03af3ac2a6d0": 11.9, + "postgres": 18.7, + "redis": 3.7 + }, + "cfs_throttled": 0.0, + "lb_queue_max": 0, + "pg_connections": 17 + } + }, + "fixed-2x0.50": { + "series": "fixed-cpu", + "repeats": 3, + "median": { + "capacity_slo": 200, + "capacity_err1": 200, + "peak": 234.7 + }, + "values": { + "capacity_slo": [ + 120, + 200, + 200 + ], + "capacity_err1": [ + 200, + 240, + 200 + ], + "peak": [ + 234.7, + 240, + 234.3 + ] + }, + "params": { + "API_REPLICAS": 2, + "API_CPUS": "0.5", + "API_MEMORY": "256M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 240, + "cpu_percent": { + "api": 78.8, + "lb": 6.6, + "loadgen-run-bc33f10e790a": 12.4, + "postgres": 17.0, + "redis": 3.8 + }, + "cfs_throttled": 0.223, + "lb_queue_max": 0, + "pg_connections": 28 + } + }, + "fixed-4x0.25": { + "series": "fixed-cpu", + "repeats": 3, + "median": { + "capacity_slo": 160, + "capacity_err1": 200, + "peak": 200 + }, + "values": { + "capacity_slo": [ + 160, + 120, + 160 + ], + "capacity_err1": [ + 200, + 200, + 160 + ], + "peak": [ + 240.6, + 200, + 194.4 + ] + }, + "params": { + "API_REPLICAS": 4, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 280, + "cpu_percent": { + "api": 87.4, + "lb": 6.6, + "loadgen-run-36ba2530fef5": 12.8, + "postgres": 17.5, + "redis": 3.5 + }, + "cfs_throttled": 0.525, + "lb_queue_max": 0, + "pg_connections": 49 + } + }, + "growth-1x0.25": { + "series": "growth", + "repeats": 1, + "median": { + "capacity_slo": 40, + "capacity_err1": 80, + "peak": 83.5 + }, + "values": { + "capacity_slo": [ + 40 + ], + "capacity_err1": [ + 80 + ], + "peak": [ + 83.5 + ] + }, + "params": { + "API_REPLICAS": 1, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 120, + "cpu_percent": { + "api": 25.7, + "lb": 4.1, + "loadgen-run-083085fa5191": 8.4, + "postgres": 7.4, + "redis": 2.6 + }, + "cfs_throttled": 1.0, + "lb_queue_max": 0, + "pg_connections": 13 + } + }, + "growth-2x0.25": { + "series": "growth", + "repeats": 1, + "median": { + "capacity_slo": 80, + "capacity_err1": 159.8, + "peak": 159.8 + }, + "values": { + "capacity_slo": [ + 80 + ], + "capacity_err1": [ + 159.8 + ], + "peak": [ + 159.8 + ] + }, + "params": { + "API_REPLICAS": 2, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 160, + "cpu_percent": { + "api": 42.6, + "lb": 5.0, + "loadgen-run-7ce67b9d067d": 9.4, + "postgres": 11.5, + "redis": 2.6 + }, + "cfs_throttled": 0.918, + "lb_queue_max": 0, + "pg_connections": 24 + } + }, + "growth-3x0.25": { + "series": "growth", + "repeats": 1, + "median": { + "capacity_slo": 80, + "capacity_err1": 160, + "peak": 177.8 + }, + "values": { + "capacity_slo": [ + 80 + ], + "capacity_err1": [ + 160 + ], + "peak": [ + 177.8 + ] + }, + "params": { + "API_REPLICAS": 3, + "API_CPUS": "0.25", + "API_MEMORY": "128M", + "API_CPUSET": "0", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 240, + "cpu_percent": { + "api": 70.6, + "lb": 6.7, + "loadgen-run-69e377d75d2b": 12.0, + "postgres": 13.5, + "redis": 3.5 + }, + "cfs_throttled": 0.912, + "lb_queue_max": 0, + "pg_connections": 35 + } + }, + "machine-2x1.00": { + "series": "machine", + "repeats": 1, + "median": { + "capacity_slo": 320, + "capacity_err1": 320, + "peak": 320 + }, + "values": { + "capacity_slo": [ + 320 + ], + "capacity_err1": [ + 320 + ], + "peak": [ + 320 + ] + }, + "params": { + "API_REPLICAS": 2, + "API_CPUS": "1.0", + "API_MEMORY": "256M", + "API_CPUSET": "0-1", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 320, + "cpu_percent": { + "api": 111.0, + "lb": 7.3, + "loadgen-run-0cfe7ec82689": 17.4, + "postgres": 19.3, + "redis": 3.8 + }, + "cfs_throttled": 0.0, + "lb_queue_max": 0, + "pg_connections": 20 + } + }, + "machine-4x1.00": { + "series": "machine", + "repeats": 1, + "median": { + "capacity_slo": 240, + "capacity_err1": 280, + "peak": 311.5 + }, + "values": { + "capacity_slo": [ + 240 + ], + "capacity_err1": [ + 280 + ], + "peak": [ + 311.5 + ] + }, + "params": { + "API_REPLICAS": 4, + "API_CPUS": "1.0", + "API_MEMORY": "256M", + "API_CPUSET": "0-1", + "DB_POOL_MAX": "10", + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 320, + "cpu_percent": { + "api": 119.8, + "lb": 8.8, + "loadgen-run-7cfbba2a71d5": 17.5, + "postgres": 21.3, + "redis": 4.0 + }, + "cfs_throttled": 0.003, + "lb_queue_max": 0, + "pg_connections": 41 + } + }, + "pool-1x1.00-p20": { + "series": "pool", + "repeats": 1, + "median": { + "capacity_slo": 200, + "capacity_err1": 280, + "peak": 280 + }, + "values": { + "capacity_slo": [ + 200 + ], + "capacity_err1": [ + 280 + ], + "peak": [ + 280 + ] + }, + "params": { + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "API_CPUSET": "0", + "DB_POOL_MAX": 20, + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 280, + "cpu_percent": { + "api": 89.7, + "lb": 6.8, + "loadgen-run-d4289cdc835a": 14.2, + "postgres": 18.6, + "redis": 3.9 + }, + "cfs_throttled": 0.0, + "lb_queue_max": 0, + "pg_connections": 21 + } + }, + "pool-1x1.00-p40": { + "series": "pool", + "repeats": 1, + "median": { + "capacity_slo": 200, + "capacity_err1": 280, + "peak": 280 + }, + "values": { + "capacity_slo": [ + 200 + ], + "capacity_err1": [ + 280 + ], + "peak": [ + 280 + ] + }, + "params": { + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "API_CPUSET": "0", + "DB_POOL_MAX": 40, + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 280, + "cpu_percent": { + "api": 87.5, + "lb": 6.6, + "loadgen-run-dfb98a6b4250": 14.0, + "postgres": 20.0, + "redis": 3.9 + }, + "cfs_throttled": 0.0, + "lb_queue_max": 0, + "pg_connections": 39.4 + } + }, + "pool-1x1.00-p5": { + "series": "pool", + "repeats": 1, + "median": { + "capacity_slo": 280, + "capacity_err1": 320, + "peak": 320 + }, + "values": { + "capacity_slo": [ + 280 + ], + "capacity_err1": [ + 320 + ], + "peak": [ + 320 + ] + }, + "params": { + "API_REPLICAS": 1, + "API_CPUS": "1.0", + "API_MEMORY": "512M", + "API_CPUSET": "0", + "DB_POOL_MAX": 5, + "LB_SERVER_MAXCONN": "0", + "LB_QUEUE_TIMEOUT": "2s" + }, + "at_peak": { + "rate": 320, + "cpu_percent": { + "api": 90.3, + "lb": 6.5, + "loadgen-run-b334fcc6230c": 13.0, + "postgres": 17.7, + "redis": 3.5 + }, + "cfs_throttled": 0.0, + "lb_queue_max": 0, + "pg_connections": 5 + } + } +} diff --git a/evidence/track1/summary.md b/evidence/track1/summary.md new file mode 100644 index 00000000..f3ea1253 --- /dev/null +++ b/evidence/track1/summary.md @@ -0,0 +1,17 @@ +| Конфигурация | Серия | Реплики × CPU, RAM | Пул / LB | Ёмкость при SLO, rps | Макс. при ошибках ≤1 %, rps | Пик полезной, rps | Повторов | +|---|---|---|---|---|---|---|---| +| admission-1x1.00-mc10 | admission | 1×1.0 CPU, 512M | pool 10, maxconn 10, queue 2s | 320/280/240 | 360/359.7/320 | 383.7/359.7/339 | 3 | +| admission-1x1.00-mc10-q250 | admission | 1×1.0 CPU, 512M | pool 10, maxconn 10, queue 250ms | 200 | 239.4 | 303.5 | 1 | +| admission-1x1.00-mc20 | admission | 1×1.0 CPU, 512M | pool 10, maxconn 20, queue 2s | 200 | 320 | 326.3 | 1 | +| admission-2x0.50-mc10 | admission | 2×0.5 CPU, 256M | pool 10, maxconn 10, queue 2s | 200 | 280 | 306.3 | 1 | +| fixed-1x1.00 | fixed-cpu | 1×1.0 CPU, 512M | pool 10, maxconn 0, queue 2s | 280/240/240 | 280/280/280 | 280/316.7/280 | 3 | +| fixed-2x0.50 | fixed-cpu | 2×0.5 CPU, 256M | pool 10, maxconn 0, queue 2s | 120/200/200 | 200/240/200 | 234.7/240/234.3 | 3 | +| fixed-4x0.25 | fixed-cpu | 4×0.25 CPU, 128M | pool 10, maxconn 0, queue 2s | 160/120/160 | 200/200/160 | 240.6/200/194.4 | 3 | +| growth-1x0.25 | growth | 1×0.25 CPU, 128M | pool 10, maxconn 0, queue 2s | 40 | 80 | 83.5 | 1 | +| growth-2x0.25 | growth | 2×0.25 CPU, 128M | pool 10, maxconn 0, queue 2s | 80 | 159.8 | 159.8 | 1 | +| growth-3x0.25 | growth | 3×0.25 CPU, 128M | pool 10, maxconn 0, queue 2s | 80 | 160 | 177.8 | 1 | +| machine-2x1.00 | machine | 2×1.0 CPU, 256M | pool 10, maxconn 0, queue 2s | 320 | 320 | 320 | 1 | +| machine-4x1.00 | machine | 4×1.0 CPU, 256M | pool 10, maxconn 0, queue 2s | 240 | 280 | 311.5 | 1 | +| pool-1x1.00-p20 | pool | 1×1.0 CPU, 512M | pool 20, maxconn 0, queue 2s | 200 | 280 | 280 | 1 | +| pool-1x1.00-p40 | pool | 1×1.0 CPU, 512M | pool 40, maxconn 0, queue 2s | 200 | 280 | 280 | 1 | +| pool-1x1.00-p5 | pool | 1×1.0 CPU, 512M | pool 5, maxconn 0, queue 2s | 280 | 320 | 320 | 1 | diff --git a/infra/haproxy/haproxy.cfg b/infra/haproxy/haproxy.cfg new file mode 100644 index 00000000..3a95b263 --- /dev/null +++ b/infra/haproxy/haproxy.cfg @@ -0,0 +1,87 @@ +# HAProxy 3.0 — единственная публичная точка входа lab3. +# Бэкенды находятся через DNS Docker (имя сервиса `api` → IP всех реплик), без sticky sessions. +# +# Обнаружение отказа реплики: +# * активная проверка GET /health/ready: inter 1s, после первой ошибки fastinter 500ms, fall 2, +# timeout check 300ms → худший случай обнаружения ≈ 1 + 0.3 + 0.5 + 0.3 ≈ 2.1 с; +# * пассивно: соединение отвергнуто → retry-on conn-failure + redispatch на другую реплику +# (только если запрос ещё не отправлен — безопасно и для неидемпотентных POST). +# * graceful shutdown реплики: SIGTERM → /health/ready = 503 на SHUTDOWN_DRAIN_SECONDS=3 с +# (> fall × fastinter) → LB выводит реплику до того, как она закроет listener. + +global + log stdout format raw local0 info + maxconn 4096 + +defaults + mode http + log global + option httplog + option dontlognull + timeout connect 300ms # сеть Docker: connect < 1 мс; к «мёртвому» IP (kill) нет RST — ждём не дольше 300 мс + timeout check 300ms # проверка не висит 1 с на убитой реплике → быстрее fall + timeout client 30s + timeout server 15s # > REQUEST_TIMEOUT_SECONDS (10 с) приложения + timeout http-request 10s + timeout http-keep-alive 10s + timeout queue "${LB_QUEUE_TIMEOUT}" # сколько запрос может ждать свободный слот maxconn в очереди LB + retries 2 + option redispatch 1 + retry-on conn-failure # НЕ empty-response/response-timeout: иначе возможен повтор выполненного POST + unique-id-format %[uuid()] # request_id ответов самого LB: случайный, без IP/портов + default-server inter 1s fastinter 500ms downinter 2s fall 2 rise 2 init-addr none resolvers docker + # Ответы, созданные самим LB, — в формате ошибок API (X-Instance-Id у них нет, это допустимо). + http-error status 503 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Нет доступных реплик API","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] hdr Retry-After 1 + http-error status 502 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Реплика API недоступна","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] + http-error status 504 content-type application/json lf-string '{"error":{"code":"unavailable","message":"Таймаут реплики API","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] + +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 3 + timeout resolve 1s + timeout retry 1s + hold valid 2s + hold obsolete 5s # исчезнувший из DNS контейнер убирается из слота через 5 с + accepted_payload_size 8192 + +frontend api + bind :8080 + # Внутренняя диагностика реплик снаружи недоступна. + http-request return status 404 content-type application/json lf-string '{"error":{"code":"not_found","message":"Объект не найден","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] if { path_beg /internal } + default_backend api + +frontend api_ttl + bind :8081 + http-request return status 404 content-type application/json lf-string '{"error":{"code":"not_found","message":"Объект не найден","request_id":"%[unique-id]"}}' hdr X-Request-Id %[unique-id] if { path_beg /internal } + default_backend api_ttl + +frontend stats + bind :8404 + stats enable + stats uri /stats + stats refresh 2s + stats show-legends + +backend api + balance roundrobin # без stickiness: состояние не хранится в реплике + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + http-reuse safe + # maxconn: не больше LB_SERVER_MAXCONN одновременных запросов на реплику, излишек ждёт в очереди LB + # (0 = без ограничения). Обоснование значения — трек 5.1 в REPORT.md. + server-template api 8 api:8080 check maxconn "${LB_SERVER_MAXCONN}" + # «Panic mode»: если readiness не проходит НИ У ОДНОЙ реплики (например, упала БД), запросы всё + # равно идут в живые процессы, и клиент получает ответ приложения (401 анониму, 503 unavailable + # в формате API, /health/live = 200), а не безликий 503 LB. Мёртвый процесс → conn-failure → 503 LB. + option allbackups + server-template apibk 8 api:8080 backup resolve-opts allow-dup-ip + +backend api_ttl + balance roundrobin + option httpchk + http-check send meth GET uri /health/ready + http-check expect status 200 + server-template ttl 2 api-ttl:8080 check + option allbackups + server-template ttlbk 2 api-ttl:8080 backup resolve-opts allow-dup-ip diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..bf743f6b --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,63 @@ +-- 0001: базовая схема lab2. Источник истины — PostgreSQL. +-- Миграция выполняется раннером в одной транзакции и регистрируется в schema_migrations. + +CREATE TABLE users ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + username text NOT NULL, + display_name text NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_key UNIQUE (username), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_len CHECK (char_length(display_name) BETWEEN 1 AND 100), + CONSTRAINT users_password_hash_kdf CHECK (password_hash LIKE '$scrypt$%') +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + title text NOT NULL, + created_by uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_len CHECK (char_length(title) BETWEEN 1 AND 100) +); + +-- Состав участников неизменен в обязательном API; position сохраняет порядок member_ids из запроса. +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users (id), + position smallint NOT NULL, + PRIMARY KEY (chat_id, user_id), + CONSTRAINT chat_members_position_key UNIQUE (chat_id, position), + CONSTRAINT chat_members_position_range CHECK (position BETWEEN 0 AND 99) +); +-- GET /chats: «чаты пользователя» — поиск по user_id. +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + chat_id uuid NOT NULL, + sender_id uuid NOT NULL, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + -- Отправитель обязан быть участником этого чата: инвариант защищён на уровне БД. + CONSTRAINT messages_sender_is_member FOREIGN KEY (chat_id, sender_id) + REFERENCES chat_members (chat_id, user_id), + CONSTRAINT messages_text_len CHECK (char_length(text) BETWEEN 1 AND 4000) +); +-- История: WHERE chat_id = $1 AND (created_at, id) > ($2, $3) ORDER BY created_at, id LIMIT n. +-- Обоснование и EXPLAIN — REPORT.md и evidence/track2/. +CREATE INDEX messages_chat_created_id_idx ON messages (chat_id, created_at, id); + +-- Серверные сессии. Хранится только SHA-256 от токена (токен — 256 бит случайности), +-- поэтому утечка дампа не даёт готовых Bearer-токенов. +CREATE TABLE sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users (id), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + CONSTRAINT sessions_token_hash_len CHECK (octet_length(token_hash) = 32), + CONSTRAINT sessions_expiry_after_creation CHECK (expires_at > created_at) +); +CREATE INDEX sessions_user_idx ON sessions (user_id); +CREATE INDEX sessions_expires_idx ON sessions (expires_at); diff --git a/migrations/0002_cache_invalidation_outbox.sql b/migrations/0002_cache_invalidation_outbox.sql new file mode 100644 index 00000000..5a6a496f --- /dev/null +++ b/migrations/0002_cache_invalidation_outbox.sql @@ -0,0 +1,8 @@ +-- 0002: транзакционный outbox инвалидации кэша сессий (уровень 4). +-- Строка пишется в той же транзакции, что и revoked_at. Если Redis недоступен в момент logout, +-- фоновый процесс API повторит инвалидацию после восстановления Redis и только затем снова +-- начнёт читать сессии из кэша. +CREATE TABLE session_cache_invalidations ( + token_hash bytea PRIMARY KEY REFERENCES sessions (token_hash) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/migrations/0003_idempotency_keys.sql b/migrations/0003_idempotency_keys.sql new file mode 100644 index 00000000..bd5e0dbd --- /dev/null +++ b/migrations/0003_idempotency_keys.sql @@ -0,0 +1,17 @@ +-- 0003 (lab3): общая идемпотентность POST /chats/{chat_id}/messages. +-- Область ключа = пользователь + метод + путь чата + ключ. Единственный идемпотентный endpoint +-- обязательного API — POST сообщения, поэтому метод и шаблон пути фиксированы, а «путь» = chat_id. +-- Миграция аддитивна (expand): код lab2 её не использует и продолжает работать при rolling update. +CREATE TABLE idempotency_keys ( + user_id uuid NOT NULL REFERENCES users (id), + chat_id uuid NOT NULL REFERENCES chats (id) ON DELETE CASCADE, + key text NOT NULL, + request_hash bytea NOT NULL, + message_id uuid NOT NULL REFERENCES messages (id) ON DELETE CASCADE, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, key), + CONSTRAINT idempotency_key_format CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + CONSTRAINT idempotency_request_hash_len CHECK (octet_length(request_hash) = 32) +); +-- Очистка ключей старше IDEMPOTENCY_TTL_SECONDS. +CREATE INDEX idempotency_keys_created_idx ON idempotency_keys (created_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..11ab1d86 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,12 @@ +# Runtime-зависимости API (точные версии, проверенные на стенде; см. REPORT.md). +starlette==1.7.0 +anyio==4.15.1 +idna==3.20 +typing_extensions==4.16.0 +uvicorn==0.54.0 +h11==0.16.0 +click==8.5.0 +psycopg[binary]==3.2.10 +psycopg-binary==3.2.10 +psycopg-pool==3.2.6 +redis==5.2.1 diff --git a/research/compose.track1.yaml b/research/compose.track1.yaml new file mode 100644 index 00000000..c2243298 --- /dev/null +++ b/research/compose.track1.yaml @@ -0,0 +1,33 @@ +# Трек 1 «Цена масштабирования»: override к compose.yaml. +# docker compose --env-file .env -f compose.yaml -f research/compose.track1.yaml ... +# Изоляция по ядрам: реплики API — только на API_CPUSET (их суммарный бюджет CPU фиксирован +# квотой API_CPUS на реплику), всё остальное (LB, PostgreSQL, Redis, генератор) — на AUX_CPUSET, +# чтобы генератор и БД не отнимали CPU у измеряемого API и наоборот. Значения задаёт +# research/track1.py для каждой конфигурации. +services: + api: + cpuset: "${API_CPUSET:-0}" + environment: + DB_POOL_MAX: ${DB_POOL_MAX:-10} + SESSION_CACHE: ${SESSION_CACHE:-redis} + lb: + cpuset: "${AUX_CPUSET:-1}" + deploy: + resources: + limits: {cpus: "1.0", memory: 256M} + postgres: + cpuset: "${AUX_CPUSET:-1}" + redis: + cpuset: "${AUX_CPUSET:-1}" + migrate: + cpuset: "${AUX_CPUSET:-1}" + loadgen: + build: research/loadgen + image: mtusi-lab3-loadgen:local + profiles: [bench] + networks: [edge] + cpuset: "${AUX_CPUSET:-1}" + read_only: true + deploy: + resources: + limits: {cpus: "1.0", memory: 512M} diff --git a/research/expand_migration_demo.sh b/research/expand_migration_demo.sh new file mode 100644 index 00000000..585cd439 --- /dev/null +++ b/research/expand_migration_demo.sh @@ -0,0 +1,41 @@ +#!/bin/sh +# Демонстрация «expand-only» миграций при rolling update (уровень 4). Шаги: +# 1. новая версия образа содержит дополнительную миграцию 0004 (временная, создаётся этим скриптом); +# 2. контейнер новой версии, запущенный ДО миграции, не становится ready (схема отстаёт) — +# LB не отправил бы на него трафик; +# 3. migrate применяет 0004 один раз; старые реплики (знают 0001–0003) остаются ready: более новая +# схема для них допустима; +# 4. rolling update на новую версию под нагрузкой — research/lb_experiments.py rolling-update. +# Вывод — evidence/lb/expand-migration.txt. После шага 4 удалите временный $MIG (БД сохранит таблицу; +# код без файла 0004 остаётся совместимым — проверяются только известные ему миграции). +set -eu +cd "$(dirname "$0")/.." +. scripts/lib.sh +OUT=evidence/lb/expand-migration.txt +mkdir -p evidence/lb +MIG=migrations/0004_demo_expand.sql +VERSION=${APP_VERSION:-expand-demo} +BUILD_CMD=${BUILD_CMD:-"docker compose --env-file $ENV_FILE build --build-arg APP_VERSION=$VERSION api"} +cat > "$MIG" <<'SQL' +-- ДЕМО (expand): новая необязательная таблица; старый код её не использует. +CREATE TABLE IF NOT EXISTS rollout_demo (id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, note text); +SQL +{ +echo "# $(now) expand-migration demo, new version $VERSION" +APP_VERSION=$VERSION $BUILD_CMD >/dev/null +echo "== 1) new-version container before migrate:" +cid=$(dc run -d --rm --no-deps api serve) +sleep 6 +echo "health/ready inside new container: $(docker exec "$cid" python -m messenger healthcheck && echo 200 || echo NOT-READY)" +docker logs "$cid" 2>&1 | grep -m1 'schema is behind' | sed 's/^/log: /' || true +docker stop -t 5 "$cid" >/dev/null +echo "== 2) migrate (one-shot):" +dc run --rm --no-deps migrate 2>/dev/null | tail -1 +echo "== 3) old replicas after migrate:" +for c in $(dc ps -q api); do + printf '%s ready=%s version=%s\n' "$(docker inspect -f '{{.Name}}' "$c")" \ + "$(docker exec "$c" python -m messenger healthcheck && echo yes || echo no)" \ + "$(docker exec "$c" python -m messenger stats | python3 -c 'import json,sys;print(json.load(sys.stdin)["version"])')" +done +} > "$OUT" 2>&1 +cat "$OUT" diff --git a/research/lb_experiments.py b/research/lb_experiments.py new file mode 100644 index 00000000..71efc010 --- /dev/null +++ b/research/lb_experiments.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +"""Эксперименты уровня 4 (stdlib, Python 3.9+): отказ каждой API-реплики под нагрузкой и rolling update. + + python3 research/lb_experiments.py failover-series [--mode stop|kill|both] + python3 research/lb_experiments.py rolling-update + +Нагрузка: closed-loop, W потоков через LB; каждый поток — свой пользователь, 1 POST сообщения +(Idempotency-Key; при ошибке/таймауте повтор С ТЕМ ЖЕ ключом и телом) на 3 GET истории. +Каждая попытка журналируется (время, операция, статус, X-Instance-Id, задержка). Итог — JSON в +evidence/lb/*.json + текстовая сводка. Токены не сохраняются. +""" +import argparse +import json +import os +import shlex +import statistics +import subprocess +import sys +import threading +import time +import uuid +from urllib.parse import quote +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "student-tests")) +from common import BASE_URL, Client, Failure, login, new_user # noqa: E402 + +OUT = ROOT / "evidence" / "lb" +DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env")) + + +def sh(*args, check=True): + return subprocess.run(list(args), cwd=ROOT, check=check, capture_output=True, text=True).stdout.strip() + + +def api_containers(): + """[(container_id, name, instance_id)] запущенных реплик сервиса api.""" + res = [] + for cid in sh(*DC, "ps", "-q", "api").split(): + name = sh("docker", "inspect", "-f", "{{.Name}}", cid).lstrip("/") + iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False) + res.append((cid, name, iid)) + return sorted(res, key=lambda x: x[1]) + + +def wait_healthy(cid, timeout=60): + t = time.monotonic() + while time.monotonic() - t < timeout: + if sh("docker", "inspect", "-f", "{{.State.Health.Status}}", cid, check=False) == "healthy": + return time.monotonic() - t + time.sleep(0.25) + raise RuntimeError("container not healthy: " + cid) + + +class Load: + def __init__(self, workers, timeout=2.0): + self.workers, self.timeout = workers, timeout + self.events, self.lock, self.stop = [], threading.Lock(), threading.Event() + boot = Client(BASE_URL) + self.actors = [] + for i in range(workers): + a = new_user(boot, "lb%d" % i); login(boot, a) + a["chat"] = boot.call("POST", "/api/v1/chats", 201, actor=a, schema="Chat", + body={"title": "lb", "member_ids": [a["id"]]})["json"] + self.actors.append(a) + self.t0 = None + self.confirmed = {} # message id -> text (подтверждено 201) + + def record(self, op, ok, status, instance, latency): + with self.lock: + self.events.append((time.monotonic() - self.t0, op, ok, status, instance, latency)) + + def worker(self, a): + c = Client(BASE_URL, timeout=self.timeout) + path = "/api/v1/chats/" + a["chat"]["id"] + "/messages" + n, pending = 0, None + while not self.stop.is_set(): + n += 1 + if n % 4 == 1: + if pending is None: + pending = (uuid.uuid4().hex, "lb-%s-%d" % (a["id"][:8], n)) + key, text = pending + op, kw = "post", dict(method="POST", path=path, expected=201, actor=a, body={"text": text}, + headers={"Idempotency-Key": key}, schema="Message") + else: + op, kw = "get", dict(method="GET", path=path + "?limit=20", expected=200, actor=a, schema="MessagePage") + t = time.monotonic() + try: + r = c.call(**kw) + self.record(op, True, r["status"], r["headers"].get("X-Instance-Id"), time.monotonic() - t) + if op == "post": + with self.lock: + self.confirmed[r["json"]["id"]] = r["json"]["text"] + pending = None + except Failure as exc: + code = str(exc).rsplit("received ", 1)[-1] if "received" in str(exc) else "conn" + self.record(op, False, code, None, time.monotonic() - t) + time.sleep(0.05) # pending остаётся: повтор с тем же ключом и телом + + def __enter__(self): + self.t0 = time.monotonic() + self.threads = [threading.Thread(target=self.worker, args=(a,), daemon=True) for a in self.actors] + [t.start() for t in self.threads] + return self + + def __exit__(self, *exc): + self.stop.set() + [t.join(timeout=10) for t in self.threads] + + def now(self): + return time.monotonic() - self.t0 + + def verify(self): + """Все подтверждённые сообщения на месте, дублей текста (неуверенных повторов) нет.""" + c = Client(BASE_URL, timeout=10) + texts, ids = [], set() + for a in self.actors: + path = "/api/v1/chats/" + a["chat"]["id"] + "/messages?limit=100" + cursor = None + while True: + q = path + ("&cursor=" + quote(cursor, safe="") if cursor else "") + page = c.call("GET", q, actor=a, schema="MessagePage")["json"] + for m in page["items"]: + texts.append(m["text"]); ids.add(m["id"]) + cursor = page["next_cursor"] + if not cursor: + break + lost = [i for i in self.confirmed if i not in ids] + return {"confirmed": len(self.confirmed), "lost_confirmed": len(lost), + "stored": len(texts), "duplicate_texts": len(texts) - len(set(texts))} + + +def window(events, a, b): + ev = [e for e in events if a <= e[0] < b] + ok = [e for e in ev if e[2]] + lat = sorted(e[5] for e in ok) + pct = (lambda p: round(1000 * lat[min(len(lat) - 1, int(p * len(lat)))], 1)) if lat else (lambda p: None) + return {"seconds": round(b - a, 2), "ok": len(ok), "errors": len(ev) - len(ok), + "rps_ok": round(len(ok) / (b - a), 1) if b > a else None, + "p50_ms": pct(0.5), "p99_ms": pct(0.99), + "by_instance": dict(sorted({i: sum(1 for e in ok if e[4] == i) for i in {e[4] for e in ok}}.items())), + "error_codes": dict(sorted({s: sum(1 for e in ev if not e[2] and e[3] == s) + for s in {e[3] for e in ev if not e[2]}}.items()))} + + +def failover_series(args): + modes = ["stop", "kill"] if args.mode == "both" else [args.mode] + results = {"measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers, + "client_timeout_s": 2.0, "phases_s": {"before": args.before, "after_action": args.after}, + "runs": []} + for mode in modes: + for idx in range(len(api_containers())): + targets = api_containers() + cid, name, iid = targets[idx] + load = Load(args.workers) + with load: + time.sleep(args.before) + t_action = load.now() + if mode == "stop": + sh("docker", "stop", "-t", "15", cid) # SIGTERM → drain → exit + else: + sh("docker", "kill", "-s", "KILL", cid) # внезапная смерть процесса + t_done = load.now() + time.sleep(args.after) + t_restore = load.now() + sh("docker", "start", cid) + healthy_after = wait_healthy(cid) + new_iid = sh("docker", "exec", cid, "cat", "/tmp/instance-id", check=False) + time.sleep(args.settle) + t_end = load.now() + ev = load.events + errs = [e for e in ev if not e[2] and e[0] >= t_action] + target_seen = [e[0] for e in ev if e[2] and e[4] == iid and e[0] >= t_action] + back_seen = [e[0] for e in ev if e[2] and e[4] == new_iid and e[0] >= t_restore] + posts_ok = sorted(e[0] for e in ev if e[2] and e[1] == "post") + gaps = [b - a for a, b in zip(posts_ok, posts_ok[1:]) if a >= t_action - 1 and b <= t_restore] + run = { + "mode": mode, "container": name, "instance_id": iid, + "action_seconds": round(t_done - t_action, 2), + "before": window(ev, 0, t_action), "during": window(ev, t_action, t_restore), + "after_restore": window(ev, t_restore, t_end), + "errors_after_action": len(errs), + "last_error_after_action_s": round(max(e[0] for e in errs) - t_action, 2) if errs else 0, + "target_last_served_after_action_s": round(max(target_seen) - t_action, 2) if target_seen else 0, + "max_gap_between_successful_writes_s": round(max(gaps), 2) if gaps else None, + "replica_healthy_after_start_s": round(healthy_after, 2), + "restarted_replica_first_served_s": round(min(back_seen) - t_restore, 2) if back_seen else None, + "data": load.verify(), + } + results["runs"].append(run) + print("%-4s %-18s errors=%-3d last_err=%5.2fs target_last=%5.2fs max_write_gap=%ss rps(before/during)=%s/%s data=%s" % ( + mode, name, run["errors_after_action"], run["last_error_after_action_s"], + run["target_last_served_after_action_s"], run["max_gap_between_successful_writes_s"], + run["before"]["rps_ok"], run["during"]["rps_ok"], run["data"]), flush=True) + time.sleep(2) + OUT.mkdir(parents=True, exist_ok=True) + (OUT / "failover-series.json").write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n") + print("saved evidence/lb/failover-series.json") + + +def rolling_update(args): + before = api_containers() + images_before = sorted({sh("docker", "inspect", "-f", "{{.Image}}", b[0])[:19] for b in before}) + load = Load(args.workers) + with load: + time.sleep(args.before) + t_a = load.now() + env = dict(os.environ) + proc = subprocess.run([str(ROOT / "scripts" / "rolling-update.sh")], cwd=ROOT, env=env, + capture_output=True, text=True) + t_b = load.now() + time.sleep(args.after) + t_end = load.now() + after = api_containers() + ev = load.events + res = { + "measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), "workers": args.workers, + "script_exit": proc.returncode, "script_seconds": round(t_b - t_a, 2), + "before_replicas": [{"container": n, "instance_id": i} for _, n, i in before], + "after_replicas": [{"container": n, "instance_id": i} for _, n, i in after], + "images_before": images_before, + "images_after": sorted({sh("docker", "inspect", "-f", "{{.Image}}", a[0])[:19] for a in after}), + "before": window(ev, 0, t_a), "during": window(ev, t_a, t_b), "after": window(ev, t_b, t_end), + "data": load.verify(), + } + OUT.mkdir(parents=True, exist_ok=True) + (OUT / "rolling-update.json").write_text(json.dumps(res, ensure_ascii=False, indent=2) + "\n") + print(json.dumps({k: res[k] for k in ("script_exit", "script_seconds", "before_replicas", "after_replicas", + "during", "data")}, ensure_ascii=False, indent=1)) + if proc.returncode: + print(proc.stderr[-2000:]) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("experiment", choices=["failover-series", "rolling-update"]) + p.add_argument("--mode", choices=["stop", "kill", "both"], default="both") + p.add_argument("--workers", type=int, default=8) + p.add_argument("--before", type=float, default=5) + p.add_argument("--after", type=float, default=10) + p.add_argument("--settle", type=float, default=5) + args = p.parse_args() + {"failover-series": failover_series, "rolling-update": rolling_update}[args.experiment](args) + + +if __name__ == "__main__": + main() diff --git a/research/loadgen/Dockerfile b/research/loadgen/Dockerfile new file mode 100644 index 00000000..cbda5623 --- /dev/null +++ b/research/loadgen/Dockerfile @@ -0,0 +1,10 @@ +# Генератор нагрузки трека 1: только stdlib Go, статический бинарник в scratch. +FROM golang:1.24.7-alpine AS build +WORKDIR /src +COPY go.mod main.go ./ +RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o /loadgen . + +FROM scratch +COPY --from=build /loadgen /loadgen +USER 65534:65534 +ENTRYPOINT ["/loadgen"] diff --git a/research/loadgen/go.mod b/research/loadgen/go.mod new file mode 100644 index 00000000..8a79712c --- /dev/null +++ b/research/loadgen/go.mod @@ -0,0 +1,3 @@ +module mtusi/loadgen + +go 1.22 diff --git a/research/loadgen/main.go b/research/loadgen/main.go new file mode 100644 index 00000000..a0abc6ef --- /dev/null +++ b/research/loadgen/main.go @@ -0,0 +1,274 @@ +// Open-loop HTTP load generator for track 1 (stdlib only, Go 1.22+). +// +// Сценарий: N синтетических пользователей, у каждого свой чат с историей; смесь запросов через LB: +// GET истории (limit=50), POST сообщения (уникальный Idempotency-Key), GET /users/me. +// Модель — открытая: запросы запускаются по расписанию (rate req/s) независимо от ответов; +// задержка считается ОТ ЗАПЛАНИРОВАННОГО момента отправки (без coordinated omission). +// Вывод — JSON-строки в stdout: {"event":"step",...} на каждую ступень rate. Токены не печатаются. +package main + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "math" + "net/http" + "os" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +type actor struct { + token, chat string +} + +var ( + target = flag.String("target", "http://lb:8080", "LB base URL") + nUsers = flag.Int("users", 64, "synthetic users (one chat each)") + seed = flag.Int("seed", 30, "messages per chat before measuring") + rates = flag.String("rates", "100,200,300", "comma-separated request rates (req/s)") + duration = flag.Duration("duration", 15*time.Second, "measured time per step") + warmup = flag.Duration("warmup", 3*time.Second, "unmeasured time per step at the same rate") + mixFlag = flag.String("mix", "history=70,post=20,me=10", "request mix, percent") + timeout = flag.Duration("timeout", 5*time.Second, "client timeout per request") + maxIn = flag.Int("maxinflight", 4000, "in-flight cap; above it a request is counted as dropped") + label = flag.String("label", "", "free-form label copied into output") + stopErr = flag.Float64("stoperr", 0.3, "stop the sweep after 2 consecutive steps with error ratio above this") +) + +var client, setupClient *http.Client + +func rnd(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + return hex.EncodeToString(b) +} + +func do(method, path string, body []byte, hdr map[string]string) (int, []byte, error) { + return doWith(client, method, path, body, hdr) +} + +func doWith(c *http.Client, method, path string, body []byte, hdr map[string]string) (int, []byte, error) { + req, err := http.NewRequest(method, *target+path, bytes.NewReader(body)) + if err != nil { + return 0, nil, err + } + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + for k, v := range hdr { + req.Header.Set(k, v) + } + resp, err := c.Do(req) + if err != nil { + return 0, nil, err + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + return resp.StatusCode, data, err +} + +func must(want int, what, method, path string, body []byte, hdr map[string]string) []byte { + code, data, err := doWith(setupClient, method, path, body, hdr) // KDF медленный: отдельный таймаут + if err != nil || code != want { + fmt.Fprintf(os.Stderr, "setup %s failed: code=%d err=%v\n", what, code, err) + os.Exit(2) + } + return data +} + +func setup() []actor { + actors := make([]actor, *nUsers) + var wg sync.WaitGroup + sem := make(chan struct{}, 4) + for i := range actors { + wg.Add(1) + go func(i int) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + user, pw := "lg_"+rnd(6), "Lg-"+rnd(12) + b, _ := json.Marshal(map[string]string{"username": user, "display_name": "load " + strconv.Itoa(i), "password": pw}) + var u struct{ ID string } + _ = json.Unmarshal(must(201, "user", "POST", "/api/v1/users", b, nil), &u) + basic := "Basic " + base64.StdEncoding.EncodeToString([]byte(user+":"+pw)) + var s struct{ Token string } + _ = json.Unmarshal(must(201, "login", "POST", "/api/v1/auth/sessions", nil, map[string]string{"Authorization": basic}), &s) + auth := map[string]string{"Authorization": "Bearer " + s.Token} + cb, _ := json.Marshal(map[string]any{"title": "load", "member_ids": []string{u.ID}}) + var c struct{ ID string } + _ = json.Unmarshal(must(201, "chat", "POST", "/api/v1/chats", cb, auth), &c) + for k := 0; k < *seed; k++ { + auth["Idempotency-Key"] = rnd(8) + mb, _ := json.Marshal(map[string]string{"text": "seed " + strconv.Itoa(k)}) + must(201, "seed", "POST", "/api/v1/chats/"+c.ID+"/messages", mb, auth) + } + actors[i] = actor{token: s.Token, chat: c.ID} + }(i) + } + wg.Wait() + return actors +} + +type sample struct { + op int + latency time.Duration // от запланированного момента + service time.Duration // от фактической отправки + status int +} + +func pct(v []float64, p float64) float64 { + if len(v) == 0 { + return math.NaN() + } + i := int(math.Ceil(p*float64(len(v)))) - 1 + if i < 0 { + i = 0 + } + return math.Round(v[i]*100) / 100 +} + +func main() { + flag.Parse() + client = &http.Client{Timeout: *timeout, Transport: &http.Transport{ + MaxIdleConns: 10000, MaxIdleConnsPerHost: 10000, IdleConnTimeout: 30 * time.Second, DisableCompression: true, + }} + setupClient = &http.Client{Timeout: 60 * time.Second} + ops := []string{"history", "post", "me"} + weights := map[string]int{} + for _, part := range strings.Split(*mixFlag, ",") { + kv := strings.SplitN(part, "=", 2) + w, _ := strconv.Atoi(kv[1]) + weights[kv[0]] = w + } + var wheel []int + for i, op := range ops { + for k := 0; k < weights[op]; k++ { + wheel = append(wheel, i) + } + } + t0 := time.Now() + actors := setup() + setupSec := time.Since(t0).Seconds() + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"event": "setup", "label": *label, "users": len(actors), "seed": *seed, "seconds": setupSec}) + + bad := 0 + for _, rs := range strings.Split(*rates, ",") { + if bad >= 2 { + break + } + rate, _ := strconv.ParseFloat(strings.TrimSpace(rs), 64) + var mu sync.Mutex + var samples []sample + var inflight, dropped, errs int64 + errKinds := map[string]int{} + total := *warmup + *duration + interval := time.Duration(float64(time.Second) / rate) + start := time.Now().Add(50 * time.Millisecond) + measureFrom := start.Add(*warmup) + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{"event": "step_start", "label": *label, "rate": rate, + "warmup_s": warmup.Seconds(), "duration_s": duration.Seconds(), "ts": float64(start.UnixNano()) / 1e9}) + var wg sync.WaitGroup + for i := 0; ; i++ { + sched := start.Add(time.Duration(i) * interval) + if sched.Sub(start) >= total { + break + } + if d := time.Until(sched); d > 0 { + time.Sleep(d) + } + measured := !sched.Before(measureFrom) + if atomic.LoadInt64(&inflight) >= int64(*maxIn) { + if measured { + atomic.AddInt64(&dropped, 1) + } + continue + } + atomic.AddInt64(&inflight, 1) + wg.Add(1) + a := actors[i%len(actors)] + op := wheel[i%len(wheel)] + go func(a actor, op int, sched time.Time, measured bool) { + defer wg.Done() + defer atomic.AddInt64(&inflight, -1) + hdr := map[string]string{"Authorization": "Bearer " + a.token} + var code int + var err error + sent := time.Now() + switch op { + case 0: + code, _, err = do("GET", "/api/v1/chats/"+a.chat+"/messages?limit=50", nil, hdr) + case 1: + hdr["Idempotency-Key"] = rnd(8) + code, _, err = do("POST", "/api/v1/chats/"+a.chat+"/messages", []byte(`{"text":"load message"}`), hdr) + case 2: + code, _, err = do("GET", "/api/v1/users/me", nil, hdr) + } + done := time.Now() + if !measured { + return + } + want := 200 + if op == 1 { + want = 201 + } + mu.Lock() + defer mu.Unlock() + if err != nil || code != want { + errs++ + k := strconv.Itoa(code) + if err != nil { + k = "transport" + if strings.Contains(err.Error(), "Timeout") || strings.Contains(err.Error(), "deadline") { + k = "timeout" + } + } + errKinds[k]++ + return + } + samples = append(samples, sample{op, done.Sub(sched), done.Sub(sent), code}) + }(a, op, sched, measured) + } + wg.Wait() + lat := make([]float64, 0, len(samples)) + svc := make([]float64, 0, len(samples)) + perOp := map[string][]float64{} + for _, s := range samples { + ms := float64(s.latency.Microseconds()) / 1000 + lat = append(lat, ms) + svc = append(svc, float64(s.service.Microseconds())/1000) + perOp[ops[s.op]] = append(perOp[ops[s.op]], ms) + } + sort.Float64s(lat) + sort.Float64s(svc) + opStats := map[string]any{} + for k, v := range perOp { + sort.Float64s(v) + opStats[k] = map[string]any{"ok": len(v), "p50_ms": pct(v, .5), "p99_ms": pct(v, .99)} + } + attempted := float64(len(samples)) + float64(errs) + float64(dropped) + if (float64(errs)+float64(dropped))/math.Max(attempted, 1) > *stopErr { + bad++ + } else { + bad = 0 + } + _ = json.NewEncoder(os.Stdout).Encode(map[string]any{ + "event": "step", "label": *label, "rate": rate, "duration_s": duration.Seconds(), + "attempted": attempted, "ok": len(samples), "errors": errs, "dropped": dropped, "error_kinds": errKinds, + "throughput_ok_rps": math.Round(float64(len(samples))/duration.Seconds()*10) / 10, + "error_ratio": math.Round((float64(errs)+float64(dropped))/math.Max(attempted, 1)*1e4) / 1e4, + "latency_ms": map[string]float64{"p50": pct(lat, .5), "p90": pct(lat, .9), "p99": pct(lat, .99), "p999": pct(lat, .999), "max": pct(lat, 1)}, + "service_ms": map[string]float64{"p50": pct(svc, .5), "p99": pct(svc, .99)}, + "per_op": opStats, + }) + } +} diff --git a/research/track1.py b/research/track1.py new file mode 100644 index 00000000..71c5708f --- /dev/null +++ b/research/track1.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +"""Трек 1 «Цена масштабирования» (stdlib, Python 3.9+). Оркестратор серий нагрузки. + + python3 research/track1.py run ... # fixed-cpu | growth | machine | admission | all + python3 research/track1.py report # сводка + графики из evidence/track1/*.json + +Для каждой конфигурации: пересоздать реплики с заданными CPU/RAM/числом (compose.yaml + +research/compose.track1.yaml), дождаться, пока LB видит все реплики UP, запустить генератор +(research/loadgen, open-loop, ступени rate) внутри сети стенда и параллельно собирать: +CPU%/RAM контейнеров (docker stats), очередь/сессии HAProxy (stats CSV), состояния соединений +PostgreSQL (pg_stat_activity), счётчики пула psycopg в каждой реплике (до/после серии). +Результат — evidence/track1/.json. Токенов и паролей в выводе нет. +""" +import argparse +import csv +import io +import json +import os +import re +import shlex +import statistics +import subprocess +import sys +import threading +import time +import urllib.request +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +OUT = ROOT / "evidence" / "track1" +BASE_FILES = os.environ.get("COMPOSE_FILE", "compose.yaml:research/compose.track1.yaml") +DC = shlex.split(os.environ.get("COMPOSE", "docker compose --env-file .env")) +STATS_URL = os.environ.get("LB_STATS_URL", "http://127.0.0.1:8404/stats;csv") +SLO_P99_MS, SLO_ERR = 250.0, 0.01 + +# Ступени offered rate (req/s). Генератор останавливает серию после 2 ступеней с >30 % ошибок. +RATES = "40,80,120,160,200,240,280,320,360,400,460,520,600,700,800" +CONFIGS = { + # Серия A: фиксированный суммарный бюджет API = 1 ядро и 512 MiB (KDF-параллелизм 4) делим на 1/2/4. + "fixed-1x1.00": dict(series="fixed-cpu", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4), + "fixed-2x0.50": dict(series="fixed-cpu", API_REPLICAS=2, API_CPUS="0.5", API_MEMORY="256M", KDF_CONCURRENCY=2), + "fixed-4x0.25": dict(series="fixed-cpu", API_REPLICAS=4, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1), + # Серия B: рост ресурсов — добавляем реплики по 0.25 ядра/128 MiB (4x0.25 = fixed-4x0.25). + "growth-1x0.25": dict(series="growth", API_REPLICAS=1, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1), + "growth-2x0.25": dict(series="growth", API_REPLICAS=2, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1), + "growth-3x0.25": dict(series="growth", API_REPLICAS=3, API_CPUS="0.25", API_MEMORY="128M", KDF_CONCURRENCY=1), + # Серия C: снять изоляцию ядер — API делит всю машину (2 vCPU) с LB/БД/генератором. + "machine-2x1.00": dict(series="machine", API_REPLICAS=2, API_CPUS="1.0", API_MEMORY="256M", KDF_CONCURRENCY=2, + API_CPUSET="0-1", AUX_CPUSET="0-1"), + "machine-4x1.00": dict(series="machine", API_REPLICAS=4, API_CPUS="1.0", API_MEMORY="256M", KDF_CONCURRENCY=2, + API_CPUSET="0-1", AUX_CPUSET="0-1"), + # Серия P: размер пула соединений при 1x1.00 (p10 = fixed-1x1.00). + "pool-1x1.00-p5": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=5), + "pool-1x1.00-p20": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=20), + "pool-1x1.00-p40": dict(series="pool", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", KDF_CONCURRENCY=4, DB_POOL_MAX=40), + # Серия D: улучшение — ограничение конкуренции на LB (maxconn на реплику; mc0 = fixed-1x1.00). + "admission-1x1.00-mc10": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", + KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=10), + "admission-1x1.00-mc20": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", + KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=20), + "admission-1x1.00-mc10-q250": dict(series="admission", API_REPLICAS=1, API_CPUS="1.0", API_MEMORY="512M", + KDF_CONCURRENCY=4, LB_SERVER_MAXCONN=10, LB_QUEUE_TIMEOUT="250ms"), + "admission-2x0.50-mc10": dict(series="admission", API_REPLICAS=2, API_CPUS="0.5", API_MEMORY="256M", + KDF_CONCURRENCY=2, LB_SERVER_MAXCONN=10), +} +DEFAULTS = dict(API_CPUSET="0", AUX_CPUSET="1", DB_POOL_MAX="10", SESSION_CACHE="redis", + LB_SERVER_MAXCONN="0", LB_QUEUE_TIMEOUT="2s") + + +def run(args, env=None, check=True, capture=True, timeout=900): + return subprocess.run(args, cwd=ROOT, env=env, check=check, timeout=timeout, + stdout=subprocess.PIPE if capture else None, stderr=subprocess.PIPE, text=True) + + +def cfg_env(cfg): + env = dict(os.environ) + env.setdefault("COMPOSE_FILE", BASE_FILES) + for k, v in {**DEFAULTS, **cfg}.items(): + if k.isupper(): + env[k] = str(v) + return env + + +def lb_rows(): + with urllib.request.urlopen(STATS_URL, timeout=2) as r: + text = r.read().decode().lstrip("# ") + return list(csv.DictReader(io.StringIO(text))) + + +def wait_lb(n, timeout=90): + t = time.monotonic() + while time.monotonic() - t < timeout: + try: + up = [r for r in lb_rows() if r["pxname"] == "api" and r["svname"].startswith("api") + and not r["svname"].startswith("apibk") and r["status"] == "UP"] + if len(up) == n: + return + except OSError: + pass + time.sleep(1) + raise SystemExit("LB does not see %d UP replicas" % n) + + +def pool_stats(env): + res = {} + for cid in run(DC + ["ps", "-q", "api"], env).stdout.split(): + out = run(["docker", "exec", cid, "python", "-m", "messenger", "stats"], check=False).stdout + try: + d = json.loads(out) + res[d["instance_id"]] = {"pool": d["pool"], "cache": d["cache"], "requests": d["app"]["requests"]} + except (ValueError, KeyError): + pass + return res + + +class Sampler: + """docker stats (CPU%, RAM), HAProxy CSV и pg_stat_activity раз в ~1 с с отметками времени.""" + + ANSI = re.compile(r"\x1b\[[0-9;]*[A-Za-z]") + + def __init__(self, env): + self.env, self.stop = env, threading.Event() + self.docker, self.lb, self.pg, self.throttle = [], [], [], [] + + def _docker(self): + p = subprocess.Popen(["docker", "stats", "--format", "{{.Name}}\t{{.CPUPerc}}\t{{.MemUsage}}"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, cwd=ROOT) + try: + for line in p.stdout: + if self.stop.is_set(): + break + line = self.ANSI.sub("", line).strip() + parts = line.split("\t") + if len(parts) == 3 and parts[1].endswith("%") and parts[0].startswith("mtusi-lab3-"): + mem = parts[2].split("/")[0].strip() + mib = float(re.sub(r"[^0-9.]", "", mem) or 0) * (1024 if "GiB" in mem else 1 / 1024 if "KiB" in mem else 1) + self.docker.append((time.time(), parts[0].replace("mtusi-lab3-", ""), float(parts[1][:-1]), mib)) + finally: + p.kill() + + @staticmethod + def _throttle(cid): + for path in ("/sys/fs/cgroup/cpu/docker/%s/cpu.stat" % cid, "/sys/fs/cgroup/system.slice/docker-%s.scope/cpu.stat" % cid): + try: + d = dict(line.split() for line in open(path)) + return int(d.get("nr_periods", 0)), int(d.get("nr_throttled", 0)) + except OSError: + continue + return None + + def _poll(self): + ids = [c for c in run(DC + ["ps", "-q", "--no-trunc", "api"], self.env, check=False).stdout.split()] + while not self.stop.is_set(): + t = time.time() + thr = [self._throttle(c) for c in ids] + if all(thr) and thr: + self.throttle.append((t, sum(x[0] for x in thr), sum(x[1] for x in thr))) + try: + rows = lb_rows() + api = [r for r in rows if r["pxname"] == "api" and r["svname"] not in ("FRONTEND",)] + be = next(r for r in api if r["svname"] == "BACKEND") + self.lb.append((t, int(be["qcur"] or 0), int(be["scur"] or 0), + sum(int(r["qcur"] or 0) for r in api if r["svname"].startswith("api")))) + except (OSError, StopIteration, ValueError): + pass + self.stop.wait(1.0) + + PG_SQL = ("SELECT 'S', extract(epoch FROM now()), coalesce(state, '?'), count(*) FROM pg_stat_activity" + " WHERE datname = 'messenger' AND backend_type = 'client backend' AND pid <> pg_backend_pid() GROUP BY 3" + " UNION ALL SELECT 'W', extract(epoch FROM now()), coalesce(wait_event_type || ':' || wait_event, 'CPU'), count(*)" + " FROM pg_stat_activity WHERE datname = 'messenger' AND state = 'active' AND pid <> pg_backend_pid() GROUP BY 3") + + def _pg(self): + """ОДНА долгоживущая сессия psql с \\watch 1 (не docker exec каждую секунду — он сам грузит хост).""" + p = subprocess.Popen(DC + ["exec", "-T", "postgres", "psql", "-U", "messenger", "-d", "messenger", "-At", "-F", "|"], + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, + cwd=ROOT, env=self.env) + p.stdin.write(self.PG_SQL + " \\watch 1\n"); p.stdin.flush() + snap, cur_ts = {}, None + try: + for line in p.stdout: + if self.stop.is_set(): + break + parts = line.strip().split("|") + if len(parts) != 4: + continue + kind, ts, key, n = parts + ts = float(ts) + if cur_ts is not None and ts != cur_ts: + self.pg.append((cur_ts, snap)); snap = {} + cur_ts = ts + snap[("state:" if kind == "S" else "wait:") + key] = int(n) + finally: + p.kill() + + def __enter__(self): + self.threads = [threading.Thread(target=self._docker, daemon=True), threading.Thread(target=self._poll, daemon=True), + threading.Thread(target=self._pg, daemon=True)] + [t.start() for t in self.threads] + return self + + def __exit__(self, *exc): + self.stop.set() + + def window(self, a, b): + cpu, mem = {}, {} + for t, name, c, m in self.docker: + if a <= t <= b: + key = "api" if re.fullmatch(r"api-\d+", name) else re.sub(r"-\d+$", "", name) + cpu.setdefault((key, name), []).append(c) + mem.setdefault(key, []).append(m) + agg = {} + for (key, name), vals in cpu.items(): + agg.setdefault(key, 0.0) + agg[key] += statistics.mean(vals) + lb = [x for x in self.lb if a <= x[0] <= b] + th = [x for x in self.throttle if a <= x[0] <= b] + throttled = (round((th[-1][2] - th[0][2]) / max(1, th[-1][1] - th[0][1]), 3) if len(th) >= 2 else None) + pg = [x[1] for x in self.pg if a <= x[0] <= b] + return { + "cpu_percent_mean": {k: round(v, 1) for k, v in sorted(agg.items())}, + "mem_mib_max": {k: round(max(v), 1) for k, v in sorted(mem.items())}, + "lb_queue_max": max((x[1] + x[3] for x in lb), default=None), + "lb_sessions_mean": round(statistics.mean(x[2] for x in lb), 1) if lb else None, + "pg_active_mean": round(statistics.mean(p.get("state:active", 0) for p in pg), 2) if pg else None, + "pg_wait_events_mean": ({k[5:]: round(sum(p.get(k, 0) for p in pg) / len(pg), 2) + for k in sorted({k for p in pg for k in p if k.startswith("wait:")})} if pg else None), + "api_cfs_throttled_period_ratio": throttled, + "pg_connections_mean": round(statistics.mean(sum(v for k, v in p.items() if k.startswith("state:")) for p in pg), 1) if pg else None, + } + + +def run_config(name, cfg, args, rep=1): + if rep > 1: + name = "%s-r%d" % (name, rep) + env = cfg_env(cfg) + print("== %s %s" % (name, {k: v for k, v in cfg.items() if k.isupper()}), flush=True) + run(DC + ["up", "-d", "--wait", "--force-recreate", "lb", "api"], env, timeout=300) + wait_lb(int(cfg["API_REPLICAS"])) + time.sleep(3) + pools_before = pool_stats(env) + rates = cfg.get("RATES", args.rates) + cmd = DC + ["--profile", "bench", "run", "--rm", "-T", "loadgen", "-rates", rates, + "-duration", "%ds" % args.duration, "-warmup", "%ds" % args.warmup, + "-users", str(args.users), "-seed", str(args.seed), "-label", name] + steps, starts = [], {} + with Sampler(env) as sampler: + p = subprocess.Popen(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + for line in p.stdout: + try: + ev = json.loads(line) + except ValueError: + continue + if ev["event"] == "step_start": + starts[ev["rate"]] = ev["ts"] + ev["warmup_s"] + elif ev["event"] == "step": + a = starts[ev["rate"]] + ev["resources"] = sampler.window(a, a + ev["duration_s"]) + ev["slo_ok"] = ev["latency_ms"]["p99"] <= SLO_P99_MS and ev["error_ratio"] <= SLO_ERR + steps.append(ev) + print(" rate %5.0f ok %6.1f rps err %5.1f%% p50 %7.1f p99 %8.1f ms cpu %s pgActive %s lbQ %s" % ( + ev["rate"], ev["throughput_ok_rps"], 100 * ev["error_ratio"], ev["latency_ms"]["p50"], + ev["latency_ms"]["p99"], ev["resources"]["cpu_percent_mean"], ev["resources"]["pg_active_mean"], + ev["resources"]["lb_queue_max"]), flush=True) + elif ev["event"] == "setup": + print(" setup %.1fs" % ev["seconds"], flush=True) + p.wait() + if p.returncode: + print(p.stderr.read()[-1500:]) + pools_after = pool_stats(env) + ok = [s for s in steps if s["slo_ok"]] + result = { + "config": name, "repeat": rep, "series": cfg["series"], "params": {**DEFAULTS, **{k: v for k, v in cfg.items() if k.isupper()}}, + "measured_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "slo": {"p99_ms": SLO_P99_MS, "error_ratio": SLO_ERR}, + "capacity_at_slo_rps": max((s["throughput_ok_rps"] for s in ok), default=0), + "peak_ok_rps": max((s["throughput_ok_rps"] for s in steps), default=0), + "steps": steps, "pool_before": pools_before, "pool_after": pools_after, + "loadgen": {"users": args.users, "seed": args.seed, "duration_s": args.duration, "warmup_s": args.warmup, + "mix": "history=70,post=20,me=10", "model": "open-loop, latency from scheduled time"}, + } + OUT.mkdir(parents=True, exist_ok=True) + (OUT / (name + ".json")).write_text(json.dumps(result, ensure_ascii=False, indent=1) + "\n") + print(" => capacity@SLO %.1f rps, peak %.1f rps" % (result["capacity_at_slo_rps"], result["peak_ok_rps"]), flush=True) + + +def report(args): + from track1_report import build # noqa: E402 (рядом в research/) + build(OUT) + + +def main(): + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("action", choices=["run", "report", "list"]) + p.add_argument("targets", nargs="*", default=["all"]) + p.add_argument("--rates", default=RATES) + p.add_argument("--duration", type=int, default=10) + p.add_argument("--warmup", type=int, default=3) + p.add_argument("--users", type=int, default=48) + p.add_argument("--seed", type=int, default=10) + p.add_argument("--rep", type=int, default=1, help="номер повтора (сохраняется как -r.json)") + args = p.parse_args() + if args.action == "list": + for k, v in CONFIGS.items(): + print(k, v) + return + if args.action == "report": + sys.path.insert(0, str(Path(__file__).parent)) + return report(args) + names = [n for n, c in CONFIGS.items() if "all" in args.targets or c["series"] in args.targets or n in args.targets] + for n in names: + run_config(n, CONFIGS[n], args, args.rep) + # вернуть штатную конфигурацию стенда + run(DC + ["up", "-d", "--wait", "--force-recreate", "lb", "api"], cfg_env({"API_REPLICAS": 2}), timeout=300) + + +if __name__ == "__main__": + main() diff --git a/research/track1_report.py b/research/track1_report.py new file mode 100644 index 00000000..60ea162a --- /dev/null +++ b/research/track1_report.py @@ -0,0 +1,208 @@ +"""Сводка и графики трека 1 из evidence/track1/*.json (stdlib; SVG без внешних библиотек). + +Графики: одна ось Y на график, линии 2 px, маркеры ≥ 8 px разной формы (вторичное кодирование), +подписи серий у конца линии + легенда, всплывающие подсказки через , таблица в summary.md. +""" +import json +import statistics +from pathlib import Path + +# Категориальная палитра (validated: dataviz reference palette, slots 1–4, light surface). +SERIES = ["#2a78d6", "#eb6834", "#1baf7a", "#eda100"] +SURFACE, INK, INK2, GRID = "#fcfcfb", "#0b0b0b", "#52514e", "#e4e3df" +SHAPES = ["circle", "square", "triangle", "diamond"] + + +def load(out): + runs = {} + for f in sorted(Path(out).glob("*.json")): + if f.name.startswith("summary"): + continue + d = json.loads(f.read_text()) + base = d["config"].rsplit("-r", 1)[0] if d.get("repeat", 1) > 1 else d["config"] + runs.setdefault(base, []).append(d) + return runs + + +def metrics(d): + steps = d["steps"] + err_ok = [s for s in steps if s["error_ratio"] <= 0.01] + return { + "capacity_slo": d["capacity_at_slo_rps"], + "capacity_err1": max((s["throughput_ok_rps"] for s in err_ok), default=0), + "peak": d["peak_ok_rps"], + } + + +def marker(shape, x, y, color, tip): + t = "<title>%s" % tip + if shape == "circle": + return '%s' % (x, y, color, SURFACE, t) + if shape == "square": + return '%s' % (x - 4.5, y - 4.5, color, SURFACE, t) + if shape == "triangle": + return '%s' % ( + x, y - 5.5, x + 5.5, y + 4.5, x - 5.5, y + 4.5, color, SURFACE, t) + return '%s' % ( + x, y - 6, x + 6, y, x, y + 6, x - 6, y, color, SURFACE, t) + + +def line_chart(path, title, subtitle, series, xlabel, ylabel, ymax=None, diagonal=False, refline=None): + """series: [(label, [(x, y, tip)])].""" + W, H, L, R, T, B = 760, 400, 64, 190, 64, 52 + xs = [p[0] for _, pts in series for p in pts] + ys = [p[1] for _, pts in series for p in pts] + xmax = max(xs) * 1.05 + ymax = ymax or max(ys) * 1.12 + sx = lambda v: L + (W - L - R) * v / xmax # noqa: E731 + sy = lambda v: H - B - (H - T - B) * v / ymax # noqa: E731 + o = ['' % (W, H), + '' % SURFACE, + '%s' % (L, INK, title), + '%s' % (L, INK2, subtitle)] + step = 10 ** len(str(int(ymax / 5))) / 10 + step = next(s for s in (step, step * 2, step * 2.5, step * 5, step * 10) if ymax / s <= 6) + v = 0 + while v <= ymax: + o.append('' % (L, W - R, sy(v), sy(v), GRID)) + o.append('%g' % (L - 8, sy(v) + 4, INK2, v)) + v += step + xstep = next(s for s in (50, 100, 200, 250, 500) if xmax / s <= 9) + v = 0 + while v <= xmax: + o.append('%g' % (sx(v), H - B + 18, INK2, v)) + v += xstep + o.append('' % (L, W - R, H - B, H - B, INK2)) + o.append('%s' % ((L + W - R) / 2, H - 12, INK2, xlabel)) + o.append('%s' % ( + (T + H - B) / 2, INK2, (T + H - B) / 2, ylabel)) + if refline: + val, text = refline + o.append('' % (L, W - R, sy(val), sy(val), INK2)) + o.append('%s' % (W - R - 4, sy(val) - 5, INK2, text)) + if diagonal: + m = min(xmax, ymax) + o.append('' % ( + sx(0), sy(0), sx(m), sy(m), INK2)) + o.append('пунктир: полезная = поданная' % (sx(m * 0.92) - 8, sy(m * 0.92), INK2)) + for i, (label, pts) in enumerate(series): + c = SERIES[i % len(SERIES)] + pts = sorted(pts) + o.append('' % ( + c, " ".join("%.1f,%.1f" % (sx(x), sy(y)) for x, y, _ in pts))) + for x, y, tip in pts: + o.append(marker(SHAPES[i % 4], sx(x), sy(y), c, "%s — %s" % (label, tip))) + # легенда сверху справа + lx = W - R + 10 + for i, (label, _) in enumerate(series): + y = T + 4 + i * 18 + o.append(marker(SHAPES[i % 4], lx + 6, y, SERIES[i % len(SERIES)], label)) + o.append('%s' % (lx + 16, y + 4, INK2, label)) + o.append("") + Path(path).write_text("\n".join(o) + "\n") + + +def bar_chart(path, title, subtitle, groups, ylabel): + """groups: [(label, value, tip)] — одна серия, одна ось.""" + W, H, L, R, T, B = 760, 360, 64, 24, 64, 70 + ymax = max(v for _, v, _ in groups) * 1.18 + n = len(groups) + bw = (W - L - R) / n + sy = lambda v: H - B - (H - T - B) * v / ymax # noqa: E731 + o = ['' % (W, H), + '' % SURFACE, + '%s' % (L, INK, title), + '%s' % (L, INK2, subtitle)] + step = next(s for s in (25, 50, 100, 200) if ymax / s <= 6) + v = 0 + while v <= ymax: + o.append('' % (L, W - R, sy(v), sy(v), GRID)) + o.append('%g' % (L - 8, sy(v) + 4, INK2, v)) + v += step + for i, (label, val, tip) in enumerate(groups): + x = L + i * bw + bw * 0.22 + w = bw * 0.56 + y = sy(val) + h = H - B - y + r = min(4, w / 2) + o.append('%s' % ( + x, H - B, y + r, x, y, x + r, y, x + w - r, x + w, y, x + w, y + r, H - B, SERIES[0], tip)) + o.append('%g' % (x + w / 2, y - 6, INK, val)) + for k, part in enumerate(label.split("\n")): + o.append('%s' % ( + x + w / 2, H - B + 16 + 13 * k, INK2, part)) + o.append('' % (L, W - R, H - B, H - B, INK2)) + o.append('%s' % ( + (T + H - B) / 2, INK2, (T + H - B) / 2, ylabel)) + o.append("") + Path(path).write_text("\n".join(o) + "\n") + + +def build(out): + out = Path(out) + runs = load(out) + table, summary = [], {} + for name, ds in sorted(runs.items()): + ms = [metrics(d) for d in ds] + agg = {k: statistics.median(m[k] for m in ms) for k in ms[0]} + spread = {k: [m[k] for m in ms] for k in ms[0]} + p = ds[0]["params"] + sat = max(ds[0]["steps"], key=lambda s: s["throughput_ok_rps"]) + res = sat["resources"] + summary[name] = {"series": ds[0]["series"], "repeats": len(ds), "median": agg, "values": spread, + "params": {k: p[k] for k in ("API_REPLICAS", "API_CPUS", "API_MEMORY", "API_CPUSET", "DB_POOL_MAX", + "LB_SERVER_MAXCONN", "LB_QUEUE_TIMEOUT") if k in p}, + "at_peak": {"rate": sat["rate"], "cpu_percent": res["cpu_percent_mean"], + "cfs_throttled": res.get("api_cfs_throttled_period_ratio"), + "lb_queue_max": res["lb_queue_max"], "pg_connections": res["pg_connections_mean"]}} + table.append("| %s | %s | %s×%s CPU, %s | pool %s, maxconn %s, queue %s | %s | %s | %s | %s |" % ( + name, ds[0]["series"], p["API_REPLICAS"], p["API_CPUS"], p["API_MEMORY"], p.get("DB_POOL_MAX"), + p.get("LB_SERVER_MAXCONN"), p.get("LB_QUEUE_TIMEOUT"), + "/".join("%g" % v for v in spread["capacity_slo"]), "/".join("%g" % v for v in spread["capacity_err1"]), + "/".join("%g" % v for v in spread["peak"]), len(ds))) + (out / "summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=1) + "\n") + head = ("| Конфигурация | Серия | Реплики × CPU, RAM | Пул / LB | Ёмкость при SLO, rps | Макс. при ошибках ≤1 %, rps | " + "Пик полезной, rps | Повторов |\n|---|---|---|---|---|---|---|---|") + (out / "summary.md").write_text(head + "\n" + "\n".join(table) + "\n") + + def curve(name, field): + """Медиана по повторам для каждой ступени rate (повторы могут остановиться на разных ступенях).""" + by_rate = {} + for d in runs[name]: + for s in d["steps"]: + by_rate.setdefault(s["rate"], []).append(s) + pts = [] + for rate, ss in sorted(by_rate.items()): + ok = statistics.median(s["throughput_ok_rps"] for s in ss) + p99 = statistics.median(s["latency_ms"]["p99"] for s in ss) + err = statistics.median(s["error_ratio"] for s in ss) + pts.append((rate, ok if field == "ok" else p99, "%g req/s: полезная %.0f rps, ошибок %.1f %%, p99 %.0f мс (медиана %d прогонов)" % ( + rate, ok, 100 * err, p99, len(ss)))) + return pts + + fixed = [("1 × 1.0 CPU", "fixed-1x1.00"), ("2 × 0.5 CPU", "fixed-2x0.50"), ("4 × 0.25 CPU", "fixed-4x0.25")] + fixed = [f for f in fixed if f[1] in runs] + if fixed: + line_chart(out / "fixed-cpu-goodput.svg", "Фиксированный бюджет API (1 CPU, 512 MiB): дробление на реплики", + "Полезная пропускная способность (2xx), медиана 3 прогонов; open-loop, смесь 70/20/10", + [(l, curve(n, "ok")) for l, n in fixed], "Поданная нагрузка, req/s", "Полезная, rps", diagonal=True) + line_chart(out / "fixed-cpu-p99.svg", "Фиксированный бюджет API: хвост задержки", + "p99 успешных ответов от запланированного момента, медиана 3 прогонов", + [(l, [(x, min(y, 2500), t) for x, y, t in curve(n, "p99")]) for l, n in fixed], + "Поданная нагрузка, req/s", "p99, мс (обрезано на 2500)", ymax=2600, refline=(250, "SLO 250 мс")) + adm = [("без лимита (maxconn 0)", "fixed-1x1.00"), ("maxconn 10", "admission-1x1.00-mc10"), + ("maxconn 20", "admission-1x1.00-mc20"), ("maxconn 10, queue 250 мс", "admission-1x1.00-mc10-q250")] + adm = [a for a in adm if a[1] in runs] + if len(adm) > 1: + line_chart(out / "admission-goodput.svg", "maxconn на LB против перегрузки (1 × 1.0 CPU)", + "Полезная пропускная способность; пул 10; без лимита и maxconn 10 — медиана 3 прогонов", + [(l, curve(n, "ok")) for l, n in adm], "Поданная нагрузка, req/s", "Полезная, rps", diagonal=True) + bars = [("growth-1x0.25", "1×0.25\n(0.25 CPU)"), ("growth-2x0.25", "2×0.25\n(0.5 CPU)"), ("growth-3x0.25", "3×0.25\n(0.75 CPU)"), + ("fixed-4x0.25", "4×0.25\n(1 CPU)"), ("fixed-1x1.00", "1×1.0\n(1 CPU)"), ("machine-2x1.00", "2×1.0\n(2 vCPU хоста)"), + ("machine-4x1.00", "4×1.0\n(2 vCPU хоста)")] + bars = [(summary[n]["median"]["peak"], l, n) for n, l in bars if n in summary] + if bars: + bar_chart(out / "growth-peak.svg", "Рост ресурсов: пик полезной пропускной способности", + "Ступени 0.25 CPU на изолированном ядре, затем вся машина (2 vCPU, общие с БД/LB/генератором)", + [(l, round(v), "%s: пик %.0f rps" % (n, v)) for v, l, n in bars], "Пик полезной, rps") + print((out / "summary.md").read_text()) diff --git a/scripts/collect-evidence.ps1 b/scripts/collect-evidence.ps1 new file mode 100644 index 00000000..bcbe6bf3 --- /dev/null +++ b/scripts/collect-evidence.ps1 @@ -0,0 +1,95 @@ +# Windows PowerShell 5.1+ equivalent of scripts/collect-evidence.sh (lab3). +# Starts the Compose stand (LB + API replicas + migrate + PostgreSQL + Redis), runs every check and writes +# outputs to evidence\ (no tokens/passwords). Stops/restarts containers during failover/restart tests. +# powershell -NoProfile -ExecutionPolicy Bypass -File scripts\collect-evidence.ps1 [-WithTrack1] [-SkipFailures] +param([switch]$WithTrack1, [switch]$SkipFailures) +$ErrorActionPreference = 'Continue' +$root = Split-Path -Parent $PSScriptRoot +Set-Location $root +$env:PYTHONUTF8 = '1' +$env:PYTHONIOENCODING = 'utf-8' +# IPv4 explicitly: on Windows 'localhost' resolves to ::1 first; the LB listens on 127.0.0.1 only. +$env:NO_PROXY = '127.0.0.1,localhost' +$B = 'http://127.0.0.1:8080' +$env:BASE_URL = $B +$env:TTL_BASE_URL = 'http://127.0.0.1:8081' +$ev = Join-Path $root 'evidence' +New-Item -ItemType Directory -Force -Path $ev | Out-Null + +# Full path of the Python APPLICATION: a bare 'py' would resolve to a PowerShell function first. +$pyApp = Get-Command py -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1 +$pythonApp = Get-Command python -CommandType Application -ErrorAction SilentlyContinue | Select-Object -First 1 +if ($pyApp) { $script:PyExe = $pyApp.Source; $script:PyPre = @('-3') } +elseif ($pythonApp) { $script:PyExe = $pythonApp.Source; $script:PyPre = @() } +else { throw 'Python 3.9+ not found (install from python.org, enable "Add to PATH")' } +function Invoke-Python { & $script:PyExe @script:PyPre @args } +function Dc { & docker compose --env-file .env @args } +$hookStop = if ($IsLinux -or $IsMacOS) { 'scripts/stop-one.sh' } else { 'scripts\stop-one.cmd' } +$hookRestart = if ($IsLinux -or $IsMacOS) { 'scripts/restart.sh' } else { 'scripts\restart.cmd' } +$startReplicas = Join-Path $PSScriptRoot 'start-replicas.ps1' + +function Get-Stamp { + $commit = 'n/a' + if (Get-Command git -ErrorAction SilentlyContinue) { + $c = (& git rev-parse --short HEAD 2>$null) + if ($LASTEXITCODE -eq 0 -and $c) { $commit = $c } + } + '# ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' commit=' + $commit + ' host=' + [Environment]::OSVersion.VersionString + ' cpus=' + [Environment]::ProcessorCount +} +function Invoke-Step([string]$name, [string]$title, [scriptblock]$block) { + Write-Host ('>>> ' + $name) -ForegroundColor Cyan + $global:LASTEXITCODE = 0 + $lines = @(& $block 2>&1 | ForEach-Object { "$_" }) + $code = $global:LASTEXITCODE + Set-Content -Path (Join-Path $ev ($name + '.txt')) -Encoding UTF8 -Value (@((Get-Stamp), ('$ ' + $title)) + $lines + @('exit=' + $code)) + $lines | Select-Object -Last 2 | ForEach-Object { Write-Host (' ' + $_) } + if ($code -ne 0) { Write-Host (' exit=' + $code + ' (see evidence\' + $name + '.txt)') -ForegroundColor Yellow } +} + +& docker version --format '{{.Server.Version}}' *> $null +if ($LASTEXITCODE -ne 0) { throw 'Docker is not running: start Docker Desktop first' } +& (Join-Path $PSScriptRoot 'init-secrets.ps1') +Set-Location $root +Write-Host '>>> docker compose up (build, wait for healthchecks)' -ForegroundColor Cyan +Dc --profile ttl up -d --build --wait +if ($LASTEXITCODE -ne 0) { Dc logs --tail=80; throw 'docker compose up failed' } + +Invoke-Step 'stack' 'docker version; compose images; ps' { & docker version --format 'docker {{.Server.Version}}'; Dc version; Dc --profile ttl images; Dc --profile ttl ps -a } +Invoke-Step 'instances' 'X-Instance-Id of 6 requests via LB' { + 1..6 | ForEach-Object { (Invoke-WebRequest -UseBasicParsing "$B/health/live").Headers['X-Instance-Id'] } +} +Invoke-Step 'make-check' 'python scripts/check_contract.py' { Invoke-Python scripts/check_contract.py } +Invoke-Step 'smoke' "python tests/smoke.py --base-url $B" { Invoke-Python tests/smoke.py --base-url $B } +Invoke-Step 'test-api' 'python student-tests/test_api.py' { Invoke-Python student-tests/test_api.py } +Invoke-Step 'test-idempotency' 'python student-tests/test_idempotency.py' { Invoke-Python student-tests/test_idempotency.py } +Invoke-Step 'test-expiry' 'python student-tests/test_expiry.py # api-ttl via LB :8081' { Invoke-Python student-tests/test_expiry.py } +Invoke-Step 'test-cache-race' 'docker compose run api: python student-tests/test_cache_race.py' { + Dc run --rm --no-deps -T -v ((Join-Path $root 'student-tests') + ':/app/student-tests:ro') --entrypoint python api student-tests/test_cache_race.py +} +Set-Content -Path (Join-Path $ev 'failover-hook.txt') -Encoding UTF8 -Value @() +foreach ($i in 1, 2) { + Invoke-Step ('failover-' + $i) ('python tests/resilience.py --mode failover --action-script ' + $hookStop) { + Invoke-Python tests/resilience.py --mode failover --action-script $hookStop --base-url $B + } + Invoke-Step ('restore-' + $i) 'scripts\start-replicas.ps1' { & $startReplicas } +} +Set-Content -Path (Join-Path $ev 'restart-hook.txt') -Encoding UTF8 -Value @() +Invoke-Step 'persistence' ('python tests/resilience.py --mode persistence --action-script ' + $hookRestart) { + Invoke-Python tests/resilience.py --mode persistence --action-script $hookRestart --base-url $B +} +if (-not $SkipFailures) { + Invoke-Step 'test-failures' 'python student-tests/test_failures.py # stops/starts redis and postgres' { Invoke-Python student-tests/test_failures.py } +} +Invoke-Step 'migrations-rerun' 'docker compose run --rm --no-deps migrate' { Dc run --rm --no-deps migrate } +Invoke-Step 'lb-failover-series' 'python research/lb_experiments.py failover-series --mode both' { Invoke-Python research/lb_experiments.py failover-series --mode both } +Write-Host 'NOTE: rolling update uses scripts/rolling-update.sh (POSIX sh). On Windows run it from Git Bash/WSL.' -ForegroundColor Yellow +Invoke-Step 'load' 'python tests/load.py --duration 30 --concurrency 4' { + Invoke-Python tests/load.py --base-url $B --duration 30 --concurrency 4 --output evidence/load.json +} +if ($WithTrack1) { + $env:COMPOSE_FILE = 'compose.yaml;research/compose.track1.yaml' + Invoke-Step 'track1' 'python research/track1.py run all' { Dc --profile bench build loadgen; Invoke-Python research/track1.py run all } + Remove-Item Env:COMPOSE_FILE + Invoke-Step 'track1-report' 'python research/track1.py report' { Invoke-Python research/track1.py report } +} +Write-Host 'Done: evidence collected in evidence\' -ForegroundColor Green diff --git a/scripts/collect-evidence.sh b/scripts/collect-evidence.sh new file mode 100644 index 00000000..438b210e --- /dev/null +++ b/scripts/collect-evidence.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Полный прогон доказательств lab3 на Compose-стенде; вывод — evidence/ (без токенов и паролей). +# Останавливает/перезапускает контейнеры стенда (failover, restart, отказы БД/Redis, rolling update). +# Трек 5.1 (≈1 ч) запускается отдельно: make track1 (или SKIP_TRACK1=0 ./scripts/collect-evidence.sh). +set -eu +cd "$(dirname "$0")/.." +. scripts/lib.sh +PY=${PYTHON:-python3} +B=${BASE_URL:-http://127.0.0.1:8080} +export BASE_URL=$B TTL_BASE_URL=${TTL_BASE_URL:-http://127.0.0.1:8081} +mkdir -p evidence +stamp() { echo "# $(now) commit=$(git rev-parse --short HEAD 2>/dev/null || echo n/a) host=$(uname -srm) cpus=$(getconf _NPROCESSORS_ONLN)"; } +run() { name=$1; shift; { stamp; echo "\$ $*"; "$@" 2>&1; echo "exit=$?"; } > "evidence/$name.txt" || true; tail -2 "evidence/$name.txt"; } + +./scripts/init-secrets.sh +dc --profile ttl up -d --build --wait +{ stamp; docker version --format 'docker {{.Server.Version}}'; dc version; dc --profile ttl images; dc --profile ttl ps; state; } > evidence/stack.txt 2>&1 +run instances sh -c "for i in 1 2 3 4 5 6; do curl -s -o /dev/null -D - $B/health/live | grep -i x-instance-id; done" +run make-check make check +run smoke $PY tests/smoke.py --base-url "$B" +run test-api $PY student-tests/test_api.py +run test-idempotency $PY student-tests/test_idempotency.py +run test-expiry $PY student-tests/test_expiry.py +run test-cache-race make test-cache-race +: > evidence/failover-hook.txt +run failover-1 $PY tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$B" +run restore-1 ./scripts/start-replicas.sh +run failover-2 $PY tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url "$B" +run restore-2 ./scripts/start-replicas.sh +: > evidence/restart-hook.txt +run persistence $PY tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url "$B" +run test-failures $PY student-tests/test_failures.py +run migrations-rerun dc run --rm --no-deps migrate +run lb-failover-series $PY research/lb_experiments.py failover-series --mode both +: > evidence/rolling-update.txt +run lb-rolling-update $PY research/lb_experiments.py rolling-update +run load $PY tests/load.py --base-url "$B" --duration 30 --concurrency 4 --output evidence/load.json +if [ "${SKIP_TRACK1:-1}" = 0 ]; then + run track1 env COMPOSE_FILE=compose.yaml:research/compose.track1.yaml $PY research/track1.py run all + run track1-report $PY research/track1.py report +fi +echo "evidence collected in evidence/" diff --git a/scripts/init-secrets.ps1 b/scripts/init-secrets.ps1 new file mode 100644 index 00000000..639d03c2 --- /dev/null +++ b/scripts/init-secrets.ps1 @@ -0,0 +1,36 @@ +# Windows PowerShell 5.1+ version of scripts/init-secrets.sh. +# Creates local synthetic secrets in secrets\ and .env (both git-ignored). +# Idempotent: existing files are NOT overwritten. Values are never printed. +$ErrorActionPreference = 'Stop' +$root = Split-Path -Parent $PSScriptRoot +Set-Location $root +New-Item -ItemType Directory -Force -Path (Join-Path $root 'secrets'), (Join-Path $root 'backups') | Out-Null +$ascii = New-Object System.Text.ASCIIEncoding + +function New-Hex { + $bytes = New-Object byte[] 24 + $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() + $rng.GetBytes($bytes) + ($bytes | ForEach-Object { $_.ToString('x2') }) -join '' +} +function Set-SecretFile([string]$name, [string]$value) { + $path = Join-Path (Join-Path $root 'secrets') $name + if (-not (Test-Path $path)) { + # No BOM, no trailing CR/LF: the value is read verbatim inside Linux containers. + [System.IO.File]::WriteAllText($path, $value, $ascii) + } +} +function Get-SecretFile([string]$name) { + ([System.IO.File]::ReadAllText((Join-Path (Join-Path $root 'secrets') $name))).Trim() +} + +Set-SecretFile 'postgres_password' (New-Hex) +Set-SecretFile 'redis_password' (New-Hex) +$pg = Get-SecretFile 'postgres_password' +$rd = Get-SecretFile 'redis_password' +Set-SecretFile 'database_url' ("postgresql://messenger:" + $pg + "@postgres:5432/messenger") +Set-SecretFile 'cache_url' ("redis://:" + $rd + "@redis:6379/0") +if (-not (Test-Path (Join-Path $root '.env'))) { + Copy-Item (Join-Path $root '.env.example') (Join-Path $root '.env') +} +Write-Output 'secrets\ and .env are ready (values not shown).' diff --git a/scripts/init-secrets.sh b/scripts/init-secrets.sh new file mode 100644 index 00000000..b3333ee8 --- /dev/null +++ b/scripts/init-secrets.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Создаёт локальные синтетические secrets для стенда (каталог secrets/ в .gitignore). +# Идемпотентно: существующие файлы НЕ перезаписываются (иначе пароль разойдётся с уже +# инициализированным volume PostgreSQL). Значения не печатаются. +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets backups +chmod 700 secrets +gen() { python3 -c 'import secrets; print(secrets.token_hex(24))'; } +if [ ! -f secrets/postgres_password ]; then gen > secrets/postgres_password; fi +if [ ! -f secrets/redis_password ]; then gen > secrets/redis_password; fi +PG=$(cat secrets/postgres_password) +RD=$(cat secrets/redis_password) +[ -f secrets/database_url ] || printf 'postgresql://messenger:%s@postgres:5432/messenger\n' "$PG" > secrets/database_url +[ -f secrets/cache_url ] || printf 'redis://:%s@redis:6379/0\n' "$RD" > secrets/cache_url +# Файлы монтируются в контейнеры как /run/secrets/*; процесс API работает под uid 10001, +# поэтому файл должен быть читаем внутри контейнера. Каталог secrets/ закрыт (700) на хосте. +chmod 644 secrets/* +[ -f .env ] || cp .env.example .env +echo "secrets/ and .env are ready (values not shown)." diff --git a/scripts/lib.sh b/scripts/lib.sh new file mode 100644 index 00000000..0d3526d8 --- /dev/null +++ b/scripts/lib.sh @@ -0,0 +1,19 @@ +# Общие функции хуков (POSIX sh). Подключается: . "$(dirname "$0")/lib.sh" +cd "$(dirname "$0")/.." +ENV_FILE=${ENV_FILE:-.env} +dc() { docker compose --env-file "$ENV_FILE" "$@"; } +now() { date -u +%FT%TZ; } +# Состояние всех контейнеров проекта: сервис, имя, статус, StartedAt, PID, alias реплики API. +state() { + for cid in $(dc ps -a -q); do + docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Name}} status={{.State.Status}} started={{.State.StartedAt}} pid={{.State.Pid}}' "$cid" | + sed 's# /# #' + done | sort +} +# Контейнер API-реплики по её X-Instance-Id (alias пишется репликой в /tmp/instance-id). +container_of() { + for cid in $(dc ps -q api); do + if [ "$(docker exec "$cid" cat /tmp/instance-id 2>/dev/null)" = "$1" ]; then echo "$cid"; return 0; fi + done + return 1 +} diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..8226b992 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper: tests\resilience.py starts the hook directly; .cmd runs without a shell. +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0restart.ps1" +exit /b %ERRORLEVEL% diff --git a/scripts/restart.ps1 b/scripts/restart.ps1 new file mode 100644 index 00000000..7b45e311 --- /dev/null +++ b/scripts/restart.ps1 @@ -0,0 +1,31 @@ +# Windows PowerShell version of scripts/restart.sh (persistence hook): restart LB, ALL API replicas, +# PostgreSQL and Redis keeping volumes. State before/after -> evidence\restart-hook.txt +$ErrorActionPreference = 'Continue' +$root = Split-Path -Parent $PSScriptRoot +Set-Location $root +$envFile = if ($env:ENV_FILE) { $env:ENV_FILE } else { '.env' } +$log = Join-Path $root 'evidence\restart-hook.txt' +function Dc { & docker compose --env-file $envFile @args 2>$null | Out-Null; if ($LASTEXITCODE -ne 0) { throw ('docker compose ' + ($args -join ' ') + ' failed') } } +function Get-State { + foreach ($cid in (& docker compose --env-file $envFile ps -a -q)) { + & docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Name}} status={{.State.Status}} started={{.State.StartedAt}} pid={{.State.Pid}}' $cid + } + & docker volume ls --format 'volume {{.Name}}' | Where-Object { $_ -match 'pgdata' } +} +try { + Add-Content -Path $log -Encoding UTF8 -Value ('== ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' before restart') + Add-Content -Path $log -Encoding UTF8 -Value (Get-State) + $ttl = @(& docker compose --env-file $envFile ps -q api-ttl 2>$null).Count -gt 0 + Dc stop lb api + if ($ttl) { Dc stop api-ttl } + Dc restart postgres redis + Dc start api + if ($ttl) { Dc start api-ttl } + Dc start lb + Add-Content -Path $log -Encoding UTF8 -Value ('== ' + (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') + ' after restart') + Add-Content -Path $log -Encoding UTF8 -Value (Get-State) + exit 0 +} catch { + [Console]::Error.WriteLine($_.Exception.Message) + exit 1 +} diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..b6f086ed --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Persistence hook: перезапускает LB, ВСЕ API-реплики и все хранилища (PostgreSQL, Redis), +# сохраняя volumes. Никаких down -v / FLUSH / повторного seed. Миграции повторно не нужны. +# Состояние контейнеров до/после (StartedAt, PID) — в evidence/restart-hook.txt. +set -eu +. "$(dirname "$0")/lib.sh" +LOG=${RESTART_EVIDENCE:-evidence/restart-hook.txt} +{ echo "== $(now) before restart"; state; docker volume ls --format 'volume {{.Name}}' | grep pgdata; } >> "$LOG" +running_ttl=$(dc ps -q api-ttl 2>/dev/null || true) +dc stop lb api >/dev/null 2>&1 +[ -z "$running_ttl" ] || dc stop api-ttl >/dev/null 2>&1 +dc restart postgres redis >/dev/null 2>&1 +dc start api >/dev/null 2>&1 +[ -z "$running_ttl" ] || dc start api-ttl >/dev/null 2>&1 +dc start lb >/dev/null 2>&1 +{ echo "== $(now) after restart"; state; docker volume ls --format 'volume {{.Name}}' | grep pgdata; } >> "$LOG" diff --git a/scripts/rolling-update.sh b/scripts/rolling-update.sh new file mode 100644 index 00000000..62c228e6 --- /dev/null +++ b/scripts/rolling-update.sh @@ -0,0 +1,35 @@ +#!/bin/sh +# Rolling update API-реплик без остановки сервиса (Compose). +# 1. собрать новый образ (старые контейнеры продолжают работать на старом image id); +# 2. применить миграции ОДИН раз (one-shot migrate). Миграции обязаны быть expand-only +# (совместимы с работающей версией), см. REPORT.md; +# 3. для каждой старой реплики: поднять одну новую (--scale N+1 --no-recreate), дождаться +# healthy, затем остановить старую через SIGTERM (drain 3 с, LB выводит её) и удалить. +# Число готовых реплик во время замены не опускается ниже N. Журнал — evidence/rolling-update.txt. +set -eu +. "$(dirname "$0")/lib.sh" +LOG=${ROLLING_EVIDENCE:-evidence/rolling-update.txt} +VERSION=${APP_VERSION:-$(date -u +%Y%m%d%H%M%S)} +BUILD_CMD=${BUILD_CMD:-"docker compose --env-file $ENV_FILE build --build-arg APP_VERSION=$VERSION api"} +wait_healthy() { # $1 = container id + i=0 + while [ "$(docker inspect -f '{{.State.Health.Status}}' "$1")" != healthy ]; do + i=$((i + 1)); [ $i -le 120 ] || { echo "replica $1 not healthy" >&2; return 1; } + sleep 0.5 + done +} +old=$(dc ps -q api) +n=$(echo "$old" | wc -w) +{ echo "== $(now) rolling update start: $n replicas, new version $VERSION"; for c in $old; do docker inspect -f 'old {{.Name}} image={{.Image}}' "$c"; done; } >> "$LOG" +$BUILD_CMD >/dev/null +dc run --rm --no-deps migrate >> "$LOG" 2>&1 +for c in $old; do + before=$(dc ps -q api) + dc up -d --no-deps --no-recreate --scale api=$((n + 1)) api >/dev/null 2>&1 + new=$(for x in $(dc ps -q api); do echo "$before" | grep -q "$x" || echo "$x"; done) + wait_healthy "$new" + echo "$(now) new $(docker inspect -f '{{.Name}} image={{.Image}}' "$new") healthy; stopping old $(docker inspect -f '{{.Name}}' "$c")" >> "$LOG" + docker stop -t 15 "$c" >/dev/null + docker rm "$c" >/dev/null +done +{ echo "== $(now) rolling update done"; for c in $(dc ps -q api); do docker inspect -f 'now {{.Name}} image={{.Image}} health={{.State.Health.Status}}' "$c"; done; } >> "$LOG" diff --git a/scripts/start-replicas.ps1 b/scripts/start-replicas.ps1 new file mode 100644 index 00000000..bb6a5fa0 --- /dev/null +++ b/scripts/start-replicas.ps1 @@ -0,0 +1,14 @@ +# Windows PowerShell version of scripts/start-replicas.sh: start stopped API replicas, wait for healthy. +$ErrorActionPreference = 'Continue' +$root = Split-Path -Parent $PSScriptRoot +Set-Location $root +$envFile = if ($env:ENV_FILE) { $env:ENV_FILE } else { '.env' } +$ids = @(& docker compose --env-file $envFile ps -a -q api) +foreach ($cid in $ids) { & docker start $cid | Out-Null } +for ($i = 0; $i -lt 120; $i++) { + $bad = @($ids | Where-Object { (& docker inspect -f '{{.State.Health.Status}}' $_) -ne 'healthy' }).Count + if ($bad -eq 0) { break } + Start-Sleep -Milliseconds 500 +} +foreach ($cid in $ids) { & docker inspect -f '{{.Name}} {{.State.Status}} {{.State.Health.Status}}' $cid } +if ($bad -ne 0) { exit 1 } diff --git a/scripts/start-replicas.sh b/scripts/start-replicas.sh new file mode 100644 index 00000000..233eac12 --- /dev/null +++ b/scripts/start-replicas.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Восстанавливает остановленные API-реплики (после stop-one) и ждёт их healthcheck (≤60 с). +set -eu +. "$(dirname "$0")/lib.sh" +for cid in $(dc ps -a -q api); do docker start "$cid" >/dev/null; done +i=0 +while [ $i -lt 120 ]; do + bad=0 + for cid in $(dc ps -a -q api); do + [ "$(docker inspect -f '{{.State.Health.Status}}' "$cid")" = healthy ] || bad=1 + done + [ $bad -eq 0 ] && break + i=$((i + 1)); sleep 0.5 +done +state | grep '^api ' +[ $bad -eq 0 ] diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..141ec220 --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,4 @@ +@echo off +rem Windows wrapper: tests\resilience.py starts the hook directly; .cmd runs without a shell. +powershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0stop-one.ps1" +exit /b %ERRORLEVEL% diff --git a/scripts/stop-one.ps1 b/scripts/stop-one.ps1 new file mode 100644 index 00000000..850cad2c --- /dev/null +++ b/scripts/stop-one.ps1 @@ -0,0 +1,30 @@ +# Windows PowerShell version of scripts/stop-one.sh (failover hook). Stops EXACTLY ONE API replica whose +# X-Instance-Id is in $env:TARGET_INSTANCE_ID and leaves it stopped. DB/Redis/LB/volumes untouched. +# Restore afterwards: scripts\start-replicas.ps1 +$ErrorActionPreference = 'Continue' +$root = Split-Path -Parent $PSScriptRoot +Set-Location $root +$envFile = if ($env:ENV_FILE) { $env:ENV_FILE } else { '.env' } +$log = Join-Path $root 'evidence\failover-hook.txt' +if (-not $env:TARGET_INSTANCE_ID) { [Console]::Error.WriteLine('TARGET_INSTANCE_ID is required'); exit 2 } +function Get-State { + foreach ($cid in (& docker compose --env-file $envFile ps -a -q)) { + & docker inspect -f '{{index .Config.Labels "com.docker.compose.service"}} {{.Name}} status={{.State.Status}} started={{.State.StartedAt}} pid={{.State.Pid}}' $cid + } +} +$target = $null +foreach ($cid in (& docker compose --env-file $envFile ps -q api)) { + $id = (& docker exec $cid cat /tmp/instance-id 2>$null) + if ($id -and $id.Trim() -eq $env:TARGET_INSTANCE_ID) { $target = $cid } +} +if (-not $target) { [Console]::Error.WriteLine('no running api replica with this id'); exit 1 } +$name = (& docker inspect -f '{{.Name}}' $target).TrimStart('/') +$stamp = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') +Add-Content -Path $log -Encoding UTF8 -Value ("== $stamp stop-one target=" + $env:TARGET_INSTANCE_ID + " container=$name") +Add-Content -Path $log -Encoding UTF8 -Value (Get-State) +& docker stop -t 15 $target | Out-Null +if ($LASTEXITCODE -ne 0) { exit 1 } +$stamp = (Get-Date).ToUniversalTime().ToString('yyyy-MM-ddTHH:mm:ssZ') +Add-Content -Path $log -Encoding UTF8 -Value ("== $stamp after stop") +Add-Content -Path $log -Encoding UTF8 -Value (Get-State) +exit 0 diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..486b6408 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# Failover hook: останавливает РОВНО ОДНУ API-реплику — ту, чей X-Instance-Id передан тестом в +# TARGET_INSTANCE_ID, — и оставляет её остановленной. БД, Redis, LB и volumes не трогает. +# docker stop → SIGTERM → реплика 3 с отвечает 503 на /health/ready (LB выводит её), затем закрывает +# listener и дожидается запросов в полёте. Возврат ≈ 4 с (< 10 с). Восстановление: scripts/start-replicas.sh +set -eu +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +. "$(dirname "$0")/lib.sh" +LOG=${FAILOVER_EVIDENCE:-evidence/failover-hook.txt} +cid=$(container_of "$TARGET_INSTANCE_ID") || { echo "no running api replica with id $TARGET_INSTANCE_ID" >&2; exit 1; } +name=$(docker inspect -f '{{.Name}}' "$cid" | tr -d /) +{ echo "== $(now) stop-one target=$TARGET_INSTANCE_ID container=$name"; state; } >> "$LOG" +docker stop -t 15 "$cid" >/dev/null +{ echo "== $(now) after stop"; state; } >> "$LOG" diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..fe0a525f --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1,2 @@ +"""MTUSI messenger API, lab3.""" +__version__ = "3.0.0" diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..77e77076 --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,125 @@ +"""Точка входа: python -m messenger [serve|migrate|healthcheck|stats].""" +from __future__ import annotations + +import asyncio +import json +import logging +import os +import sys +import time +import urllib.request + +_STD = set(vars(logging.LogRecord("", 0, "", 0, "", (), None))) | {"message", "asctime"} + + +class JsonFormatter(logging.Formatter): + """Структурированный лог в stdout. Токены, пароли, Authorization и тела запросов не логируются.""" + + def format(self, record: logging.LogRecord) -> str: + data = {"ts": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime(record.created)) + "Z", + "level": record.levelname.lower(), "logger": record.name, "msg": record.getMessage()} + data.update({k: v for k, v in vars(record).items() if k not in _STD and not k.startswith("_")}) + if record.exc_info: + data["exc"] = self.formatException(record.exc_info).splitlines()[-1] + return json.dumps(data, ensure_ascii=False, default=str) + + +def setup_logging() -> None: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO").upper(), handlers=[handler], force=True) + logging.getLogger("uvicorn.access").disabled = True # свой access-log без query-строк + + +def serve() -> int: + import uvicorn + + from .app import build_app + from .config import load + + settings = load() + app = build_app(settings) + state = app.state.messenger + log = logging.getLogger("messenger.shutdown") + + class DrainingServer(uvicorn.Server): + """Graceful shutdown для работы за LB. + 1-й SIGTERM/SIGINT: readiness → 503 и `Connection: close`, запросы ещё обслуживаются; + через SHUTDOWN_DRAIN_SECONDS (≥ времени обнаружения LB) uvicorn перестаёт принимать + соединения и дожидается запросов в полёте. Повторный сигнал — немедленный выход.""" + + drain_started = None + + def handle_exit(self, sig, frame): + if self.drain_started is None and settings.shutdown_drain_seconds > 0 and not self.should_exit: + self.drain_started = time.monotonic() + state.draining = True + log.info("drain started", extra={"signal": sig, "instance_id": settings.instance_id, + "drain_seconds": settings.shutdown_drain_seconds}) + return + super().handle_exit(sig, frame) + + async def on_tick(self, counter): + if (self.drain_started is not None and not self.should_exit + and time.monotonic() - self.drain_started >= settings.shutdown_drain_seconds): + log.info("drain finished; closing listener", extra={"instance_id": settings.instance_id}) + self.should_exit = True + return await super().on_tick(counter) + + config = uvicorn.Config(app, host=settings.host, port=settings.port, log_config=None, + access_log=False, server_header=False, proxy_headers=False, + # keep-alive дольше, чем idle-пул HAProxy: соединение первым закрывает LB, + # а не реплика (иначе гонка «LB шлёт запрос в закрываемое соединение»). + timeout_keep_alive=int(os.environ.get("KEEPALIVE_SECONDS", "30")), + timeout_graceful_shutdown=10, lifespan="on", + backlog=int(os.environ.get("BACKLOG", "2048"))) + server = DrainingServer(config) + server.run() + return 0 if server.started else 1 + + +def migrate() -> int: + from .config import load + from .db import migrate_with_wait + + applied = asyncio.run(migrate_with_wait(load())) + print(json.dumps({"applied": applied})) + return 0 + + +def healthcheck() -> int: + port = os.environ.get("APP_PORT", "8080") + path = sys.argv[2] if len(sys.argv) > 2 else "/health/ready" + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}{path}", timeout=2) as resp: + return 0 if resp.status == 200 else 1 + except Exception: # noqa: BLE001 + return 1 + + +def stats() -> int: + port = os.environ.get("APP_PORT", "8080") + with urllib.request.urlopen(f"http://127.0.0.1:{port}/internal/stats", timeout=2) as resp: + print(resp.read().decode()) + return 0 + + +def main() -> int: + setup_logging() + command = sys.argv[1] if len(sys.argv) > 1 else "serve" + commands = {"serve": serve, "migrate": migrate, "healthcheck": healthcheck, "stats": stats} + if command not in commands: + print("usage: python -m messenger [serve|migrate|healthcheck|stats]", file=sys.stderr) + return 2 + try: + return commands[command]() + except Exception as exc: # noqa: BLE001 + from .config import ConfigError + if isinstance(exc, ConfigError): + logging.getLogger("messenger").error("configuration error: %s", exc) + return 2 + raise + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..955de809 --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,620 @@ +"""HTTP API мессенджера (lab3): Starlette + psycopg 3 (async pool) + Redis/Valkey-кэш сессий. + +Реплика stateless: всё, от чего зависит корректность (сессии, идемпотентность, данные), лежит +в PostgreSQL; Redis — только кэш. Любую реплику можно остановить или заменить.""" +from __future__ import annotations + +import asyncio +import base64 +import binascii +import contextlib +import hashlib +import json +import os +import re +import logging +import time +import uuid +from pathlib import Path + +from psycopg import errors as pg_errors +from starlette.applications import Starlette +from starlette.exceptions import HTTPException +from starlette.requests import Request +from starlette.responses import JSONResponse, Response +from starlette.routing import Route + +from . import db +from .config import Settings +from .errors import ApiError, invalid, not_found, unauthorized_basic, unauthorized_bearer, unavailable +from .security import PasswordHasher, looks_like_token, new_token, token_hash +from .sessions import Principal, SessionCache +from .validation import (USERNAME_RE, decode_cursor, encode_cursor, parse_json_object, parse_limit, + parse_uuid, single_param, text_field) + +log = logging.getLogger("messenger.http") + + +# -------------------------------------------------------------------------------------------- +# Ответы и обвязка +# -------------------------------------------------------------------------------------------- + +def request_id(request: Request) -> str: + rid = getattr(request.state, "request_id", None) + if not rid: + rid = uuid.uuid4().hex + request.state.request_id = rid + return rid + + +def json_response(request: Request, payload, status: int = 200, headers: dict | None = None) -> JSONResponse: + h = {"X-Request-Id": request_id(request), "Cache-Control": "no-store"} + h.update(headers or {}) + return JSONResponse(payload, status_code=status, headers=h) + + +def error_response(request: Request, err: ApiError) -> JSONResponse: + rid = request_id(request) + body = {"error": {"code": err.public_code, "message": err.message, "request_id": rid}} + return json_response(request, body, err.status, err.headers) + + +class RequestContextMiddleware: + """Чистый ASGI middleware: X-Request-Id в каждом ответе (включая 404/405 роутера) и access-log + без query-строки, заголовков и тела.""" + + def __init__(self, app, instance_id: str = "api", state=None): + self.app = app + self.instance_id = instance_id.encode() + self.state = state + + async def __call__(self, scope, receive, send): + if scope["type"] != "http": + return await self.app(scope, receive, send) + rid = uuid.uuid4().hex + scope.setdefault("state", {})["request_id"] = rid + started = time.monotonic() + status_holder = {} + + async def send_wrapper(message): + if message["type"] == "http.response.start": + status_holder["status"] = message["status"] + headers = list(message.get("headers", [])) + if not any(k.lower() == b"x-request-id" for k, _ in headers): + headers.append((b"x-request-id", rid.encode())) + headers.append((b"x-instance-id", self.instance_id)) + if self.state is not None and self.state.draining: + # Во время drain просим клиента/LB не переиспользовать соединение с этой репликой. + headers.append((b"connection", b"close")) + message["headers"] = headers + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + finally: + if self.state is not None: + self.state.stats["requests"] += 1 + log.info("request", extra={"request_id": rid, "method": scope.get("method"), + "path": scope.get("path"), "status": status_holder.get("status"), + "duration_ms": round((time.monotonic() - started) * 1000, 2)}) + + +def endpoint(handler): + """Единая обработка ошибок и общий таймаут запроса.""" + async def wrapper(request: Request): + state: AppState = request.app.state.messenger + try: + async with asyncio.timeout(state.settings.request_timeout_seconds): + return await handler(request, state) + except ApiError as err: + return error_response(request, err) + except Exception as exc: # noqa: BLE001 + if db.is_unavailable(exc) or isinstance(exc, TimeoutError): + log.warning("dependency unavailable", extra={"request_id": request_id(request), + "error": type(exc).__name__}) + return error_response(request, unavailable()) + log.exception("unhandled error", extra={"request_id": request_id(request)}) + return error_response(request, ApiError("internal_error")) + wrapper.__name__ = handler.__name__ + return wrapper + + +async def read_body(request: Request, limit: int) -> bytes: + declared = request.headers.get("content-length") + if declared is not None: + if not declared.isdigit(): + raise invalid("Некорректный Content-Length") + if int(declared) > limit: + raise ApiError("payload_too_large") + chunks, size = [], 0 + async for chunk in request.stream(): + size += len(chunk) + if size > limit: + raise ApiError("payload_too_large") + chunks.append(chunk) + return b"".join(chunks) + + +async def read_json(request: Request, state: "AppState", allowed: set[str], required: set[str]) -> dict: + ctype = request.headers.get("content-type", "").split(";")[0].strip().lower() + body = await read_body(request, state.settings.max_body_bytes) + if ctype != "application/json": + raise invalid("Ожидается Content-Type: application/json") + return parse_json_object(body, allowed, required) + + +def user_json(row) -> dict: + return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], + "created_at": db.iso(row["created_at"])} + + +def chat_json(row) -> dict: + return {"id": str(row["id"]), "title": row["title"], + "member_ids": [str(m) for m in row["member_ids"]], "created_at": db.iso(row["created_at"])} + + +def message_json(row) -> dict: + return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), + "text": row["text"], "created_at": db.iso(row["created_at"])} + + +# -------------------------------------------------------------------------------------------- +# Состояние приложения и аутентификация +# -------------------------------------------------------------------------------------------- + +class AppState: + def __init__(self, settings: Settings): + self.settings = settings + self.pool = db.make_pool(settings) + self.cache = SessionCache(settings) + self.hasher = PasswordHasher(settings.scrypt_log2_n, settings.scrypt_r, settings.scrypt_p, + settings.kdf_concurrency) + self.ready = False # схема проверена, пул открыт + self.draining = False # получен SIGTERM: readiness = 503, LB выводит реплику + self.stop = asyncio.Event() + self.tasks: list[asyncio.Task] = [] + self.stats = {"requests": 0, "inflight": 0, "idempotent_replays": 0, "idempotency_conflicts": 0} + + +async def authenticate(request: Request, state: AppState) -> Principal: + """Только Authorization: Bearer . X-User-Id и Basic игнорируются/отклоняются.""" + header = request.headers.get("authorization", "") + scheme, _, token = header.partition(" ") + token = token.strip() + if scheme.lower() != "bearer" or not looks_like_token(token): + raise unauthorized_bearer() + th = token_hash(token) + principal = await state.cache.get(th) + if principal is not None: + return principal + async with state.pool.connection() as conn: + row = await (await conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at" + " FROM sessions s JOIN users u ON u.id = s.user_id" + " WHERE s.token_hash = %s AND s.revoked_at IS NULL AND s.expires_at > clock_timestamp()", + (th,))).fetchone() + if row is None: + raise unauthorized_bearer() + principal = Principal(user=user_json(row), expires_at=row["expires_at"].timestamp(), token_hash=th) + await state.cache.put(principal) + return principal + + +def parse_basic(header: str) -> tuple[str, str]: + scheme, _, value = header.partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise unauthorized_basic() + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise unauthorized_basic() from None + username, sep, password = decoded.partition(":") # двоеточие допустимо только в пароле + if not sep or not username.isascii() or not USERNAME_RE.fullmatch(username) or not 12 <= len(password) <= 128: + raise unauthorized_basic() + return username, password + + +# -------------------------------------------------------------------------------------------- +# Handlers +# -------------------------------------------------------------------------------------------- + +@endpoint +async def live(request: Request, state: AppState): + return json_response(request, {"status": "ok"}) + + +@endpoint +async def ready(request: Request, state: AppState): + # Критичная зависимость синхронного пути — только PostgreSQL. Redis — кэш: при его потере + # API продолжает работу через источник истины, поэтому readiness от него не зависит. + if not state.ready or state.draining: + raise unavailable() + try: + async with asyncio.timeout(1.5): + async with state.pool.connection() as conn: + await conn.execute("SELECT 1") + except Exception: # noqa: BLE001 + raise unavailable() from None + return json_response(request, {"status": "ok"}) + + +@endpoint +async def internal_stats(request: Request, state: AppState): + """Только для исследования/диагностики изнутри контейнера; LB отдаёт 404 на /internal/*.""" + if request.client is None or request.client.host not in {"127.0.0.1", "::1"}: + raise not_found() + pool = state.pool.get_stats() + return json_response(request, {"instance_id": state.settings.instance_id, + "version": os.environ.get("APP_VERSION", "dev"), "ready": state.ready, + "draining": state.draining, "app": state.stats, + "cache": state.cache.stats, "pool": pool}) + + +@endpoint +async def create_user(request: Request, state: AppState): + data = await read_json(request, state, {"username", "display_name", "password"}, + {"username", "display_name", "password"}) + username = data["username"] + if not isinstance(username, str) or not USERNAME_RE.fullmatch(username): + raise invalid("username должен соответствовать [a-z0-9_]{3,32}") + display_name = text_field(data, "display_name", 1, 100) + password = text_field(data, "password", 12, 128) + async with state.pool.connection() as conn: + taken = await (await conn.execute("SELECT 1 FROM users WHERE username = %s", (username,))).fetchone() + if taken: + raise ApiError("conflict", "username уже занят") + password_hash = await state.hasher.hash(password) + try: + async with state.pool.connection() as conn: + row = await (await conn.execute( + "INSERT INTO users (username, display_name, password_hash) VALUES (%s, %s, %s)" + " RETURNING id, username, display_name, created_at", + (username, display_name, password_hash))).fetchone() + except pg_errors.UniqueViolation: + # Гонка регистрации: проверку выше прошли оба запроса, уникальный индекс пропустил один. + log.info("registration race resolved by unique constraint", extra={"request_id": request_id(request)}) + raise ApiError("conflict", "username уже занят") from None + return json_response(request, user_json(row), 201) + + +@endpoint +async def me(request: Request, state: AppState): + principal = await authenticate(request, state) + return json_response(request, principal.user) + + +@endpoint +async def create_session(request: Request, state: AppState): + username, password = parse_basic(request.headers.get("authorization", "")) + body = await read_body(request, 1024) + if body: + raise invalid("POST /auth/sessions не принимает тело") + async with state.pool.connection() as conn: + row = await (await conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,))).fetchone() + ok = await state.hasher.verify(password, row["password_hash"] if row else None) + if not ok: + raise unauthorized_basic() + token = new_token() + th = token_hash(token) + async with state.pool.connection() as conn: + srow = await (await conn.execute( + "INSERT INTO sessions (token_hash, user_id, expires_at)" + " VALUES (%s, %s, clock_timestamp() + make_interval(secs => %s)) RETURNING expires_at", + (th, row["id"], state.settings.session_ttl_seconds))).fetchone() + user = user_json(row) + await state.cache.put(Principal(user=user, expires_at=srow["expires_at"].timestamp(), token_hash=th)) + return json_response(request, {"token": token, "token_type": "Bearer", + "expires_at": db.iso(srow["expires_at"]), "user": user}, 201) + + +@endpoint +async def delete_session(request: Request, state: AppState): + principal = await authenticate(request, state) + th = principal.token_hash + async with state.pool.connection() as conn: + async with conn.transaction(): + revoked = await (await conn.execute( + "UPDATE sessions SET revoked_at = clock_timestamp()" + " WHERE token_hash = %s AND revoked_at IS NULL RETURNING 1", (th,))).fetchone() + if revoked is None: + raise unauthorized_bearer() + if state.cache.enabled: + await conn.execute( + "INSERT INTO session_cache_invalidations (token_hash) VALUES (%s) ON CONFLICT DO NOTHING", + (th,)) + # COMMIT в PostgreSQL уже сделал отзыв окончательным; Redis лишь догоняет источник истины. + if state.cache.enabled and await state.cache.invalidate(th): + with contextlib.suppress(Exception): + async with state.pool.connection() as conn: + await conn.execute("DELETE FROM session_cache_invalidations WHERE token_hash = %s", (th,)) + return Response(status_code=204, headers={"X-Request-Id": request_id(request)}) + + +@endpoint +async def create_chat(request: Request, state: AppState): + principal = await authenticate(request, state) + data = await read_json(request, state, {"title", "member_ids"}, {"title", "member_ids"}) + title = text_field(data, "title", 1, 100) + raw_members = data["member_ids"] + if not isinstance(raw_members, list) or not 1 <= len(raw_members) <= 100: + raise invalid("member_ids: массив из 1–100 UUID") + members = [parse_uuid(m, "member_id") for m in raw_members] + if len(set(members)) != len(members): + raise invalid("member_ids содержит дубликаты") + caller = uuid.UUID(principal.user["id"]) + if caller not in members: + raise invalid("member_ids должен содержать создателя чата") + async with state.pool.connection() as conn: + try: + async with conn.transaction(): # атомарно: чат и весь состав или ничего + chat = await (await conn.execute( + "INSERT INTO chats (title, created_by) VALUES (%s, %s) RETURNING id, title, created_at", + (title, caller))).fetchone() + await conn.execute( + "INSERT INTO chat_members (chat_id, user_id, position)" + " SELECT %s, m.user_id, m.ord - 1 FROM unnest(%s::uuid[]) WITH ORDINALITY AS m(user_id, ord)", + (chat["id"], members)) + except pg_errors.ForeignKeyViolation: + raise invalid("member_ids содержит неизвестного пользователя") from None + chat["member_ids"] = members + return json_response(request, chat_json(chat), 201) + + +_CHAT_LIST_SQL = ( + "SELECT c.id, c.title, c.created_at," + " (SELECT array_agg(x.user_id ORDER BY x.position) FROM chat_members x WHERE x.chat_id = c.id) AS member_ids" + " FROM chat_members m JOIN chats c ON c.id = m.chat_id" + " WHERE m.user_id = %s {after}" + " ORDER BY c.created_at, c.id LIMIT %s") + + +@endpoint +async def list_chats(request: Request, state: AppState): + principal = await authenticate(request, state) + caller = uuid.UUID(principal.user["id"]) + limit = parse_limit(request.query_params) + cursor = single_param(request.query_params, "cursor") + async with state.pool.connection() as conn: + if cursor is not None: + ts, last_id = decode_cursor(cursor, "chats", caller) + rows = await (await conn.execute(_CHAT_LIST_SQL.format(after="AND (c.created_at, c.id) > (%s, %s)"), + (caller, ts, last_id, limit + 1))).fetchall() + else: + rows = await (await conn.execute(_CHAT_LIST_SQL.format(after=""), (caller, limit + 1))).fetchall() + items = rows[:limit] + next_cursor = (encode_cursor("chats", caller, items[-1]["created_at"], items[-1]["id"]) + if len(rows) > limit else None) + return json_response(request, {"items": [chat_json(r) for r in items], "next_cursor": next_cursor}) + + +@endpoint +async def get_chat(request: Request, state: AppState): + principal = await authenticate(request, state) + chat_id = parse_uuid(request.path_params["chat_id"], "chat_id") + async with state.pool.connection() as conn: + row = await (await conn.execute( + "SELECT c.id, c.title, c.created_at, array_agg(m.user_id ORDER BY m.position) AS member_ids" + " FROM chats c JOIN chat_members m ON m.chat_id = c.id" + " WHERE c.id = %s AND EXISTS (SELECT 1 FROM chat_members a WHERE a.chat_id = c.id AND a.user_id = %s)" + " GROUP BY c.id", (chat_id, uuid.UUID(principal.user["id"])))).fetchone() + if row is None: + raise not_found() # нет или чужой — одинаковый ответ + return json_response(request, chat_json(row)) + + +async def _is_member(conn, chat_id: uuid.UUID, user_id: uuid.UUID) -> bool: + row = await (await conn.execute("SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, user_id))).fetchone() + return row is not None + + +IDEMPOTENCY_KEY_RE = re.compile(r"[A-Za-z0-9._:-]{1,128}") + + +def idempotency_key(request: Request) -> str: + values = request.headers.getlist("idempotency-key") + if len(values) != 1 or not IDEMPOTENCY_KEY_RE.fullmatch(values[0]): + raise invalid("Требуется заголовок Idempotency-Key: [A-Za-z0-9._:-]{1,128}") + return values[0] + + +def request_fingerprint(body: dict) -> bytes: + """SHA-256 канонического JSON: порядок ключей и пробелы не влияют на равенство запросов.""" + canonical = json.dumps(body, sort_keys=True, ensure_ascii=False, separators=(",", ":")) + return hashlib.sha256(canonical.encode("utf-8")).digest() + + +class _Replay(Exception): + """Ключ уже занят: откатить свою вставку и прочитать сохранённый результат.""" + + +@endpoint +async def create_message(request: Request, state: AppState): + principal = await authenticate(request, state) + chat_id = parse_uuid(request.path_params["chat_id"], "chat_id") + sender = uuid.UUID(principal.user["id"]) + try: + key = idempotency_key(request) + data = await read_json(request, state, {"text"}, {"text"}) + text = text_field(data, "text", 1, 4000) + except ApiError as err: + if err.status == 400: + # Посторонний не должен по 400/404 узнавать о существовании чата. + async with state.pool.connection() as conn: + if not await _is_member(conn, chat_id, sender): + raise not_found() from None + raise + fingerprint = request_fingerprint({"text": text}) + ttl = state.settings.idempotency_ttl_seconds + async with state.pool.connection() as conn: + try: + # Одна транзакция: сообщение и запись ключа фиксируются вместе или не фиксируются вовсе. + # Конкурентный INSERT того же ключа (любая реплика) ждёт COMMIT/ROLLBACK первого на + # уникальном индексе: после COMMIT получает конфликт, после ROLLBACK — проходит сам. + async with conn.transaction(): + await conn.execute( + "DELETE FROM idempotency_keys WHERE user_id = %s AND chat_id = %s AND key = %s" + " AND created_at < clock_timestamp() - make_interval(secs => %s)", + (sender, chat_id, key, ttl)) + row = await (await conn.execute( + "INSERT INTO messages (chat_id, sender_id, text)" + " SELECT %(c)s, %(u)s, %(t)s" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %(c)s AND user_id = %(u)s)" + " RETURNING id, chat_id, sender_id, text, created_at", + {"c": chat_id, "u": sender, "t": text})).fetchone() + if row is None: + raise not_found() # откат: ошибка доступа не фиксируется как результат ключа + claimed = await (await conn.execute( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_hash, message_id)" + " VALUES (%s, %s, %s, %s, %s) ON CONFLICT DO NOTHING RETURNING 1", + (sender, chat_id, key, fingerprint, row["id"]))).fetchone() + if claimed is None: + raise _Replay() + except _Replay: + # Повтор: снова проверены сессия (authenticate) и доступ (членство), затем — сохранённый ответ. + stored = await (await conn.execute( + "SELECT k.request_hash, m.id, m.chat_id, m.sender_id, m.text, m.created_at" + " FROM idempotency_keys k JOIN messages m ON m.id = k.message_id" + " WHERE k.user_id = %s AND k.chat_id = %s AND k.key = %s" + " AND EXISTS (SELECT 1 FROM chat_members a WHERE a.chat_id = k.chat_id AND a.user_id = k.user_id)", + (sender, chat_id, key))).fetchone() + if stored is None: + raise not_found() from None + if bytes(stored["request_hash"]) != fingerprint: + state.stats["idempotency_conflicts"] += 1 + raise ApiError("conflict", "Idempotency-Key уже использован с другим запросом") from None + state.stats["idempotent_replays"] += 1 + return json_response(request, message_json(stored), 201, {"Idempotent-Replayed": "true"}) + return json_response(request, message_json(row), 201) + + +_HISTORY_SQL = ( + "SELECT id, chat_id, sender_id, text, created_at FROM messages" + " WHERE chat_id = %s {after} ORDER BY created_at, id LIMIT %s") + + +@endpoint +async def list_messages(request: Request, state: AppState): + principal = await authenticate(request, state) + chat_id = parse_uuid(request.path_params["chat_id"], "chat_id") + async with state.pool.connection() as conn: + if not await _is_member(conn, chat_id, uuid.UUID(principal.user["id"])): + raise not_found() + limit = parse_limit(request.query_params) + cursor = single_param(request.query_params, "cursor") + if cursor is not None: + ts, last_id = decode_cursor(cursor, "messages", chat_id) + rows = await (await conn.execute(_HISTORY_SQL.format(after="AND (created_at, id) > (%s, %s)"), + (chat_id, ts, last_id, limit + 1))).fetchall() + else: + rows = await (await conn.execute(_HISTORY_SQL.format(after=""), (chat_id, limit + 1))).fetchall() + items = rows[:limit] + next_cursor = (encode_cursor("messages", chat_id, items[-1]["created_at"], items[-1]["id"]) + if len(rows) > limit else None) + return json_response(request, {"items": [message_json(r) for r in items], "next_cursor": next_cursor}) + + +# -------------------------------------------------------------------------------------------- +# Фоновые задачи и сборка приложения +# -------------------------------------------------------------------------------------------- + +async def wait_for_schema(state: AppState) -> None: + """Реплика не мигрирует схему сама (это делает one-shot сервис migrate) и не становится ready, + пока в БД не применены все миграции, которые знает этот код.""" + delay = 0.5 + while not state.stop.is_set(): + try: + ok, missing = await db.schema_status(state.pool) + if ok: + state.ready = True + log.info("schema compatible; replica ready", extra={"instance_id": state.settings.instance_id}) + return + log.warning("schema is behind this code; waiting for migrate", extra={"missing": missing}) + except Exception as exc: # noqa: BLE001 — БД ещё недоступна или нет schema_migrations + log.warning("schema check failed; retrying", extra={"error": type(exc).__name__}) + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(state.stop.wait(), timeout=delay) + delay = min(delay * 2, 5) + + +async def expired_sessions_cleanup(state: AppState) -> None: + """Удаляет сессии, истёкшие более суток назад, пачками (не держит долгих блокировок).""" + while not state.stop.is_set(): + with contextlib.suppress(Exception): + async with state.pool.connection() as conn: + await conn.execute( + "DELETE FROM sessions WHERE token_hash IN (SELECT token_hash FROM sessions" + " WHERE expires_at < clock_timestamp() - interval '1 day' LIMIT 1000)") + await conn.execute( + "DELETE FROM idempotency_keys WHERE ctid IN (SELECT ctid FROM idempotency_keys" + " WHERE created_at < clock_timestamp() - make_interval(secs => %s) LIMIT 5000)", + (state.settings.idempotency_ttl_seconds,)) + with contextlib.suppress(asyncio.TimeoutError): + await asyncio.wait_for(state.stop.wait(), timeout=300) + + +def build_app(settings: Settings) -> Starlette: + state = AppState(settings) + + @contextlib.asynccontextmanager + async def lifespan(app: Starlette): + if settings.migrate_on_start: + applied = await db.migrate_with_wait(settings) + log.info("migrations checked", extra={"applied": applied}) + await state.pool.open(wait=False) + with contextlib.suppress(OSError): + Path(settings.instance_id_file).write_text(settings.instance_id + "\n") + state.tasks.append(asyncio.create_task(wait_for_schema(state))) + if state.cache.enabled: + state.tasks.append(asyncio.create_task(state.cache.maintenance_loop(state.pool, state.stop))) + state.tasks.append(asyncio.create_task(expired_sessions_cleanup(state))) + log.info("api started", extra={"instance_id": settings.instance_id, "version": os.environ.get("APP_VERSION", "dev"), + "cache": state.cache.enabled, + "session_ttl_seconds": settings.session_ttl_seconds}) + try: + yield + finally: + state.ready = False + state.stop.set() + for task in state.tasks: + task.cancel() + await asyncio.gather(*state.tasks, return_exceptions=True) + await state.cache.close() + await state.pool.close() + + async def http_exception(request: Request, exc: HTTPException): + if exc.status_code == 405: + err = ApiError("method_not_allowed", "Метод не поддерживается", headers=dict(exc.headers or {})) + elif exc.status_code == 404: + err = ApiError("not_found") + else: + err = ApiError("invalid_request") + return error_response(request, err) + + async def server_error(request: Request, exc: Exception): + return error_response(request, ApiError("internal_error")) + + routes = [ + Route("/health/live", live, methods=["GET"]), + Route("/health/ready", ready, methods=["GET"]), + Route("/internal/stats", internal_stats, methods=["GET"]), + Route("/api/v1/users", create_user, methods=["POST"]), + Route("/api/v1/users/me", me, methods=["GET"]), + Route("/api/v1/auth/sessions", create_session, methods=["POST"]), + Route("/api/v1/auth/sessions/current", delete_session, methods=["DELETE"]), + Route("/api/v1/chats", create_chat, methods=["POST"]), + Route("/api/v1/chats", list_chats, methods=["GET"]), + Route("/api/v1/chats/{chat_id}", get_chat, methods=["GET"]), + Route("/api/v1/chats/{chat_id}/messages", create_message, methods=["POST"]), + Route("/api/v1/chats/{chat_id}/messages", list_messages, methods=["GET"]), + ] + app = Starlette(routes=routes, lifespan=lifespan, + exception_handlers={HTTPException: http_exception, Exception: server_error}) + app.add_middleware(RequestContextMiddleware, instance_id=settings.instance_id, state=state) + app.state.messenger = state + return app + + +__all__ = ["build_app"] diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..129f598a --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,132 @@ +"""Конфигурация из окружения. Секреты читаются из *_FILE (Docker/Compose secrets).""" +from __future__ import annotations + +import os +import secrets +from dataclasses import dataclass +from pathlib import Path + + +class ConfigError(RuntimeError): + pass + + +def _secret(name: str, required: bool) -> str | None: + """Значение NAME или содержимое файла NAME_FILE (приоритет у файла).""" + path = os.environ.get(name + "_FILE") + if path: + try: + value = Path(path).read_text(encoding="utf-8").strip() + except OSError as exc: + raise ConfigError(f"cannot read {name}_FILE") from exc + if not value: + raise ConfigError(f"{name}_FILE is empty") + return value + value = os.environ.get(name, "").strip() + if value: + return value + if required: + raise ConfigError(f"{name} or {name}_FILE is required") + return None + + +def _int(name: str, default: int, lo: int, hi: int) -> int: + raw = os.environ.get(name) + if raw is None or raw == "": + return default + try: + value = int(raw) + except ValueError as exc: + raise ConfigError(f"{name} must be an integer") from exc + if not lo <= value <= hi: + raise ConfigError(f"{name} must be in [{lo}, {hi}]") + return value + + +def _float(name: str, default: float, lo: float, hi: float) -> float: + raw = os.environ.get(name) + if raw is None or raw == "": + return default + try: + value = float(raw) + except ValueError as exc: + raise ConfigError(f"{name} must be a number") from exc + if not lo <= value <= hi: + raise ConfigError(f"{name} must be in [{lo}, {hi}]") + return value + + +@dataclass(frozen=True) +class Settings: + host: str + port: int + env: str + database_url: str + cache_url: str | None + session_ttl_seconds: int + session_cache_ttl_seconds: int + cache_timeout_seconds: float + cache_retry_seconds: float + db_pool_min: int + db_pool_max: int + db_pool_timeout_seconds: float + db_connect_timeout_seconds: int + db_statement_timeout_ms: int + request_timeout_seconds: float + max_body_bytes: int + kdf_concurrency: int + scrypt_log2_n: int + scrypt_r: int + scrypt_p: int + migrate_on_start: bool + migrate_wait_seconds: int + instance_id: str + instance_id_file: str + idempotency_ttl_seconds: int + shutdown_drain_seconds: float + + @property + def cache_enabled(self) -> bool: + return self.cache_url is not None + + +def load() -> Settings: + store = os.environ.get("SESSION_STORE", "postgres").strip() + if store != "postgres": + # Источник истины сессий — всегда PostgreSQL; Redis/Valkey — только кэш (см. REPORT.md). + raise ConfigError("SESSION_STORE must be 'postgres' (Redis/Valkey is a cache via CACHE_URL)") + cache_mode = os.environ.get("SESSION_CACHE", "auto").strip() + if cache_mode not in {"auto", "redis", "off"}: + raise ConfigError("SESSION_CACHE must be auto, redis or off") + cache_url = None if cache_mode == "off" else _secret("CACHE_URL", required=cache_mode == "redis") + pool_max = _int("DB_POOL_MAX", 10, 1, 100) + return Settings( + host=os.environ.get("APP_HOST", "0.0.0.0"), + port=_int("APP_PORT", 8080, 1, 65535), + env=os.environ.get("APP_ENV", "lab"), + database_url=_secret("DATABASE_URL", required=True) or "", + cache_url=cache_url, + session_ttl_seconds=_int("SESSION_TTL_SECONDS", 3600, 1, 30 * 24 * 3600), + session_cache_ttl_seconds=_int("SESSION_CACHE_TTL_SECONDS", 60, 1, 3600), + cache_timeout_seconds=_float("CACHE_TIMEOUT_SECONDS", 0.2, 0.01, 5), + cache_retry_seconds=_float("CACHE_RETRY_SECONDS", 2.0, 0.1, 60), + db_pool_min=min(_int("DB_POOL_MIN", 1, 0, 100), pool_max), + db_pool_max=pool_max, + db_pool_timeout_seconds=_float("DB_POOL_TIMEOUT_SECONDS", 2.0, 0.1, 60), + db_connect_timeout_seconds=_int("DB_CONNECT_TIMEOUT_SECONDS", 3, 1, 60), + db_statement_timeout_ms=_int("DB_STATEMENT_TIMEOUT_MS", 5000, 100, 600000), + request_timeout_seconds=_float("REQUEST_TIMEOUT_SECONDS", 10.0, 1, 120), + max_body_bytes=_int("MAX_BODY_BYTES", 65536, 1024, 1048576), + kdf_concurrency=_int("KDF_CONCURRENCY", 4, 1, 64), + scrypt_log2_n=_int("SCRYPT_LOG2_N", 15, 14, 20), + scrypt_r=_int("SCRYPT_R", 8, 1, 32), + scrypt_p=_int("SCRYPT_P", 3, 1, 16), + # lab3: миграции выполняет отдельный one-shot сервис `migrate`; реплики их только проверяют. + migrate_on_start=os.environ.get("MIGRATE_ON_START", "false").lower() in {"1", "true", "yes"}, + migrate_wait_seconds=_int("MIGRATE_WAIT_SECONDS", 60, 0, 600), + # Непрозрачный alias реплики: случайный на каждый запуск процесса, без hostname/IP. + instance_id=os.environ.get("INSTANCE_ID") or "api-" + secrets.token_hex(4), + instance_id_file=os.environ.get("INSTANCE_ID_FILE", "/tmp/instance-id"), + idempotency_ttl_seconds=_int("IDEMPOTENCY_TTL_SECONDS", 86400, 60, 30 * 24 * 3600), + shutdown_drain_seconds=_float("SHUTDOWN_DRAIN_SECONDS", 3.0, 0, 60), + ) diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..7437dd2b --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,129 @@ +"""Пул соединений PostgreSQL, версионируемые миграции и классификация ошибок БД.""" +from __future__ import annotations + +import asyncio +import hashlib +import logging +import time +from datetime import datetime, timezone +from pathlib import Path + +import psycopg +from psycopg import errors as pg_errors +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool, PoolTimeout + +from .config import Settings + +log = logging.getLogger("messenger.db") + +MIGRATIONS_DIR = Path(__file__).resolve().parents[2] / "migrations" +MIGRATION_LOCK_ID = 2026_0202 # pg_advisory_lock: параллельные реплики не применяют миграции дважды + + +def connection_kwargs(settings: Settings) -> dict: + options = ( + f"-c statement_timeout={settings.db_statement_timeout_ms} " + "-c lock_timeout=2000 " + "-c idle_in_transaction_session_timeout=10000 " + "-c TimeZone=UTC" + ) + return { + "connect_timeout": settings.db_connect_timeout_seconds, + "application_name": "messenger-api", + "options": options, + "row_factory": dict_row, + } + + +def make_pool(settings: Settings) -> AsyncConnectionPool: + return AsyncConnectionPool( + conninfo=settings.database_url, + kwargs=connection_kwargs(settings), + min_size=settings.db_pool_min, + max_size=settings.db_pool_max, + timeout=settings.db_pool_timeout_seconds, # ожидание свободного соединения + max_waiting=200, # очередь ожидания ограничена + max_idle=300, + reconnect_timeout=30, + check=AsyncConnectionPool.check_connection, # после рестарта PG битые соединения отбрасываются + open=False, + name="messenger", + ) + + +def is_unavailable(exc: BaseException) -> bool: + """Ошибки, означающие «зависимость недоступна/перегружена», а не баг приложения.""" + if isinstance(exc, (PoolTimeout, psycopg.OperationalError, asyncio.TimeoutError, ConnectionError)): + return True + return isinstance(exc, (pg_errors.QueryCanceled, pg_errors.LockNotAvailable, + pg_errors.AdminShutdown, pg_errors.CannotConnectNow)) + + +def iso(ts: datetime) -> str: + """RFC 3339 UTC с микросекундами и суффиксом Z (точность timestamptz в PostgreSQL).""" + return ts.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + + +def _migration_files() -> list[tuple[str, str, str]]: + result = [] + for path in sorted(MIGRATIONS_DIR.glob("[0-9][0-9][0-9][0-9]_*.sql")): + sql = path.read_text(encoding="utf-8") + result.append((path.stem, sql, hashlib.sha256(sql.encode()).hexdigest())) + return result + + +async def migrate(settings: Settings) -> list[str]: + """Применяет недостающие миграции. Повторный запуск — no-op; изменённая применённая миграция — + ошибка (миграции неизменяемы, исправления — новой версией).""" + applied_now: list[str] = [] + async with await psycopg.AsyncConnection.connect( + settings.database_url, autocommit=True, **connection_kwargs(settings)) as conn: + await conn.execute("SELECT pg_advisory_lock(%s)", (MIGRATION_LOCK_ID,)) + try: + await conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + " version text PRIMARY KEY, checksum text NOT NULL," + " applied_at timestamptz NOT NULL DEFAULT clock_timestamp())") + rows = await (await conn.execute("SELECT version, checksum FROM schema_migrations")).fetchall() + done = {r["version"]: r["checksum"] for r in rows} + for version, sql, checksum in _migration_files(): + if version in done: + if done[version] != checksum: + raise RuntimeError(f"applied migration {version} was modified") + continue + async with conn.transaction(): + await conn.execute(sql) + await conn.execute( + "INSERT INTO schema_migrations (version, checksum) VALUES (%s, %s)", + (version, checksum)) + applied_now.append(version) + log.info("migration applied", extra={"version": version}) + finally: + await conn.execute("SELECT pg_advisory_unlock(%s)", (MIGRATION_LOCK_ID,)) + return applied_now + + +async def schema_status(pool) -> tuple[bool, list[str]]: + """Совместима ли схема с этим кодом: применены все известные ему миграции с теми же checksum. + Более новые миграции (expand-фаза следующей версии) допустимы.""" + async with pool.connection() as conn: + rows = await (await conn.execute( + "SELECT version, checksum FROM schema_migrations")).fetchall() + done = {r["version"]: r["checksum"] for r in rows} + missing = [v for v, _, c in _migration_files() if done.get(v) != c] + return not missing, missing + + +async def migrate_with_wait(settings: Settings) -> list[str]: + deadline = time.monotonic() + settings.migrate_wait_seconds + delay = 0.5 + while True: + try: + return await migrate(settings) + except psycopg.OperationalError: + if time.monotonic() >= deadline: + raise + log.warning("database not reachable yet; retrying migrations") + await asyncio.sleep(delay) + delay = min(delay * 2, 5) diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..856d98fd --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,67 @@ +"""Единый формат ошибок API: {"error":{"code","message","request_id"}}.""" +from __future__ import annotations + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + +_STATUS = { + "invalid_request": 400, + "unauthorized": 401, + "not_found": 404, + "method_not_allowed": 405, + "conflict": 409, + "payload_too_large": 413, + "unsupported_media_type": 415, + "rate_limited": 429, + "internal_error": 500, + "unavailable": 503, +} + +_DEFAULT_MESSAGE = { + "invalid_request": "Некорректный запрос", + "unauthorized": "Требуется действительная аутентификация", + "not_found": "Объект не найден", + "conflict": "Конфликт состояния", + "payload_too_large": "Слишком большой запрос", + "unsupported_media_type": "Неподдерживаемый тип содержимого", + "rate_limited": "Слишком много запросов", + "internal_error": "Внутренняя ошибка", + "unavailable": "Сервис временно недоступен", +} + + +class ApiError(Exception): + def __init__(self, code: str, message: str | None = None, *, challenge: str | None = None, + headers: dict[str, str] | None = None): + super().__init__(code) + self.code = code + self.status = _STATUS[code] + self.message = message or _DEFAULT_MESSAGE.get(code, "Ошибка") + self.headers = dict(headers or {}) + if challenge: + self.headers["WWW-Authenticate"] = challenge + + @property + def public_code(self) -> str: + # 405 не входит в enum контракта: отдаём invalid_request с корректным HTTP-статусом. + return "invalid_request" if self.code == "method_not_allowed" else self.code + + +def invalid(message: str) -> ApiError: + return ApiError("invalid_request", message) + + +def unauthorized_bearer() -> ApiError: + return ApiError("unauthorized", challenge=BEARER_CHALLENGE) + + +def unauthorized_basic() -> ApiError: + return ApiError("unauthorized", "Неверные учётные данные", challenge=BASIC_CHALLENGE) + + +def not_found() -> ApiError: + return ApiError("not_found") + + +def unavailable() -> ApiError: + return ApiError("unavailable", headers={"Retry-After": "1"}) diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 00000000..638fafa6 --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,93 @@ +"""Password KDF (scrypt из OpenSSL через hashlib) и opaque-токены сессий. + +Пароль хешируется медленным memory-hard KDF с солью: цель — сделать перебор украденных +hash дорогим. Токен сессии уже содержит 256 бит случайности, перебирать его бессмысленно, +поэтому для индекса в БД/кэше достаточно быстрого SHA-256 — это разные задачи. +""" +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import hmac +import secrets + +_PREFIX = "$scrypt$" + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii").rstrip("=") + + +def _unb64(text: str) -> bytes: + return base64.b64decode(text + "=" * (-len(text) % 4), validate=True) + + +class PasswordHasher: + """Формат PHC-подобный: $scrypt$ln=15,r=8,p=3$$. Параметры хранятся в строке, + поэтому их можно усилить позднее без потери старых hash.""" + + def __init__(self, log2_n: int, r: int, p: int, concurrency: int): + self.log2_n, self.r, self.p = log2_n, r, p + # Ограничиваем одновременные KDF: каждый вызов занимает 128*r*N байт RAM (32 MiB при ln=15). + self._slots = asyncio.Semaphore(concurrency) + self._dummy: str | None = None + + def _maxmem(self, log2_n: int, r: int, p: int) -> int: + return 128 * r * (2 ** log2_n) * 2 + 1024 * 1024 * p + + def _derive(self, password: str, salt: bytes, log2_n: int, r: int, p: int) -> bytes: + return hashlib.scrypt(password.encode("utf-8"), salt=salt, n=2 ** log2_n, r=r, p=p, + maxmem=self._maxmem(log2_n, r, p), dklen=32) + + def hash_sync(self, password: str) -> str: + salt = secrets.token_bytes(16) + digest = self._derive(password, salt, self.log2_n, self.r, self.p) + return f"{_PREFIX}ln={self.log2_n},r={self.r},p={self.p}${_b64(salt)}${_b64(digest)}" + + def verify_sync(self, password: str, encoded: str) -> bool: + try: + if not encoded.startswith(_PREFIX): + return False + params, salt_b64, hash_b64 = encoded[len(_PREFIX):].split("$") + fields = dict(item.split("=", 1) for item in params.split(",")) + log2_n, r, p = int(fields["ln"]), int(fields["r"]), int(fields["p"]) + if not (10 <= log2_n <= 22 and 1 <= r <= 32 and 1 <= p <= 16): + return False + expected = _unb64(hash_b64) + actual = self._derive(password, _unb64(salt_b64), log2_n, r, p) + except (ValueError, KeyError): + return False + return hmac.compare_digest(actual, expected) + + async def hash(self, password: str) -> str: + async with self._slots: + return await asyncio.to_thread(self.hash_sync, password) + + async def verify(self, password: str, encoded: str | None) -> bool: + """Для неизвестного пользователя проверяем фиктивный hash: время ответа не выдаёт, + существует ли username.""" + if encoded is None: + if self._dummy is None: + self._dummy = await self.hash("dummy-password-for-timing") + encoded, password = self._dummy, password + "\x00never" + async with self._slots: + await asyncio.to_thread(self.verify_sync, password, encoded) + return False + async with self._slots: + return await asyncio.to_thread(self.verify_sync, password, encoded) + + +TOKEN_BYTES = 32 # 256 бит энтропии + + +def new_token() -> str: + return secrets.token_urlsafe(TOKEN_BYTES) # 43 символа base64url + + +def token_hash(token: str) -> bytes: + return hashlib.sha256(token.encode("ascii")).digest() + + +def looks_like_token(token: str) -> bool: + return len(token) == 43 and all(c.isascii() and (c.isalnum() or c in "-_") for c in token) diff --git a/src/messenger/sessions.py b/src/messenger/sessions.py new file mode 100644 index 00000000..0eae17ac --- /dev/null +++ b/src/messenger/sessions.py @@ -0,0 +1,185 @@ +"""Сессии: PostgreSQL — источник истины, Redis/Valkey — read-through кэш с TTL. + +Политика (подробно — REPORT.md): +* ключ sess:v1: хранит {user, expires_at}; TTL = min(SESSION_CACHE_TTL_SECONDS, + остаток жизни сессии) — кэш никогда не переживает срок сессии; +* logout: revoked_at + строка outbox в одной транзакции PostgreSQL, затем в Redis атомарно + «надгробие» sess:v1:revoked: и DEL ключа. Заполнение кэша — Lua SET … только если + надгробия нет, поэтому конкурентный read-through не вернёт отозванную сессию в кэш; +* Redis недоступен → аутентификация идёт в PostgreSQL (fail-open только к источнику истины, + не к анонимному доступу); после ошибки кэш не читается, пока фоновый цикл не проверит Redis + и не дообработает outbox инвалидаций; +* flush/restart Redis = холодный кэш: сессии остаются в PostgreSQL. +""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +from dataclasses import dataclass + +from psycopg_pool import AsyncConnectionPool +from redis import asyncio as aioredis +from redis.exceptions import RedisError + +from .config import Settings + +log = logging.getLogger("messenger.sessions") + +KEY = "sess:v1:" +TOMBSTONE = "sess:v1:revoked:" + +# KEYS[1]=session key, KEYS[2]=tombstone; ARGV[1]=value, ARGV[2]=ttl ms +_SET_IF_NOT_REVOKED = """ +if redis.call('EXISTS', KEYS[2]) == 1 then return 0 end +redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2]) +return 1 +""" +# KEYS[1]=session key, KEYS[2]=tombstone; ARGV[1]=tombstone ttl ms +_INVALIDATE = """ +redis.call('SET', KEYS[2], '1', 'PX', ARGV[1]) +redis.call('DEL', KEYS[1]) +return 1 +""" + + +@dataclass +class Principal: + user: dict + expires_at: float # epoch seconds + token_hash: bytes + + +class SessionCache: + def __init__(self, settings: Settings): + self.settings = settings + self.client: aioredis.Redis | None = None + self._set_script = None + self._invalidate_script = None + # degraded=True: кэш не читаем (было исключение Redis или есть недообработанный outbox). + self.degraded = True + self._next_probe = 0.0 + self.stats = {"hits": 0, "misses": 0, "errors": 0} + if settings.cache_url: + t = settings.cache_timeout_seconds + self.client = aioredis.from_url( + settings.cache_url, socket_timeout=t, socket_connect_timeout=t, + retry_on_timeout=False, health_check_interval=0, max_connections=50, + decode_responses=False) + self._set_script = self.client.register_script(_SET_IF_NOT_REVOKED) + self._invalidate_script = self.client.register_script(_INVALIDATE) + + @property + def enabled(self) -> bool: + return self.client is not None + + def _tombstone_ms(self) -> int: + # Надгробие живёт дольше любой записи кэша и любого запроса в полёте. + return int((self.settings.session_cache_ttl_seconds + self.settings.request_timeout_seconds + 5) * 1000) + + def _fail(self, op: str, exc: Exception) -> None: + self.stats["errors"] += 1 + if not self.degraded: + log.warning("cache unavailable; falling back to PostgreSQL", + extra={"op": op, "error": type(exc).__name__}) + self.degraded = True + self._next_probe = time.monotonic() + self.settings.cache_retry_seconds + + async def get(self, th: bytes) -> Principal | None: + if not self.client or self.degraded: + return None + try: + raw = await self.client.get(KEY + th.hex()) + except (RedisError, OSError, asyncio.TimeoutError) as exc: + self._fail("get", exc) + return None + if raw is None: + self.stats["misses"] += 1 + return None + try: + data = json.loads(raw) + principal = Principal(user=data["user"], expires_at=float(data["exp"]), token_hash=th) + except (ValueError, KeyError, TypeError): + return None + if principal.expires_at <= time.time(): + return None + self.stats["hits"] += 1 + return principal + + async def put(self, p: Principal) -> None: + if not self.client or self.degraded: + return + ttl_ms = int(min(self.settings.session_cache_ttl_seconds, p.expires_at - time.time()) * 1000) + if ttl_ms < 1: + return + value = json.dumps({"user": p.user, "exp": p.expires_at}, separators=(",", ":")) + try: + await self._set_script(keys=[KEY + p.token_hash.hex(), TOMBSTONE + p.token_hash.hex()], + args=[value, ttl_ms]) + except (RedisError, OSError, asyncio.TimeoutError) as exc: + self._fail("put", exc) + + async def invalidate(self, th: bytes) -> bool: + if not self.client: + return True + try: + await self._invalidate_script(keys=[KEY + th.hex(), TOMBSTONE + th.hex()], + args=[self._tombstone_ms()]) + return True + except (RedisError, OSError, asyncio.TimeoutError) as exc: + self._fail("invalidate", exc) + return False + + async def ping(self) -> bool: + if not self.client: + return False + try: + return bool(await self.client.ping()) + except (RedisError, OSError, asyncio.TimeoutError): + return False + + async def drain_outbox(self, pool: AsyncConnectionPool) -> int: + """Повторяет инвалидации из session_cache_invalidations. True-результат: outbox пуст.""" + drained = 0 + while True: + async with pool.connection() as conn: + rows = await (await conn.execute( + "SELECT token_hash FROM session_cache_invalidations ORDER BY created_at LIMIT 500" + )).fetchall() + if not rows: + return drained + done = [] + for row in rows: + th = bytes(row["token_hash"]) + if not await self.invalidate(th): + raise ConnectionError("cache still unavailable") + done.append(th) + async with pool.connection() as conn: + await conn.execute("DELETE FROM session_cache_invalidations WHERE token_hash = ANY(%s)", (done,)) + drained += len(done) + + async def maintenance_loop(self, pool: AsyncConnectionPool, stop: asyncio.Event) -> None: + """Возвращает кэш в работу только после успешного PING и пустого outbox.""" + while not stop.is_set(): + if self.client and time.monotonic() >= self._next_probe: + try: + if await self.ping(): + drained = await self.drain_outbox(pool) + if self.degraded: + log.info("cache available again", extra={"replayed_invalidations": drained}) + self.degraded = False + else: + self._fail("probe", ConnectionError("ping failed")) + except ConnectionError as exc: + self._fail("drain", exc) + except Exception as exc: # noqa: BLE001 — БД недоступна: состояние кэша не меняем + log.warning("outbox drain skipped", extra={"error": type(exc).__name__}) + try: + await asyncio.wait_for(stop.wait(), timeout=1.0) + except asyncio.TimeoutError: + pass + + async def close(self) -> None: + if self.client: + await self.client.aclose() diff --git a/src/messenger/validation.py b/src/messenger/validation.py new file mode 100644 index 00000000..c6cd17d7 --- /dev/null +++ b/src/messenger/validation.py @@ -0,0 +1,119 @@ +"""Строгая валидация входа: лишние поля, неверные типы и malformed JSON — 400.""" +from __future__ import annotations + +import base64 +import json +import re +import uuid +from datetime import datetime + +from .errors import ApiError, invalid + +USERNAME_RE = re.compile(r"[a-z0-9_]{3,32}") + + +def _no_duplicates(pairs): + obj = {} + for key, value in pairs: + if key in obj: + raise ValueError("duplicate key") + obj[key] = value + return obj + + +def _reject_constant(name): + raise ValueError("non-standard JSON constant " + name) + + +def parse_json_object(raw: bytes, allowed: set[str], required: set[str]) -> dict: + if not raw: + raise invalid("Ожидается JSON-объект") + try: + data = json.loads(raw.decode("utf-8"), object_pairs_hook=_no_duplicates, + parse_constant=_reject_constant) + except (UnicodeDecodeError, ValueError, RecursionError): + raise invalid("Некорректный JSON") from None + if not isinstance(data, dict): + raise invalid("Ожидается JSON-объект") + extra = set(data) - allowed + if extra: + raise invalid("Лишние поля: " + ", ".join(sorted(extra))[:200]) + missing = required - set(data) + if missing: + raise invalid("Отсутствуют поля: " + ", ".join(sorted(missing))) + return data + + +def text_field(data: dict, name: str, lo: int, hi: int) -> str: + value = data.get(name) + if not isinstance(value, str): + raise invalid(f"Поле {name} должно быть строкой") + if not lo <= len(value) <= hi: # len() в Python — число Unicode code points + raise invalid(f"Поле {name}: длина {lo}–{hi} символов") + if lo > 0 and not value.strip(): + raise invalid(f"Поле {name} не может состоять только из пробелов") + if "\x00" in value: + raise invalid(f"Поле {name} содержит недопустимый символ") + try: + value.encode("utf-8") + except UnicodeEncodeError: # одиночные суррогаты из \uD800-escape + raise invalid(f"Поле {name} содержит некорректный Unicode") from None + return value + + +def parse_uuid(value, what: str = "UUID") -> uuid.UUID: + if not isinstance(value, str): + raise invalid(f"Некорректный {what}") + try: + parsed = uuid.UUID(value) + except ValueError: + raise invalid(f"Некорректный {what}") from None + # Принимаем только каноническую 36-символьную запись (без {} и urn:). + if len(value) != 36: + raise invalid(f"Некорректный {what}") + return parsed + + +def parse_limit(params) -> int: + values = params.getlist("limit") + if not values: + return 50 + if len(values) > 1 or not values[0].isascii() or not values[0].isdigit() or len(values[0]) > 3: + raise invalid("limit должен быть целым 1–100") + limit = int(values[0]) + if not 1 <= limit <= 100: + raise invalid("limit должен быть целым 1–100") + return limit + + +def single_param(params, name: str) -> str | None: + values = params.getlist(name) + if len(values) > 1: + raise invalid(f"Параметр {name} указан несколько раз") + return values[0] if values else None + + +# --- keyset cursor ------------------------------------------------------------------------- +# Курсор непрозрачен для клиента: base64url(JSON) с видом списка и областью (чат/пользователь). +# Подделка курсора не даёт доступа: область всё равно проверяется, а позиция лишь фильтр. + +def encode_cursor(kind: str, scope: uuid.UUID, created_at: datetime, item_id: uuid.UUID) -> str: + payload = json.dumps({"v": 1, "k": kind, "s": str(scope), "t": created_at.isoformat(), "i": str(item_id)}, + separators=(",", ":")).encode() + return base64.urlsafe_b64encode(payload).decode().rstrip("=") + + +def decode_cursor(cursor: str, kind: str, scope: uuid.UUID) -> tuple[datetime, uuid.UUID]: + try: + if not cursor or len(cursor) > 512 or not cursor.isascii(): + raise ValueError + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)) + data = json.loads(raw) + if data.get("v") != 1 or data.get("k") != kind or data.get("s") != str(scope): + raise ValueError + ts = datetime.fromisoformat(data["t"]) + if ts.tzinfo is None: + raise ValueError + return ts, uuid.UUID(data["i"]) + except (ValueError, KeyError, TypeError, AttributeError, UnicodeError): + raise ApiError("invalid_request", "Некорректный cursor") from None diff --git a/student-tests/common.py b/student-tests/common.py new file mode 100644 index 00000000..05518e3e --- /dev/null +++ b/student-tests/common.py @@ -0,0 +1,111 @@ +"""Общая обвязка собственных проверок студента (stdlib, Python 3.9+). + +Переиспользует HTTP-клиент публичных тестов (tests/client.py), не изменяя его. Команды управления +стендом и доступа к SQL задаются переменными окружения; по умолчанию — Compose из этого репозитория. +""" +import base64 +import json +import os +import shlex +import subprocess +import sys +import time +import traceback +import uuid +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tests")) + +from client import Client, Failure, check # noqa: E402 + +DC = os.environ.get("COMPOSE", "docker compose --env-file .env") +PSQL = os.environ.get("PSQL", DC + " exec -T postgres psql -U messenger -d messenger") +COMMANDS = { + "stop_db": os.environ.get("STOP_DB", DC + " stop postgres"), + "start_db": os.environ.get("START_DB", DC + " start postgres"), + "stop_cache": os.environ.get("STOP_CACHE", DC + " stop redis"), + "start_cache": os.environ.get("START_CACHE", DC + " start redis"), +} +BASE_URL = os.environ.get("BASE_URL", "http://127.0.0.1:8080") +TTL_BASE_URL = os.environ.get("TTL_BASE_URL", "http://127.0.0.1:8081") +TTL_SECONDS = float(os.environ.get("TTL_SECONDS", "2")) + + +def sh(name_or_cmd, timeout=90): + cmd = COMMANDS.get(name_or_cmd, name_or_cmd) + subprocess.run(shlex.split(cmd), cwd=ROOT, check=True, timeout=timeout, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + +def sql(query): + """Одна SQL-команда через psql (-At); возвращает строки результата. Только синтетические данные.""" + out = subprocess.run(shlex.split(PSQL) + ["-X", "-At", "-v", "ON_ERROR_STOP=1", "-c", query], + cwd=ROOT, check=True, capture_output=True, text=True, timeout=60) + return [line for line in out.stdout.splitlines() if line] + + +def basic(username, password): + return "Basic " + base64.b64encode((username + ":" + password).encode("utf-8")).decode() + + +def new_user(c, label="s", password=None): + username = ("st_" + label + "_" + uuid.uuid4().hex[:10])[:32] + password = password or ("Pw-" + uuid.uuid4().hex) + user = c.call("POST", "/api/v1/users", 201, schema="User", + body={"username": username, "display_name": "Тест " + label, "password": password})["json"] + return {"id": user["id"], "user": user, "username": username, "password": password} + + +def login(c, actor, expected=201): + r = c.call("POST", "/api/v1/auth/sessions", expected, headers={"Authorization": basic(actor["username"], actor["password"])}, + schema="Session" if expected == 201 else None) + if expected == 201: + actor["token"] = r["json"]["token"] + actor["expires_at"] = r["json"]["expires_at"] + return r + + +def wait_until(probe, timeout=60, interval=0.5): + deadline = time.monotonic() + timeout + last = None + while time.monotonic() < deadline: + try: + if probe(): + return time.monotonic() + except Failure as exc: + last = exc + time.sleep(interval) + raise Failure("condition not reached in %ss (%s)" % (timeout, last)) + + +class Suite: + def __init__(self, name): + self.name, self.results = name, [] + + def case(self, title): + def decorator(fn): + self.results.append((title, fn)) + return fn + return decorator + + def run(self): + passed = failed = 0 + print("## " + self.name) + for title, fn in self.results: + start = time.monotonic() + try: + note = fn() + passed += 1 + print("PASS %-62s %6.2fs%s" % (title, time.monotonic() - start, (" " + note) if note else "")) + except Exception as exc: # noqa: BLE001 + failed += 1 + print("FAIL %-62s %s" % (title, exc)) + if os.environ.get("DEBUG"): + traceback.print_exc() + print("summary: %d passed, %d failed" % (passed, failed)) + raise SystemExit(1 if failed else 0) + + +__all__ = ["Client", "Failure", "check", "sh", "sql", "basic", "new_user", "login", "wait_until", "Suite", + "BASE_URL", "TTL_BASE_URL", "TTL_SECONDS", "json"] diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..0b2caf68 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,238 @@ +#!/usr/bin/env python3 +"""Негативные и контрактные проверки уровня 3/4: аутентификация, KDF, валидация, ACL, пагинация, +атомарность/rollback создания чата, гонка регистрации, параметризация SQL.""" +import hashlib +import threading +import uuid +from urllib.parse import quote + +from common import BASE_URL, Client, Suite, basic, check, login, new_user, sql + +c = Client(BASE_URL) +s = Suite("API, auth, validation, ACL, transactions (" + BASE_URL + ")") +alice = new_user(c, "alice"); login(c, alice) +bob = new_user(c, "bob"); login(c, bob) +eve = new_user(c, "eve"); login(c, eve) +chat = c.call("POST", "/api/v1/chats", 201, actor=alice, schema="Chat", + body={"title": "student-tests", "member_ids": [alice["id"], bob["id"]]})["json"] +CHAT = "/api/v1/chats/" + chat["id"] + + +@s.case("password stored only as scrypt KDF hash; User has no secret fields") +def _(): + row = sql("SELECT password_hash FROM users WHERE id = '%s'" % alice["id"])[0] + check(row.startswith("$scrypt$ln=15,r=8,p=3$"), "unexpected hash format") + check(alice["password"] not in row, "plaintext password in DB") + check(set(alice["user"]) == {"id", "username", "display_name", "created_at"}, "User leaks fields") + other = new_user(c, "same", password=alice["password"]) + h2 = sql("SELECT password_hash FROM users WHERE id = '%s'" % other["id"])[0] + check(h2 != row, "same password produced same hash: salt missing") + + +@s.case("wrong password and unknown user: identical 401 + Basic challenge") +def _(): + r1 = c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic(alice["username"], "wrong-password-123")}) + r2 = c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": basic("no_such_user_x", "wrong-password-123")}) + check(r1["json"]["error"]["message"] == r2["json"]["error"]["message"], "messages differ") + check(r1["headers"]["WWW-Authenticate"] == 'Basic realm="messenger", charset="UTF-8"', "challenge mismatch") + + +@s.case("malformed Basic / Bearer on /auth/sessions / body on login") +def _(): + for value in ("Basic", "Basic !!!", "Basic " + "bm9jb2xvbg==", "Bearer " + "x" * 43): + c.call("POST", "/api/v1/auth/sessions", 401, headers={"Authorization": value}) + c.call("POST", "/api/v1/auth/sessions", 401) + c.call("POST", "/api/v1/auth/sessions", 400, body={"username": alice["username"]}, + headers={"Authorization": basic(alice["username"], alice["password"])}) + + +@s.case("password with ':' and non-ASCII UTF-8 works (RFC 7617)") +def _(): + u = new_user(c, "colon", password="пароль:с:двоеточием-😀") + login(c, u) + me = c.call("GET", "/api/v1/users/me", actor=u, schema="User")["json"] + check(me["id"] == u["id"], "wrong user") + + +@s.case("opaque 256-bit token, unique per login, DB stores only SHA-256") +def _(): + a, b = dict(alice), dict(alice) + login(c, a); login(c, b) + check(a["token"] != b["token"] and len(a["token"]) == 43, "token not unique/opaque") + check(a["token"] not in (alice["id"], alice["username"]), "token derived from identity") + digest = hashlib.sha256(a["token"].encode()).hexdigest() + check(sql("SELECT count(*) FROM sessions WHERE token_hash = decode('%s','hex')" % digest) == ["1"], "hash not found") + check(sql("SELECT count(*) FROM information_schema.columns WHERE table_name = 'sessions'" + " AND data_type IN ('text', 'character varying')") == ["0"], "sessions has a text column for raw token") + + +@s.case("only Bearer accepted; Basic and X-User-Id never authenticate") +def _(): + c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": basic(alice["username"], alice["password"])}) + c.call("GET", "/api/v1/chats", 401, headers={"X-User-Id": alice["id"]}) + c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer " + "A" * 43}) + c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer short"}) + me = c.call("GET", "/api/v1/users/me", actor=bob, headers={"X-User-Id": alice["id"]}, schema="User")["json"] + check(me["id"] == bob["id"], "X-User-Id overrode session") + + +@s.case("logout revokes only the current session; repeat -> 401") +def _(): + u = new_user(c, "logout"); first = dict(u); second = dict(u) + login(c, first); login(c, second) + c.call("GET", "/api/v1/users/me", actor=first, schema="User") # прогреть кэш + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=first) + for path in ("/api/v1/users/me", "/api/v1/chats"): + c.call("GET", path, 401, actor=first) + c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=first) + c.call("GET", "/api/v1/users/me", actor=second, schema="User") + check(sql("SELECT count(*) FROM sessions WHERE user_id='%s' AND revoked_at IS NOT NULL" % u["id"]) == ["1"], + "revocation not persisted") + + +@s.case("strict JSON validation -> 400") +def _(): + base = {"username": "valid_" + uuid.uuid4().hex[:8], "display_name": "X", "password": "p" * 12} + bad = [dict(base, extra=1), dict(base, username="UPPER"), dict(base, username="ab"), + dict(base, display_name=" "), dict(base, display_name="x" * 101), dict(base, password="p" * 11), + dict(base, password="p" * 129), dict(base, display_name=5), {"username": base["username"]}] + for body in bad: + c.call("POST", "/api/v1/users", 400, body=body) + c.call("POST", "/api/v1/users", 400, raw=b'{"username": ', headers={"Content-Type": "application/json"}) + c.call("POST", "/api/v1/users", 400, raw=b'[1,2]', headers={"Content-Type": "application/json"}) + c.call("POST", "/api/v1/users", 400, raw=b'{"username":"a","username":"b"}', headers={"Content-Type": "application/json"}) + c.call("POST", "/api/v1/users", 400, raw=b'username=x', headers={"Content-Type": "application/x-www-form-urlencoded"}) + c.call("POST", "/api/v1/users", 413, raw=b'{"x":"' + b"a" * 70000 + b'"}', headers={"Content-Type": "application/json"}) + + +@s.case("message text: 4000 code points OK, 4001 / whitespace / sender_id -> 400") +def _(): + msg = c.message(alice, chat, "😀" * 4000)["json"] + check(len(msg["text"]) == 4000, "text truncated") + for body in ({"text": "a" * 4001}, {"text": " \n\t "}, {"text": "x", "sender_id": bob["id"]}, {"text": None}, {}): + c.call("POST", CHAT + "/messages", 400, actor=alice, body=body) + + +@s.case("SQL is parameterized: injection payloads stored verbatim") +def _(): + payload = "'); DROP TABLE messages; -- \" \\ %s $1" + msg = c.message(alice, chat, payload)["json"] + check(msg["text"] == payload, "payload altered") + c.call("GET", CHAT + "/messages?limit=1", actor=alice, schema="MessagePage") + c.call("GET", "/api/v1/chats/" + quote("' OR 1=1 --", safe=""), 400, actor=alice) + + +@s.case("ACL: outsider gets 404 everywhere (even for invalid body)") +def _(): + c.call("GET", CHAT, 404, actor=eve) + c.call("GET", CHAT + "/messages", 404, actor=eve) + c.call("POST", CHAT + "/messages", 404, actor=eve, body={"text": "x"}) + c.call("POST", CHAT + "/messages", 404, actor=eve, body={"text": ""}) + c.call("GET", "/api/v1/chats/" + str(uuid.uuid4()), 404, actor=alice) + c.call("GET", "/api/v1/chats/not-a-uuid", 400, actor=alice) + got = c.call("GET", CHAT, actor=bob, schema="Chat")["json"] + check(got["member_ids"] == [alice["id"], bob["id"]], "member order/content changed") + + +@s.case("create chat validation: duplicates, caller missing, unknown member") +def _(): + c.call("POST", "/api/v1/chats", 400, actor=alice, body={"title": "d", "member_ids": [alice["id"], alice["id"]]}) + c.call("POST", "/api/v1/chats", 400, actor=alice, body={"title": "d", "member_ids": [bob["id"]]}) + c.call("POST", "/api/v1/chats", 400, actor=alice, body={"title": "d", "member_ids": []}) + c.call("POST", "/api/v1/chats", 400, actor=alice, body={"title": " ", "member_ids": [alice["id"]]}) + + +@s.case("compound create-chat rolls back on FK failure (no orphan chat)") +def _(): + title = "rollback-" + uuid.uuid4().hex + before = sql("SELECT count(*) FROM chats")[0] + c.call("POST", "/api/v1/chats", 400, actor=alice, + body={"title": title, "member_ids": [alice["id"], bob["id"], str(uuid.uuid4())]}) + check(sql("SELECT count(*) FROM chats WHERE title = '%s'" % title) == ["0"], "orphan chat committed") + check(sql("SELECT count(*) FROM chat_members m LEFT JOIN chats c ON c.id = m.chat_id WHERE c.id IS NULL") == ["0"], + "orphan membership") + return "chats before=%s, failed insert left 0 rows" % before + + +@s.case("registration race: 20 parallel same username -> exactly one 201") +def _(): + username = "race_" + uuid.uuid4().hex[:12] + statuses, lock = [], threading.Lock() + barrier = threading.Barrier(20) + + def worker(): + local = Client(BASE_URL, timeout=30) + barrier.wait() + r = local.call("POST", "/api/v1/users", [201, 409], + body={"username": username, "display_name": "R", "password": "Race-password-123"}) + with lock: + statuses.append(r["status"]) + threads = [threading.Thread(target=worker) for _ in range(20)] + [t.start() for t in threads]; [t.join() for t in threads] + check(sorted(statuses) == [201] + [409] * 19, "statuses: %s" % sorted(statuses)) + check(sql("SELECT count(*) FROM users WHERE username = '%s'" % username) == ["1"], "duplicate rows") + return "201x1, 409x19, rows=1" + + +@s.case("keyset pagination: 120 msgs, varying limit, no gaps/duplicates") +def _(): + room = c.call("POST", "/api/v1/chats", 201, actor=alice, schema="Chat", + body={"title": "pages", "member_ids": [alice["id"], bob["id"]]})["json"] + path = "/api/v1/chats/" + room["id"] + "/messages" + sent = [c.message(alice if i % 2 else bob, room, "m%03d" % i)["json"] for i in range(120)] + got, cursor, limits, n = [], None, [7, 1, 50, 100, 3], 0 + while True: + q = "?limit=%d" % limits[n % len(limits)] + ("&cursor=" + quote(cursor, safe="") if cursor else "") + page = c.call("GET", path + q, actor=bob, schema="MessagePage")["json"] + got.extend(page["items"]); cursor = page["next_cursor"]; n += 1 + if cursor is None: + break + check([m["id"] for m in got] == [m["id"] for m in sent], "order/completeness mismatch") + first = c.call("GET", path + "?limit=2", actor=bob, schema="MessagePage")["json"] + c.call("GET", CHAT + "/messages?cursor=" + quote(first["next_cursor"], safe=""), 400, actor=alice) + for q in ("?limit=0", "?limit=101", "?limit=abc", "?limit=-1", "?limit=1&limit=2", "?cursor=garbage", "?cursor="): + c.call("GET", path + q, 400, actor=bob) + default = c.call("GET", path, actor=bob, schema="MessagePage")["json"] + check(len(default["items"]) == 50 and default["next_cursor"], "default limit must be 50") + return "%d pages" % n + + +@s.case("chat list: only own chats, ordered, cursor bound to user") +def _(): + u = new_user(c, "lister"); login(c, u) + made = [c.call("POST", "/api/v1/chats", 201, actor=u, schema="Chat", + body={"title": "c%d" % i, "member_ids": [u["id"]]})["json"]["id"] for i in range(3)] + got, cursor = [], None + while True: + page = c.call("GET", "/api/v1/chats?limit=1" + ("&cursor=" + quote(cursor, safe="") if cursor else ""), + actor=u, schema="ChatPage")["json"] + got += [x["id"] for x in page["items"]]; cursor = page["next_cursor"] + if cursor is None: + break + foreign = cursor + check(got == made, "chat list mismatch") + c.call("GET", "/api/v1/chats?cursor=" + quote(foreign, safe=""), 400, actor=alice) + + +@s.case("read-after-write: confirmed message visible to next read") +def _(): + m = c.message(bob, chat, "raw-" + uuid.uuid4().hex)["json"] + items, cursor = [], None + while True: + page = c.call("GET", CHAT + "/messages?limit=100" + ("&cursor=" + quote(cursor, safe="") if cursor else ""), + actor=alice, schema="MessagePage")["json"] + items += page["items"]; cursor = page["next_cursor"] + if not cursor: + break + check(items[-1] == m, "last message is not the just-written one") + + +@s.case("unknown route 404 / wrong method 405 in error format") +def _(): + c.call("GET", "/api/v1/nope", 404) + r = c.call("PUT", "/api/v1/users", 405, body={}) + check(r["json"]["error"]["code"] == "invalid_request", "405 code") + + +if __name__ == "__main__": + s.run() diff --git a/student-tests/test_cache_race.py b/student-tests/test_cache_race.py new file mode 100644 index 00000000..31710364 --- /dev/null +++ b/student-tests/test_cache_race.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""In-process проверки политики кэша сессий против настоящего Redis/Valkey (нужен CACHE_URL[_FILE]). + +Сценарий гонки: запрос A прочитал сессию из PostgreSQL ДО logout, а записать её в кэш пытается ПОСЛЕ +logout. Без защиты отозванный токен жил бы в кэше до SESSION_CACHE_TTL_SECONDS. Защита — «надгробие» +и атомарный Lua SET-if-not-revoked. Запуск в Compose: make test-cache-race. +""" +import asyncio +import os +import secrets +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) +os.environ.setdefault("DATABASE_URL", "postgresql://unused") # БД этому тесту не нужна + +from messenger.config import load # noqa: E402 +from messenger.sessions import KEY, TOMBSTONE, Principal, SessionCache # noqa: E402 + + +async def main(): + settings = load() + if not settings.cache_enabled: + raise SystemExit("CACHE_URL or CACHE_URL_FILE is required") + cache = SessionCache(settings) + cache.degraded = False + user = {"id": "00000000-0000-4000-8000-000000000000", "username": "cache_test", "display_name": "t", + "created_at": "2026-01-01T00:00:00.000000Z"} + failures = 0 + + def report(ok, title, note=""): + nonlocal failures + failures += 0 if ok else 1 + print(("PASS " if ok else "FAIL ") + title + ((" " + note) if note else "")) + + th = secrets.token_bytes(32) + p = Principal(user=user, expires_at=time.time() + 3600, token_hash=th) + await cache.put(p) + report((await cache.get(th)) is not None, "read-through put makes entry readable") + ttl = await cache.client.pttl(KEY + th.hex()) + report(0 < ttl <= settings.session_cache_ttl_seconds * 1000, "entry TTL capped by SESSION_CACHE_TTL_SECONDS", + "pttl=%dms" % ttl) + + await cache.invalidate(th) # logout + report((await cache.get(th)) is None, "invalidate removes entry") + await cache.put(p) # «опоздавший» read-through после logout + report((await cache.client.exists(KEY + th.hex())) == 0, "late put after logout is refused by tombstone") + tomb = await cache.client.pttl(TOMBSTONE + th.hex()) + report(tomb > settings.session_cache_ttl_seconds * 1000, "tombstone outlives any cache entry", "pttl=%dms" % tomb) + + th2 = secrets.token_bytes(32) + await cache.put(Principal(user=user, expires_at=time.time() + 1.2, token_hash=th2)) + ttl2 = await cache.client.pttl(KEY + th2.hex()) + report(0 < ttl2 <= 1200, "entry never outlives session expiry", "pttl=%dms" % ttl2) + await asyncio.sleep(1.3) + report((await cache.get(th2)) is None, "expired session not served from cache") + + th3 = secrets.token_bytes(32) + await cache.put(Principal(user=user, expires_at=time.time() + 3600, token_hash=th3)) + cache._fail("test", ConnectionError("simulated")) + report((await cache.get(th3)) is None, "degraded mode: cache is not read until outbox drained") + + await cache.client.delete(KEY + th.hex(), TOMBSTONE + th.hex(), KEY + th3.hex()) + await cache.close() + print("summary: %s" % ("all passed" if not failures else "%d failed" % failures)) + raise SystemExit(1 if failures else 0) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 00000000..07aa6509 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +"""Истечение сессии при коротком SESSION_TTL_SECONDS (экземпляр api-ttl, профиль compose `ttl`). + +Проверяет и путь через кэш Redis (запись кэша прогрета сразу после входа), и путь через PostgreSQL: +TTL записи в кэше = min(SESSION_CACHE_TTL_SECONDS, остаток сессии), поэтому кэш не продлевает сессию. +""" +import datetime as dt +import time + +from common import TTL_BASE_URL, TTL_SECONDS, Client, Suite, check, login, new_user, sql + +c = Client(TTL_BASE_URL) +s = Suite("session expiry, SESSION_TTL_SECONDS=%g (%s)" % (TTL_SECONDS, TTL_BASE_URL)) + + +@s.case("expires_at = now + SESSION_TTL_SECONDS") +def _(): + u = new_user(c, "ttl"); login(c, u) + left = dt.datetime.fromisoformat(u["expires_at"][:-1] + "+00:00") - dt.datetime.now(dt.timezone.utc) + check(0 < left.total_seconds() <= TTL_SECONDS + 0.5, "unexpected TTL: %s" % left) + return "remaining %.2fs" % left.total_seconds() + + +@s.case("token valid before expiry, 401 Bearer after (cache warmed)") +def _(): + u = new_user(c, "ttl2"); login(c, u) + c.call("GET", "/api/v1/users/me", actor=u, schema="User") # заполняет кэш + c.call("GET", "/api/v1/chats", actor=u, schema="ChatPage") + time.sleep(TTL_SECONDS + 1.0) + r = c.call("GET", "/api/v1/users/me", 401, actor=u) + check(r["headers"]["WWW-Authenticate"].startswith("Bearer"), "challenge") + c.call("DELETE", "/api/v1/auth/sessions/current", 401, actor=u) + rows = sql("SELECT (expires_at < clock_timestamp())::text, (revoked_at IS NULL)::text FROM sessions s" + " JOIN users u ON u.id = s.user_id WHERE u.username = '%s'" % u["username"]) + check(rows == ["true|true"], "expiry must come from expires_at, not deletion/revocation: %s" % rows) + + +@s.case("re-login after expiry issues a new valid session") +def _(): + u = new_user(c, "ttl3"); login(c, u) + old = u["token"] + time.sleep(TTL_SECONDS + 1.0) + c.call("GET", "/api/v1/users/me", 401, actor=u) + login(c, u) + check(u["token"] != old, "token reused") + c.call("GET", "/api/v1/users/me", actor=u, schema="User") + + +if __name__ == "__main__": + s.run() diff --git a/student-tests/test_failures.py b/student-tests/test_failures.py new file mode 100644 index 00000000..156081a6 --- /dev/null +++ b/student-tests/test_failures.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Отказы зависимостей: Redis (кэш) и PostgreSQL (источник истины). + +ВНИМАНИЕ: тест останавливает и снова запускает контейнеры redis и postgres (по умолчанию через +docker compose; команды переопределяются STOP_DB/START_DB/STOP_CACHE/START_CACHE). Volumes не трогает. +""" +import time +import uuid + +from common import BASE_URL, Client, Failure, Suite, basic, check, login, new_user, sh, sql, wait_until + +c = Client(BASE_URL, timeout=10) +s = Suite("dependency failures (" + BASE_URL + ")") +alice = new_user(c, "fa"); login(c, alice) +bob = new_user(c, "fb"); login(c, bob) +chat = c.call("POST", "/api/v1/chats", 201, actor=alice, schema="Chat", + body={"title": "failures", "member_ids": [alice["id"], bob["id"]]})["json"] +PATH = "/api/v1/chats/" + chat["id"] + "/messages" + + +def timed(fn): + t = time.monotonic(); r = fn(); return r, time.monotonic() - t + + +@s.case("Redis down: API bypasses cache via PostgreSQL, no anonymous trust") +def _(): + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") # запись в кэше есть + victim = new_user(c, "fv"); login(c, victim) + c.call("GET", "/api/v1/users/me", actor=victim, schema="User") + sh("stop_cache") + try: + _, took = timed(lambda: c.call("GET", "/api/v1/users/me", actor=alice, schema="User")) + check(took < 2.0, "fallback too slow: %.2fs" % took) + c.call("GET", "/health/ready", schema="Health") # кэш не критичен + c.message(alice, chat, "while redis is down") + fresh = new_user(c, "fn"); login(c, fresh) # вход без Redis + c.call("GET", "/api/v1/users/me", actor=fresh, schema="User") + c.call("GET", "/api/v1/users/me", 401) # аноним — 401 + c.call("GET", "/api/v1/users/me", 401, headers={"Authorization": "Bearer " + "B" * 43}) + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=victim) # logout при мёртвом Redis + c.call("GET", "/api/v1/users/me", 401, actor=victim) + pending = sql("SELECT count(*) FROM session_cache_invalidations")[0] + finally: + sh("start_cache") + # После возврата Redis старая запись кэша victim не должна «воскресить» отозванный токен. + wait_until(lambda: sql("SELECT count(*) FROM session_cache_invalidations") == ["0"], timeout=30) + for _ in range(10): + c.call("GET", "/api/v1/users/me", 401, actor=victim) + time.sleep(0.3) + c.call("GET", "/api/v1/users/me", actor=alice, schema="User") + return "fallback latency %.3fs; outbox rows during outage=%s, drained after recovery" % (took, pending) + + +@s.case("PostgreSQL down: bounded 503 in error format; anonymous still 401") +def _(): + before = len(c.call("GET", PATH + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"]) + sh("stop_db") + observed = {} + try: + c.call("GET", "/health/live", schema="Health") + r, took = timed(lambda: c.call("GET", "/health/ready", 503)) + observed["ready"] = round(took, 2) + for name, call in { + "post_message": lambda: c.call("POST", PATH, 503, actor=alice, body={"text": "db down"}, + headers={"Idempotency-Key": uuid.uuid4().hex}), + "history": lambda: c.call("GET", PATH, 503, actor=bob), + "login": lambda: c.call("POST", "/api/v1/auth/sessions", 503, + headers={"Authorization": basic(alice["username"], alice["password"])}), + "register": lambda: c.call("POST", "/api/v1/users", 503, + body={"username": "x" + uuid.uuid4().hex[:10], "display_name": "x", "password": "p" * 12}), + }.items(): + r, took = timed(call) + check(r["json"]["error"]["code"] == "unavailable", name + ": wrong code") + check(took < 5.0, name + ": not bounded (%.2fs)" % took) + check("psycopg" not in r["json"]["error"]["message"].lower(), "leaks internals") + observed[name] = round(took, 2) + c.call("GET", "/api/v1/users/me", 401) # без токена — 401 + r = c.call("GET", "/api/v1/chats", [401, 503], headers={"Authorization": "Bearer " + "C" * 43}) + check(r["status"] != 200, "unknown token accepted") + finally: + sh("start_db") + t0 = time.monotonic() + wait_until(lambda: c.call("GET", "/health/ready", [200, 503])["status"] == 200, timeout=90) + recovered = time.monotonic() - t0 + after = c.call("GET", PATH + "?limit=100", actor=bob, schema="MessagePage")["json"]["items"] + check(len(after) == before, "data changed across DB outage") + c.message(alice, chat, "after db recovery") + return "latency(s)=%s; ready again in %.1fs" % (observed, recovered) + + +if __name__ == "__main__": + try: + s.run() + except Failure as exc: + print("FAIL", exc) + raise SystemExit(1) diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..6adaa751 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Общая идемпотентность POST /chats/{id}/messages через LB и несколько реплик. + +Ключ — заголовок Idempotency-Key, область: пользователь + метод + путь чата + ключ, срок ≥ 24 ч. +Проверки: повтор и конкурентный повтор через разные реплики → ровно одно сообщение; другой body → 409; +области ключа; формат ключа; неуспешный запрос не «занимает» ключ; повтор снова проверяет сессию и доступ. +""" +import json +import threading +import uuid +from collections import Counter + +from common import BASE_URL, Client, Suite, check, login, new_user, sql + +c = Client(BASE_URL) +s = Suite("idempotency across replicas (" + BASE_URL + ")") +alice = new_user(c, "ia"); login(c, alice) +bob = new_user(c, "ib"); login(c, bob) +eve = new_user(c, "ie"); login(c, eve) + + +def chat_of(*actors): + return c.call("POST", "/api/v1/chats", 201, actor=actors[0], schema="Chat", + body={"title": "idem", "member_ids": [a["id"] for a in actors]})["json"] + + +chat = chat_of(alice, bob) +PATH = "/api/v1/chats/" + chat["id"] + "/messages" + + +def post(actor, key, body=None, raw=None, expected=201, client=None, path=PATH): + h = {"Idempotency-Key": key} if key is not None else {} + if raw is not None: + h["Content-Type"] = "application/json" + return (client or c).call("POST", path, expected, actor=actor, body=body, raw=raw, headers=h, + schema="Message" if expected == 201 else None) + + +def count_text(text): + return int(sql("SELECT count(*) FROM messages WHERE text = '%s'" % text.replace("'", "''"))[0]) + + +@s.case("sequential replays: same 201 + same Message from different replicas") +def _(): + key, text = uuid.uuid4().hex, "seq-" + uuid.uuid4().hex + first = post(alice, key, {"text": text}) + instances = {first["headers"]["X-Instance-Id"]} + for _ in range(10): + r = post(alice, key, {"text": text}) + check(r["json"] == first["json"], "replay differs") + check(r["headers"].get("Idempotent-Replayed") == "true", "replay not marked") + instances.add(r["headers"]["X-Instance-Id"]) + check(len(instances) >= 2, "replays were not served by >=2 replicas: %s" % instances) + check(count_text(text) == 1, "duplicate message") + return "replicas=%d, rows=1" % len(instances) + + +@s.case("30 concurrent POSTs, same key + body, via LB -> exactly one message") +def _(): + key, text = uuid.uuid4().hex, "conc-" + uuid.uuid4().hex + results, lock, barrier = [], threading.Lock(), threading.Barrier(30) + + def worker(): + local = Client(BASE_URL, timeout=15) # отдельное соединение → LB распределяет по репликам + barrier.wait() + r = post(alice, key, {"text": text}, client=local) + with lock: + results.append((r["status"], json.dumps(r["json"], sort_keys=True), r["headers"]["X-Instance-Id"])) + threads = [threading.Thread(target=worker) for _ in range(30)] + [t.start() for t in threads]; [t.join() for t in threads] + check(len(results) == 30 and all(st == 201 for st, _, _ in results), "not all 201") + check(len({b for _, b, _ in results}) == 1, "different Message bodies returned") + per_instance = Counter(i for _, _, i in results) + check(len(per_instance) >= 2, "concurrency did not span replicas: %s" % dict(per_instance)) + check(count_text(text) == 1, "duplicates: %d" % count_text(text)) + return "per-replica responses %s, rows=1" % dict(per_instance) + + +@s.case("concurrent same key, two different bodies -> one message, others 201-same or 409") +def _(): + key, a, b = uuid.uuid4().hex, "race-a-" + uuid.uuid4().hex, "race-b-" + uuid.uuid4().hex + out, lock, barrier = [], threading.Lock(), threading.Barrier(20) + + def worker(i): + local = Client(BASE_URL, timeout=15) + barrier.wait() + r = post(alice, key, {"text": a if i % 2 else b}, client=local, expected=[201, 409]) + with lock: + out.append((i % 2, r["status"], r["json"])) + threads = [threading.Thread(target=worker, args=(i,)) for i in range(20)] + [t.start() for t in threads]; [t.join() for t in threads] + rows = count_text(a) + count_text(b) + check(rows == 1, "expected exactly one message, got %d" % rows) + winner = a if count_text(a) else b + for parity, status, body in out: + mine = a if parity else b + check(status == (201 if mine == winner else 409), "status %d for %s" % (status, "winner" if mine == winner else "loser")) + return "201x%d, 409x%d" % (sum(1 for o in out if o[1] == 201), sum(1 for o in out if o[1] == 409)) + + +@s.case("after commit: different body -> 409; reordered/whitespace JSON -> replay") +def _(): + key, text = uuid.uuid4().hex, "fmt-" + uuid.uuid4().hex + first = post(alice, key, {"text": text}) + r = post(alice, key, {"text": text + "!"}, expected=409) + check(r["json"]["error"]["code"] == "conflict", "409 code") + raw = ('{ \n "text" : ' + json.dumps(text) + ' \n}').encode() + check(post(alice, key, raw=raw)["json"] == first["json"], "semantically equal JSON not replayed") + + +@s.case("scope: user + chat + key (same key elsewhere is independent)") +def _(): + key = "shared-key:" + uuid.uuid4().hex[:8] + m1 = post(alice, key, {"text": "scope-1"})["json"] + m2 = post(bob, key, {"text": "scope-1"})["json"] # другой пользователь + other = chat_of(alice, bob) + m3 = post(alice, key, {"text": "scope-1"}, path="/api/v1/chats/" + other["id"] + "/messages")["json"] + check(len({m1["id"], m2["id"], m3["id"]}) == 3, "keys leaked across scope") + check(m2["sender_id"] == bob["id"], "sender") + + +@s.case("key format: missing/empty/129 chars/invalid -> 400; 128 chars ok") +def _(): + for key in (None, "", "a" * 129, "bad key", "k%y", "a/b", "k\"q"): + post(alice, key, {"text": "x"}, expected=400) + post(alice, "A" * 120 + "._:-z019", {"text": "len-128"}) + + +@s.case("failed attempt does not claim the key (400/404 not recorded)") +def _(): + key = uuid.uuid4().hex + post(alice, key, {"text": " "}, expected=400) + m = post(alice, key, {"text": "after-400"})["json"] + check(m["text"] == "after-400", "key was claimed by failed request") + key2 = uuid.uuid4().hex + post(eve, key2, {"text": "intruder"}, expected=404) # посторонний + post(alice, key2, {"text": "after-404"}) + + +@s.case("replay re-checks session and access") +def _(): + u = dict(alice); login(c, u) + key, text = uuid.uuid4().hex, "relogin-" + uuid.uuid4().hex + post(u, key, {"text": text}) + c.call("DELETE", "/api/v1/auth/sessions/current", 204, actor=u) + post(u, key, {"text": text}, expected=401) + post(eve, key, {"text": text}, expected=404) # чужой ключ/чат не раскрывается + check(post(alice, key, {"text": text})["json"]["text"] == text, "new session must replay") + + +@s.case("key older than IDEMPOTENCY_TTL_SECONDS is reusable (24 h window)") +def _(): + key, text = uuid.uuid4().hex, "ttl-" + uuid.uuid4().hex + first = post(alice, key, {"text": text})["json"] + sql("UPDATE idempotency_keys SET created_at = created_at - interval '25 hours' WHERE key = '%s'" % key) + second = post(alice, key, {"text": text})["json"] + check(second["id"] != first["id"], "expired key must create a new message") + check(count_text(text) == 2, "rows") + + +if __name__ == "__main__": + s.run()