Author SHA1 Message Date
koichakaev 937dd0f7a9 Восстановить README с условием лабораторной
Вернуть исходную методичку задания без изменений, как требует CONTRIBUTING.md.
2026-11-06 19:19:17 +03:00
koichakaev 48b8446800 Добавить графики и результаты теста нагрузки
Добавить графики масштабирования и исходные данные rate-controlled испытаний.
2026-11-06 19:15:16 +03:00
koichakaev aa33e40ee5 Добавить результаты нагрузочных проверок
Сохранить доказательства выполнения лабораторной работы 3.
2026-11-06 19:14:21 +03:00
koichakaev 73e45f60f7 Добавить пользовательские проверки 2026-11-06 19:13:01 +03:00
koichakaev 45fd5772ff Добавить парный эксперимент 8 против 12 реплик 2026-11-06 19:12:47 +03:00
koichakaev 41edfbc66b Добавить scripts отказа и трека масштабирования 2026-11-06 19:12:34 +03:00
koichakaev 77dae0ec87 Добавить схему и таблицу идемпотентности 2026-11-06 19:12:14 +03:00
koichakaev cc51070a9c Добавить конфигурацию HAProxy 2026-11-06 19:11:59 +03:00
koichakaev c1e59428e9 Добавить приложение, Compose и отчёт 2026-11-06 19:11:20 +03:00
koichakaev 56e1e89416 ЛР 3: реализация балансировки и отчёт по масштабированию 2026-11-06 19:10:45 +03:00
30 changed files with 3823 additions and 45 deletions
+9 -11
View File
@@ -1,13 +1,11 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
POSTGRES_USER=messenger
POSTGRES_PASSWORD=change-me-before-running
POSTGRES_DB=messenger
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
DB_POOL_MAX=10
DB_POOL_MIN=1
API_CPUS=1.0
API_MEMORY=512m
CURSOR_SECRET=replace-with-a-random-secret-of-at-least-32-bytes
LB_CPUS=0.25
+1
View File
@@ -30,3 +30,4 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.pytest_cache/
+11
View File
@@ -0,0 +1,11 @@
FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PIP_DISABLE_PIP_VERSION_CHECK=1 PIP_DEFAULT_TIMEOUT=60 PIP_RETRIES=3
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
COPY --chown=app:app server.py ./server.py
COPY --chown=app:app migrations ./migrations
USER 10001:10001
EXPOSE 8080
HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"
CMD ["python", "server.py"]
+15 -12
View File
@@ -1,30 +1,33 @@
PYTHON ?= python3
BASE_URL ?= http://localhost:8080
BASE_URL ?= http://127.0.0.1:8080
CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence test-failover
.PHONY: help check validate test load test-persistence test-failover test-idempotency track1 track1-offered track1-offered-scale rolling-update
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "check, validate, test, load, test-persistence, test-failover, test-idempotency, track1, track1-offered, track1-offered-scale, rolling-update"
check:
$(PYTHON) scripts/check_contract.py
validate:
$(PYTHON) scripts/validate_openapi.py
test:
$(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS)
load:
$(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS)
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/restart.sh"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT=scripts/stop-one.sh"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
test-idempotency:
$(PYTHON) student-tests/idempotency_race.py --base-url "$(BASE_URL)" $(TLS_ARGS)
track1:
$(PYTHON) scripts/track1_experiment.py
track1-offered:
$(PYTHON) scripts/track1_offered_load.py
track1-offered-scale:
$(PYTHON) scripts/track1_offered_scale.py
rolling-update:
./scripts/rolling-update.sh
+86 -22
View File
@@ -1,37 +1,101 @@
# Отчёт — лабораторная 3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Койчакаев Багавдин, БВТ2301.
- Целевая оценка: 5; законченный трек: 1 — цена масштабирования.
- Состав реализации: API ЛР2, HAProxy, минимум две API-реплики, общие PostgreSQL и Valkey, конкурентная идемпотентность в PostgreSQL.
- Внешние источники: контракт и материалы курса. Использованы Python 3.12.8, psycopg 3.2.3, argon2-cffi 23.1.0, redis-py 5.2.1; при подготовке использовался Codex.
## Запуск из чистого clone
## Запуск
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
```sh
cp .env.example .env
# задайте POSTGRES_PASSWORD и CURSOR_SECRET (случайный ключ минимум 32 байта)
docker compose --env-file .env config --quiet
docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
make check
make validate
make test BASE_URL=http://127.0.0.1:8080
make test-idempotency BASE_URL=http://127.0.0.1:8080
make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080
make test-failover ACTION_SCRIPT=scripts/stop-one.sh BASE_URL=http://127.0.0.1:8080
```
Приложение слушает только внутреннюю сеть Compose. Единственный опубликованный порт — loopback порт HAProxy 8080. Миграции запускает одноразовый сервис `migrate` перед API, под advisory lock PostgreSQL. Первая схема сохраняется совместимой с базой ЛР2; миграция `002_idempotency` добавляет таблицу ключей и индекс срока хранения.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
HTTP client → HAProxy :8080 → API × 2+ → PostgreSQL (messages, sessions, idempotency)
↘ общий Valkey (TTL lookup hint; PostgreSQL остаётся источником истины)
```
## Доказательства критериев
HAProxy round-robin балансирует запросы между адресами DNS `api`, повторно разрешая адрес каждые 2 секунды. Проверка `GET /health/ready` выполняется раз в секунду; две ошибки исключают backend, два успеха возвращают его. Connect timeout — 1 секунда, timeout ожидания/ответа — 10 секунд. API слушает `0.0.0.0:8080` внутри сети и на SIGTERM прекращает `serve_forever`; Compose ждёт до 10 секунд для штатного завершения.
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
Сессия проверяется PostgreSQL на каждом запросе; значение Valkey лишь ускоряет lookup и не авторизует пользователя без живой строки в PostgreSQL. Общий `CURSOR_SECRET` нужен, чтобы cursor истории, созданный одной репликой, принимался другой. Секрет и пароль БД находятся только в локальном `.env`, игнорируемом Git и Docker build context.
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Для каждого POST сообщения обязателен ключ `[A-Za-z0-9._:-]{1,128}`. Уникальная область — пользователь, метод, путь чата и ключ; body хешируется после нормализации JSON, а пропущенный `attachment_ids` равен `[]`. `pg_advisory_xact_lock` сериализует одновременные запросы одной области во всех API-процессах; PostgreSQL хранит отпечаток body и точный JSON результата вместе с сообщением в одной транзакции. Совпавший запрос возвращает тот же `201` и Message; отличающийся body — `409`; запись можно использовать заново спустя 24 часа. Ошибка/откат транзакции не оставляет частично зарегистрированный ключ. Каждый API-ответ включая ошибку содержит непустой `X-Request-Id` и идентификатор процесса в `X-Instance-Id`.
## Эксперименты и отказы
### Роли и версии
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Роль | Service | Версия | Health/volume/порты |
|---|---|---|---|
| API | `api` (Compose scale) | Python 3.12.8 slim Bookworm, pinned digest в Dockerfile | `/health/ready`, без host-порта, без локального состояния |
| Балансировщик | `haproxy` | 3.0.8 Alpine | HAProxy backend checks, единственный порт `127.0.0.1:8080` |
| База | `postgres` | 16.4 Bookworm | `pg_isready`, общий volume `postgres_data`, внутренний порт 5432 |
| Кэш сессий | `valkey` | 8.0.1 Alpine | `PING`, volume `valkey_data`, AOF `everysec`, внутренний порт 6379 |
| Миграции | `migrate` | тот же образ API | одноразовый контейнер; выполняется до запуска реплик |
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
CPU/RAM limits заданы в compose для API, HAProxy, PostgreSQL и Valkey. `api` не имеет `container_name`, поэтому штатное горизонтальное масштабирование поддерживается.
## Ограничения и следующие шаги
## Проверки уровня 3 и 4
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Публичный smoke проверяет обязательный API через HAProxy. Дополнительный параллельный тест запускает 32 запроса с одним ключом и одним body одновременно; все ответы вернули один ID сообщения, наблюдались две реплики. Persistence-сценарий перезапустил API, HAProxy, PostgreSQL и Valkey без удаления volumes; тот же Bearer-токен прочитал те же данные. Три failover-прогона остановили API-реплику по очереди и повторили сценарий после восстановления. Каждый наблюдал две реплики до отказа; новые записи восстановились за 2,902 с, 1,543 с и 1,447 с; ошибок было 0 во всех трёх прогонах (113, 125 и 114 успешных записей); сообщение до отказа и все подтверждённые записи остались доступны, дубли отсутствовали. В конце теста остановленная реплика оставалась остановленной; Compose восстановил её перед следующим прогоном и после финальной проверки.
Rolling replacement под нагрузкой выполняется скриптом `scripts/rolling-update.sh`: пул сначала увеличивается до трёх, затем контейнеры заменяются по одному с сохранением активного backend. Под нагрузкой 60 секунд, concurrency 16: 50 291 успешная запись (836,949 RPS), 24 ошибки из 50 315 запросов (0,048%), p50/p95/p99 = 6,118/55,938/85,729 мс. После завершения пул восстановлен до двух здоровых реплик. Метрики сохранены в `evidence/rolling-update-load.json`; успешность запросов при замене не абсолютная, что отражено в ограничениях.
## Трек 1 — цена масштабирования
**Гипотеза.** При равном суммарном CPU/RAM перенос ресурса с одного API на несколько реплик улучшит RPS только до насыщения общей PostgreSQL или пула соединений. При фиксированной мощности на реплику добавление API в какой-то момент перестанет повышать throughput; увеличение пула сверх полезного уровня добавит конкуренцию за БД, а не пропускную способность.
**Методика.** `scripts/track1_experiment.py` пересоздаёт API при каждом профиле, ждёт readiness и 5 секунд стабилизации, затем трижды измеряет закрытый цикл POST-сообщений по 30 секунд с concurrency 16. Перед каждым sample очищаются только учебные таблицы `messages` и `idempotency_keys`, затем выполняется `ANALYZE`; порядок вариантов перемешивается детерминированным seed. Сессии/пользователи остаются в PostgreSQL. Фиксированный бюджет: 2 vCPU и 1 GiB RAM суммарно для API, разделённые между 1/2/4 репликами. Серия роста: 0,5 vCPU и 256 MiB на реплику для 1/2/4/8 реплик. Отдельное сравнение при четырёх репликах меняет `DB_POOL_MAX` 5/10/20. PostgreSQL ограничен 2 CPU и 1 GiB; HAProxy — 0,25 CPU и 128 MiB. Записываются успешные RPS, ошибки, p50/p95/p99, CPU/RAM компонентов и число сообщений. Сводки и исходные данные — `evidence/track1-results.json`, график — `evidence/track1-rps.svg`.
Чтобы определить точку насыщения, `scripts/track1_offered_load.py` подаёт фиксированный поток 250/500/750/1000 POST RPS на восемь API-реплик (0,5 CPU / 256 MiB каждая) по три раза в перемешанном порядке; HTTP-клиент переиспользуется в потоках, число in-flight ограничено, Docker stats во время нагрузочного окна не опрашивается. Throughput рассчитан по фактическому времени вместе с drain. Затем `scripts/track1_offered_scale.py` делает три парных прогона 8/12 реплик при предложенных 1000 RPS. Между повторами масштабирование возвращается к восьми репликам, чтобы освободить DB-соединения до очистки синтетических таблиц. Все исходные измерения лежат в `evidence/offered-load-results.json` и `evidence/offered-scale-results.json`; график rate sweep — `evidence/offered-load.svg`.
| Профиль | Реплики | Пул/реплика | медиана RPS (min–max) | медиана ошибок | p50/p95/p99, ms | API CPU mean/peak, % | API RAM peak, MiB |
|---|---:|---:|---:|---:|---:|---:|---:|
| fixed-1 | 1 | 10 | 418,5 (389,7–456,3) | 0,017% | 32,3/53,0/82,4 | 174,3/190,2 | 61,9 |
| fixed-2 | 2 | 10 | 698,6 (653,6–749,2) | 0% | 8,0/62,4/92,9 | 74,0/109,5 | 60,9 |
| fixed-4 | 4 | 10 | 837,7 (590,7–900,3) | 0,007% | 5,2/57,5/101,9 | 38,2/53,3 | 62,4 |
| scaled-1 | 1 | 10 | 103,7 (102,1–108,8) | 0,064% | 111,1/313,3/484,9 | 47,8/53,3 | 83,5 |
| scaled-2 | 2 | 10 | 274,6 (244,8–276,3) | 0,048% | 22,9/183,7/290,4 | 37,6/52,9 | 140,0 |
| scaled-4 | 4 | 10 | 680,7 (635,3–875,7) | 0% | 6,0/71,0/123,1 | 40,8/53,2 | 79,2 |
| scaled-8 | 8 | 10 | 764,2 (655,4–787,6) | 0% | 9,5/70,6/111,2 | 29,0/50,1 | 82,0 |
| pool-5 | 4 | 5 | 921,2 (837,9–1061,3) | 0,003% | 5,3/49,2/89,8 | 40,4/53,3 | 62,6 |
| pool-10 | 4 | 10 | 808,9 (611,5–969,2) | 0,020% | 5,5/66,7/108,3 | 38,9/51,9 | 99,5 |
| pool-20 | 4 | 20 | 951,3 (573,5–1044,2) | 0,003% | 5,4/45,3/89,2 | 41,7/52,5 | 63,1 |
Среда: macOS 26.5.1 arm64, 8 host CPU; Docker Engine 29.8.2; Docker VM 8 CPU / 4 106 605 444 байта RAM (~3 918 MiB); Python клиента 3.14.6; Python API 3.12.8; PostgreSQL 16.4. Закрытая серия длилась 30 секунд, rate-controlled серия — 15 секунд на sample. Во всех замерах JSON фиксирует размер синтетических таблиц; перед sample `messages` и `idempotency_keys` очищались, остальные пользовательские данные сохранялись. Closed-loop percentiles подвержены coordinated omission; rate-controlled серия имеет короткие окна и оценивает только эту среду. Все результаты ограничены одним Docker-host; потеря узла одновременно затрагивает API, HAProxy, PostgreSQL и локальные volumes.
| Предложено, RPS | Медиана успешных RPS (min–max) | Медиана ошибок | Медиана p95, ms |
|---:|---:|---:|---:|
| 250 | 249,6 (249,6–250,0) | 0% | 9,9 |
| 500 | 499,6 (419,6–499,9) | 0% | 41,6 |
| 750 | 745,9 (693,2–748,7) | 0,018% | 100,9 |
| 1000 | 810,1 (553,2–879,3) | 2,753% | 1 611,3 |
Парное сравнение при 1000 предложенных RPS:
| API-реплик | Успешные RPS, медиана (min–max) | p95, медиана, ms | Ошибки POST за 3 повтора | Диагностика PostgreSQL |
|---:|---:|---:|---:|---|
| 8 | 727,0 (584,4–851,5) | 1 504,7 | 874 / 45 000 | `SELECT` прошёл |
| 12 | 555,8 (546,0–774,9) | 1 986,4 | 12 / 45 000 | Все 3 запроса отклонены: `too many clients already` |
**Поиск насыщения и улучшение.** В rate-controlled серии 250 и 500 предложенных RPS полностью принимались без ошибок (медианный p95 9,9 и 41,6 мс). При 750 RPS медиана успешной пропускной способности составила 745,9 RPS, p95 — 100,9 мс. При 1000 RPS throughput вырос лишь до 810,1 RPS, медианная доля ошибок достигла 2,75%, а p95 — 1 611 мс. Это показывает knee общей цепочки между 750 и 1000 RPS; повторяемость ограничена тремя короткими прогонами, поэтому это не универсальная производственная ёмкость.
Компонентное ограничение установлено отдельной проверкой DB-пула. При `DB_POOL_MAX=10` восемь реплик могут открыть до 80 соединений, а двенадцать — до 120. Фактический `max_connections` PostgreSQL равен 100. В трёх прогонах с 12 репликами и нагрузкой 1000 RPS, помимо роста задержки, PostgreSQL трижды отказал даже в диагностическом `SELECT` с `FATAL: sorry, too many clients already`; после возврата к восьми репликам запросы к БД снова проходили. Наблюдение сохранено в `evidence/db-saturation-observation.txt`, а метрики сравнения — `evidence/offered-scale-results.json`. Таким образом, насыщение пула/БД возникает между восемью и двенадцатью репликами при текущем лимите пула; дальнейшее увеличение API без снижения размера пула или повышения DB connection budget ухудшает доступность общих соединений. 8→12 RPS-диапазоны перекрываются, поэтому причинный вывод основан на подтверждённом исчерпании connection slots, а не только на падении медианного throughput.
Подтверждённое улучшение при фиксированном суммарном бюджете 2 vCPU / 1 GiB: переход fixed-1→fixed-2 поднял медиану с 418,5 до 698,6 RPS (+66,9%); диапазоны трёх повторов не пересекаются (389,7–456,3 и 653,6–749,2 RPS). p50 снизился с 32,3 до 8,0 мс, p95 изменился с 53,0 до 62,4 мс. Серии DB_POOL_MAX=5/10/20 при низкой конкуренции и HAProxy 0,25/0,5 CPU имеют перекрывающиеся диапазоны; их эффект не заявляется как отдельное улучшение. Все измерения ограничены одной Docker VM и конкретным синтетическим профилем.
## Результаты и ограничения
Офлайн структурная проверка, API smoke, гонка идемпотентности, перезапуск и failover пройдены; подробный stdout сохранён в `evidence/lab3-checks.txt`. Полная OpenAPI/JSON Schema проверка требует `openapi-spec-validator==0.7.2` из `requirements-dev.txt`; установить зависимость в этой среде не удалось из-за недоступного DNS PyPI. HTTP используется только на loopback для синтетических данных. Эта схема выдерживает отказ одного API-процесса при работающих HAProxy, host, PostgreSQL и Valkey; она не выдерживает потерю Docker-host, HAProxy или PostgreSQL. При отказе Valkey действующие сессии проверяются по PostgreSQL. Отдельной HA базы данных и второго узла нет. Под нагрузкой при rolling replacement замерены 24 ошибки на 50 315 запросов (0,048%); для повторения именно операции создания сообщения клиент должен повторить запрос с тем же `Idempotency-Key`. Другие POST-операции не обещают прозрачную идемпотентность.
+79
View File
@@ -0,0 +1,79 @@
services:
migrate:
build: .
command: ["python", "server.py"]
environment:
RUN_MIGRATIONS_ONLY: "1"
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
CACHE_URL: redis://valkey:6379/0
depends_on:
postgres: { condition: service_healthy }
valkey: { condition: service_healthy }
restart: "no"
api:
build: .
restart: unless-stopped
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
CURSOR_SECRET: ${CURSOR_SECRET:?Set CURSOR_SECRET in .env}
DATABASE_URL: postgresql://${POSTGRES_USER:-messenger}:${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}@postgres:5432/${POSTGRES_DB:-messenger}
CACHE_URL: redis://valkey:6379/0
DB_POOL_MAX: ${DB_POOL_MAX:-10}
DB_POOL_MIN: ${DB_POOL_MIN:-1}
depends_on:
migrate: { condition: service_completed_successfully }
valkey: { condition: service_healthy }
cpus: ${API_CPUS:-1.0}
mem_limit: ${API_MEMORY:-512m}
stop_grace_period: 10s
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 2s
timeout: 3s
retries: 2
start_period: 10s
postgres:
image: postgres:16.4-bookworm
restart: unless-stopped
environment:
POSTGRES_USER: ${POSTGRES_USER:-messenger}
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env}
POSTGRES_DB: ${POSTGRES_DB:-messenger}
volumes: ["postgres_data:/var/lib/postgresql/data"]
cpus: 2.0
mem_limit: 1g
healthcheck:
test: ["CMD-SHELL", "pg_isready -U \"$$POSTGRES_USER\" -d \"$$POSTGRES_DB\""]
interval: 3s
timeout: 3s
retries: 20
valkey:
image: valkey/valkey:8.0.1-alpine
restart: unless-stopped
command: ["valkey-server", "--appendonly", "yes", "--appendfsync", "everysec"]
volumes: ["valkey_data:/data"]
cpus: 0.25
mem_limit: 128m
healthcheck:
test: ["CMD", "valkey-cli", "ping"]
interval: 3s
timeout: 3s
retries: 10
haproxy:
image: haproxy:3.0.8-alpine
restart: unless-stopped
ports: ["127.0.0.1:${APP_PORT:-8080}:8080"]
volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"]
depends_on: [api]
cpus: ${LB_CPUS:-0.25}
mem_limit: 128m
healthcheck:
test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"]
interval: 10s
timeout: 3s
retries: 3
volumes:
postgres_data:
valkey_data:
+110
View File
@@ -0,0 +1,110 @@
PostgreSQL connection saturation observation at 12 API replicas; no credentials or message content recorded.
$ docker compose --env-file .env ps api
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab3-api-10 sha256:4833b59a6d4c8406b779432843914b72e61ee92aecd4325e4d034e6839f2ff57 "python server.py" api 45 minutes ago Up 45 minutes (healthy) 8080/tcp
mtusi2026lab3-api-11 mtusi2026lab3-api "python server.py" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi2026lab3-api-12 mtusi2026lab3-api "python server.py" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi2026lab3-api-13 mtusi2026lab3-api "python server.py" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi2026lab3-api-14 mtusi2026lab3-api "python server.py" api About a minute ago Up About a minute (healthy) 8080/tcp
mtusi2026lab3-api-3 sha256:5f93cf319bad7c0e5f360f38e3e3f07708f2357101f6c5804f0194b1a02377cd "python server.py" api 50 minutes ago Up 50 minutes (healthy) 8080/tcp
mtusi2026lab3-api-4 sha256:5f93cf319bad7c0e5f360f38e3e3f07708f2357101f6c5804f0194b1a02377cd "python server.py" api 50 minutes ago Up 50 minutes (healthy) 8080/tcp
mtusi2026lab3-api-5 sha256:5f93cf319bad7c0e5f360f38e3e3f07708f2357101f6c5804f0194b1a02377cd "python server.py" api 50 minutes ago Up 50 minutes (healthy) 8080/tcp
mtusi2026lab3-api-6 sha256:5f93cf319bad7c0e5f360f38e3e3f07708f2357101f6c5804f0194b1a02377cd "python server.py" api 50 minutes ago Up 50 minutes (healthy) 8080/tcp
mtusi2026lab3-api-7 sha256:4833b59a6d4c8406b779432843914b72e61ee92aecd4325e4d034e6839f2ff57 "python server.py" api 45 minutes ago Up 45 minutes (healthy) 8080/tcp
mtusi2026lab3-api-8 sha256:4833b59a6d4c8406b779432843914b72e61ee92aecd4325e4d034e6839f2ff57 "python server.py" api 45 minutes ago Up 45 minutes (healthy) 8080/tcp
mtusi2026lab3-api-9 sha256:4833b59a6d4c8406b779432843914b72e61ee92aecd4325e4d034e6839f2ff57 "python server.py" api 45 minutes ago Up 45 minutes (healthy) 8080/tcp
exit_code=0
$ docker compose --env-file .env exec -T postgres psql -U messenger -d messenger -Atc SHOW max_connections; SELECT count(*) FROM pg_stat_activity;
psql: error: connection to server on socket "/var/run/postgresql/.s.PGSQL.5432" failed: FATAL: sorry, too many clients already
exit_code=2
$ compose scale to 8 API replicas
Container mtusi2026lab3-postgres-1 Running
Container mtusi2026lab3-valkey-1 Running
Container mtusi2026lab3-haproxy-1 Running
Container mtusi2026lab3-api-6 Recreate
Container mtusi2026lab3-api-12 Stopping
Container mtusi2026lab3-api-3 Recreate
Container mtusi2026lab3-api-11 Stopping
Container mtusi2026lab3-api-7 Recreate
Container mtusi2026lab3-api-13 Stopping
Container mtusi2026lab3-api-5 Recreate
Container mtusi2026lab3-api-14 Stopping
Container mtusi2026lab3-api-10 Recreate
Container mtusi2026lab3-api-4 Recreate
Container mtusi2026lab3-api-9 Recreate
Container mtusi2026lab3-api-8 Recreate
Container mtusi2026lab3-api-11 Stopped
Container mtusi2026lab3-api-11 Removing
Container mtusi2026lab3-api-11 Removed
Container mtusi2026lab3-api-13 Stopped
Container mtusi2026lab3-api-13 Removing
Container mtusi2026lab3-api-13 Removed
Container mtusi2026lab3-api-14 Stopped
Container mtusi2026lab3-api-14 Removing
Container mtusi2026lab3-api-14 Removed
Container mtusi2026lab3-api-12 Stopped
Container mtusi2026lab3-api-12 Removing
Container mtusi2026lab3-api-12 Removed
Container mtusi2026lab3-api-3 Recreated
Container mtusi2026lab3-api-7 Recreated
Container mtusi2026lab3-api-9 Recreated
Container mtusi2026lab3-api-4 Recreated
Container mtusi2026lab3-api-6 Recreated
Container mtusi2026lab3-api-8 Recreated
Container mtusi2026lab3-api-10 Recreated
Container mtusi2026lab3-api-5 Recreated
Container mtusi2026lab3-postgres-1 Waiting
Container mtusi2026lab3-valkey-1 Waiting
Container mtusi2026lab3-valkey-1 Healthy
Container mtusi2026lab3-postgres-1 Healthy
Container mtusi2026lab3-migrate-1 Starting
Container mtusi2026lab3-migrate-1 Started
Container mtusi2026lab3-valkey-1 Waiting
Container mtusi2026lab3-migrate-1 Waiting
Container mtusi2026lab3-valkey-1 Healthy
Container mtusi2026lab3-migrate-1 Exited
Container mtusi2026lab3-api-5 Starting
Container mtusi2026lab3-api-5 Started
Container mtusi2026lab3-api-3 Starting
Container mtusi2026lab3-api-3 Started
Container mtusi2026lab3-api-6 Starting
Container mtusi2026lab3-api-6 Started
Container mtusi2026lab3-api-7 Starting
Container mtusi2026lab3-api-7 Started
Container mtusi2026lab3-api-4 Starting
Container mtusi2026lab3-api-4 Started
Container mtusi2026lab3-api-8 Starting
Container mtusi2026lab3-api-8 Started
Container mtusi2026lab3-api-9 Starting
Container mtusi2026lab3-api-9 Started
Container mtusi2026lab3-api-10 Starting
Container mtusi2026lab3-api-10 Started
Container mtusi2026lab3-valkey-1 Waiting
Container mtusi2026lab3-haproxy-1 Waiting
Container mtusi2026lab3-migrate-1 Waiting
Container mtusi2026lab3-api-5 Waiting
Container mtusi2026lab3-api-3 Waiting
Container mtusi2026lab3-api-6 Waiting
Container mtusi2026lab3-api-7 Waiting
Container mtusi2026lab3-api-4 Waiting
Container mtusi2026lab3-api-8 Waiting
Container mtusi2026lab3-api-9 Waiting
Container mtusi2026lab3-api-10 Waiting
Container mtusi2026lab3-postgres-1 Waiting
Container mtusi2026lab3-haproxy-1 Healthy
Container mtusi2026lab3-valkey-1 Healthy
Container mtusi2026lab3-postgres-1 Healthy
Container mtusi2026lab3-migrate-1 Exited
Container mtusi2026lab3-api-5 Healthy
Container mtusi2026lab3-api-3 Healthy
Container mtusi2026lab3-api-6 Healthy
Container mtusi2026lab3-api-7 Healthy
Container mtusi2026lab3-api-4 Healthy
Container mtusi2026lab3-api-8 Healthy
Container mtusi2026lab3-api-9 Healthy
Container mtusi2026lab3-api-10 Healthy
exit_code=0
$ query PostgreSQL connection limit and count after scale down
100
15
exit_code=0
+153
View File
@@ -0,0 +1,153 @@
$ docker compose --env-file .env config --quiet
exit_code=0
$ docker compose --env-file .env up -d --build --scale api=2 --wait --wait-timeout 180
#1 [internal] load local bake definitions
#1 reading from stdin 1.06kB done
#1 DONE 0.0s
#2 [api internal] load build definition from Dockerfile
#2 transferring dockerfile: 785B done
#2 DONE 0.0s
#3 [api internal] load metadata for docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
#3 DONE 1.9s
#4 [api internal] load .dockerignore
#4 transferring context: 196B done
#4 DONE 0.0s
#5 [migrate internal] load build context
#5 transferring context: 194B done
#5 DONE 0.0s
#6 [migrate 1/6] FROM docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0
#6 resolve docker.io/library/python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 done
#6 DONE 0.0s
#7 [migrate 2/6] WORKDIR /app
#7 CACHED
#8 [migrate 4/6] RUN pip install --no-cache-dir -r requirements.txt && groupadd --gid 10001 app && useradd --uid 10001 --gid app --no-create-home --shell /usr/sbin/nologin app
#8 CACHED
#9 [migrate 3/6] COPY requirements.txt .
#9 CACHED
#10 [migrate 5/6] COPY --chown=app:app server.py ./server.py
#10 CACHED
#11 [migrate 6/6] COPY --chown=app:app migrations ./migrations
#11 CACHED
#12 [api] exporting to image
#12 exporting layers done
#12 exporting manifest sha256:6d456f82ad1a9f5b8ce4b3a64cab8b843c36f037fcf79696a82db3ee58a97133 done
#12 exporting config sha256:019ccb0a5711c492cb5dbcc7b6c8d1cd8996a1f4ea28579e93493499ea6176f1 done
#12 exporting attestation manifest sha256:7a889781683085054b961e9bca3897ccc36a5174d3c5e020c834341f2c857ca7 done
#12 exporting manifest list sha256:b95f98ffcfcb7cd1b3a5c187f0e0d5d39d078a2f74e151032735e5fa16c230dd done
#12 naming to docker.io/library/mtusi2026lab3-api:latest done
#12 unpacking to docker.io/library/mtusi2026lab3-api:latest done
#12 DONE 0.1s
#13 [migrate] exporting to image
#13 exporting layers done
#13 exporting manifest sha256:fd5e9154b8af2d17e52adf4b864ac3474ba1d0907ea6381808e7e10cf8e2e461 done
#13 exporting config sha256:682c09802432b2cc1f0e66f0658ac7eef8f82eeeca28fb344054cc2eaf9fb73b done
#13 exporting attestation manifest sha256:a4ab08e7eb901351bc35d3da999b1f46c58a723be0a20bec35af0c904af7f180 done
#13 exporting manifest list sha256:d80d6a8bd4dd17f178cc13b321dd7467c69425f5fcb35aeda2ebeed1ce056e68 done
#13 naming to docker.io/library/mtusi2026lab3-migrate:latest done
#13 unpacking to docker.io/library/mtusi2026lab3-migrate:latest done
#13 DONE 0.1s
#14 [api] resolving provenance for metadata file
#14 DONE 0.0s
#15 [migrate] resolving provenance for metadata file
#15 DONE 0.0s
Image mtusi2026lab3-migrate Building
Image mtusi2026lab3-api Building
Image mtusi2026lab3-migrate Built
Image mtusi2026lab3-api Built
Container mtusi2026lab3-valkey-1 Running
Container mtusi2026lab3-haproxy-1 Running
Container mtusi2026lab3-postgres-1 Running
Container mtusi2026lab3-api-6 Stopping
Container mtusi2026lab3-api-7 Stopping
Container mtusi2026lab3-api-3 Recreate
Container mtusi2026lab3-api-8 Stopping
Container mtusi2026lab3-api-5 Stopping
Container mtusi2026lab3-api-4 Recreate
Container mtusi2026lab3-api-10 Stopping
Container mtusi2026lab3-api-9 Stopping
Container mtusi2026lab3-api-3 Recreated
Container mtusi2026lab3-api-4 Recreated
Container mtusi2026lab3-api-10 Stopped
Container mtusi2026lab3-api-10 Removing
Container mtusi2026lab3-api-5 Stopped
Container mtusi2026lab3-api-5 Removing
Container mtusi2026lab3-api-10 Removed
Container mtusi2026lab3-api-7 Stopped
Container mtusi2026lab3-api-7 Removing
Container mtusi2026lab3-api-5 Removed
Container mtusi2026lab3-api-8 Stopped
Container mtusi2026lab3-api-8 Removing
Container mtusi2026lab3-api-7 Removed
Container mtusi2026lab3-api-9 Stopped
Container mtusi2026lab3-api-9 Removing
Container mtusi2026lab3-api-8 Removed
Container mtusi2026lab3-api-6 Stopped
Container mtusi2026lab3-api-6 Removing
Container mtusi2026lab3-api-9 Removed
Container mtusi2026lab3-api-6 Removed
Container mtusi2026lab3-postgres-1 Waiting
Container mtusi2026lab3-valkey-1 Waiting
Container mtusi2026lab3-postgres-1 Healthy
Container mtusi2026lab3-valkey-1 Healthy
Container mtusi2026lab3-migrate-1 Starting
Container mtusi2026lab3-migrate-1 Started
Container mtusi2026lab3-valkey-1 Waiting
Container mtusi2026lab3-migrate-1 Waiting
Container mtusi2026lab3-migrate-1 Exited
Container mtusi2026lab3-valkey-1 Healthy
Container mtusi2026lab3-api-4 Starting
Container mtusi2026lab3-api-4 Started
Container mtusi2026lab3-api-3 Starting
Container mtusi2026lab3-api-3 Started
Container mtusi2026lab3-haproxy-1 Waiting
Container mtusi2026lab3-migrate-1 Waiting
Container mtusi2026lab3-api-4 Waiting
Container mtusi2026lab3-api-3 Waiting
Container mtusi2026lab3-postgres-1 Waiting
Container mtusi2026lab3-valkey-1 Waiting
Container mtusi2026lab3-valkey-1 Healthy
Container mtusi2026lab3-haproxy-1 Healthy
Container mtusi2026lab3-migrate-1 Exited
Container mtusi2026lab3-postgres-1 Healthy
Container mtusi2026lab3-api-4 Healthy
Container mtusi2026lab3-api-3 Healthy
exit_code=0
$ .venv/bin/python scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
exit_code=0
$ .venv/bin/python scripts/validate_openapi.py
Traceback (most recent call last):
File "/Users/koichakaev/Desktop/учеба/ВНП/mtusi2026lab3/scripts/validate_openapi.py", line 5, in <module>
from openapi_spec_validator import validate
ModuleNotFoundError: No module named 'openapi_spec_validator'
exit_code=1
$ .venv/bin/python tests/smoke.py --base-url http://127.0.0.1:8080
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
exit_code=0
$ .venv/bin/python tests/idempotency_race.py --base-url http://127.0.0.1:8080
PASS: 32 concurrent retries, one message, 2 API instances observed
exit_code=0
$ docker compose --env-file .env ps
NAME IMAGE COMMAND SERVICE CREATED STATUS PORTS
mtusi2026lab3-api-3 mtusi2026lab3-api "python server.py" api 30 seconds ago Up 27 seconds (healthy) 8080/tcp
mtusi2026lab3-api-4 mtusi2026lab3-api "python server.py" api 30 seconds ago Up 27 seconds (healthy) 8080/tcp
mtusi2026lab3-haproxy-1 haproxy:3.0.8-alpine "docker-entrypoint.s…" haproxy 42 minutes ago Up 42 minutes (healthy) 127.0.0.1:8080->8080/tcp
mtusi2026lab3-postgres-1 postgres:16.4-bookworm "docker-entrypoint.s…" postgres 21 hours ago Up About an hour (healthy) 5432/tcp
mtusi2026lab3-valkey-1 valkey/valkey:8.0.1-alpine "docker-entrypoint.s…" valkey 21 hours ago Up About an hour (healthy) 6379/tcp
exit_code=0
NOTE: Full OpenAPI validator unavailable: openapi-spec-validator is absent and PyPI DNS is unavailable; offline contract check passed.
+14
View File
@@ -0,0 +1,14 @@
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
.venv/bin/python tests/smoke.py --base-url "http://127.0.0.1:8080"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
.venv/bin/python tests/idempotency_race.py --base-url "http://127.0.0.1:8080"
PASS: 32 concurrent retries, one message, 2 API instances observed
.venv/bin/python tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://127.0.0.1:8080"
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
.venv/bin/python tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://127.0.0.1:8080"
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 114, "failed_attempts": 0, "first_recovery_seconds": 1.447}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
NOTE: scripts/validate_openapi.py not run: openapi-spec-validator==0.7.2 is declared in requirements-dev.txt but pip install was blocked by unavailable DNS for pypi.org. Offline structural check passed.
+371
View File
@@ -0,0 +1,371 @@
{
"method": "Rate-controlled POST sweep; 15s per sample; offered rate 250/500/750/1000 RPS; three randomized repeats; 8 API replicas at 0.5 CPU/256 MiB, DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; max in-flight 8192; per-thread HTTP client reuse; synthetic message/idempotency tables reset per sample; no Docker stats polling during load.",
"summary": [
{
"offered_rate_rps": 250,
"completed_success_rps_total_elapsed_median": 249.622,
"completed_success_rps_total_elapsed_min": 249.559,
"completed_success_rps_total_elapsed_max": 249.959,
"success_ratio_median": 1.0,
"p95_ms_median": 9.931,
"effective_success_ratio_offered_median": 1.0,
"skipped_overload_median": 0
},
{
"offered_rate_rps": 500,
"completed_success_rps_total_elapsed_median": 499.579,
"completed_success_rps_total_elapsed_min": 419.571,
"completed_success_rps_total_elapsed_max": 499.857,
"success_ratio_median": 1.0,
"p95_ms_median": 41.601,
"effective_success_ratio_offered_median": 1.0,
"skipped_overload_median": 0
},
{
"offered_rate_rps": 750,
"completed_success_rps_total_elapsed_median": 745.875,
"completed_success_rps_total_elapsed_min": 693.24,
"completed_success_rps_total_elapsed_max": 748.66,
"success_ratio_median": 0.999822,
"p95_ms_median": 100.889,
"effective_success_ratio_offered_median": 0.999822,
"skipped_overload_median": 0
},
{
"offered_rate_rps": 1000,
"completed_success_rps_total_elapsed_median": 810.128,
"completed_success_rps_total_elapsed_min": 553.163,
"completed_success_rps_total_elapsed_max": 879.333,
"success_ratio_median": 0.972467,
"p95_ms_median": 1611.346,
"effective_success_ratio_offered_median": 0.972467,
"skipped_overload_median": 0
}
],
"samples": [
{
"variant": "offered-500",
"sample": 1,
"offered_rate_rps": 500,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 7501,
"scheduled_requests": 7501,
"successful_requests": 7501,
"failed_requests": 0,
"failure_types": {},
"skipped_overload": 0,
"offered_rps_actual": 500.067,
"submitted_rps_actual": 500.067,
"completed_success_rps_total_elapsed": 499.579,
"success_ratio": 1.0,
"effective_success_ratio_offered": 1.0,
"latency_successful_ms": {
"p50": 3.277,
"p95": 41.601,
"p99": 71.014
},
"drain_seconds_after_offer_window": 0.015,
"message_rows_after": 7501
},
{
"variant": "offered-750",
"sample": 1,
"offered_rate_rps": 750,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 11251,
"scheduled_requests": 11251,
"successful_requests": 11249,
"failed_requests": 2,
"failure_types": {
"Failure": 2
},
"skipped_overload": 0,
"offered_rps_actual": 750.067,
"submitted_rps_actual": 750.067,
"completed_success_rps_total_elapsed": 748.66,
"success_ratio": 0.999822,
"effective_success_ratio_offered": 0.999822,
"latency_successful_ms": {
"p50": 3.819,
"p95": 100.889,
"p99": 1021.492
},
"drain_seconds_after_offer_window": 0.026,
"message_rows_after": 11249
},
{
"variant": "offered-1000",
"sample": 1,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14976,
"failed_requests": 24,
"failure_types": {
"Failure": 24
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 810.128,
"success_ratio": 0.9984,
"effective_success_ratio_offered": 0.9984,
"latency_successful_ms": {
"p50": 109.324,
"p95": 1611.346,
"p99": 2175.384
},
"drain_seconds_after_offer_window": 3.486,
"message_rows_after": 14976
},
{
"variant": "offered-250",
"sample": 1,
"offered_rate_rps": 250,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 3751,
"scheduled_requests": 3751,
"successful_requests": 3751,
"failed_requests": 0,
"failure_types": {},
"skipped_overload": 0,
"offered_rps_actual": 250.067,
"submitted_rps_actual": 250.067,
"completed_success_rps_total_elapsed": 249.959,
"success_ratio": 1.0,
"effective_success_ratio_offered": 1.0,
"latency_successful_ms": {
"p50": 2.744,
"p95": 9.931,
"p99": 48.516
},
"drain_seconds_after_offer_window": 0.006,
"message_rows_after": 3751
},
{
"variant": "offered-750",
"sample": 2,
"offered_rate_rps": 750,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 11251,
"scheduled_requests": 11251,
"successful_requests": 11251,
"failed_requests": 0,
"failure_types": {},
"skipped_overload": 0,
"offered_rps_actual": 750.067,
"submitted_rps_actual": 750.067,
"completed_success_rps_total_elapsed": 745.875,
"success_ratio": 1.0,
"effective_success_ratio_offered": 1.0,
"latency_successful_ms": {
"p50": 5.657,
"p95": 79.469,
"p99": 164.156
},
"drain_seconds_after_offer_window": 0.084,
"message_rows_after": 11251
},
{
"variant": "offered-1000",
"sample": 2,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14283,
"failed_requests": 717,
"failure_types": {
"Failure": 717
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 879.333,
"success_ratio": 0.9522,
"effective_success_ratio_offered": 0.9522,
"latency_successful_ms": {
"p50": 150.455,
"p95": 1296.214,
"p99": 2320.975
},
"drain_seconds_after_offer_window": 1.243,
"message_rows_after": 14283
},
{
"variant": "offered-250",
"sample": 2,
"offered_rate_rps": 250,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 3751,
"scheduled_requests": 3751,
"successful_requests": 3751,
"failed_requests": 0,
"failure_types": {},
"skipped_overload": 0,
"offered_rps_actual": 250.067,
"submitted_rps_actual": 250.067,
"completed_success_rps_total_elapsed": 249.559,
"success_ratio": 1.0,
"effective_success_ratio_offered": 1.0,
"latency_successful_ms": {
"p50": 2.689,
"p95": 8.215,
"p99": 44.123
},
"drain_seconds_after_offer_window": 0.031,
"message_rows_after": 3751
},
{
"variant": "offered-500",
"sample": 2,
"offered_rate_rps": 500,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 7501,
"scheduled_requests": 7501,
"successful_requests": 7501,
"failed_requests": 0,
"failure_types": {},
"skipped_overload": 0,
"offered_rps_actual": 500.067,
"submitted_rps_actual": 500.067,
"completed_success_rps_total_elapsed": 499.857,
"success_ratio": 1.0,
"effective_success_ratio_offered": 1.0,
"latency_successful_ms": {
"p50": 3.567,
"p95": 34.969,
"p99": 65.8
},
"drain_seconds_after_offer_window": 0.006,
"message_rows_after": 7501
},
{
"variant": "offered-750",
"sample": 3,
"offered_rate_rps": 750,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 11251,
"scheduled_requests": 11251,
"successful_requests": 11236,
"failed_requests": 15,
"failure_types": {
"Failure": 15
},
"skipped_overload": 0,
"offered_rps_actual": 750.067,
"submitted_rps_actual": 750.067,
"completed_success_rps_total_elapsed": 693.24,
"success_ratio": 0.998667,
"effective_success_ratio_offered": 0.998667,
"latency_successful_ms": {
"p50": 8.339,
"p95": 858.144,
"p99": 1593.896
},
"drain_seconds_after_offer_window": 1.208,
"message_rows_after": 11236
},
{
"variant": "offered-250",
"sample": 3,
"offered_rate_rps": 250,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 3751,
"scheduled_requests": 3751,
"successful_requests": 3751,
"failed_requests": 0,
"failure_types": {},
"skipped_overload": 0,
"offered_rps_actual": 250.067,
"submitted_rps_actual": 250.067,
"completed_success_rps_total_elapsed": 249.622,
"success_ratio": 1.0,
"effective_success_ratio_offered": 1.0,
"latency_successful_ms": {
"p50": 2.685,
"p95": 17.715,
"p99": 52.77
},
"drain_seconds_after_offer_window": 0.027,
"message_rows_after": 3751
},
{
"variant": "offered-500",
"sample": 3,
"offered_rate_rps": 500,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 7501,
"scheduled_requests": 7501,
"successful_requests": 7493,
"failed_requests": 8,
"failure_types": {
"Failure": 8
},
"skipped_overload": 0,
"offered_rps_actual": 500.067,
"submitted_rps_actual": 500.067,
"completed_success_rps_total_elapsed": 419.571,
"success_ratio": 0.998933,
"effective_success_ratio_offered": 0.998933,
"latency_successful_ms": {
"p50": 9.042,
"p95": 1812.817,
"p99": 2541.843
},
"drain_seconds_after_offer_window": 2.859,
"message_rows_after": 7493
},
{
"variant": "offered-1000",
"sample": 3,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14587,
"failed_requests": 413,
"failure_types": {
"Failure": 413
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 553.163,
"success_ratio": 0.972467,
"effective_success_ratio_offered": 0.972467,
"latency_successful_ms": {
"p50": 535.721,
"p95": 2011.397,
"p99": 4616.569
},
"drain_seconds_after_offer_window": 11.37,
"message_rows_after": 14587
}
]
}
+33
View File
@@ -0,0 +1,33 @@
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="500" viewBox="0 0 900 500">
<rect width="100%" height="100%" fill="#fff"/>
<text x="450" y="30" text-anchor="middle" font-family="sans-serif" font-size="20">Насыщение при росте предложенной нагрузки</text>
<line x1="90" y1="415.0" x2="810" y2="415.0" stroke="#ddd"/>
<text x="78" y="420.0" text-anchor="end" font-family="sans-serif" font-size="12">0</text>
<line x1="90" y1="326.2" x2="810" y2="326.2" stroke="#ddd"/>
<text x="78" y="331.2" text-anchor="end" font-family="sans-serif" font-size="12">250</text>
<line x1="90" y1="237.5" x2="810" y2="237.5" stroke="#ddd"/>
<text x="78" y="242.5" text-anchor="end" font-family="sans-serif" font-size="12">500</text>
<line x1="90" y1="148.8" x2="810" y2="148.8" stroke="#ddd"/>
<text x="78" y="153.8" text-anchor="end" font-family="sans-serif" font-size="12">750</text>
<line x1="90" y1="60.0" x2="810" y2="60.0" stroke="#ddd"/>
<text x="78" y="65.0" text-anchor="end" font-family="sans-serif" font-size="12">1000</text>
<text x="90.0" y="439" text-anchor="middle" font-family="sans-serif" font-size="12">250</text>
<text x="330.0" y="439" text-anchor="middle" font-family="sans-serif" font-size="12">500</text>
<text x="570.0" y="439" text-anchor="middle" font-family="sans-serif" font-size="12">750</text>
<text x="810.0" y="439" text-anchor="middle" font-family="sans-serif" font-size="12">1000</text>
<polyline points="90.0,326.4 330.0,237.6 570.0,150.2 810.0,127.4" fill="none" stroke="#1769aa" stroke-width="3"/>
<polyline points="90.0,413.2 330.0,407.6 570.0,397.1 810.0,129.0" fill="none" stroke="#d1495b" stroke-width="3"/>
<circle cx="90.0" cy="326.4" r="5" fill="#1769aa"/><text x="90.0" y="314.4" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#1769aa">250</text>
<circle cx="330.0" cy="237.6" r="5" fill="#1769aa"/><text x="330.0" y="225.6" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#1769aa">500</text>
<circle cx="570.0" cy="150.2" r="5" fill="#1769aa"/><text x="570.0" y="138.2" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#1769aa">746</text>
<circle cx="810.0" cy="127.4" r="5" fill="#1769aa"/><text x="810.0" y="115.4" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#1769aa">810</text>
<circle cx="90.0" cy="413.2" r="5" fill="#d1495b"/><text x="90.0" y="432.2" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#d1495b">10ms</text>
<circle cx="330.0" cy="407.6" r="5" fill="#d1495b"/><text x="330.0" y="426.6" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#d1495b">42ms</text>
<circle cx="570.0" cy="397.1" r="5" fill="#d1495b"/><text x="570.0" y="416.1" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#d1495b">101ms</text>
<circle cx="810.0" cy="129.0" r="5" fill="#d1495b"/><text x="810.0" y="148.0" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#d1495b">1611ms</text>
<text x="450.0" y="464" text-anchor="middle" font-family="sans-serif" font-size="13">Предложенная нагрузка, RPS</text>
<text x="22" y="237.5" text-anchor="middle" transform="rotate(-90 22 237.5)" font-family="sans-serif" font-size="13">Успешные RPS за offer+drain</text>
<text x="878" y="237.5" text-anchor="middle" transform="rotate(90 878 237.5)" font-family="sans-serif" font-size="13">p95, ms</text>
<line x1="90" y1="484" x2="115" y2="484" stroke="#1769aa" stroke-width="3"/><text x="123" y="488" font-family="sans-serif" font-size="12">Успешные RPS</text>
<line x1="250" y1="484" x2="275" y2="484" stroke="#d1495b" stroke-width="3"/><text x="283" y="488" font-family="sans-serif" font-size="12">p95 успешных запросов</text>
</svg>

After

Width:  |  Height:  |  Size: 3.7 KiB

+209
View File
@@ -0,0 +1,209 @@
{
"method": "Randomized paired blocks at offered 1000 RPS, 15s per sample; three repeats; compare 8 versus 12 replicas at fixed 0.5 CPU/256 MiB per API replica; DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; HAProxy server-template supports 16 backends; scale to 8 before each DB reset to release idle connections, then scale to target; synthetic messages and idempotency keys truncated per sample; HTTP client reused per worker.",
"summary": [
{
"replicas": 8,
"offered_rate_rps": 1000,
"completed_success_rps_total_elapsed_median": 726.979,
"completed_success_rps_min": 584.377,
"completed_success_rps_max": 851.508,
"success_ratio_median": 0.981267,
"p95_ms_median": 1504.722,
"error_count_total": 874,
"message_rows_after_median": 14719
},
{
"replicas": 12,
"offered_rate_rps": 1000,
"completed_success_rps_total_elapsed_median": 555.8,
"completed_success_rps_min": 545.991,
"completed_success_rps_max": 774.91,
"success_ratio_median": 0.999733,
"p95_ms_median": 1986.412,
"error_count_total": 12,
"message_rows_after_median": null
}
],
"samples": [
{
"variant": "offered-1000",
"sample": 1,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14719,
"failed_requests": 281,
"failure_types": {
"Failure": 281
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 851.508,
"success_ratio": 0.981267,
"effective_success_ratio_offered": 0.981267,
"latency_successful_ms": {
"p50": 206.696,
"p95": 1307.155,
"p99": 1984.206
},
"drain_seconds_after_offer_window": 2.286,
"message_rows_after": 14719,
"message_rows_after_error": null,
"replicas": 8
},
{
"variant": "offered-1000",
"sample": 1,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14996,
"failed_requests": 4,
"failure_types": {
"Failure": 4
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 774.91,
"success_ratio": 0.999733,
"effective_success_ratio_offered": 0.999733,
"latency_successful_ms": {
"p50": 400.696,
"p95": 1664.779,
"p99": 2378.651
},
"drain_seconds_after_offer_window": 4.352,
"message_rows_after": null,
"message_rows_after_error": "psql: error: connection to server on socket \"/var/run/postgresql/.s.PGSQL.5432\" failed: FATAL: sorry, too many clients already",
"replicas": 12
},
{
"variant": "offered-1000",
"sample": 2,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14804,
"failed_requests": 196,
"failure_types": {
"Failure": 196
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 584.377,
"success_ratio": 0.986933,
"effective_success_ratio_offered": 0.986933,
"latency_successful_ms": {
"p50": 496.284,
"p95": 2007.63,
"p99": 2684.382
},
"drain_seconds_after_offer_window": 10.333,
"message_rows_after": 14804,
"message_rows_after_error": null,
"replicas": 8
},
{
"variant": "offered-1000",
"sample": 2,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14999,
"failed_requests": 1,
"failure_types": {
"Failure": 1
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 545.991,
"success_ratio": 0.999933,
"effective_success_ratio_offered": 0.999933,
"latency_successful_ms": {
"p50": 605.99,
"p95": 2589.505,
"p99": 3420.92
},
"drain_seconds_after_offer_window": 12.471,
"message_rows_after": null,
"message_rows_after_error": "psql: error: connection to server on socket \"/var/run/postgresql/.s.PGSQL.5432\" failed: FATAL: sorry, too many clients already",
"replicas": 12
},
{
"variant": "offered-1000",
"sample": 3,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14993,
"failed_requests": 7,
"failure_types": {
"Failure": 7
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 555.8,
"success_ratio": 0.999533,
"effective_success_ratio_offered": 0.999533,
"latency_successful_ms": {
"p50": 717.667,
"p95": 1986.412,
"p99": 2959.931
},
"drain_seconds_after_offer_window": 11.976,
"message_rows_after": null,
"message_rows_after_error": "psql: error: connection to server on socket \"/var/run/postgresql/.s.PGSQL.5432\" failed: FATAL: sorry, too many clients already",
"replicas": 12
},
{
"variant": "offered-1000",
"sample": 3,
"offered_rate_rps": 1000,
"duration_seconds": 15,
"concurrency_workers": 512,
"max_inflight_requests": 8192,
"offered_slots": 15000,
"scheduled_requests": 15000,
"successful_requests": 14603,
"failed_requests": 397,
"failure_types": {
"Failure": 397
},
"skipped_overload": 0,
"offered_rps_actual": 1000.0,
"submitted_rps_actual": 1000.0,
"completed_success_rps_total_elapsed": 726.979,
"success_ratio": 0.973533,
"effective_success_ratio_offered": 0.973533,
"latency_successful_ms": {
"p50": 291.159,
"p95": 1504.722,
"p99": 2242.259
},
"drain_seconds_after_offer_window": 5.087,
"message_rows_after": 14603,
"message_rows_after_error": null,
"replicas": 8
}
]
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 60.0,
"elapsed_seconds": 60.088,
"concurrency": 16,
"successful_requests": 50291,
"failed_requests": 24,
"error_ratio": 0.000477,
"successful_rps": 836.949,
"successful_latency_ms": {
"p50": 6.118,
"p95": 55.938,
"p99": 85.729
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
File diff suppressed because it is too large Load Diff
+36
View File
@@ -0,0 +1,36 @@
<svg xmlns="http://www.w3.org/2000/svg" width="900" height="460" viewBox="0 0 900 460">
<rect width="100%" height="100%" fill="white"/>
<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="20">API scaling: successful requests per second</text>
<line x1="90" y1="365.0" x2="830" y2="365.0" stroke="#ddd"/>
<text x="80" y="369.0" text-anchor="end" font-family="sans-serif" font-size="12">0.0</text>
<line x1="90" y1="287.5" x2="830" y2="287.5" stroke="#ddd"/>
<text x="80" y="291.5" text-anchor="end" font-family="sans-serif" font-size="12">209.4</text>
<line x1="90" y1="210.0" x2="830" y2="210.0" stroke="#ddd"/>
<text x="80" y="214.0" text-anchor="end" font-family="sans-serif" font-size="12">418.8</text>
<line x1="90" y1="132.5" x2="830" y2="132.5" stroke="#ddd"/>
<text x="80" y="136.5" text-anchor="end" font-family="sans-serif" font-size="12">628.2</text>
<line x1="90" y1="55.0" x2="830" y2="55.0" stroke="#ddd"/>
<text x="80" y="59.0" text-anchor="end" font-family="sans-serif" font-size="12">837.7</text>
<circle cx="238.0" cy="210.1" r="6" fill="#2563eb"/>
<text x="238.0" y="200.1" text-anchor="middle" font-family="sans-serif" font-size="12">418.5</text>
<circle cx="386.0" cy="106.5" r="6" fill="#2563eb"/>
<text x="386.0" y="96.5" text-anchor="middle" font-family="sans-serif" font-size="12">698.6</text>
<circle cx="534.0" cy="55.0" r="6" fill="#2563eb"/>
<text x="534.0" y="45.0" text-anchor="middle" font-family="sans-serif" font-size="12">837.7</text>
<polyline points="238.0,210.1 386.0,106.5 534.0,55.0" fill="none" stroke="#2563eb" stroke-width="3"/>
<circle cx="238.0" cy="326.6" r="6" fill="#f97316"/>
<text x="238.0" y="316.6" text-anchor="middle" font-family="sans-serif" font-size="12">103.7</text>
<circle cx="386.0" cy="263.4" r="6" fill="#f97316"/>
<text x="386.0" y="253.4" text-anchor="middle" font-family="sans-serif" font-size="12">274.6</text>
<circle cx="534.0" cy="113.1" r="6" fill="#f97316"/>
<text x="534.0" y="103.1" text-anchor="middle" font-family="sans-serif" font-size="12">680.7</text>
<circle cx="682.0" cy="82.2" r="6" fill="#f97316"/>
<text x="682.0" y="72.2" text-anchor="middle" font-family="sans-serif" font-size="12">764.2</text>
<polyline points="238.0,326.6 386.0,263.4 534.0,113.1 682.0,82.2" fill="none" stroke="#f97316" stroke-width="3"/>
<text x="238.0" y="390" text-anchor="middle" font-family="sans-serif" font-size="14">1 replicas</text>
<text x="386.0" y="390" text-anchor="middle" font-family="sans-serif" font-size="14">2 replicas</text>
<text x="534.0" y="390" text-anchor="middle" font-family="sans-serif" font-size="14">4 replicas</text>
<text x="682.0" y="390" text-anchor="middle" font-family="sans-serif" font-size="14">8 replicas</text>
<line x1="130" y1="420" x2="160" y2="420" stroke="#2563eb" stroke-width="3"/><text x="168" y="424" font-family="sans-serif" font-size="13">Fixed aggregate budget</text>
<line x1="430" y1="420" x2="460" y2="420" stroke="#f97316" stroke-width="3"/><text x="468" y="424" font-family="sans-serif" font-size="13">Increased aggregate budget</text>
</svg>

After

Width:  |  Height:  |  Size: 3.0 KiB

+33
View File
@@ -0,0 +1,33 @@
global
log stdout format raw local0
maxconn 2048
hard-stop-after 8s
defaults
log global
mode http
option httplog
timeout connect 1s
timeout client 10s
timeout server 10s
timeout http-request 5s
retries 1
option redispatch
option httpchk GET /health/ready
http-check expect status 200
resolvers docker
nameserver embedded 127.0.0.11:53
resolve_retries 3
timeout resolve 1s
timeout retry 1s
hold valid 2s
hold obsolete 2s
frontend public_api
bind :8080
default_backend messenger_api
backend messenger_api
balance roundrobin
server-template api 1-16 api:8080 check inter 1s fall 2 rise 2 resolvers docker init-addr libc,none
+39
View File
@@ -0,0 +1,39 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name varchar(100) NOT NULL CHECK (length(btrim(display_name)) > 0),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title varchar(100) NOT NULL CHECK (length(btrim(title)) > 0),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_by_user ON chat_members(user_id, chat_id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT,
text varchar(4000) NOT NULL CHECK (length(btrim(text)) > 0),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX messages_history_keyset ON messages(chat_id, created_at, id);
-- PostgreSQL is authoritative for session durability and revocation. Valkey only
-- keeps a short-lived lookup hint; a cache hit is still checked against this row.
CREATE TABLE sessions (
token_hash char(64) PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE INDEX sessions_expiry ON sessions(expires_at);
+11
View File
@@ -0,0 +1,11 @@
CREATE TABLE idempotency_keys (
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
method varchar(8) NOT NULL CHECK (method = 'POST'),
resource_path text NOT NULL,
key varchar(128) NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash char(64) NOT NULL,
response_body jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
PRIMARY KEY (user_id, method, resource_path, key)
);
CREATE INDEX idempotency_expiry ON idempotency_keys(created_at);
+9
View File
@@ -0,0 +1,9 @@
argon2-cffi==23.1.0
argon2-cffi-bindings==26.1.0
cffi==2.1.1
pycparser==3.12
psycopg[binary,pool]==3.2.3
psycopg-binary==3.2.3
psycopg-pool==3.3.3
redis==5.2.1
typing_extensions==4.16.0
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
# Restart every stateful dependency and all API processes without deleting volumes.
docker compose --env-file .env restart api haproxy postgres valkey
+25
View File
@@ -0,0 +1,25 @@
#!/bin/sh
set -eu
compose="docker compose --env-file .env"
# Add one ready replica before removing any old container.
$compose up -d --scale api=3 --wait --wait-timeout 120 api
python3 - <<'PY'
import time, urllib.request
end=time.monotonic()+60
while time.monotonic()<end:
try:
urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=1).read()
break
except Exception:
time.sleep(.25)
else:
raise SystemExit('LB did not become ready')
PY
# Remove one old instance at a time while another healthy backend is present.
for id in $(docker compose --env-file .env ps -q api | head -n 2); do
docker stop --time 3 "$id" >/dev/null
docker rm "$id" >/dev/null
docker compose --env-file .env up -d --scale api=3 --wait --wait-timeout 120 api
sleep 3
done
docker compose --env-file .env up -d --scale api=2 --wait --wait-timeout 120 api
+14
View File
@@ -0,0 +1,14 @@
#!/bin/sh
set -eu
: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}"
# X-Instance-Id is the container hostname (the Docker short ID). Resolve it to
# exactly one scaled Compose container and stop only that process.
for id in $(docker compose --env-file .env ps -q api); do
hostname=$(docker inspect --format '{{.Config.Hostname}}' "$id")
if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then
docker stop --time 3 "$id" >/dev/null
exit 0
fi
done
echo "No running api container matches TARGET_INSTANCE_ID" >&2
exit 1
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""Repeatable local scaling study: fixed aggregate budget, then increased budget."""
import json, os, platform, random, subprocess, sys, time, urllib.request
from statistics import median
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
COMPOSE = ["docker", "compose", "--env-file", ".env"]
OUT = ROOT / "evidence"
LOAD = [sys.executable, "tests/load.py", "--base-url", "http://127.0.0.1:8080", "--duration", os.getenv("BENCH_SECONDS", "30"), "--concurrency", os.getenv("BENCH_CONCURRENCY", "16")]
def run(args, env=None, check=True):
subprocess.run(args, cwd=ROOT, env=env, check=check)
def docker_usage():
ids=[]
for service in ("api","postgres","haproxy"):
found=subprocess.run(COMPOSE+["ps","-q",service],cwd=ROOT,text=True,capture_output=True,check=True).stdout.split()
ids.extend(found)
if not ids: return {}
raw=subprocess.run(["docker","stats","--no-stream","--format","{{json .}}",*ids],cwd=ROOT,text=True,capture_output=True,check=True).stdout
grouped={"api":{"cpu_percent":[],"memory_mib":[]},"postgres":{"cpu_percent":[],"memory_mib":[]},"haproxy":{"cpu_percent":[],"memory_mib":[]}}
for line in raw.splitlines():
try: item=json.loads(line)
except ValueError: continue
name=item.get("Name","")
role="api" if "-api-" in name else "postgres" if "-postgres-" in name else "haproxy" if "-haproxy-" in name else None
if not role: continue
try: grouped[role]["cpu_percent"].append(float(item["CPUPerc"].rstrip("%")))
except (KeyError,ValueError): pass
try:
size=item["MemUsage"].split("/")[0].strip()
if size.endswith("GiB"): value=float(size[:-3])*1024
elif size.endswith("MiB"): value=float(size[:-3])
elif size.endswith("KiB"): value=float(size[:-3])/1024
else: value=float(size[:-1])/(1024*1024)
grouped[role]["memory_mib"].append(value)
except (KeyError,ValueError,IndexError): pass
return grouped
def row_count():
env={}
for line in (ROOT/".env").read_text().splitlines():
if "=" in line:
k,v=line.split("=",1); env[k]=v
cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-Atc","SELECT count(*) FROM messages"]
result=subprocess.run(cmd,cwd=ROOT,text=True,capture_output=True,check=True)
return int(result.stdout.strip())
def reset_synthetic_messages():
"""Use an identical empty message/idempotency dataset for every sample."""
env={}
for line in (ROOT/".env").read_text().splitlines():
if "=" in line:
k,v=line.split("=",1); env[k]=v
cmd=COMPOSE+["exec","-T","postgres","psql","-U",env.get("POSTGRES_USER","messenger"),"-d",env.get("POSTGRES_DB","messenger"),"-v","ON_ERROR_STOP=1","-c","TRUNCATE TABLE messages, idempotency_keys; ANALYZE messages; ANALYZE idempotency_keys;"]
subprocess.run(cmd,cwd=ROOT,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL,check=True)
def benchmark_load(dest, env):
reset_synthetic_messages()
before=row_count(); samples=[]
proc=subprocess.Popen(LOAD+["--output",str(dest)],cwd=ROOT,env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True)
while proc.poll() is None:
stats=docker_usage()
if stats: samples.append(stats)
time.sleep(1)
output=proc.communicate()[0]
print(output,end="")
if proc.returncode: raise subprocess.CalledProcessError(proc.returncode,LOAD)
after=row_count()
flat={}
for role in ("api","postgres","haproxy"):
cpus=[x for sample in samples for x in sample[role]["cpu_percent"]]
mems=[x for sample in samples for x in sample[role]["memory_mib"]]
flat[role]={"cpu_percent_mean":round(sum(cpus)/len(cpus),2) if cpus else None,"cpu_percent_peak":round(max(cpus),2) if cpus else None,"memory_mib_peak":round(max(mems),2) if mems else None}
return {"message_rows_before":before,"message_rows_after":after,"docker_stats_samples":len(samples),"container_resource_usage":flat}
def wait_ready(timeout=90):
deadline=time.monotonic()+timeout
while time.monotonic()<deadline:
try:
with urllib.request.urlopen("http://127.0.0.1:8080/health/ready", timeout=1) as r:
if r.status==200: return
except Exception: pass
time.sleep(.25)
raise RuntimeError("LB did not become ready in 90 seconds")
def variant(name, replicas, cpus, memory, db_pool=10, sample=1):
env=os.environ.copy(); env.update(API_CPUS=str(cpus), API_MEMORY=memory, DB_POOL_MAX=str(db_pool))
run(COMPOSE+["up","-d","--build","--scale",f"api={replicas}","--wait","--wait-timeout","180"],env)
wait_ready()
time.sleep(5) # identical settling interval after topology change
dest=OUT/(name+f"-sample{sample}.json")
stats=benchmark_load(dest,env)
item=json.loads(dest.read_text()); item.update({"variant":name,"replicas":replicas,"aggregate_api_cpu_limit":round(replicas*cpus,3),"api_memory_limit_per_replica":memory,"db_pool_max_per_replica":db_pool,**stats})
dest.write_text(json.dumps(item,ensure_ascii=False,indent=2)+"\n")
return item
def main():
OUT.mkdir(exist_ok=True)
samples=[]
repeats=int(os.getenv("BENCH_REPEATS", "3"))
cases=[]
# Two vCPU / 1 GiB total API budget, distributed equally across replicas.
cases += [(f"fixed-{n}",n,c,m,10) for n,c,m in [(1,2,"1g"),(2,1,"512m"),(4,.5,"256m")]]
# More total budget at 0.5 CPU / 256 MiB per replica.
cases += [(f"scaled-{n}",n,.5,"256m",10) for n in (1,2,4,8)]
# Tuning comparison at identical four-replica CPU/RAM to find a supported improvement.
cases += [(f"pool-{pool}",4,.5,"256m",pool) for pool in (5,10,20)]
rng=random.Random(20261105)
for sample in range(1,repeats+1):
block=cases[:]; rng.shuffle(block)
for name,n,c,m,pool in block:
row=variant(name,n,c,m,pool,sample)
row["sample"]=sample; row["db_pool_max_per_replica"]=pool
samples.append(row)
medians=[]
for name,n,c,m,pool in cases:
group=[r for r in samples if r["variant"]==name]
combined={"variant":name,"replicas":n,"aggregate_api_cpu_limit":round(n*c,3),"api_memory_limit_per_replica":m,"db_pool_max_per_replica":pool}
for metric in ("successful_rps",): combined[metric]=round(median(r[metric] for r in group),3)
for q in ("p50","p95","p99"):
combined.setdefault("successful_latency_ms",{})[q]=round(median(r["successful_latency_ms"][q] for r in group),3)
combined["error_ratio"]=round(median(r["error_ratio"] for r in group),6)
combined["successful_requests"]=int(median(r["successful_requests"] for r in group))
combined["failed_requests"]=int(median(r["failed_requests"] for r in group))
combined["message_rows_before"]=int(median(r["message_rows_before"] for r in group))
combined["message_rows_after"]=int(median(r["message_rows_after"] for r in group))
combined["container_resource_usage"]={}
for role in ("api","postgres","haproxy"):
combined["container_resource_usage"][role]={key:round(median(r["container_resource_usage"][role][key] for r in group if r["container_resource_usage"][role][key] is not None),2) if any(r["container_resource_usage"][role][key] is not None for r in group) else None for key in ("cpu_percent_mean","cpu_percent_peak","memory_mib_peak")}
medians.append(combined)
payload={"method":"closed-loop POST, 5s stabilization after each topology change; each sample starts after TRUNCATE of synthetic messages/idempotency_keys and ANALYZE; deterministic randomized blocks; median summary plus every raw sample","duration_seconds":float(os.getenv("BENCH_SECONDS","30")),"concurrency":int(os.getenv("BENCH_CONCURRENCY","16")),"repeat_count":repeats,"random_seed":20261105,"host_platform":platform.platform(),"host_cpu_count":os.cpu_count(),"docker_server_version":subprocess.run(["docker","version","--format","{{.Server.Version}}"],cwd=ROOT,text=True,capture_output=True,check=True).stdout.strip(),"results":medians,"samples":samples}
(OUT/"track1-results.json").write_text(json.dumps(payload,ensure_ascii=False,indent=2)+"\n")
make_svg([r for r in medians if r["variant"].startswith(("fixed-","scaled-"))],OUT/"track1-rps.svg")
print(OUT/"track1-results.json")
def make_svg(rows,path):
# Small dependency-free chart for the report; data values remain in JSON.
groups=[("Fixed aggregate budget",[r for r in rows if r["variant"].startswith("fixed-")]),("Increased aggregate budget",[r for r in rows if r["variant"].startswith("scaled-")])]
w,h=900,460; left,top,plotw,ploth=90,55,740,310
vmax=max((r["successful_rps"] for r in rows),default=1) or 1
out=[f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}">','<rect width="100%" height="100%" fill="white"/>','<text x="450" y="28" text-anchor="middle" font-family="sans-serif" font-size="20">API scaling: successful requests per second</text>']
for tick in range(5):
y=top+ploth-tick*ploth/4; val=vmax*tick/4
out += [f'<line x1="{left}" y1="{y:.1f}" x2="{left+plotw}" y2="{y:.1f}" stroke="#ddd"/>',f'<text x="{left-10}" y="{y+4:.1f}" text-anchor="end" font-family="sans-serif" font-size="12">{val:.1f}</text>']
colors=["#2563eb","#f97316"]
for gi,(label,items) in enumerate(groups):
pts=[]
for i,r in enumerate(items):
x=left+plotw*({1:1,2:2,4:3,8:4}[r["replicas"]])/5; y=top+ploth-(r["successful_rps"]/vmax)*ploth
pts.append(f"{x:.1f},{y:.1f}"); out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="6" fill="{colors[gi]}"/>'); out.append(f'<text x="{x:.1f}" y="{y-10:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12">{r["successful_rps"]:.1f}</text>')
out.append(f'<polyline points="{" ".join(pts)}" fill="none" stroke="{colors[gi]}" stroke-width="3"/>')
positions={1:1,2:2,4:3,8:4}
for n,pos in positions.items():
x=left+plotw*pos/5; out.append(f'<text x="{x:.1f}" y="{top+ploth+25}" text-anchor="middle" font-family="sans-serif" font-size="14">{n} replicas</text>')
out += [f'<line x1="130" y1="420" x2="160" y2="420" stroke="{colors[0]}" stroke-width="3"/><text x="168" y="424" font-family="sans-serif" font-size="13">{groups[0][0]}</text>',f'<line x1="430" y1="420" x2="460" y2="420" stroke="{colors[1]}" stroke-width="3"/><text x="468" y="424" font-family="sans-serif" font-size="13">{groups[1][0]}</text>','</svg>']
path.write_text("\n".join(out)+"\n")
if __name__=="__main__": main()
+188
View File
@@ -0,0 +1,188 @@
#!/usr/bin/env python3
"""Rate-controlled synthetic write test to locate the throughput knee."""
from concurrent.futures import ThreadPoolExecutor, as_completed
import json
import math
import random
import statistics
import subprocess
import threading
import time
from pathlib import Path
import sys
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / "tests"))
sys.path.insert(0, str(ROOT / "scripts"))
from client import Client
import track1_experiment as bench
OUT = ROOT / "evidence"
DURATION = 15
RATES = (250, 500, 750, 1000)
REPEATS = 3
def percentile(values, p):
values = sorted(values)
if not values:
return None
return round(1000 * values[max(0, math.ceil(p * len(values)) - 1)], 3)
def run_sample(actor, chat, rate, repeat, reset=True):
if reset:
bench.reset_synthetic_messages()
latencies = []
failures = []
lock = threading.Lock()
local = threading.local()
start = time.monotonic()
deadline = start + DURATION
def request(index, due):
client = getattr(local, "client", None)
if client is None:
client = Client("http://127.0.0.1:8080", timeout=8)
local.client = client
sent = time.monotonic()
try:
client.message(actor, chat, f"offered-{rate}-{repeat}-{index}", key=f"offered-{rate}-{repeat}-{index}")
result = (time.monotonic() - sent, None)
except Exception as exc:
result = (None, f"{type(exc).__name__}: {str(exc)[:120]}")
with lock:
if result[0] is not None:
latencies.append(result[0])
else:
failures.append(result[1])
submitted = 0
offered_slots = 0
skipped_overload = 0
due = start
next_prune_at = start + .05
max_inflight = 8192
with ThreadPoolExecutor(max_workers=512) as pool:
futures = []
while due < deadline:
offered_slots += 1
if time.monotonic() >= next_prune_at:
futures = [future for future in futures if not future.done()]
next_prune_at = time.monotonic() + .05
if len(futures) < max_inflight:
futures.append(pool.submit(request, submitted, due))
submitted += 1
else:
skipped_overload += 1
due += 1 / rate
remaining = due - time.monotonic()
if remaining > 0:
time.sleep(remaining)
for future in as_completed(futures):
future.result()
elapsed = time.monotonic() - start
counts = {}
for failure in failures:
prefix = failure.split(":", 1)[0]
counts[prefix] = counts.get(prefix, 0) + 1
row_count_error = None
try:
after = bench.row_count()
except subprocess.CalledProcessError as exc:
after = None
row_count_error = (exc.stderr or "PostgreSQL row-count query failed").strip()[-300:]
return {
"variant": f"offered-{rate}", "sample": repeat, "offered_rate_rps": rate,
"duration_seconds": DURATION, "concurrency_workers": 512,
"max_inflight_requests": max_inflight, "offered_slots": offered_slots,
"scheduled_requests": submitted, "successful_requests": len(latencies),
"failed_requests": len(failures), "failure_types": counts,
"skipped_overload": skipped_overload,
"offered_rps_actual": round(offered_slots / DURATION, 3),
"submitted_rps_actual": round(submitted / DURATION, 3),
"completed_success_rps_total_elapsed": round(len(latencies) / elapsed, 3),
"success_ratio": round(len(latencies) / submitted, 6) if submitted else None,
"effective_success_ratio_offered": round(len(latencies) / offered_slots, 6) if offered_slots else None,
"latency_successful_ms": {f"p{p}": percentile(latencies, p/100) for p in (50,95,99)},
"drain_seconds_after_offer_window": round(max(0, elapsed - DURATION), 3),
"message_rows_after": after,
"message_rows_after_error": row_count_error,
}
def main():
OUT.mkdir(exist_ok=True)
env = os.environ.copy()
env.update(API_CPUS="0.5", API_MEMORY="256m", DB_POOL_MAX="10", LB_CPUS="0.25")
bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env)
bench.wait_ready()
time.sleep(5)
client = Client("http://127.0.0.1:8080")
actor, _, _, chat = client.bootstrap()
rng = random.Random(20261106)
samples = []
for repeat in range(1, REPEATS + 1):
rates = list(RATES)
rng.shuffle(rates)
for rate in rates:
row = run_sample(actor, chat, rate, repeat)
samples.append(row)
print(json.dumps(row, ensure_ascii=False), flush=True)
summary = []
for rate in RATES:
group = [row for row in samples if row["offered_rate_rps"] == rate]
summary.append({
"offered_rate_rps": rate,
"completed_success_rps_total_elapsed_median": round(statistics.median(row["completed_success_rps_total_elapsed"] for row in group), 3),
"completed_success_rps_total_elapsed_min": min(row["completed_success_rps_total_elapsed"] for row in group),
"completed_success_rps_total_elapsed_max": max(row["completed_success_rps_total_elapsed"] for row in group),
"success_ratio_median": round(statistics.median(row["success_ratio"] for row in group), 6),
"p95_ms_median": round(statistics.median(row["latency_successful_ms"]["p95"] for row in group), 3),
"effective_success_ratio_offered_median": round(statistics.median(row["effective_success_ratio_offered"] for row in group), 6),
"skipped_overload_median": int(statistics.median(row["skipped_overload"] for row in group)),
})
result = {
"method": "Rate-controlled POST sweep; 15s per sample; offered rate 250/500/750/1000 RPS; three randomized repeats; 8 API replicas at 0.5 CPU/256 MiB, DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; max in-flight 8192; per-thread HTTP client reuse; synthetic message/idempotency tables reset per sample; no Docker stats polling during load.",
"summary": summary, "samples": samples,
}
(OUT / "offered-load-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
make_plot(summary, OUT / "offered-load.svg")
print(json.dumps(summary, ensure_ascii=False, indent=2))
def make_plot(rows, path):
w, h = 900, 500
left, right, top, bottom = 90, 90, 60, 85
pw, ph = w-left-right, h-top-bottom
max_rps, max_p95 = 1000, 2000
rps = [r["completed_success_rps_total_elapsed_median"] for r in rows]
p95 = [r["p95_ms_median"] for r in rows]
xs = [left + i*pw/(len(rows)-1) for i in range(len(rows))]
yr = [top + ph*(1-v/max_rps) for v in rps]
yp = [top + ph*(1-v/max_p95) for v in p95]
out = [f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="{h}" viewBox="0 0 {w} {h}">',
'<rect width="100%" height="100%" fill="#fff"/>',
'<text x="450" y="30" text-anchor="middle" font-family="sans-serif" font-size="20">Насыщение при росте предложенной нагрузки</text>']
for tick in (0,250,500,750,1000):
y=top+ph*(1-tick/max_rps)
out += [f'<line x1="{left}" y1="{y:.1f}" x2="{left+pw}" y2="{y:.1f}" stroke="#ddd"/>',
f'<text x="{left-12}" y="{y+5:.1f}" text-anchor="end" font-family="sans-serif" font-size="12">{tick}</text>']
for x, row in zip(xs, rows):
out += [f'<text x="{x:.1f}" y="{top+ph+24}" text-anchor="middle" font-family="sans-serif" font-size="12">{row["offered_rate_rps"]}</text>']
out += [f'<polyline points="{" ".join(f"{x:.1f},{y:.1f}" for x,y in zip(xs,yr))}" fill="none" stroke="#1769aa" stroke-width="3"/>',
f'<polyline points="{" ".join(f"{x:.1f},{y:.1f}" for x,y in zip(xs,yp))}" fill="none" stroke="#d1495b" stroke-width="3"/>']
for x,y,value in zip(xs,yr,rps):
out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="5" fill="#1769aa"/><text x="{x:.1f}" y="{y-12:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#1769aa">{value:.0f}</text>')
for x,y,value in zip(xs,yp,p95):
out.append(f'<circle cx="{x:.1f}" cy="{y:.1f}" r="5" fill="#d1495b"/><text x="{x:.1f}" y="{y+19:.1f}" text-anchor="middle" font-family="sans-serif" font-size="12" fill="#d1495b">{value:.0f}ms</text>')
out += [f'<text x="{left+pw/2:.1f}" y="{h-36}" text-anchor="middle" font-family="sans-serif" font-size="13">Предложенная нагрузка, RPS</text>',
f'<text x="22" y="{top+ph/2:.1f}" text-anchor="middle" transform="rotate(-90 22 {top+ph/2:.1f})" font-family="sans-serif" font-size="13">Успешные RPS за offer+drain</text>',
f'<text x="{w-22}" y="{top+ph/2:.1f}" text-anchor="middle" transform="rotate(90 {w-22} {top+ph/2:.1f})" font-family="sans-serif" font-size="13">p95, ms</text>',
f'<line x1="{left}" y1="{h-16}" x2="{left+25}" y2="{h-16}" stroke="#1769aa" stroke-width="3"/><text x="{left+33}" y="{h-12}" font-family="sans-serif" font-size="12">Успешные RPS</text>',
f'<line x1="{left+160}" y1="{h-16}" x2="{left+185}" y2="{h-16}" stroke="#d1495b" stroke-width="3"/><text x="{left+193}" y="{h-12}" font-family="sans-serif" font-size="12">p95 успешных запросов</text>',
'</svg>']
path.write_text('\n'.join(out)+'\n')
if __name__ == "__main__":
main()
+78
View File
@@ -0,0 +1,78 @@
#!/usr/bin/env python3
"""Rate-controlled paired scale test for 8 versus 12 API replicas."""
import json
import os
import random
import statistics
import time
from pathlib import Path
import track1_experiment as bench
from track1_offered_load import run_sample
from client import Client
ROOT = Path(__file__).resolve().parents[1]
OUT = ROOT / "evidence"
REPEATS = 3
RATES = (1000,)
REPLICAS = (8, 12)
def main():
OUT.mkdir(exist_ok=True)
env = os.environ.copy()
env.update(API_CPUS="0.5", API_MEMORY="256m", DB_POOL_MAX="10", LB_CPUS="0.25")
bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env)
bench.wait_ready()
time.sleep(5)
client = Client("http://127.0.0.1:8080")
actor, _, _, chat = client.bootstrap()
rng = random.Random(20261107)
samples = []
last_replica_count = 8
for repeat in range(1, REPEATS + 1):
block = [(replicas, rate) for replicas in REPLICAS for rate in RATES]
rng.shuffle(block)
for replicas, rate in block:
if last_replica_count != 8:
bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", "api=8", "--wait", "--wait-timeout", "180"], env)
bench.wait_ready()
time.sleep(5)
last_replica_count = 8
bench.reset_synthetic_messages()
if replicas != last_replica_count:
bench.run(bench.COMPOSE + ["up", "-d", "--build", "--scale", f"api={replicas}", "--wait", "--wait-timeout", "180"], env)
bench.wait_ready()
time.sleep(5)
last_replica_count = replicas
row = run_sample(actor, chat, rate, repeat, reset=False)
row["replicas"] = replicas
row["sample"] = repeat
samples.append(row)
print(json.dumps(row, ensure_ascii=False), flush=True)
summary = []
for replicas in REPLICAS:
for rate in RATES:
group = [r for r in samples if r["replicas"] == replicas and r["offered_rate_rps"] == rate]
summary.append({
"replicas": replicas,
"offered_rate_rps": rate,
"completed_success_rps_total_elapsed_median": round(statistics.median(r["completed_success_rps_total_elapsed"] for r in group), 3),
"completed_success_rps_min": min(r["completed_success_rps_total_elapsed"] for r in group),
"completed_success_rps_max": max(r["completed_success_rps_total_elapsed"] for r in group),
"success_ratio_median": round(statistics.median(r["success_ratio"] for r in group), 6),
"p95_ms_median": round(statistics.median(r["latency_successful_ms"]["p95"] for r in group), 3),
"error_count_total": sum(r["failed_requests"] for r in group),
"message_rows_after_median": int(statistics.median(r["message_rows_after"] for r in group if r["message_rows_after"] is not None)) if any(r["message_rows_after"] is not None for r in group) else None,
})
result = {
"method": "Randomized paired blocks at offered 1000 RPS, 15s per sample; three repeats; compare 8 versus 12 replicas at fixed 0.5 CPU/256 MiB per API replica; DB_POOL_MAX=10, PostgreSQL 2 CPU/1 GiB, HAProxy 0.25 CPU; HAProxy server-template supports 16 backends; scale to 8 before each DB reset to release idle connections, then scale to target; synthetic messages and idempotency keys truncated per sample; HTTP client reused per worker.",
"summary": summary,
"samples": samples,
}
(OUT / "offered-scale-results.json").write_text(json.dumps(result, ensure_ascii=False, indent=2) + "\n")
print(json.dumps(summary, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
+317
View File
@@ -0,0 +1,317 @@
#!/usr/bin/env python3
"""PostgreSQL-backed messenger API for MTUSI lab 2."""
from __future__ import annotations
import base64, binascii, hashlib, json, logging, os, re, secrets, signal, socket, threading, time, uuid
from datetime import datetime, timedelta, timezone
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import parse_qs, urlsplit
import psycopg
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool, PoolTimeout
import redis
from argon2 import PasswordHasher, exceptions as argon_exc
LOG = logging.getLogger("messenger")
USERNAME_RE = re.compile(r"\A[a-z0-9_]{3,32}\Z")
POOL: ConnectionPool | None = None
CACHE: redis.Redis | None = None
PASSWORDS = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
MAX_BODY_BYTES = 1024 * 1024
CURSOR_SECRET = os.getenv("CURSOR_SECRET", "local-only-change-this-cursor-secret").encode("utf-8")
INSTANCE_ID = os.getenv("API_INSTANCE_ID") or socket.gethostname() or str(uuid.uuid4())
IDEMPOTENCY_KEY_RE = re.compile(r"\A[A-Za-z0-9._:-]{1,128}\Z")
def utcnow() -> datetime: return datetime.now(timezone.utc)
def stamp(value: datetime) -> str: return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace("+00:00", "Z")
def new_id() -> str: return str(uuid.uuid4())
class ApiError(Exception):
def __init__(self, status, code, message): self.status, self.code, self.message = status, code, message
def invalid(message="Некорректный запрос"): raise ApiError(400, "invalid_request", message)
def unavailable(): raise ApiError(503, "unavailable", "Зависимость временно недоступна")
def db():
assert POOL is not None
return POOL.connection()
def migrate():
migrations = Path(__file__).parent / "migrations"
with db() as conn:
conn.execute("SELECT pg_advisory_xact_lock(2026102802)")
conn.execute("CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, applied_at timestamptz NOT NULL DEFAULT now())")
for migration in sorted(migrations.glob("*.sql")):
version = migration.stem
if conn.execute("SELECT 1 FROM schema_migrations WHERE version=%s", (version,)).fetchone() is None:
conn.execute(migration.read_text(encoding="utf-8"))
conn.execute("INSERT INTO schema_migrations(version) VALUES (%s)", (version,))
def as_user(row): return {"id": str(row["id"]), "username": row["username"], "display_name": row["display_name"], "created_at": stamp(row["created_at"])}
def as_chat(row, members): return {"id": str(row["id"]), "title": row["title"], "member_ids": [str(x) for x in members], "created_at": stamp(row["created_at"])}
def as_message(row): return {"id": str(row["id"]), "chat_id": str(row["chat_id"]), "sender_id": str(row["sender_id"]), "text": row["text"], "created_at": stamp(row["created_at"])}
def require_text(value, field, maximum):
if not isinstance(value, str) or not value.strip() or len(value) > maximum: invalid(f"Поле {field} должно содержать от 1 до {maximum} символов и не быть пустым")
try: value.encode("utf-8")
except UnicodeEncodeError: invalid(f"Поле {field} должно содержать корректный Unicode")
return value
def strict_object(value, allowed, required):
if not isinstance(value, dict) or set(value)-allowed or required-set(value): invalid("Тело запроса содержит неверные поля")
return value
def parse_uuid(value, label="UUID"):
try: return str(uuid.UUID(value))
except (ValueError, TypeError, AttributeError): invalid(f"Некорректный {label}")
def cursor_encode(scope, row):
obj = {"scope":scope,"created_at":stamp(row["created_at"]),"id":str(row["id"])}
raw = json.dumps(obj,separators=(",",":"),ensure_ascii=True).encode()
sig = hashlib.sha256(CURSOR_SECRET+raw).digest()[:12]
return base64.urlsafe_b64encode(raw+sig).decode().rstrip("=")
def cursor_decode(value, scope):
try:
raw = base64.urlsafe_b64decode(value+"="*(-len(value)%4)); payload,sig=raw[:-12],raw[-12:]
if not secrets.compare_digest(sig,hashlib.sha256(CURSOR_SECRET+payload).digest()[:12]): invalid("Некорректный cursor")
obj=json.loads(payload)
if set(obj)!={"scope","created_at","id"} or obj["scope"]!=scope: invalid("Cursor относится к другому списку")
dt=datetime.fromisoformat(obj["created_at"].replace("Z","+00:00")); return dt,parse_uuid(obj["id"])
except (ValueError,TypeError,KeyError,binascii.Error): invalid("Некорректный cursor")
class Handler(BaseHTTPRequestHandler):
protocol_version="HTTP/1.0"
def setup(self): super().setup(); self.connection.settimeout(3)
def log_message(self, fmt,*args): LOG.info("%s %s %s",self.command,urlsplit(self.path).path,args[1] if len(args)>1 else "")
def do_GET(self): self.dispatch()
def do_POST(self): self.dispatch()
def do_DELETE(self): self.dispatch()
def send_json(self,status,payload,headers=None):
body=json.dumps(payload,ensure_ascii=False,separators=(",",":")).encode()
self.send_response(status); self.send_header("Content-Type","application/json; charset=utf-8"); self.send_header("Content-Length",str(len(body)))
self.send_header("X-Request-Id",self.request_id); self.send_header("X-Instance-Id",INSTANCE_ID)
for k,v in (headers or {}).items(): self.send_header(k,v)
self.end_headers(); self.wfile.write(body)
def empty(self,status,headers=None):
self.send_response(status); self.send_header("Content-Length","0"); self.send_header("X-Request-Id",self.request_id); self.send_header("X-Instance-Id",INSTANCE_ID)
for k,v in (headers or {}).items(): self.send_header(k,v)
self.end_headers()
def error(self,e): self.send_json(e.status,{"error":{"code":e.code,"message":e.message,"request_id":self.request_id}},getattr(e,"headers",None))
def read_json(self):
if self.headers.get_content_type()!="application/json": raise ApiError(415,"unsupported_media_type","Ожидается Content-Type: application/json")
try: n=int(self.headers.get("Content-Length",""))
except ValueError: invalid("Некорректный Content-Length")
if n<0: invalid("Некорректный Content-Length")
if n>MAX_BODY_BYTES: raise ApiError(413,"payload_too_large","Тело запроса слишком велико")
raw=self.rfile.read(n)
if len(raw)!=n: invalid("Тело запроса передано не полностью")
try: val=json.loads(raw.decode("utf-8"))
except (UnicodeError,json.JSONDecodeError): invalid("Тело запроса должно быть корректным JSON UTF-8")
if not isinstance(val,dict): invalid("Тело запроса должно быть JSON-объектом")
return val
def challenge(self,scheme):
e=ApiError(401,"unauthorized","Нет действительной сессии"); e.headers={"WWW-Authenticate":scheme}; raise e
def authenticate(self):
header=self.headers.get("Authorization","")
if not header.startswith("Bearer "): self.challenge('Bearer realm="messenger"')
token=header[7:].strip()
if not token or len(token)>256: self.challenge('Bearer realm="messenger"')
digest=hashlib.sha256(token.encode()).hexdigest(); cache_key="sess:"+digest
uid=None
if CACHE:
try: uid=CACHE.get(cache_key)
except redis.RedisError: pass
with db() as conn:
# Valkey's expiring user ID is a lookup hint. PostgreSQL validates it
# against the live session on every request, so missed cache deletion
# during an outage can never keep a logged-out token valid.
row=None
if uid:
row=conn.execute("SELECT u.id,u.username,u.display_name,u.created_at,s.expires_at FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=%s AND s.user_id=%s AND s.expires_at>now()",(digest,uid)).fetchone()
if not row:
row=conn.execute("SELECT u.id,u.username,u.display_name,u.created_at,s.expires_at FROM sessions s JOIN users u ON u.id=s.user_id WHERE s.token_hash=%s AND s.expires_at>now()",(digest,)).fetchone()
if not row: self.challenge('Bearer realm="messenger"')
if CACHE and str(row["id"])!=uid:
try: CACHE.setex(cache_key,max(1,int((row["expires_at"]-utcnow()).total_seconds())),str(row["id"]))
except redis.RedisError: pass
return as_user(row),digest
def dispatch(self):
self.request_id=secrets.token_hex(16)
try:
split=urlsplit(self.path); path=split.path; q=parse_qs(split.query,keep_blank_values=True)
if path in ("/health/live","/health/ready"):
if self.command!="GET" or q: invalid("Health endpoint принимает только GET без параметров")
if path.endswith("ready"):
try:
with db() as c: c.execute("SELECT 1")
except Exception: unavailable()
self.send_json(200,{"status":"ok"}); return
if path=="/api/v1/users" and self.command=="POST": self.register(); return
if path=="/api/v1/auth/sessions" and self.command=="POST": self.login(); return
if path=="/api/v1/auth/sessions/current" and self.command=="DELETE":
actor,digest=self.authenticate()
with db() as c: c.execute("DELETE FROM sessions WHERE token_hash=%s",(digest,))
if CACHE:
try: CACHE.delete("sess:"+digest)
except redis.RedisError: pass
self.empty(204); return
actor,_=self.authenticate()
if path=="/api/v1/users/me" and self.command=="GET": self.no_query(q); self.send_json(200,actor); return
if path=="/api/v1/chats":
if self.command=="POST": self.no_query(q); self.create_chat(actor); return
if self.command=="GET": self.list_chats(actor,q); return
m=re.fullmatch(r"/api/v1/chats/([^/]+)(?:/(messages))?",path)
if m:
cid=parse_uuid(m.group(1),"chat_id")
if m.group(2)=="messages":
if self.command=="POST": self.create_message(actor,cid); return
if self.command=="GET": self.list_messages(actor,cid,q); return
if self.command=="GET":
self.no_query(q)
with db() as c:
row=c.execute("SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE c.id=%s AND m.user_id=%s",(cid,actor["id"])).fetchone()
if not row: raise ApiError(404,"not_found","Объект не найден")
ids=c.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id",(cid,)).fetchall()
self.send_json(200,as_chat(row,[x["user_id"] for x in ids])); return
raise ApiError(404,"not_found","Объект не найден")
except ApiError as e: self.error(e)
except (psycopg.Error,PoolTimeout,redis.RedisError) as e: LOG.warning("dependency error request_id=%s type=%s",self.request_id,type(e).__name__); self.error(ApiError(503,"unavailable","Зависимость временно недоступна"))
except (BrokenPipeError,ConnectionResetError): pass
except Exception: LOG.exception("request failed request_id=%s",self.request_id); self.error(ApiError(500,"internal_error","Внутренняя ошибка сервера"))
def no_query(self,q):
if q: invalid("Этот endpoint не принимает query-параметры")
def register(self):
b=strict_object(self.read_json(),{"username","display_name","password"},{"username","display_name","password"})
if not isinstance(b["username"],str) or not USERNAME_RE.fullmatch(b["username"]): invalid("username должен соответствовать [a-z0-9_]{3,32}")
name=require_text(b["display_name"],"display_name",100); password=b["password"]
if not isinstance(password,str) or not 12<=len(password)<=128: invalid("password должен содержать от 12 до 128 символов")
uid=new_id(); hashed=PASSWORDS.hash(password)
try:
with db() as c: row=c.execute("INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING id,username,display_name,created_at",(uid,b["username"],name,hashed)).fetchone()
except psycopg.errors.UniqueViolation: raise ApiError(409,"conflict","username уже занят")
self.send_json(201,as_user(row))
def login(self):
if self.headers.get("Content-Length") not in (None, "0", ""): invalid("POST /auth/sessions не принимает body")
h=self.headers.get("Authorization","")
try:
if not h.startswith("Basic "): raise ValueError()
userpass=base64.b64decode(h[6:],validate=True).decode("utf-8"); username,password=userpass.split(":",1)
except (ValueError,UnicodeError,binascii.Error): self.challenge('Basic realm="messenger", charset="UTF-8"')
with db() as c: row=c.execute("SELECT id,username,display_name,created_at,password_hash FROM users WHERE username=%s",(username,)).fetchone()
# Verify a fixed dummy hash for unknown users to reduce timing-based enumeration.
try: ok=PASSWORDS.verify(row["password_hash"] if row else DUMMY_HASH,password)
except (argon_exc.VerificationError,ValueError): ok=False
if not row or not ok: self.challenge('Basic realm="messenger", charset="UTF-8"')
token=secrets.token_urlsafe(32); digest=hashlib.sha256(token.encode()).hexdigest(); ttl=int(os.getenv("SESSION_TTL_SECONDS","3600")); expires=utcnow()+timedelta(seconds=ttl)
with db() as c: c.execute("INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)",(digest,row["id"],expires))
if CACHE:
try: CACHE.setex("sess:"+digest,ttl,str(row["id"]))
except redis.RedisError: pass
self.send_json(201,{"token":token,"token_type":"Bearer","expires_at":stamp(expires),"user":as_user(row)})
def create_chat(self,actor):
b=strict_object(self.read_json(),{"title","member_ids"},{"title","member_ids"}); title=require_text(b["title"],"title",100); ids=b["member_ids"]
if not isinstance(ids,list) or not 1<=len(ids)<=100: invalid("member_ids должен содержать от 1 до 100 UUID")
ids=[parse_uuid(v,"member_id") for v in ids]
if len(set(ids))!=len(ids): invalid("member_ids не должен содержать повторов")
if actor["id"] not in ids: invalid("Создатель чата должен входить в member_ids")
cid=new_id()
try:
with db() as c:
known=c.execute("SELECT id FROM users WHERE id=ANY(%s)",(ids,)).fetchall()
if len(known)!=len(ids): invalid("Все участники чата должны быть зарегистрированы")
row=c.execute("INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at",(cid,title)).fetchone()
with c.cursor() as cursor:
cursor.executemany("INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)",[(cid,u) for u in ids])
except psycopg.errors.ForeignKeyViolation: invalid("Все участники чата должны быть зарегистрированы")
self.send_json(201,as_chat(row,ids))
def parse_page(self,q,scope):
if set(q)-{"limit","cursor"} or any(len(v)!=1 for v in q.values()): invalid("Неверные параметры пагинации")
raw=q.get("limit",["50"])[0]
if not re.fullmatch(r"[0-9]{1,3}",raw): invalid("limit должен быть целым числом от 1 до 100")
limit=int(raw)
if not 1<=limit<=100: invalid("limit должен быть целым числом от 1 до 100")
cur=cursor_decode(q["cursor"][0],scope) if "cursor" in q and q["cursor"][0] else None
if "cursor" in q and not q["cursor"][0]: invalid("cursor не должен быть пустым")
return limit,cur
def list_chats(self,actor,q):
limit,cur=self.parse_page(q,"chats"); params=[actor["id"]]; clause=""
if cur: clause=" AND (c.created_at,c.id)>(%s,%s)"; params.extend(cur)
with db() as c:
rows=c.execute("SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id WHERE m.user_id=%s"+clause+" ORDER BY c.created_at,c.id LIMIT %s",(*params,limit+1)).fetchall()
items=[]
for row in rows[:limit]:
members=c.execute("SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id",(row["id"],)).fetchall(); items.append(as_chat(row,[m["user_id"] for m in members]))
self.send_json(200,{"items":items,"next_cursor":cursor_encode("chats",rows[limit-1]) if len(rows)>limit else None})
def create_message(self,actor,cid):
self.no_query(parse_qs(urlsplit(self.path).query,keep_blank_values=True))
key=self.headers.get("Idempotency-Key", "")
if not IDEMPOTENCY_KEY_RE.fullmatch(key): invalid("Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}")
with db() as c:
if not c.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",(cid,actor["id"])).fetchone(): raise ApiError(404,"not_found","Объект не найден")
b=strict_object(self.read_json(),{"text","attachment_ids"},{"text"}); text=require_text(b["text"],"text",4000)
attachment_ids=b.get("attachment_ids", [])
if attachment_ids != []: invalid("Вложения не поддерживаются в лабораторной 3")
canonical=json.dumps({"text":text,"attachment_ids":[]},ensure_ascii=False,sort_keys=True,separators=(",",":"))
request_hash=hashlib.sha256(canonical.encode("utf-8")).hexdigest()
resource_path=f"/api/v1/chats/{cid}/messages"
scope=f"{actor['id']}\nPOST\n{resource_path}\n{key}"
try:
with db() as c:
# One transaction-scoped lock serializes equal keys across API replicas.
c.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s,0))",(scope,))
old=c.execute("SELECT request_hash,response_body FROM idempotency_keys WHERE user_id=%s AND method='POST' AND resource_path=%s AND key=%s AND created_at>now()-interval '24 hours'",(actor["id"],resource_path,key)).fetchone()
if old:
if old["request_hash"]!=request_hash: raise ApiError(409,"conflict","Idempotency-Key уже использован с другим содержимым")
payload=old["response_body"]
else:
c.execute("DELETE FROM idempotency_keys WHERE user_id=%s AND method='POST' AND resource_path=%s AND key=%s",(actor["id"],resource_path,key))
row=c.execute("INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) RETURNING id,chat_id,sender_id,text,created_at",(new_id(),cid,actor["id"],text)).fetchone()
payload=as_message(row)
c.execute("INSERT INTO idempotency_keys(user_id,method,resource_path,key,request_hash,response_body) VALUES (%s,'POST',%s,%s,%s,%s::jsonb)",(actor["id"],resource_path,key,request_hash,json.dumps(payload,ensure_ascii=False,separators=(",",":"))))
except psycopg.errors.ForeignKeyViolation: raise ApiError(404,"not_found","Объект не найден")
self.send_json(201,payload)
def list_messages(self,actor,cid,q):
with db() as c:
if not c.execute("SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s",(cid,actor["id"])).fetchone(): raise ApiError(404,"not_found","Объект не найден")
limit,cur=self.parse_page(q,"messages:"+cid); params=[cid]; clause=""
if cur: clause=" AND (created_at,id)>(%s,%s)"; params.extend(cur)
with db() as c: rows=c.execute("SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s"+clause+" ORDER BY created_at,id LIMIT %s",(*params,limit+1)).fetchall()
self.send_json(200,{"items":[as_message(r) for r in rows[:limit]],"next_cursor":cursor_encode("messages:"+cid,rows[limit-1]) if len(rows)>limit else None})
class Server(ThreadingHTTPServer):
daemon_threads=True; allow_reuse_address=True
def main():
global POOL,CACHE,DUMMY_HASH
logging.basicConfig(level=os.getenv("LOG_LEVEL","INFO").upper(),format="%(asctime)s %(levelname)s %(message)s")
pool_max=int(os.getenv("DB_POOL_MAX","10"))
if not 1<=pool_max<=50: raise SystemExit("DB_POOL_MAX must be between 1 and 50")
POOL=ConnectionPool(os.getenv("DATABASE_URL","postgresql://messenger:messenger@postgres:5432/messenger"),min_size=int(os.getenv("DB_POOL_MIN","1")),max_size=pool_max,timeout=3,kwargs={"row_factory":dict_row},open=True)
CACHE=redis.Redis.from_url(os.getenv("CACHE_URL","redis://valkey:6379/0"),socket_connect_timeout=0.5,socket_timeout=0.5,decode_responses=True)
DUMMY_HASH=PASSWORDS.hash("dummy-password-do-not-use-123")
deadline=time.monotonic()+60
while True:
try: migrate(); break
except (psycopg.Error, PoolTimeout):
if time.monotonic()>deadline: raise
LOG.info("waiting for PostgreSQL"); time.sleep(1)
if os.getenv("RUN_MIGRATIONS_ONLY") == "1":
POOL.close()
return
try: CACHE.ping()
except redis.RedisError: LOG.warning("Valkey unavailable; session validation will use PostgreSQL")
host=os.getenv("APP_HOST","0.0.0.0"); port=int(os.getenv("APP_PORT","8080")); server=Server((host,port),Handler)
def stop(signum,_frame): LOG.info("received signal %s; stopping",signum); threading.Thread(target=server.shutdown,daemon=True).start()
signal.signal(signal.SIGTERM,stop); signal.signal(signal.SIGINT,stop)
try: LOG.info("server listening on %s:%s",host,port); server.serve_forever(poll_interval=.2)
finally: server.server_close(); POOL.close(); LOG.info("server stopped")
if __name__=="__main__": main()
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env python3
"""Stop/start Valkey and PostgreSQL with volumes preserved; verify bounded failures and recovery."""
import subprocess
import time
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run
ROOT = Path(__file__).resolve().parents[1]
def compose(action, service):
subprocess.run(["docker", "compose", "--env-file", ".env", action, service], cwd=ROOT,
check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
def main():
args=arguments("Check session fallback and database dependency failure")
c=Client(args.base_url,args.ca_file,timeout=8)
actor=c.user("dependency")
compose("stop","valkey")
try:
c.call("GET","/health/ready",200,schema="Health")
me=c.call("GET","/api/v1/users/me",200,actor=actor,schema="User")["json"]
check(me["id"]==actor["id"],"session failed while Valkey was down")
finally:
compose("start","valkey")
compose("stop","postgres")
try:
c.call("GET","/health/ready",503)
c.call("GET","/api/v1/users/me",503,actor=actor)
finally:
compose("start","postgres")
deadline=time.monotonic()+60
while True:
try:
c.call("GET","/health/ready",200,schema="Health")
break
except Failure:
check(time.monotonic()<deadline,"PostgreSQL did not recover within 60 seconds")
time.sleep(.5)
c.call("GET","/api/v1/users/me",200,actor=actor,schema="User")
print("PASS: Valkey outage fell back to PostgreSQL; PostgreSQL outage returned bounded 503 and recovered with the same session")
if __name__=="__main__": run(main)
+23
View File
@@ -0,0 +1,23 @@
#!/usr/bin/env python3
"""Concurrent duplicate POSTs through LB; assert one stable message result."""
from concurrent.futures import ThreadPoolExecutor
import uuid
from client import Client, arguments, check, run
def main():
args=arguments('Parallel retry test through the shared load balancer')
base=Client(args.base_url,args.ca_file,timeout=10)
alice,bob,eve,chat=base.bootstrap(); key='race:'+uuid.uuid4().hex
body={'text':'Параллельная отправка'}
def send(_):
c=Client(args.base_url,args.ca_file,timeout=10)
return c.call('POST',f"/api/v1/chats/{chat['id']}/messages",201,actor=alice,body=body,headers={'Idempotency-Key':key},schema='Message')
with ThreadPoolExecutor(max_workers=16) as pool: replies=list(pool.map(send,range(32)))
ids={r['json']['id'] for r in replies}; instances={r['headers']['X-Instance-Id'] for r in replies}
check(len(ids)==1,'concurrent retries created more than one message')
check(len(instances)>=2,'test did not observe requests served by two API replicas')
history=base.call('GET',f"/api/v1/chats/{chat['id']}/messages?limit=100",actor=bob,schema='MessagePage')['json']['items']
check(sum(m['id'] in ids for m in history)==1,'message is not visible exactly once through LB')
print(f'PASS: 32 concurrent retries, one message, {len(instances)} API instances observed')
if __name__=='__main__': run(main)
+54
View File
@@ -0,0 +1,54 @@
#!/usr/bin/env python3
"""Level-4 checks: concurrent unique registration and atomic chat creation."""
import concurrent.futures
import json
import os
import subprocess
import uuid
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
def db_count(table):
# Fixed table allowlist: user input is never interpolated here.
assert table in {"users", "chats", "chat_members", "messages"}
output = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
"psql", "-X", "-U", "messenger", "-d", "messenger", "-At", "-c",
"SELECT count(*) FROM " + table], cwd=ROOT, check=True, text=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout
return int(output.strip())
def main():
args = arguments("Check registration race and transaction rollback")
c = Client(args.base_url, args.ca_file)
alice, bob, _, _ = c.bootstrap()
username = "race_" + uuid.uuid4().hex[:16]
body = {"username": username, "display_name": "Гонка регистрации", "password": "Test-password-" + uuid.uuid4().hex}
def register(_):
try:
return c.call("POST", "/api/v1/users", [201, 409], body=body)["status"]
except Failure:
raise
with concurrent.futures.ThreadPoolExecutor(max_workers=12) as pool:
statuses = list(pool.map(register, range(12)))
check(statuses.count(201) == 1 and statuses.count(409) == 11,
"username race must create exactly one user and return 409 to all other requests")
before = db_count("chats")
after = c.call("POST", "/api/v1/chats", 400, actor=alice,
body={"title": "rollback probe", "member_ids": [alice["id"], str(uuid.uuid4())]})
check(after["status"] == 400 and db_count("chats") == before,
"invalid multi-step chat creation left a partial chat row")
plan = subprocess.run(["docker", "compose", "--env-file", ".env", "exec", "-T", "postgres",
"psql", "-X", "-U", "messenger", "-d", "messenger", "-At", "-c",
"EXPLAIN (ANALYZE, BUFFERS) SELECT id FROM messages WHERE chat_id=md5('chat-0')::uuid ORDER BY created_at,id LIMIT 50"],
cwd=ROOT, check=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE).stdout.splitlines()
print(json.dumps({"registration_race": {"201": statuses.count(201), "409": statuses.count(409)},
"chat_rollback": "PASS", "history_index_plan": plan}, ensure_ascii=False, indent=2))
if __name__ == "__main__": run(main)
+20
View File
@@ -0,0 +1,20 @@
#!/usr/bin/env python3
"""Run only with SESSION_TTL_SECONDS=2 in the Compose environment."""
import time
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tests"))
from client import Client, Failure, arguments, check, run
def main():
args=arguments("Check persisted session expiry with a short TTL")
c=Client(args.base_url,args.ca_file)
actor=c.user("ttl")
c.call("GET","/api/v1/users/me",200,actor=actor,schema="User")
time.sleep(2.2)
c.call("GET","/api/v1/users/me",401,actor=actor)
print("PASS: session expired at the configured two-second TTL")
if __name__=="__main__": run(main)