From 369c5c1660f9e90c1fd5be82b346cc847022add3 Mon Sep 17 00:00:00 2001 From: Kondrashov Nikita Date: Sun, 15 Nov 2026 22:00:21 +0300 Subject: [PATCH] [+] add project --- .env.example | 15 +- .github/pull_request_template.md | 80 ++- Dockerfile | 15 + Makefile | 21 +- README.md | 74 ++- REPORT.md | 106 +++- app/__init__.py | 0 app/main.py | 748 +++++++++++++++++++++++ app/migrate.py | 35 ++ app/migrations/001_initial.sql | 54 ++ app/migrations/002_idempotency.sql | 12 + app/settings.py | 54 ++ compose.yaml | 138 +++++ evidence/benchmark/fixed-r1-1p00-1.json | 28 + evidence/benchmark/fixed-r1-1p00-2.json | 28 + evidence/benchmark/fixed-r1-1p00-3.json | 28 + evidence/benchmark/fixed-r2-0p50-1.json | 28 + evidence/benchmark/fixed-r2-0p50-2.json | 28 + evidence/benchmark/fixed-r2-0p50-3.json | 28 + evidence/benchmark/fixed-r4-0p25-1.json | 28 + evidence/benchmark/fixed-r4-0p25-2.json | 28 + evidence/benchmark/fixed-r4-0p25-3.json | 28 + evidence/benchmark/growth-r2-1p00-1.json | 28 + evidence/benchmark/growth-r2-1p00-2.json | 28 + evidence/benchmark/growth-r2-1p00-3.json | 28 + evidence/benchmark/growth-r4-0p50-1.json | 28 + evidence/benchmark/growth-r4-0p50-2.json | 28 + evidence/benchmark/growth-r4-0p50-3.json | 28 + evidence/benchmark/machine.json | 5 + evidence/benchmark/summary.csv | 6 + evidence/benchmark/summary.json | 82 +++ evidence/benchmark/throughput.svg | 20 + evidence/failover-api1.json | 16 + evidence/failover-api2.json | 16 + evidence/rolling-restart-load.json | 17 + infra/haproxy.cfg | 30 + lab.json | 2 +- requirements.txt | 5 + scripts/benchmark.py | 250 ++++++++ scripts/restart.sh | 24 + scripts/rolling-restart.sh | 28 + scripts/stop-one.sh | 17 + tests/README.md | 10 + tests/grade4.py | 53 ++ tests/replica_failover.py | 146 +++++ 45 files changed, 2442 insertions(+), 57 deletions(-) create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/main.py create mode 100644 app/migrate.py create mode 100644 app/migrations/001_initial.sql create mode 100644 app/migrations/002_idempotency.sql create mode 100644 app/settings.py create mode 100644 compose.yaml create mode 100644 evidence/benchmark/fixed-r1-1p00-1.json create mode 100644 evidence/benchmark/fixed-r1-1p00-2.json create mode 100644 evidence/benchmark/fixed-r1-1p00-3.json create mode 100644 evidence/benchmark/fixed-r2-0p50-1.json create mode 100644 evidence/benchmark/fixed-r2-0p50-2.json create mode 100644 evidence/benchmark/fixed-r2-0p50-3.json create mode 100644 evidence/benchmark/fixed-r4-0p25-1.json create mode 100644 evidence/benchmark/fixed-r4-0p25-2.json create mode 100644 evidence/benchmark/fixed-r4-0p25-3.json create mode 100644 evidence/benchmark/growth-r2-1p00-1.json create mode 100644 evidence/benchmark/growth-r2-1p00-2.json create mode 100644 evidence/benchmark/growth-r2-1p00-3.json create mode 100644 evidence/benchmark/growth-r4-0p50-1.json create mode 100644 evidence/benchmark/growth-r4-0p50-2.json create mode 100644 evidence/benchmark/growth-r4-0p50-3.json create mode 100644 evidence/benchmark/machine.json create mode 100644 evidence/benchmark/summary.csv create mode 100644 evidence/benchmark/summary.json create mode 100644 evidence/benchmark/throughput.svg create mode 100644 evidence/failover-api1.json create mode 100644 evidence/failover-api2.json create mode 100644 evidence/rolling-restart-load.json create mode 100644 infra/haproxy.cfg create mode 100644 requirements.txt create mode 100644 scripts/benchmark.py create mode 100755 scripts/restart.sh create mode 100755 scripts/rolling-restart.sh create mode 100755 scripts/stop-one.sh create mode 100644 tests/grade4.py create mode 100644 tests/replica_failover.py diff --git a/.env.example b/.env.example index d32b9be0..9a5a4e07 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,8 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# Compose settings for the local synthetic test environment. +LB_PORT=8080 SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 +API_CPUS=0.50 +API_MEMORY=256M +DB_POOL_MAX_SIZE=8 diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index e6e63eb2..6bec0ca1 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,70 @@ ## Лабораторная 3 -Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Автор / группа: Кондрашов Никита / БПИ2304 +Целевая оценка (3/4/5): 5 +Трек 5: трек 1 — цена масштабирования API -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: Перенесены пользователи, чаты, участники, сообщения, keyset-пагинация и Bearer-сессии из предыдущей лабораторной. Добавлены PostgreSQL, отдельный запуск миграций, Redis как общий кэш профилей, HAProxy и две API-реплики. Идемпотентность сообщений хранится в PostgreSQL: конкурентные запросы с одинаковым ключом создают одно сообщение, а повтор ключа с другим телом возвращает 409. Добавлены health/readiness checks, graceful shutdown, failover и последовательный перезапуск реплик. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +Команды запуска из чистого clone (выполнять из корня `mtusi2026lab3`): + +```sh +[ -f .env ] || cp .env.example .env +mkdir -p secrets +[ -f secrets/database_password ] || openssl rand -hex 32 > secrets/database_password +chmod 600 secrets/database_password + +docker compose --env-file .env config --quiet +docker compose --env-file .env build api migrate +docker compose --env-file .env up -d postgres redis +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --scale api=2 api load_balancer +docker compose --env-file .env ps +``` + +Дождаться статуса `healthy`, затем проверить сервис: + +```sh +curl --fail http://localhost:8080/health/ready +make check +make validate +make test +make test-grade4 +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` + +Проверка отказа каждой API-реплики: + +```sh +for c in $(docker compose ps -q api); do + docker inspect --format '{{.Config.Hostname}}' "$c" +done +``` + +Подставить один из выведенных hostname в `TARGET_INSTANCE_ID`, восстановить две реплики после теста и повторить со вторым: + +```sh +make test-failover-target TARGET_INSTANCE_ID=ВСТАВЬТЕ_HOSTNAME OUTPUT=evidence/failover-api1.json +docker compose --env-file .env up -d --scale api=2 api +make test-failover-target TARGET_INSTANCE_ID=ВСТАВЬТЕ_HOSTNAME OUTPUT=evidence/failover-api2.json +docker compose --env-file .env up -d --scale api=2 api +``` + +Для проверки rolling restart запустить `make load OUTPUT=evidence/rolling-restart-load.json` и `make rolling-restart` одновременно в двух терминалах. Эксперимент трека 5: + +```sh +python3 scripts/benchmark.py --duration 10 --concurrency 8 --repeats 3 +``` + +Результаты проверок и ссылка на REPORT.md: `make check`, `make validate`, Compose config, `make test`, `make test-grade4`, `make test-persistence`, отказы обеих реплик и rolling restart под нагрузкой выполнены. Smoke, контракт, конкурирующая идемпотентность и persistence прошли. Восстановление записей после отказов — 2,463 и 2,529 с; при rolling restart за 30 секунд выполнено 11 200 успешных запросов и зарегистрирована 1 ошибка. Подробные результаты и ограничения — в [REPORT.md](../REPORT.md), итоговый вывод проверок — в [evidence/acceptance.log](../evidence/acceptance.log). Результаты трека масштабирования — в [evidence/benchmark/summary.csv](../evidence/benchmark/summary.csv) и [графике](../evidence/benchmark/throughput.svg): при фиксированном бюджете 1 CPU/512 MiB throughput составил 319/267/186 RPS для 1/2/4 реплик; при удвоении бюджета для двух реплик — 564 RPS, дальнейшее увеличение до четырёх реплик дало 568 RPS. + +Ограничения / известные ошибки: Локальный single-node Compose. PostgreSQL, HAProxy и Docker host не реплицированы; их отказ не проверялся. Нагрузочный тест closed-loop и характеризует только этот стенд и профиль. + +Источники и использование ИИ: использованы контракт и тесты лабораторной, документация Docker Compose и HAProxy. ИИ использовался при исправлении ошибок, проверке изменений и подготовке документации. + +- [x] Обязательный минимум лабораторной реализован; межлабораторные переходы описаны в REPORT.md. +- [x] Контракт и публичные тесты не ослаблялись. +- [x] `make check`, `make validate`, `make test` и дополнительные проверки выполнены; вывод сохранён в evidence. +- [x] Проверены persistence, отказ каждой API-реплики и rolling restart под нагрузкой. +- [x] Секреты находятся в локальных игнорируемых файлах; отчёт и тестовые доказательства не содержат токенов. +- [x] README задания сохранён; REPORT.md описывает запуск, архитектуру, эксперименты и ограничения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..64cbc8b5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +WORKDIR /srv/messenger +COPY requirements.txt . +RUN python -m pip install --no-cache-dir -r requirements.txt +COPY app ./app + +USER 10001:10001 + +EXPOSE 8080 +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1", "--timeout-keep-alive", "5", "--timeout-graceful-shutdown", "10"] diff --git a/Makefile b/Makefile index 5d86d171..9df2f02a 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,15 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence test-failover +OUTPUT ?= +OUTPUT_ARGS = $(if $(OUTPUT),--output "$(OUTPUT)",) + +.PHONY: help check validate test load test-grade4 test-persistence test-failover test-failover-target rolling-restart help: - @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "check: offline structure; validate: full OpenAPI; test: public smoke; test-grade4: shared idempotency and replica distribution" + @echo "load: synthetic writes; test-persistence/test-failover: ACTION_SCRIPT hooks; rolling-restart: serial API restarts" + @echo "test-failover-target: TARGET_INSTANCE_ID= OUTPUT=" check: $(PYTHON) scripts/check_contract.py @@ -19,7 +24,10 @@ test: $(PYTHON) tests/smoke.py --base-url "$(BASE_URL)" $(TLS_ARGS) load: - $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(TLS_ARGS) + $(PYTHON) tests/load.py --base-url "$(BASE_URL)" $(OUTPUT_ARGS) $(TLS_ARGS) + +test-grade4: + $(PYTHON) tests/grade4.py --base-url "$(BASE_URL)" $(TLS_ARGS) test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) @@ -28,3 +36,10 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-failover-target: + @test -n "$(TARGET_INSTANCE_ID)" || (echo "Set TARGET_INSTANCE_ID from X-Instance-Id"; exit 2) + $(PYTHON) tests/replica_failover.py --target-id "$(TARGET_INSTANCE_ID)" --action-script scripts/stop-one.sh $(OUTPUT_ARGS) --base-url "$(BASE_URL)" $(TLS_ARGS) + +rolling-restart: + ./scripts/rolling-restart.sh diff --git a/README.md b/README.md index 10381de4..fab9d72c 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Запустить несколько взаимозаменяемых экземпляров API за единой точкой входа и сохранить корректность при остановке одного из них. -Это **skeleton**, а не готовый сервер. Здесь есть задание, контракт и публичные проверки. Реализацию приложения, Dockerfile и требуемую инфраструктуру пишет студент. Язык и framework свободные; UI не обязателен. +Реализация использует FastAPI, PostgreSQL, Redis как необязательный профильный кэш и HAProxy. Единственная опубликованная точка входа — HAProxy; API-реплики и хранилища находятся во внутренней сети Compose. UI не требуется. [Карта курса](COURSE.md) · [Процесс fork/PR](CONTRIBUTING.md) · [OpenAPI](contracts/openapi.json) · [Семантика API](contracts/README.md) · [Проверки](tests/README.md) · [Отчёт](REPORT.md) · [Примеры JSON](contracts/examples.json) · [Памятка преподавателю](MAINTAINERS.md) @@ -45,25 +45,75 @@ **Трек 2. Отказ узла.** Расширьте стенд до нескольких узлов, покажите перенос API при потере узла и разберите оставшиеся SPOF (LB, DB, storage). Для заявленной защищённой зависимости покажите реальный failover и сохранность данных. Если DB не реплицирована, не заявляйте выживание её узла. -## Как начать и проверить +## Запуск стенда + +Требуются Docker Engine/Desktop с Compose v2 и Python 3.9+ на машине, с которой запускаются проверки. Образы приложений/зависимостей закреплены в `compose.yaml` и `Dockerfile`. ```sh -# Работает прямо в skeleton, Python 3.9+; приложение не запускается: -make check +[ -f .env ] || cp .env.example .env +mkdir -p secrets +[ -f secrets/database_password ] || openssl rand -hex 32 > secrets/database_password +chmod 600 secrets/database_password -# После реализации запустите свой сервер/стенд по REPORT.md. -# По умолчанию тест использует http://localhost:8080. +docker compose --env-file .env config --quiet +docker compose --env-file .env build api migrate +docker compose --env-file .env up -d postgres redis +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --scale api=2 api load_balancer + +# Дождитесь готовности и выполните публичный контракт: +until curl -fsS http://localhost:8080/health/ready; do sleep 1; done +make check make test +make test-grade4 + +# Скрипты отказов должны быть исполняемыми: +chmod +x scripts/stop-one.sh scripts/restart.sh scripts/rolling-restart.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh ``` -`.env.example`, `compose.example.yaml` и `contracts/infrastructure.json` описывают настройки и роли компонентов. **compose.example.yaml — список ролей с пустым services, не запускаемый стенд.** Реализуйте свой `compose.yaml`/`stack.yaml`, закрепите версии образов, заполните локальный `.env` и опишите bootstrap/миграции. Все зависимости поднимаются из этого репозитория; ссылка «у меня БД уже установлена» не заменяет воспроизводимость. +Для независимой проверки **каждой** реплики получите aliases, наблюдаемые как `X-Instance-Id`, затем явно выберите target. После каждого прогона восстановите обе реплики: ```sh -# Compose-вариант после реализации: -docker compose --env-file .env config --quiet -docker compose --env-file .env up -d --build -# Дождитесь /health/ready; затем make test. -# Для Swarm укажите эквивалентные build/push/deploy-команды в REPORT.md. +for c in $(docker compose ps -q api); do docker inspect --format '{{.Config.Hostname}}' "$c"; done +make test-failover-target TARGET_INSTANCE_ID=alias-1 OUTPUT=evidence/failover-api1.json +docker compose --env-file .env up -d --scale api=2 api +make test-failover-target TARGET_INSTANCE_ID=alias-2 OUTPUT=evidence/failover-api2.json +docker compose --env-file .env up -d --scale api=2 api +``` + +Уровень 4: запустите `make load OUTPUT=evidence/rolling-restart-load.json` в одном терминале и `make rolling-restart` в другом; соберите вывод обоих процессов. `make rolling-restart` поочерёдно завершает каждый процесс с SIGTERM, ожидает readiness и только затем переходит к следующей реплике. + +Эксперимент трека 5 записывает каждый запуск как отдельный JSON, сводки CSV/JSON и SVG-график: + +```sh +python3 scripts/benchmark.py --duration 10 --concurrency 8 --repeats 3 +``` + +Скрипт сравнивает 1/2/4 реплики при общем лимите API 1 CPU/512 MiB и две конфигурации с увеличенными суммарными лимитами. Увеличение API-реплик и нагрузка создают синтетические данные в учебной PostgreSQL. Не публикуйте секреты; raw output содержит только метрики. + +Обычная остановка сохраняет volumes; намеренный сброс удаляет учебную БД и Redis: + +```sh +docker compose --env-file .env down # сохранить данные +docker compose --env-file .env down -v # намеренно удалить volumes и тестовые данные +``` + +Для полной OpenAPI-валидации: + +```sh +python3 -m venv .venv +.venv/bin/python -m pip install -r requirements-dev.txt +make validate PYTHON=.venv/bin/python +``` + +`make test` проверяет контракт, а не инфраструктурную оценку. Persistence, failover, последовательный рестарт и нагрузочный трек проверяются отдельными командами; доказательства и ограничения приведены в [REPORT.md](REPORT.md). + +## Офлайн-проверка + +```sh +make check ``` diff --git a/REPORT.md b/REPORT.md index c2679005..98079660 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,109 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: не указаны. +- Целевая оценка: 5; трек 1 — цена масштабирования. +- Дата эксперимента: 2026-11-15; host Python 3.9.6, Docker 29.1.3, Compose 2.40.3. +- Библиотеки: FastAPI 0.115.6, psycopg 3.2.4, redis-py 5.2.1, argon2-cffi 23.1.0, Uvicorn 0.34.0. +- Источники: контракт/tests этого репозитория; [Compose services](https://docs.docker.com/reference/compose-file/services/), [HAProxy 3.1 Configuration Manual](https://docs.haproxy.org/3.1/configuration.html); приложение продолжает модель lab2. +- Помощь ИИ: использовалась для реализации, тестирования и подготовки отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуются Docker Engine/Desktop с Compose v2, Python 3.9+ и OpenSSL. Контейнер API использует Python 3.12.8; Compose-образы: PostgreSQL 16.6-alpine, Redis 7.4.2-alpine, HAProxy 3.1-alpine. Все теги образов заданы явно, `latest` не используется. + +```sh +[ -f .env ] || cp .env.example .env +mkdir -p secrets +[ -f secrets/database_password ] || openssl rand -hex 32 > secrets/database_password +chmod 600 secrets/database_password + +docker compose --env-file .env config --quiet +docker compose --env-file .env build api migrate +docker compose --env-file .env up -d postgres redis +docker compose --env-file .env run --rm migrate +docker compose --env-file .env up -d --scale api=2 api load_balancer +until curl -fsS http://localhost:8080/health/ready; do sleep 1; done + +make check +make test +make test-grade4 +make test-persistence ACTION_SCRIPT=scripts/restart.sh +``` + +Для failover сначала выведите фактические aliases контейнеров, затем выполните targeted-тест для каждой реплики, восстанавливая её между прогонами: + +```sh +for c in $(docker compose ps -q api); do docker inspect --format '{{.Config.Hostname}}' "$c"; done +make test-failover-target TARGET_INSTANCE_ID=alias-from-first-line OUTPUT=evidence/failover-api1.json +docker compose --env-file .env up -d --scale api=2 api +make test-failover-target TARGET_INSTANCE_ID=alias-from-second-line OUTPUT=evidence/failover-api2.json +docker compose --env-file .env up -d --scale api=2 api +``` + +Для rolling restart запустить `make load OUTPUT=evidence/rolling-restart-load.json` и одновременно `make rolling-restart`. Масштабирование не удаляет volumes. `docker compose down` останавливает стенд, сохраняя данные; только намеренный `docker compose down -v` удаляет локальные volumes и учебные данные. `.env` и `secrets/database_password` локальны и исключены из Git. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text +HTTP client -> HAProxy :8080 -> API × 2+ -> PostgreSQL (источник истины) + | -> Redis (необязательный profile cache) +``` + +Compose DNSRR публикует адреса всех API tasks; HAProxy `server-template` обновляет адреса через Docker DNS и балансирует запросы без sticky sessions. Readiness проверяет `/health/ready`; HAProxy опрашивает backend каждую секунду, исключает его после двух неуспешных проверок. Сетевой таймаут проверки — около 2 секунд; расчётная граница обнаружения — до 5 секунд. На двух целевых отказах восстановление успешных записей измерено за 2.463 и 2.529 секунды. + +Пользователи, чаты, сообщения, сессии и ключи идемпотентности хранятся в PostgreSQL. Redis — общий, persistent AOF кэш профилей; отключение/потеря его кэша не меняет корректность. Миграция `002_idempotency.sql` запускается отдельной Compose-командой до API, а не при старте каждого worker. Idempotency claim, запись сообщения и привязка результата выполняются в одной транзакции. Уникальность — `(user_id, chat_id, key)`; hash нормализованного тела блокирует повтор ключа с иным содержимым; после 24 часов ключ можно использовать заново. + +`X-Instance-Id` — hostname конкретного контейнера, стабилен в течение его жизни и позволяет операторскому hook сопоставить API-ответ с контейнером. Uvicorn закрывает процесс с graceful timeout 10 секунд; Compose даёт 12 секунд на остановку. Порты API/БД/Redis наружу не опубликованы; наружу открыт только HAProxy. Лимиты по умолчанию: API 0.5 CPU/256 MiB на реплику, PostgreSQL 0.5 CPU/384 MiB, Redis и HAProxy по 0.25 CPU/128 MiB. `container_name` не задан. + +Ограничение: PostgreSQL, Redis, HAProxy и Docker host не реплицированы; это эксперимент о масштабировании API и отказе процесса API, а не отказоустойчивость узла/БД/LB. Redis здесь только кэш, PostgreSQL остаётся SPOF и источником истины. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| Критерий | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Контракт lab3 | `make check`, `make validate`, `docker compose config --quiet` | PASS; Compose, Python syntax и OpenAPI валидны | [acceptance.log](./evidence/acceptance.log) | +| Публичный сценарий | `make test` | PASS: регистрация, сессии, чат, история, доступы и replay | [acceptance.log](./evidence/acceptance.log) | +| ≥2 API через LB | `make test-grade4` | PASS: одна сессия наблюдала два aliases | [acceptance.log](./evidence/acceptance.log) | +| Конкурентный ключ через LB | `make test-grade4` | 12 конкурентных POST с одним ключом вернули одно сообщение; другой body дал 409 | [acceptance.log](./evidence/acceptance.log) | +| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: PostgreSQL, Redis и API перезапущены; прежний токен и данные сохранились | [acceptance.log](./evidence/acceptance.log) | +| Отказ каждой реплики | `make test-failover-target ...` для каждого alias | PASS; метрики и сохранность записаны раздельно | [failover-api1.json](./evidence/failover-api1.json), [failover-api2.json](./evidence/failover-api2.json) | +| Rolling restart под нагрузкой | `make load OUTPUT=...` параллельно с `make rolling-restart` | PASS; один переходный отказ, успешные записи продолжались | [rolling-restart-load.json](./evidence/rolling-restart-load.json) | +| Трек 5: повторяемые замеры | `python3 scripts/benchmark.py --duration 10 --concurrency 8 --repeats 3` | Результаты эксперимента и выводы см. ниже | [benchmark/](./evidence/benchmark) | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +## Эксперимент трека 1 — цена масштабирования + +**Гипотеза.** При неизменных суммарных ресурсах API добавление реплик само по себе не увеличит throughput пропорционально: общий PostgreSQL и его CPU/соединения станут ограничением. После увеличения бюджета API возможен рост до насыщения общей БД или самого тестового клиента. + +**Методика.** Один и тот же closed-loop POST-сценарий из `tests/load.py`, concurrency=8, 10 секунд на запуск, 3 повтора. Сравниваются 1/2/4 API с общим бюджетом API 1 CPU/512 MiB (`1×1.00/512`, `2×0.50/256`, `4×0.25/128`); затем 2/4 API при общем бюджете 2 CPU/1 GiB. Лимиты PostgreSQL, Redis и HAProxy неизменны. Все trial идут на одном Compose host/volume: перед серией было 83,215 сообщений, 15 прогонов добавили 57,434, после серии — 140,649. На прогон создаётся новый синтетический чат. Скрипт сохраняет отдельный JSON на повтор, агрегаты JSON/CSV и SVG; `docker stats` опрашивается раз в секунду. Docker daemon: 12 CPU, 8,217,165,824 bytes RAM. CPU/RAM в таблице — максимумы секундных выборок, не средние; p50/p95/p99 — по успешным запросам. + +| Серия | API × лимит на реплику | Общий API budget | successful RPS, mean ± SD | error ratio | p50 / p95 / p99, ms | peak API CPU / RAM; peak PostgreSQL CPU | +| --- | --- | --- | ---: | ---: | --- | --- | +| Fixed | 1 × 1 CPU / 512 MiB | 1 CPU / 512 MiB | 319.059 ± 33.367 | 0% | 18.192 / 51.926 / 112.067 | 102.95% / 58.9 MiB; 31.17% | +| Fixed | 2 × 0.5 CPU / 256 MiB | 1 CPU / 512 MiB | 266.984 ± 4.411 | 0% | 10.205 / 90.475 / 104.543 | 51.56% / 59.0 MiB; 15.29% | +| Fixed | 4 × 0.25 CPU / 128 MiB | 1 CPU / 512 MiB | 185.785 ± 16.019 | 0% | 5.342 / 190.819 / 283.077 | 25.83% / 120.8 MiB; 14.06% | +| Growth | 2 × 1 CPU / 512 MiB | 2 CPU / 1 GiB | 564.067 ± 17.221 | 0% | 10.116 / 41.225 / 67.687 | 103.95% / 67.3 MiB; 30.74% | +| Growth | 4 × 0.5 CPU / 256 MiB | 2 CPU / 1 GiB | 568.154 ± 59.664 | 0% | 6.388 / 64.832 / 83.160 | 52.66% / 61.9 MiB; 42.36% | + +Все 15 запусков завершились без HTTP-ошибок. При фиксированном бюджете переход с 1 на 2/4 реплики уменьшил throughput с 319 до 267/186 RPS; у четырёх реплик API memory достигал 120.8 из 128 MiB, а p95 вырос до 191 ms. Иными словами, количество процессов само по себе не масштабирует доступный CPU/RAM. + +**Точка насыщения и улучшение.** При бюджете 2 CPU/1 GiB рост с 2 до 4 API дал только +0.7% mean RPS (564 → 568; разброс 59.7 RPS), но p95 ухудшился на 57% (41.2 → 64.8 ms). Пик PostgreSQL вырос до 42.36% одного CPU при лимите 0.5 CPU (около 85% выделенного ядра); у API также видны пики на CPU-quota. Это первый измеренный knee для данного профиля: общий DB CPU становится вероятным ограничением вместе с scheduler/worker overhead; четыре реплики больше не дают надёжного выигрыша. Краткие секундные maxima не доказывают устойчивую стопроцентную загрузку, поэтому точная производственная capacity не заявляется. + +Подтверждённое улучшение в измеренной конфигурации: на двух API увеличение общего лимита с 1 CPU/512 MiB до 2 CPU/1 GiB повысило throughput с 267 до 564 RPS (+111%), снизило p95 с 90.5 до 41.2 ms (-54%), errors остались 0. Это улучшение подтверждено только для выбранного host и closed-loop профиля. См. [raw trial JSON](./evidence/benchmark), [summary.csv](./evidence/benchmark/summary.csv), [summary.json](./evidence/benchmark/summary.json), [machine.json](./evidence/benchmark/machine.json) и [график throughput](./evidence/benchmark/throughput.svg). ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Ожидание | Наблюдение | Восстановление / данные | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Остановка API replica `30ba967428ed` | LB продолжает обслуживать через оставшуюся реплику; неопределённый POST безопасно повторяется | 110 успешных write, 1 неуспешная попытка; 7.286 успешных RPS, 0.9009% write errors; recovery 2.463 s | Подтверждённые записи сохранены, дублей 0 | +| Остановка API replica `030ce8e8117a` | То же при отказе второй реплики | 108 успешных write, 1 неуспешная попытка; 7.163 успешных RPS, 0.9174% write errors; recovery 2.529 s | Подтверждённые записи сохранены, дублей 0 | +| Последовательный рестарт 2 API под нагрузкой | Вторая реплика обслуживает, пока первая завершает процесс | 30 s, concurrency 4: 11,200 успешных, 1 ошибка; 372.715 RPS; p50/p95/p99 = 4.884/41.279/82.383 ms | Оба процесса стали healthy; подтверждённая история проверена smoke/failover тестами | +| Перезапуск state services и API | Сессия и данные доступны с прежним токеном | Persistence hook завершился PASS | Volumes сохранены, подтверждённые данные не потеряны | + +Для `/health/ready` критична только PostgreSQL: Redis служит кэшем и имеет fallback на БД. Недоступность самой PostgreSQL или хоста не выдерживается. Нагрузка `tests/load.py` — закрытая и клиент-ограниченная; один отказ/timeout может скрыть результат COMMIT, поэтому это throughput HTTP-подтверждений, а не измерение числа транзакций БД. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Нет HA для PostgreSQL, HAProxy, Redis как процесса и Docker host; отказ этих компонентов не покрыт тестами. +- Compose restart под нагрузкой доказывает graceful последовательный рестарт API-процессов, но не blue/green обновление разных версий приложения. +- Таблица ключей очищается логически при повторном использовании ключа старше 24 часов; массовая уборка не реализована. +- Benchmark — один локальный закрытый профиль и три повтора; сначала следует интерпретировать фактические сводки, прежде чем заявлять saturation point или переносить выводы на production. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..f9179666 --- /dev/null +++ b/app/main.py @@ -0,0 +1,748 @@ +import base64 +import binascii +import hashlib +import json +import logging +import os +import re +import secrets +import socket +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone +from typing import Any, Dict, List, Optional, Tuple + +import psycopg +import redis +from argon2 import PasswordHasher +from argon2.exceptions import InvalidHashError, VerificationError, VerifyMismatchError +from fastapi import Depends, FastAPI, Header, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from pydantic import BaseModel, ConfigDict, Field, field_validator +from psycopg.errors import UniqueViolation +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool, PoolTimeout +from starlette.exceptions import HTTPException as StarletteHTTPException + +from app.settings import database_connection, session_ttl_seconds + +logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO")) +logger = logging.getLogger("messenger") +PASSWORD_HASHER = PasswordHasher() +USERNAME_PATTERN = re.compile(r"^[a-z0-9_]{3,32}$") +CURSOR_VERSION = 1 +MAX_POOL_SIZE = int(os.environ.get("DB_POOL_MAX_SIZE", "10")) +if MAX_POOL_SIZE < 1 or MAX_POOL_SIZE > 50: + raise RuntimeError("DB_POOL_MAX_SIZE must be between 1 and 50") + + +class APIError(Exception): + def __init__( + self, status_code: int, code: str, message: str, challenge: Optional[str] = None + ) -> None: + self.status_code = status_code + self.code = code + self.message = message + self.challenge = challenge + + +class StrictModel(BaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + +class UserInput(StrictModel): + username: str = Field(pattern=r"^[a-z0-9_]{3,32}$") + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator("display_name") + @classmethod + def display_name_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("display_name must not be blank") + return value + + @field_validator("password") + @classmethod + def password_no_nul(cls, value: str) -> str: + if "\x00" in value: + raise ValueError("password contains an invalid character") + return value + + +class ChatInput(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: List[str] = Field(min_length=1, max_length=100) + + @field_validator("title") + @classmethod + def title_not_blank(cls, value: str) -> str: + if not value.strip(): + raise ValueError("title must not be blank") + return value + + +class MessageInput(StrictModel): + text: str = Field(min_length=1, max_length=4000) + + +def utc_string(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="microseconds").replace( + "+00:00", "Z" + ) + + +def user_json(row: Dict[str, Any]) -> Dict[str, str]: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": row["display_name"], + "created_at": utc_string(row["created_at"]), + } + + +def chat_json(row: Dict[str, Any], member_ids: List[str]) -> Dict[str, Any]: + return { + "id": str(row["id"]), + "title": row["title"], + "member_ids": member_ids, + "created_at": utc_string(row["created_at"]), + } + + +def message_json(row: Dict[str, Any]) -> Dict[str, str]: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": row["text"], + "created_at": utc_string(row["created_at"]), + } + + +def db_unavailable() -> APIError: + return APIError(503, "unavailable", "Сервис временно недоступен") + + +def get_connection(): + pool: ConnectionPool = app.state.pool + try: + with pool.connection(timeout=2) as connection: + yield connection + except (psycopg.OperationalError, PoolTimeout) as exc: + logger.warning("database connection unavailable: %s", type(exc).__name__) + raise db_unavailable() from None + + +@asynccontextmanager +async def lifespan(application: FastAPI): + session_ttl_seconds() + application.state.pool = ConnectionPool( + conninfo="", + kwargs={ + **database_connection(), + "autocommit": True, + "row_factory": dict_row, + }, + min_size=1, + max_size=MAX_POOL_SIZE, + timeout=2, + max_lifetime=1800, + open=True, + name="messenger-api", + ) + redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0") + application.state.redis = redis.Redis.from_url( + redis_url, + socket_connect_timeout=1, + socket_timeout=1, + decode_responses=True, + health_check_interval=30, + ) + application.state.instance_id = socket.gethostname() + try: + yield + finally: + application.state.redis.close() + application.state.pool.close() + + +app = FastAPI(title="Messenger API", version="1.0.0", lifespan=lifespan) + + +@app.middleware("http") +async def request_metadata(request: Request, call_next): + request_id = request.headers.get("X-Request-Id", "") + if not request_id or len(request_id) > 128: + request_id = str(uuid.uuid4()) + request.state.request_id = request_id + try: + response = await call_next(request) + except Exception: + logger.exception("unhandled request failure") + response = JSONResponse( + status_code=500, + content={ + "error": { + "code": "internal_error", + "message": "Внутренняя ошибка", + "request_id": request_id, + } + }, + ) + response.headers["X-Request-Id"] = request_id + response.headers["X-Instance-Id"] = request.app.state.instance_id + return response + + +@app.exception_handler(APIError) +async def handle_api_error(request: Request, exc: APIError): + headers = {"WWW-Authenticate": exc.challenge} if exc.challenge else None + return JSONResponse( + status_code=exc.status_code, + content={ + "error": { + "code": exc.code, + "message": exc.message, + "request_id": request.state.request_id, + } + }, + headers=headers, + ) + + +@app.exception_handler(RequestValidationError) +async def handle_validation_error(request: Request, exc: RequestValidationError): + return JSONResponse( + status_code=400, + content={ + "error": { + "code": "invalid_request", + "message": "Некорректный запрос", + "request_id": request.state.request_id, + } + }, + ) + + +@app.exception_handler(StarletteHTTPException) +async def handle_http_error(request: Request, exc: StarletteHTTPException): + status = 404 if exc.status_code == 404 else exc.status_code + code = "not_found" if status == 404 else "invalid_request" + return JSONResponse( + status_code=status, + content={ + "error": { + "code": code, + "message": "Объект не найден" if status == 404 else "Некорректный запрос", + "request_id": request.state.request_id, + } + }, + ) + + +@app.exception_handler(psycopg.Error) +async def handle_database_error(request: Request, exc: psycopg.Error): + logger.warning("database operation failed: %s", type(exc).__name__) + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "Сервис временно недоступен", + "request_id": request.state.request_id, + } + }, + ) + + +@app.exception_handler(PoolTimeout) +async def handle_pool_timeout(request: Request, exc: PoolTimeout): + return JSONResponse( + status_code=503, + content={ + "error": { + "code": "unavailable", + "message": "Сервис временно недоступен", + "request_id": request.state.request_id, + } + }, + ) + + +def auth_error(challenge: str = 'Bearer realm="messenger"') -> APIError: + return APIError(401, "unauthorized", "Требуется действительная аутентификация", challenge) + + +def parse_uuid(value: str) -> uuid.UUID: + try: + return uuid.UUID(value) + except (ValueError, AttributeError, TypeError): + raise APIError(400, "invalid_request", "Некорректный UUID") from None + + +def authenticated_user( + request: Request, + authorization: Optional[str] = Header(default=None), + connection=Depends(get_connection), +) -> Dict[str, Any]: + if not authorization or not authorization.startswith("Bearer "): + raise auth_error() + token = authorization[7:] + if not token or len(token) > 256: + raise auth_error() + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + now = datetime.now(timezone.utc) + cache = request.app.state.redis + cache_key = "session:" + token_hash.hex() + cached: Optional[Dict[str, Any]] = None + try: + raw = cache.get(cache_key) + if raw: + cached_value = json.loads(raw) + if isinstance(cached_value, dict): + cached = cached_value + else: + logger.warning("session cache entry has an invalid shape") + except (redis.RedisError, json.JSONDecodeError): + logger.warning("session cache read failed; using PostgreSQL") + try: + session = connection.execute( + """ + SELECT s.user_id, s.expires_at, s.revoked_at + FROM sessions AS s + WHERE s.token_hash = %s + """, + (token_hash,), + ).fetchone() + except psycopg.Error as exc: + logger.warning("session lookup failed: %s", type(exc).__name__) + raise db_unavailable() from None + if not session or session["revoked_at"] is not None or session["expires_at"] <= now: + try: + cache.delete(cache_key) + except redis.RedisError: + logger.warning("expired session cache invalidation failed") + raise auth_error() + # PostgreSQL remains authoritative for expiry and revocation. Redis avoids + # a user-profile query on a cache hit. + if cached and cached.get("user_id") == str(session["user_id"]): + profile = cached.get("user") + if isinstance(profile, dict) and profile.get("id") == str(session["user_id"]): + user = profile + else: + cached = None + else: + cached = None + if not cached: + profile_row = connection.execute( + """ + SELECT id, username, display_name, created_at + FROM users WHERE id = %s + """, + (session["user_id"],), + ).fetchone() + if not profile_row: + raise auth_error() + user = user_json(profile_row) + remaining = max(1, int((session["expires_at"] - now).total_seconds())) + try: + cache.set( + cache_key, + json.dumps({"user_id": str(session["user_id"]), "user": user}), + ex=remaining, + ) + except redis.RedisError: + logger.warning("session cache write failed; continuing with PostgreSQL") + request.state.token_hash = token_hash + return user + + +@app.get("/health/live") +def live() -> Dict[str, str]: + return {"status": "ok"} + + +@app.get("/health/ready") +def ready(request: Request, connection=Depends(get_connection)) -> Dict[str, str]: + try: + connection.execute("SELECT 1") + except psycopg.Error: + raise db_unavailable() from None + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +def register_user(payload: UserInput, connection=Depends(get_connection)): + user_id = uuid.uuid4() + password_hash = PASSWORD_HASHER.hash(payload.password) + try: + with connection.transaction(): + row = connection.execute( + """ + INSERT INTO users (id, username, display_name, password_hash) + VALUES (%s, %s, %s, %s) + RETURNING id, username, display_name, created_at + """, + (user_id, payload.username, payload.display_name, password_hash), + ).fetchone() + except UniqueViolation: + raise APIError(409, "conflict", "Имя пользователя уже занято") from None + except psycopg.Error as exc: + logger.warning("user registration failed: %s", type(exc).__name__) + raise db_unavailable() from None + return user_json(row) + + +def decode_basic_credentials(authorization: Optional[str]) -> Tuple[str, str]: + challenge = 'Basic realm="messenger", charset="UTF-8"' + if not authorization or not authorization.startswith("Basic "): + raise auth_error(challenge) + try: + raw = base64.b64decode(authorization[6:], validate=True).decode("utf-8") + username, password = raw.split(":", 1) + except (binascii.Error, UnicodeDecodeError, ValueError): + raise auth_error(challenge) from None + if not USERNAME_PATTERN.fullmatch(username) or not 12 <= len(password) <= 128: + raise auth_error(challenge) + return username, password + + +@app.post("/api/v1/auth/sessions", status_code=201) +def create_session( + request: Request, + authorization: Optional[str] = Header(default=None), + connection=Depends(get_connection), +): + username, password = decode_basic_credentials(authorization) + row = connection.execute( + """ + SELECT id, username, display_name, password_hash, created_at + FROM users WHERE username = %s + """, + (username,), + ).fetchone() + if not row: + raise auth_error('Basic realm="messenger", charset="UTF-8"') + try: + PASSWORD_HASHER.verify(row["password_hash"], password) + except (VerifyMismatchError, VerificationError, InvalidHashError): + raise auth_error('Basic realm="messenger", charset="UTF-8"') from None + token = secrets.token_urlsafe(32) + token_hash = hashlib.sha256(token.encode("utf-8")).digest() + expires_at = datetime.now(timezone.utc) + timedelta(seconds=session_ttl_seconds()) + try: + with connection.transaction(): + connection.execute( + """ + INSERT INTO sessions (token_hash, user_id, expires_at) + VALUES (%s, %s, %s) + """, + (token_hash, row["id"], expires_at), + ) + except psycopg.Error as exc: + logger.warning("session creation failed: %s", type(exc).__name__) + raise db_unavailable() from None + user = user_json(row) + try: + request.app.state.redis.set( + "session:" + token_hash.hex(), + json.dumps({"user_id": str(row["id"]), "user": user}), + ex=session_ttl_seconds(), + ) + except redis.RedisError: + logger.warning("session cache write failed; session remains durable in PostgreSQL") + return { + "token": token, + "token_type": "Bearer", + "expires_at": utc_string(expires_at), + "user": user, + } + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +def revoke_session( + request: Request, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + token_hash: bytes = request.state.token_hash + try: + with connection.transaction(): + result = connection.execute( + """ + UPDATE sessions SET revoked_at = clock_timestamp() + WHERE token_hash = %s AND revoked_at IS NULL + """, + (token_hash,), + ) + if result.rowcount != 1: + raise auth_error() + except APIError: + raise + except psycopg.Error as exc: + logger.warning("session revocation failed: %s", type(exc).__name__) + raise db_unavailable() from None + try: + request.app.state.redis.delete("session:" + token_hash.hex()) + except redis.RedisError: + logger.warning("revoked session cache invalidation failed") + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +def current_user(user: Dict[str, Any] = Depends(authenticated_user)): + return user + + +def member_ids_for(connection, chat_id: uuid.UUID) -> List[str]: + rows = connection.execute( + "SELECT user_id FROM chat_members WHERE chat_id = %s ORDER BY user_id", + (chat_id,), + ).fetchall() + return [str(row["user_id"]) for row in rows] + + +def require_chat_member( + connection, chat_id: uuid.UUID, user_id: uuid.UUID +) -> Dict[str, Any]: + row = connection.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats AS c JOIN chat_members AS cm ON cm.chat_id = c.id + WHERE c.id = %s AND cm.user_id = %s + """, + (chat_id, user_id), + ).fetchone() + if not row: + raise APIError(404, "not_found", "Объект не найден") + return row + + +@app.post("/api/v1/chats", status_code=201) +def create_chat( + payload: ChatInput, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + ids = [parse_uuid(member_id) for member_id in payload.member_ids] + if len(set(ids)) != len(ids) or uuid.UUID(user["id"]) not in ids: + raise APIError(400, "invalid_request", "Некорректный список участников") + chat_id = uuid.uuid4() + try: + with connection.transaction(): + found = connection.execute( + "SELECT id FROM users WHERE id = ANY(%s)", + (ids,), + ).fetchall() + if len(found) != len(ids): + raise APIError(400, "invalid_request", "Участник не существует") + chat = connection.execute( + """ + INSERT INTO chats (id, title) VALUES (%s, %s) + RETURNING id, title, created_at + """, + (chat_id, payload.title), + ).fetchone() + for member_id in ids: + connection.execute( + "INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)", + (chat_id, member_id), + ) + except APIError: + raise + except psycopg.Error as exc: + logger.warning("chat creation rolled back: %s", type(exc).__name__) + raise db_unavailable() from None + return chat_json(chat, [str(member_id) for member_id in sorted(ids)]) + + +@app.get("/api/v1/chats") +def list_chats( + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), + limit: int = 50, + cursor: Optional[str] = None, +): + if limit < 1 or limit > 100: + raise APIError(400, "invalid_request", "limit должен быть от 1 до 100") + cursor_values = decode_cursor(cursor, "chats") if cursor else None + parameters: List[Any] = [uuid.UUID(user["id"])] + clause = "" + if cursor_values: + clause = "AND (c.created_at, c.id) > (%s, %s)" + parameters.extend(cursor_values) + parameters.append(limit + 1) + rows = connection.execute( + """ + SELECT c.id, c.title, c.created_at + FROM chats c JOIN chat_members cm ON cm.chat_id = c.id + WHERE cm.user_id = %s + """ + + clause + + " ORDER BY c.created_at, c.id LIMIT %s", + parameters, + ).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + items = [ + chat_json(row, member_ids_for(connection, row["id"])) + for row in rows + ] + next_cursor = encode_cursor("chats", rows[-1]) if has_more and rows else None + return {"items": items, "next_cursor": next_cursor} + + +@app.get("/api/v1/chats/{chat_id}") +def get_chat( + chat_id: str, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + parsed_id = parse_uuid(chat_id) + row = require_chat_member(connection, parsed_id, uuid.UUID(user["id"])) + return chat_json(row, member_ids_for(connection, parsed_id)) + + +def encode_cursor(kind: str, row: Dict[str, Any]) -> str: + payload = json.dumps( + [CURSOR_VERSION, kind, str(row["created_at"]), str(row["id"])], + separators=(",", ":"), + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + +def decode_cursor(value: str, kind: str) -> Tuple[datetime, uuid.UUID]: + try: + padded = value + "=" * (-len(value) % 4) + decoded = json.loads(base64.urlsafe_b64decode(padded.encode("ascii"))) + if ( + not isinstance(decoded, list) + or len(decoded) != 4 + or decoded[0] != CURSOR_VERSION + or decoded[1] != kind + ): + raise ValueError + created_at = datetime.fromisoformat(decoded[2]) + if created_at.tzinfo is None: + raise ValueError + return created_at, uuid.UUID(decoded[3]) + except (ValueError, TypeError, UnicodeError, json.JSONDecodeError): + raise APIError(400, "invalid_request", "Некорректный cursor") from None + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +def create_message( + chat_id: str, + payload: MessageInput, + idempotency_key: Optional[str] = Header(default=None, alias="Idempotency-Key"), + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), +): + if not idempotency_key or not re.fullmatch(r"[A-Za-z0-9._:-]{1,128}", idempotency_key): + raise APIError(400, "invalid_request", "Некорректный Idempotency-Key") + parsed_chat_id = parse_uuid(chat_id) + chat = require_chat_member(connection, parsed_chat_id, uuid.UUID(user["id"])) + if not payload.text.strip(): + raise APIError(400, "invalid_request", "Сообщение не может быть пустым") + sender_id = uuid.UUID(user["id"]) + request_hash = hashlib.sha256( + json.dumps({"text": payload.text}, ensure_ascii=False, sort_keys=True).encode("utf-8") + ).digest() + try: + with connection.transaction(): + claim = connection.execute( + """ + INSERT INTO idempotency_keys + (user_id, chat_id, idempotency_key, request_hash) + VALUES (%s, %s, %s, %s) + ON CONFLICT (user_id, chat_id, idempotency_key) DO UPDATE + SET request_hash = EXCLUDED.request_hash, + message_id = NULL, + created_at = clock_timestamp() + WHERE idempotency_keys.created_at < + clock_timestamp() - interval '24 hours' + RETURNING user_id + """, + (sender_id, chat["id"], idempotency_key, request_hash), + ).fetchone() + if claim: + row = connection.execute( + """ + INSERT INTO messages (id, chat_id, sender_id, text) + VALUES (%s, %s, %s, %s) + RETURNING id, chat_id, sender_id, text, created_at + """, + (uuid.uuid4(), chat["id"], sender_id, payload.text), + ).fetchone() + result = message_json(row) + connection.execute( + """ + UPDATE idempotency_keys SET message_id = %s + WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s + """, + (row["id"], sender_id, chat["id"], idempotency_key), + ) + else: + existing = connection.execute( + """ + SELECT request_hash, message_id + FROM idempotency_keys + WHERE user_id = %s AND chat_id = %s AND idempotency_key = %s + """, + (sender_id, chat["id"], idempotency_key), + ).fetchone() + if not existing or existing["request_hash"] != request_hash: + raise APIError(409, "conflict", "Ключ идемпотентности уже использован") + row = connection.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages WHERE id = %s + """, + (existing["message_id"],), + ).fetchone() + if not row: + raise APIError(503, "unavailable", "Результат запроса временно недоступен") + result = message_json(row) + except APIError: + raise + except psycopg.Error as exc: + logger.warning("message write failed: %s", type(exc).__name__) + raise db_unavailable() from None + return result + + +@app.get("/api/v1/chats/{chat_id}/messages") +def list_messages( + chat_id: str, + user: Dict[str, Any] = Depends(authenticated_user), + connection=Depends(get_connection), + limit: int = 50, + cursor: Optional[str] = None, +): + if limit < 1 or limit > 100: + raise APIError(400, "invalid_request", "limit должен быть от 1 до 100") + parsed_chat_id = parse_uuid(chat_id) + chat = require_chat_member(connection, parsed_chat_id, uuid.UUID(user["id"])) + cursor_values = decode_cursor(cursor, str(parsed_chat_id)) if cursor else None + parameters: List[Any] = [chat["id"]] + clause = "" + if cursor_values: + clause = "AND (created_at, id) > (%s, %s)" + parameters.extend(cursor_values) + parameters.append(limit + 1) + rows = connection.execute( + """ + SELECT id, chat_id, sender_id, text, created_at + FROM messages WHERE chat_id = %s + """ + + clause + + " ORDER BY created_at, id LIMIT %s", + parameters, + ).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + items = [message_json(row) for row in rows] + next_cursor = encode_cursor(str(parsed_chat_id), rows[-1]) if has_more and rows else None + return {"items": items, "next_cursor": next_cursor} diff --git a/app/migrate.py b/app/migrate.py new file mode 100644 index 00000000..559901f2 --- /dev/null +++ b/app/migrate.py @@ -0,0 +1,35 @@ +from pathlib import Path + +import psycopg + +from app.settings import database_connection + + +def main() -> None: + migration_dir = Path(__file__).with_name("migrations") + with psycopg.connect(**database_connection(), autocommit=False) as connection: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS schema_migrations ( + version varchar(255) PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() + ) + """ + ) + for migration in sorted(migration_dir.glob("*.sql")): + already_applied = connection.execute( + "SELECT 1 FROM schema_migrations WHERE version = %s", + (migration.name,), + ).fetchone() + if already_applied: + continue + connection.execute(migration.read_text(encoding="utf-8")) + connection.execute( + "INSERT INTO schema_migrations (version) VALUES (%s)", + (migration.name,), + ) + connection.commit() + + +if __name__ == "__main__": + main() diff --git a/app/migrations/001_initial.sql b/app/migrations/001_initial.sql new file mode 100644 index 00000000..6cccb668 --- /dev/null +++ b/app/migrations/001_initial.sql @@ -0,0 +1,54 @@ +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE, + display_name varchar(100) NOT NULL, + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT users_username_format CHECK (username ~ '^[a-z0-9_]{3,32}$'), + CONSTRAINT users_display_name_nonblank CHECK (length(btrim(display_name)) > 0) +); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT chats_title_nonblank CHECK (length(btrim(title)) > 0) +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + PRIMARY KEY (chat_id, user_id) +); + +CREATE INDEX IF NOT EXISTS chat_members_by_user + ON chat_members (user_id, chat_id); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE RESTRICT, + text varchar(4000) NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT messages_text_nonblank CHECK (length(btrim(text)) > 0) +); + +CREATE INDEX IF NOT EXISTS messages_history_keyset + ON messages (chat_id, created_at, id); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash bytea PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + revoked_at timestamptz, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE INDEX IF NOT EXISTS sessions_active_by_expiry + ON sessions (expires_at) + WHERE revoked_at IS NULL; + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version varchar(255) PRIMARY KEY, + applied_at timestamptz NOT NULL DEFAULT clock_timestamp() +); diff --git a/app/migrations/002_idempotency.sql b/app/migrations/002_idempotency.sql new file mode 100644 index 00000000..550e73e7 --- /dev/null +++ b/app/migrations/002_idempotency.sql @@ -0,0 +1,12 @@ +CREATE TABLE IF NOT EXISTS idempotency_keys ( + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + idempotency_key varchar(128) NOT NULL, + request_hash bytea NOT NULL, + message_id uuid REFERENCES messages(id) ON DELETE SET NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (user_id, chat_id, idempotency_key) +); + +CREATE INDEX IF NOT EXISTS idempotency_keys_expiry + ON idempotency_keys (created_at); diff --git a/app/settings.py b/app/settings.py new file mode 100644 index 00000000..e473c91e --- /dev/null +++ b/app/settings.py @@ -0,0 +1,54 @@ +import os +from pathlib import Path +from typing import Dict, Optional + + +def read_setting( + name: str, + file_name: str, + default: Optional[str] = None, + default_file: Optional[str] = None, +) -> str: + configured_secret_path = os.environ.get(file_name) + secret_path = configured_secret_path or default_file + if secret_path and (configured_secret_path or Path(secret_path).is_file()): + try: + return Path(secret_path).read_text(encoding="utf-8").strip() + except OSError as exc: + raise RuntimeError("Unable to read configured secret file") from exc + value = os.environ.get(name, default) + if value is None: + if default_file: + raise RuntimeError( + "Required setting is missing: " + + name + + "; set " + + file_name + + " or mount the secret at " + + default_file + ) + raise RuntimeError("Required application setting is missing: " + name) + return value + + +def database_connection() -> Dict[str, object]: + return { + "host": os.environ.get("DB_HOST", "localhost"), + "port": int(os.environ.get("DB_PORT", "5432")), + "dbname": os.environ.get("DB_NAME", "messenger"), + "user": os.environ.get("DB_USER", "messenger"), + "password": read_setting( + "DB_PASSWORD", + "DB_PASSWORD_FILE", + default_file="/run/secrets/database_password", + ), + "connect_timeout": int(os.environ.get("DB_CONNECT_TIMEOUT_SECONDS", "3")), + "options": "-c statement_timeout=5000 -c lock_timeout=3000", + } + + +def session_ttl_seconds() -> int: + value = int(os.environ.get("SESSION_TTL_SECONDS", "3600")) + if value < 1: + raise RuntimeError("SESSION_TTL_SECONDS must be at least 1") + return value diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..1d4760a0 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,138 @@ +services: + postgres: + image: postgres:16.6-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/database_password + secrets: + - database_password + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 2s + retries: 20 + cpus: 0.50 + mem_limit: 384m + restart: unless-stopped + networks: + - backend + + redis: + image: redis:7.4.2-alpine + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "everysec"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 3s + timeout: 2s + retries: 20 + cpus: 0.25 + mem_limit: 128m + restart: unless-stopped + networks: + - backend + + migrate: + build: + context: . + dockerfile: Dockerfile + command: ["python", "-m", "app.migrate"] + environment: + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: messenger + DB_USER: messenger + DB_PASSWORD_FILE: /run/secrets/database_password + secrets: + - database_password + depends_on: + postgres: + condition: service_healthy + cpus: 0.50 + mem_limit: 256m + networks: + - backend + profiles: + - tools + + api: + build: + context: . + dockerfile: Dockerfile + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + DB_HOST: postgres + DB_PORT: 5432 + DB_NAME: messenger + DB_USER: messenger + DB_PASSWORD_FILE: /run/secrets/database_password + DB_POOL_MAX_SIZE: ${DB_POOL_MAX_SIZE:-8} + REDIS_URL: redis://redis:6379/0 + secrets: + - database_password + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + healthcheck: + test: + - CMD + - python + - -c + - "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)" + interval: 2s + timeout: 2s + retries: 3 + start_period: 8s + stop_grace_period: 12s + deploy: + replicas: ${API_REPLICAS:-2} + endpoint_mode: dnsrr + resources: + limits: + cpus: "${API_CPUS:-0.50}" + memory: ${API_MEMORY:-256M} + restart: unless-stopped + networks: + - backend + + load_balancer: + image: haproxy:3.1-alpine + ports: + - "${LB_PORT:-8080}:8080" + volumes: + - ./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro + depends_on: + api: + condition: service_healthy + healthcheck: + test: ["CMD", "haproxy", "-c", "-f", "/usr/local/etc/haproxy/haproxy.cfg"] + interval: 5s + timeout: 2s + retries: 3 + cpus: 0.25 + mem_limit: 128m + restart: unless-stopped + networks: + - frontend + - backend + +secrets: + database_password: + file: ./secrets/database_password + +volumes: + postgres_data: + redis_data: + +networks: + frontend: + backend: + internal: true diff --git a/evidence/benchmark/fixed-r1-1p00-1.json b/evidence/benchmark/fixed-r1-1p00-1.json new file mode 100644 index 00000000..c9e57e44 --- /dev/null +++ b/evidence/benchmark/fixed-r1-1p00-1.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.072, + "concurrency": 8, + "successful_requests": 2829, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 280.889, + "successful_latency_ms": { + "p50": 18.167, + "p95": 60.25, + "p99": 181.53 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 1, + "api_cpu_limit_each": "1.00", + "api_memory_limit_each": "512M", + "run": 1, + "resource_samples": 16, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 101.62, + "peak_sampled_api_memory_bytes": 60534292, + "peak_sampled_postgres_cpu_percent": 31.17, + "peak_sampled_postgres_memory_bytes": 94948556 +} diff --git a/evidence/benchmark/fixed-r1-1p00-2.json b/evidence/benchmark/fixed-r1-1p00-2.json new file mode 100644 index 00000000..cdde963b --- /dev/null +++ b/evidence/benchmark/fixed-r1-1p00-2.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.042, + "concurrency": 8, + "successful_requests": 3350, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 333.596, + "successful_latency_ms": { + "p50": 18.648, + "p95": 47.519, + "p99": 76.754 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 1, + "api_cpu_limit_each": "1.00", + "api_memory_limit_each": "512M", + "run": 2, + "resource_samples": 16, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 102.95, + "peak_sampled_api_memory_bytes": 61698211, + "peak_sampled_postgres_cpu_percent": 16.43, + "peak_sampled_postgres_memory_bytes": 96762593 +} diff --git a/evidence/benchmark/fixed-r1-1p00-3.json b/evidence/benchmark/fixed-r1-1p00-3.json new file mode 100644 index 00000000..743cafd3 --- /dev/null +++ b/evidence/benchmark/fixed-r1-1p00-3.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.076, + "concurrency": 8, + "successful_requests": 3453, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 342.691, + "successful_latency_ms": { + "p50": 17.762, + "p95": 48.01, + "p99": 77.918 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 1, + "api_cpu_limit_each": "1.00", + "api_memory_limit_each": "512M", + "run": 3, + "resource_samples": 16, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 102.36, + "peak_sampled_api_memory_bytes": 61750640, + "peak_sampled_postgres_cpu_percent": 16.24, + "peak_sampled_postgres_memory_bytes": 99121889 +} diff --git a/evidence/benchmark/fixed-r2-0p50-1.json b/evidence/benchmark/fixed-r2-0p50-1.json new file mode 100644 index 00000000..a64b0ac7 --- /dev/null +++ b/evidence/benchmark/fixed-r2-0p50-1.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.093, + "concurrency": 8, + "successful_requests": 2746, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 272.067, + "successful_latency_ms": { + "p50": 10.2, + "p95": 89.385, + "p99": 105.538 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 2, + "api_cpu_limit_each": "0.50", + "api_memory_limit_each": "256M", + "run": 1, + "resource_samples": 20, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 50.43, + "peak_sampled_api_memory_bytes": 60397977, + "peak_sampled_postgres_cpu_percent": 15.29, + "peak_sampled_postgres_memory_bytes": 120586240 +} diff --git a/evidence/benchmark/fixed-r2-0p50-2.json b/evidence/benchmark/fixed-r2-0p50-2.json new file mode 100644 index 00000000..e1e0a275 --- /dev/null +++ b/evidence/benchmark/fixed-r2-0p50-2.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.097, + "concurrency": 8, + "successful_requests": 2673, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 264.72, + "successful_latency_ms": { + "p50": 10.439, + "p95": 90.939, + "p99": 102.587 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 2, + "api_cpu_limit_each": "0.50", + "api_memory_limit_each": "256M", + "run": 2, + "resource_samples": 20, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 51.56, + "peak_sampled_api_memory_bytes": 61876469, + "peak_sampled_postgres_cpu_percent": 14.73, + "peak_sampled_postgres_memory_bytes": 122683392 +} diff --git a/evidence/benchmark/fixed-r2-0p50-3.json b/evidence/benchmark/fixed-r2-0p50-3.json new file mode 100644 index 00000000..60bc1fc2 --- /dev/null +++ b/evidence/benchmark/fixed-r2-0p50-3.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.107, + "concurrency": 8, + "successful_requests": 2670, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 264.165, + "successful_latency_ms": { + "p50": 9.976, + "p95": 91.101, + "p99": 105.504 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 2, + "api_cpu_limit_each": "0.50", + "api_memory_limit_each": "256M", + "run": 3, + "resource_samples": 20, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 50.66, + "peak_sampled_api_memory_bytes": 60565749, + "peak_sampled_postgres_cpu_percent": 15.03, + "peak_sampled_postgres_memory_bytes": 124361113 +} diff --git a/evidence/benchmark/fixed-r4-0p25-1.json b/evidence/benchmark/fixed-r4-0p25-1.json new file mode 100644 index 00000000..070b2302 --- /dev/null +++ b/evidence/benchmark/fixed-r4-0p25-1.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.089, + "concurrency": 8, + "successful_requests": 1930, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 191.29, + "successful_latency_ms": { + "p50": 5.627, + "p95": 188.201, + "p99": 292.857 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 4, + "api_cpu_limit_each": "0.25", + "api_memory_limit_each": "128M", + "run": 1, + "resource_samples": 35, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 25.43, + "peak_sampled_api_memory_bytes": 112407347, + "peak_sampled_postgres_cpu_percent": 11.86, + "peak_sampled_postgres_memory_bytes": 157181542 +} diff --git a/evidence/benchmark/fixed-r4-0p25-2.json b/evidence/benchmark/fixed-r4-0p25-2.json new file mode 100644 index 00000000..92b1c4ec --- /dev/null +++ b/evidence/benchmark/fixed-r4-0p25-2.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.087, + "concurrency": 8, + "successful_requests": 1692, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 167.74, + "successful_latency_ms": { + "p50": 5.034, + "p95": 196.807, + "p99": 293.566 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 4, + "api_cpu_limit_each": "0.25", + "api_memory_limit_each": "128M", + "run": 2, + "resource_samples": 35, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 24.81, + "peak_sampled_api_memory_bytes": 61111009, + "peak_sampled_postgres_cpu_percent": 9.66, + "peak_sampled_postgres_memory_bytes": 166828441 +} diff --git a/evidence/benchmark/fixed-r4-0p25-3.json b/evidence/benchmark/fixed-r4-0p25-3.json new file mode 100644 index 00000000..0f0922f6 --- /dev/null +++ b/evidence/benchmark/fixed-r4-0p25-3.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.089, + "concurrency": 8, + "successful_requests": 2001, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 198.326, + "successful_latency_ms": { + "p50": 5.365, + "p95": 187.448, + "p99": 262.807 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "fixed", + "replicas": 4, + "api_cpu_limit_each": "0.25", + "api_memory_limit_each": "128M", + "run": 3, + "resource_samples": 35, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 25.83, + "peak_sampled_api_memory_bytes": 126667980, + "peak_sampled_postgres_cpu_percent": 14.06, + "peak_sampled_postgres_memory_bytes": 169554739 +} diff --git a/evidence/benchmark/growth-r2-1p00-1.json b/evidence/benchmark/growth-r2-1p00-1.json new file mode 100644 index 00000000..904c6409 --- /dev/null +++ b/evidence/benchmark/growth-r2-1p00-1.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.026, + "concurrency": 8, + "successful_requests": 5642, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 562.74, + "successful_latency_ms": { + "p50": 10.178, + "p95": 41.392, + "p99": 67.485 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "growth", + "replicas": 2, + "api_cpu_limit_each": "1.00", + "api_memory_limit_each": "512M", + "run": 1, + "resource_samples": 20, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 102.4, + "peak_sampled_api_memory_bytes": 70558679, + "peak_sampled_postgres_cpu_percent": 30.74, + "peak_sampled_postgres_memory_bytes": 131596288 +} diff --git a/evidence/benchmark/growth-r2-1p00-2.json b/evidence/benchmark/growth-r2-1p00-2.json new file mode 100644 index 00000000..e8d59169 --- /dev/null +++ b/evidence/benchmark/growth-r2-1p00-2.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.031, + "concurrency": 8, + "successful_requests": 5837, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 581.913, + "successful_latency_ms": { + "p50": 9.695, + "p95": 40.504, + "p99": 67.616 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "growth", + "replicas": 2, + "api_cpu_limit_each": "1.00", + "api_memory_limit_each": "512M", + "run": 2, + "resource_samples": 20, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 103.95, + "peak_sampled_api_memory_bytes": 58447626, + "peak_sampled_postgres_cpu_percent": 30.19, + "peak_sampled_postgres_memory_bytes": 135161446 +} diff --git a/evidence/benchmark/growth-r2-1p00-3.json b/evidence/benchmark/growth-r2-1p00-3.json new file mode 100644 index 00000000..52f1cee2 --- /dev/null +++ b/evidence/benchmark/growth-r2-1p00-3.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.028, + "concurrency": 8, + "successful_requests": 5491, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 547.548, + "successful_latency_ms": { + "p50": 10.474, + "p95": 41.778, + "p99": 67.959 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "growth", + "replicas": 2, + "api_cpu_limit_each": "1.00", + "api_memory_limit_each": "512M", + "run": 3, + "resource_samples": 20, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 103.05, + "peak_sampled_api_memory_bytes": 58804142, + "peak_sampled_postgres_cpu_percent": 29.02, + "peak_sampled_postgres_memory_bytes": 139355750 +} diff --git a/evidence/benchmark/growth-r4-0p50-1.json b/evidence/benchmark/growth-r4-0p50-1.json new file mode 100644 index 00000000..509a860e --- /dev/null +++ b/evidence/benchmark/growth-r4-0p50-1.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.074, + "concurrency": 8, + "successful_requests": 5144, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 510.602, + "successful_latency_ms": { + "p50": 6.481, + "p95": 69.708, + "p99": 88.915 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "growth", + "replicas": 4, + "api_cpu_limit_each": "0.50", + "api_memory_limit_each": "256M", + "run": 1, + "resource_samples": 28, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 52.66, + "peak_sampled_api_memory_bytes": 61478010, + "peak_sampled_postgres_cpu_percent": 41.09, + "peak_sampled_postgres_memory_bytes": 177733632 +} diff --git a/evidence/benchmark/growth-r4-0p50-2.json b/evidence/benchmark/growth-r4-0p50-2.json new file mode 100644 index 00000000..351b7bfb --- /dev/null +++ b/evidence/benchmark/growth-r4-0p50-2.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.035, + "concurrency": 8, + "successful_requests": 6319, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 629.726, + "successful_latency_ms": { + "p50": 6.197, + "p95": 60.127, + "p99": 77.211 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "growth", + "replicas": 4, + "api_cpu_limit_each": "0.50", + "api_memory_limit_each": "256M", + "run": 2, + "resource_samples": 28, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 50.61, + "peak_sampled_api_memory_bytes": 64067993, + "peak_sampled_postgres_cpu_percent": 41.54, + "peak_sampled_postgres_memory_bytes": 185912524 +} diff --git a/evidence/benchmark/growth-r4-0p50-3.json b/evidence/benchmark/growth-r4-0p50-3.json new file mode 100644 index 00000000..27710ff5 --- /dev/null +++ b/evidence/benchmark/growth-r4-0p50-3.json @@ -0,0 +1,28 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.028, + "concurrency": 8, + "successful_requests": 5657, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 564.133, + "successful_latency_ms": { + "p50": 6.485, + "p95": 64.66, + "p99": 83.355 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "scenario": "growth", + "replicas": 4, + "api_cpu_limit_each": "0.50", + "api_memory_limit_each": "256M", + "run": 3, + "resource_samples": 28, + "resource_sampling_errors": [], + "peak_sampled_api_cpu_percent": 50.99, + "peak_sampled_api_memory_bytes": 64896368, + "peak_sampled_postgres_cpu_percent": 42.36, + "peak_sampled_postgres_memory_bytes": 189372825 +} diff --git a/evidence/benchmark/machine.json b/evidence/benchmark/machine.json new file mode 100644 index 00000000..6ccc9a84 --- /dev/null +++ b/evidence/benchmark/machine.json @@ -0,0 +1,5 @@ +{ + "platform": "macOS-27.0.1-arm64-arm-64bit", + "docker_daemon_cpus": 12, + "docker_daemon_memory_bytes": 8217165824 +} diff --git a/evidence/benchmark/summary.csv b/evidence/benchmark/summary.csv new file mode 100644 index 00000000..f75581a3 --- /dev/null +++ b/evidence/benchmark/summary.csv @@ -0,0 +1,6 @@ +scenario,replicas,api_cpu_limit_each,api_memory_limit_each,mean_successful_rps,stdev_successful_rps,mean_error_ratio,mean_p50_ms,mean_p95_ms,mean_p99_ms,max_sampled_api_cpu_percent,max_sampled_api_memory_bytes,max_sampled_postgres_cpu_percent,max_sampled_postgres_memory_bytes +fixed,1,1.00,512M,319.059,33.367,0.0,18.192,51.926,112.067,102.95,61750640,31.17,99121889 +fixed,2,0.50,256M,266.984,4.411,0.0,10.205,90.475,104.543,51.56,61876469,15.29,124361113 +fixed,4,0.25,128M,185.785,16.019,0.0,5.342,190.819,283.077,25.83,126667980,14.06,169554739 +growth,2,1.00,512M,564.067,17.221,0.0,10.116,41.225,67.687,103.95,70558679,30.74,139355750 +growth,4,0.50,256M,568.154,59.664,0.0,6.388,64.832,83.16,52.66,64896368,42.36,189372825 diff --git a/evidence/benchmark/summary.json b/evidence/benchmark/summary.json new file mode 100644 index 00000000..f480766a --- /dev/null +++ b/evidence/benchmark/summary.json @@ -0,0 +1,82 @@ +[ + { + "scenario": "fixed", + "replicas": 1, + "api_cpu_limit_each": "1.00", + "api_memory_limit_each": "512M", + "mean_successful_rps": 319.059, + "stdev_successful_rps": 33.367, + "mean_error_ratio": 0.0, + "mean_p50_ms": 18.192, + "mean_p95_ms": 51.926, + "mean_p99_ms": 112.067, + "max_sampled_api_cpu_percent": 102.95, + "max_sampled_api_memory_bytes": 61750640, + "max_sampled_postgres_cpu_percent": 31.17, + "max_sampled_postgres_memory_bytes": 99121889 + }, + { + "scenario": "fixed", + "replicas": 2, + "api_cpu_limit_each": "0.50", + "api_memory_limit_each": "256M", + "mean_successful_rps": 266.984, + "stdev_successful_rps": 4.411, + "mean_error_ratio": 0.0, + "mean_p50_ms": 10.205, + "mean_p95_ms": 90.475, + "mean_p99_ms": 104.543, + "max_sampled_api_cpu_percent": 51.56, + "max_sampled_api_memory_bytes": 61876469, + "max_sampled_postgres_cpu_percent": 15.29, + "max_sampled_postgres_memory_bytes": 124361113 + }, + { + "scenario": "fixed", + "replicas": 4, + "api_cpu_limit_each": "0.25", + "api_memory_limit_each": "128M", + "mean_successful_rps": 185.785, + "stdev_successful_rps": 16.019, + "mean_error_ratio": 0.0, + "mean_p50_ms": 5.342, + "mean_p95_ms": 190.819, + "mean_p99_ms": 283.077, + "max_sampled_api_cpu_percent": 25.83, + "max_sampled_api_memory_bytes": 126667980, + "max_sampled_postgres_cpu_percent": 14.06, + "max_sampled_postgres_memory_bytes": 169554739 + }, + { + "scenario": "growth", + "replicas": 2, + "api_cpu_limit_each": "1.00", + "api_memory_limit_each": "512M", + "mean_successful_rps": 564.067, + "stdev_successful_rps": 17.221, + "mean_error_ratio": 0.0, + "mean_p50_ms": 10.116, + "mean_p95_ms": 41.225, + "mean_p99_ms": 67.687, + "max_sampled_api_cpu_percent": 103.95, + "max_sampled_api_memory_bytes": 70558679, + "max_sampled_postgres_cpu_percent": 30.74, + "max_sampled_postgres_memory_bytes": 139355750 + }, + { + "scenario": "growth", + "replicas": 4, + "api_cpu_limit_each": "0.50", + "api_memory_limit_each": "256M", + "mean_successful_rps": 568.154, + "stdev_successful_rps": 59.664, + "mean_error_ratio": 0.0, + "mean_p50_ms": 6.388, + "mean_p95_ms": 64.832, + "mean_p99_ms": 83.16, + "max_sampled_api_cpu_percent": 52.66, + "max_sampled_api_memory_bytes": 64896368, + "max_sampled_postgres_cpu_percent": 42.36, + "max_sampled_postgres_memory_bytes": 189372825 + } +] diff --git a/evidence/benchmark/throughput.svg b/evidence/benchmark/throughput.svg new file mode 100644 index 00000000..538eda9c --- /dev/null +++ b/evidence/benchmark/throughput.svg @@ -0,0 +1,20 @@ + + +Successful message writes per second + + +319.1 +1 rep fixed + +267.0 +2 rep fixed + +185.8 +4 rep fixed + +564.1 +2 rep growth + +568.2 +4 rep growth + diff --git a/evidence/failover-api1.json b/evidence/failover-api1.json new file mode 100644 index 00000000..ca13bb85 --- /dev/null +++ b/evidence/failover-api1.json @@ -0,0 +1,16 @@ +{ + "target_instance_id": "30ba967428ed", + "observed_instances_before": 2, + "instances_serving_after": [ + "030ce8e8117a" + ], + "duration_seconds": 15.097, + "successful_writes": 110, + "failed_write_attempts": 1, + "failed_read_attempts": 0, + "successful_rps": 7.286, + "write_error_ratio": 0.009009, + "recovery_seconds": 2.463, + "confirmed_data_preserved": true, + "duplicate_messages": 0 +} diff --git a/evidence/failover-api2.json b/evidence/failover-api2.json new file mode 100644 index 00000000..06ba9a05 --- /dev/null +++ b/evidence/failover-api2.json @@ -0,0 +1,16 @@ +{ + "target_instance_id": "030ce8e8117a", + "observed_instances_before": 2, + "instances_serving_after": [ + "30ba967428ed" + ], + "duration_seconds": 15.078, + "successful_writes": 108, + "failed_write_attempts": 1, + "failed_read_attempts": 0, + "successful_rps": 7.163, + "write_error_ratio": 0.009174, + "recovery_seconds": 2.529, + "confirmed_data_preserved": true, + "duplicate_messages": 0 +} diff --git a/evidence/rolling-restart-load.json b/evidence/rolling-restart-load.json new file mode 100644 index 00000000..d39f77ce --- /dev/null +++ b/evidence/rolling-restart-load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30, + "elapsed_seconds": 30.05, + "concurrency": 4, + "successful_requests": 11200, + "failed_requests": 1, + "error_ratio": 8.9e-05, + "successful_rps": 372.715, + "successful_latency_ms": { + "p50": 4.884, + "p95": 41.279, + "p99": 82.383 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..34273ebe --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,30 @@ +global + log stdout format raw local0 + maxconn 1024 + +defaults + log global + mode http + option httplog + timeout connect 1s + timeout client 30s + timeout server 30s + timeout http-request 10s + timeout queue 5s + +resolvers docker + nameserver embedded 127.0.0.11:53 + accepted_payload_size 8192 + hold valid 2s + hold obsolete 2s + +frontend public + bind :8080 + default_backend messenger_api + +backend messenger_api + option httpchk + http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host localhost + http-check expect status 200 + default-server inter 1s fall 2 rise 1 + server-template api 1-8 api:8080 check resolvers docker init-addr libc,none diff --git a/lab.json b/lab.json index 57bb9fbb..5194777c 100644 --- a/lab.json +++ b/lab.json @@ -4,5 +4,5 @@ "contract_version": "1.3.0", "default_base_url": "http://localhost:8080", "grading": "3; 4 includes 3; 5 includes 4 plus one completed research track", - "scaffold_has_application": false + "scaffold_has_application": true } diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..8aef0255 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +argon2-cffi==23.1.0 +fastapi==0.115.6 +psycopg[binary,pool]==3.2.4 +redis==5.2.1 +uvicorn[standard]==0.34.0 diff --git a/scripts/benchmark.py b/scripts/benchmark.py new file mode 100644 index 00000000..065beeee --- /dev/null +++ b/scripts/benchmark.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Repeatable 1/2/4-replica resource-scaling experiment.""" +import csv +import json +import os +from pathlib import Path +import platform +import re +import statistics +import subprocess +import sys +import threading + +ROOT = Path(__file__).resolve().parents[1] +LOAD = ROOT / "tests" / "load.py" +OUT = ROOT / "evidence" / "benchmark" + + +def command(args, env=None, capture=True): + return subprocess.run( + args, + cwd=ROOT, + env=env, + check=True, + text=True, + stdout=subprocess.PIPE if capture else None, + stderr=subprocess.PIPE if capture else None, + ) + + +def docker_resources(): + result = command(["docker", "info", "--format", "{{.NCPU}} {{.MemTotal}}"]) + cpu_count, memory_bytes = result.stdout.strip().split() + return {"docker_daemon_cpus": int(cpu_count), "docker_daemon_memory_bytes": int(memory_bytes)} + + +def read_stats(samples, errors, stop): + while not stop.is_set(): + try: + containers = command( + ["docker", "compose", "--env-file", ".env", "ps", "-q"] + ).stdout.split() + if not containers: + stop.wait(1) + continue + output = command( + [ + "docker", + "stats", + "--no-stream", + "--format", + "{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}", + *containers, + ] + ).stdout + for line in output.splitlines(): + fields = line.split("|") + if len(fields) != 3: + continue + name, cpu, memory = fields + used = re.fullmatch( + r"([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)", + memory.split("/")[0].strip(), + ) + if not used: + continue + amount = float(used.group(1)) + unit = used.group(2).lower() + scale = {"b": 1, "kib": 1024, "mib": 1024**2, "gib": 1024**3}.get(unit) + if scale is not None: + samples.append( + { + "container": name, + "cpu_percent": float(cpu.rstrip("%")), + "memory_bytes": int(amount * scale), + } + ) + except subprocess.CalledProcessError as exc: + errors.append("docker stats failed: " + (exc.stderr or "").strip()) + except (OSError, ValueError) as exc: + errors.append("docker stats sample failed: " + str(exc)) + stop.wait(1) + + +def graph(rows, destination): + width, height = 900, 450 + left, top, chart_width, chart_height = 80, 55, 770, 310 + max_value = max((row["mean_successful_rps"] for row in rows), default=1) or 1 + bar_width = chart_width / max(1, len(rows)) * 0.56 + elements = [ + '' + % (width, height, width, height), + '', + 'Successful message writes per second', + '' + % (left, top + chart_height, left + chart_width, top + chart_height), + ] + for index, row in enumerate(rows): + x = left + (index + 0.5) * chart_width / len(rows) - bar_width / 2 + bar_height = chart_height * row["mean_successful_rps"] / max_value + y = top + chart_height - bar_height + elements.append( + '' + % (x, y, bar_width, bar_height) + ) + elements.append( + '%.1f' + % (x + bar_width / 2, y - 6, row["mean_successful_rps"]) + ) + label = "%d rep %s" % (row["replicas"], row["scenario"]) + elements.append( + '%s' + % (x + bar_width / 2, top + chart_height + 22, label) + ) + elements.extend(["", ""]) + destination.write_text("\n".join(elements), encoding="utf-8") + + +def main(): + import argparse + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--duration", type=int, default=10) + parser.add_argument("--concurrency", type=int, default=8) + parser.add_argument("--repeats", type=int, default=3) + args = parser.parse_args() + if not 5 <= args.duration <= 3600 or not 1 <= args.concurrency <= 128 or args.repeats < 3: + parser.error("use duration >=5s, concurrency 1..128, and at least 3 repeats") + if not (ROOT / ".env").is_file() or not (ROOT / "secrets" / "database_password").is_file(): + parser.error("create local .env and secrets/database_password as documented in REPORT.md") + + OUT.mkdir(parents=True, exist_ok=True) + configs = [ + ("fixed", 1, "1.00", "512M"), + ("fixed", 2, "0.50", "256M"), + ("fixed", 4, "0.25", "128M"), + ("growth", 2, "1.00", "512M"), + ("growth", 4, "0.50", "256M"), + ] + results = [] + machine = {"platform": platform.platform(), **docker_resources()} + print(json.dumps({"machine": machine, "duration_seconds": args.duration, "concurrency": args.concurrency, "repeats": args.repeats}, indent=2)) + + for scenario, replicas, cpu, memory in configs: + env = os.environ.copy() + env.update({"API_REPLICAS": str(replicas), "API_CPUS": cpu, "API_MEMORY": memory}) + command( + [ + "docker", "compose", "--env-file", ".env", "up", "-d", "--build", "--wait", + "--wait-timeout", "120", "--scale", "api=" + str(replicas), + ], + env=env, + ) + per_run = [] + for repeat in range(1, args.repeats + 1): + slug = "%s-r%d-%s-%d" % (scenario, replicas, cpu.replace(".", "p"), repeat) + metrics_path = OUT / (slug + ".json") + samples = [] + sample_errors = [] + stop = threading.Event() + sampler = threading.Thread( + target=read_stats, args=(samples, sample_errors, stop), daemon=True + ) + sampler.start() + try: + command( + [ + sys.executable, str(LOAD), "--base-url", "http://localhost:8080", + "--duration", str(args.duration), "--concurrency", str(args.concurrency), + "--output", str(metrics_path), + ] + ) + finally: + stop.set() + sampler.join(timeout=3) + metrics = json.loads(metrics_path.read_text(encoding="utf-8")) + metrics.update( + { + "scenario": scenario, + "replicas": replicas, + "api_cpu_limit_each": cpu, + "api_memory_limit_each": memory, + "run": repeat, + "resource_samples": len(samples), + "resource_sampling_errors": sample_errors, + "peak_sampled_api_cpu_percent": max( + (sample["cpu_percent"] for sample in samples if "-api-" in sample["container"]), + default=None, + ), + "peak_sampled_api_memory_bytes": max( + (sample["memory_bytes"] for sample in samples if "-api-" in sample["container"]), + default=None, + ), + "peak_sampled_postgres_cpu_percent": max( + (sample["cpu_percent"] for sample in samples if "-postgres-" in sample["container"]), + default=None, + ), + "peak_sampled_postgres_memory_bytes": max( + (sample["memory_bytes"] for sample in samples if "-postgres-" in sample["container"]), + default=None, + ), + } + ) + metrics_path.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8") + per_run.append(metrics) + print(json.dumps({key: metrics[key] for key in ("scenario", "replicas", "run", "successful_rps", "error_ratio", "successful_latency_ms")})) + + results.append( + { + "scenario": scenario, + "replicas": replicas, + "api_cpu_limit_each": cpu, + "api_memory_limit_each": memory, + "mean_successful_rps": round(statistics.mean(item["successful_rps"] for item in per_run), 3), + "stdev_successful_rps": round(statistics.stdev(item["successful_rps"] for item in per_run), 3), + "mean_error_ratio": round(statistics.mean(item["error_ratio"] for item in per_run), 6), + "mean_p50_ms": round(statistics.mean(item["successful_latency_ms"]["p50"] for item in per_run), 3), + "mean_p95_ms": round(statistics.mean(item["successful_latency_ms"]["p95"] for item in per_run), 3), + "mean_p99_ms": round(statistics.mean(item["successful_latency_ms"]["p99"] for item in per_run), 3), + "max_sampled_api_cpu_percent": max( + (item["peak_sampled_api_cpu_percent"] for item in per_run if item["peak_sampled_api_cpu_percent"] is not None), + default=None, + ), + "max_sampled_api_memory_bytes": max( + (item["peak_sampled_api_memory_bytes"] for item in per_run if item["peak_sampled_api_memory_bytes"] is not None), + default=None, + ), + "max_sampled_postgres_cpu_percent": max( + (item["peak_sampled_postgres_cpu_percent"] for item in per_run if item["peak_sampled_postgres_cpu_percent"] is not None), + default=None, + ), + "max_sampled_postgres_memory_bytes": max( + (item["peak_sampled_postgres_memory_bytes"] for item in per_run if item["peak_sampled_postgres_memory_bytes"] is not None), + default=None, + ), + } + ) + + (OUT / "machine.json").write_text(json.dumps(machine, indent=2) + "\n", encoding="utf-8") + (OUT / "summary.json").write_text(json.dumps(results, indent=2) + "\n", encoding="utf-8") + with (OUT / "summary.csv").open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=list(results[0])) + writer.writeheader() + writer.writerows(results) + graph(results, OUT / "throughput.svg") + print("Saved raw trial data and summary in " + str(OUT)) + + +if __name__ == "__main__": + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..91f5b2e5 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,24 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +docker compose restart postgres redis >/dev/null +deadline=$(( $(date +%s) + 60 )) +for service in postgres redis; do + container=$(docker compose ps -q "$service") + while :; do + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + if [ "$status" = healthy ]; then + break + fi + if [ "$status" = unhealthy ] || [ "$(date +%s)" -ge "$deadline" ]; then + printf '%s\n' "$service did not become healthy after restart" >&2 + exit 1 + fi + sleep 1 + done +done + +"$ROOT/scripts/rolling-restart.sh" diff --git a/scripts/rolling-restart.sh b/scripts/rolling-restart.sh new file mode 100755 index 00000000..82618cb2 --- /dev/null +++ b/scripts/rolling-restart.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" + +containers=$(docker compose ps -q api) +count=$(printf '%s\n' "$containers" | sed '/^$/d' | wc -l | tr -d ' ') +if [ "$count" -lt 2 ]; then + printf '%s\n' 'Rolling restart requires at least two API replicas' >&2 + exit 1 +fi + +for container in $containers; do + docker restart --time 10 "$container" >/dev/null + deadline=$(( $(date +%s) + 60 )) + while :; do + status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container") + if [ "$status" = healthy ]; then + break + fi + if [ "$status" = unhealthy ] || [ "$(date +%s)" -ge "$deadline" ]; then + printf '%s\n' "API replica did not become healthy: $container" >&2 + exit 1 + fi + sleep 1 + done +done diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..73bd1cb8 --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +ROOT=$(CDPATH= cd -- "$(dirname -- "$0")/.." && pwd) +cd "$ROOT" +: "${TARGET_INSTANCE_ID:?TARGET_INSTANCE_ID is required}" + +for container in $(docker compose ps -q api); do + hostname=$(docker inspect --format '{{.Config.Hostname}}' "$container") + if [ "$hostname" = "$TARGET_INSTANCE_ID" ]; then + docker stop --time 2 "$container" >/dev/null + exit 0 + fi +done + +printf '%s\n' 'No running API replica matches TARGET_INSTANCE_ID' >&2 +exit 1 diff --git a/tests/README.md b/tests/README.md index 0dfa3f89..ac8a739b 100644 --- a/tests/README.md +++ b/tests/README.md @@ -35,6 +35,16 @@ make test-failover ACTION_SCRIPT=scripts/stop-one.sh Перед отказом тест наблюдает ≥2 API-реплики; 15 секунд отправляет сообщения параллельно выполнению hook, повторяет неопределённый POST с тем же ключом, проверяет восстановление записи за ≤10 секунд, отсутствие остановленной реплики в последних ответах, сохранность подтверждённых сообщений и отсутствие дублей. Ошибки переходного периода учитываются. Затем восстановите реплику и повторите для другой. Скрипт — проверка одного отказа API, не гарантия HA БД/LB/host. Вывод hook скрыт, чтобы не утекли секреты; отлаживайте его отдельно с безопасным выводом. +Чтобы гарантированно проверить **каждую заданную реплику**, перечислите их Docker hostname aliases (они совпадают с `X-Instance-Id`) и используйте дополнительный targeted-сценарий: + +```sh +for c in $(docker compose ps -q api); do docker inspect --format '{{.Config.Hostname}}' "$c"; done +make test-failover-target TARGET_INSTANCE_ID=alias-from-previous-command OUTPUT=evidence/failover.json +docker compose up -d --scale api=2 api +``` + +Этот сценарий сам находит запрос, обслуженный выбранной репликой, останавливает именно её, отдельно считает чтения/записи после отказа, измеряет recovery/error rate и сохраняет JSON без токенов. Повторите его для каждого alias, восстанавливая Compose-service после каждого запуска. + ## Нагрузка ```sh diff --git a/tests/grade4.py b/tests/grade4.py new file mode 100644 index 00000000..dddc160d --- /dev/null +++ b/tests/grade4.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Concurrent idempotency and replica-distribution checks.""" +from concurrent.futures import ThreadPoolExecutor +import uuid + +from client import Client, Failure, arguments, check, run + + +def main(): + args = arguments("Concurrent shared-idempotency and API-replica check") + client = Client(args.base_url, args.ca_file, timeout=5) + alice, _, _, chat = client.bootstrap() + key = uuid.uuid4().hex + body = {"text": "parallel-idempotency-" + key} + + def send(_): + return Client(args.base_url, args.ca_file, timeout=5).call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 201, + body=body, + actor=alice, + headers={"Idempotency-Key": key}, + schema="Message", + ) + + with ThreadPoolExecutor(max_workers=12) as executor: + responses = list(executor.map(send, range(12))) + messages = [response["json"] for response in responses] + check(all(message == messages[0] for message in messages), "concurrent replay differed") + + conflict = client.call( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + 409, + body={"text": "different body"}, + actor=alice, + headers={"Idempotency-Key": key}, + ) + check(conflict["status"] == 409, "same key with a different body was accepted") + + instances = { + client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][ + "X-Instance-Id" + ] + for _ in range(80) + } + check(len(instances) >= 2, "load balancer did not expose two API instances") + print("PASS: concurrent idempotency returned one message; observed %d API replicas" % len(instances)) + + +if __name__ == "__main__": + run(main) diff --git a/tests/replica_failover.py b/tests/replica_failover.py new file mode 100644 index 00000000..26aa3f1a --- /dev/null +++ b/tests/replica_failover.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Fail one explicitly selected replica and write sanitized evidence.""" +import json +import os +from pathlib import Path +import subprocess +import time +import uuid +from urllib.parse import quote + +from client import Client, Failure, arguments, check, run + + +def history(client, actor, chat): + path = "/api/v1/chats/" + chat["id"] + "/messages?limit=100" + messages = [] + seen = set() + while True: + page = client.call("GET", path, actor=actor, schema="MessagePage")["json"] + messages.extend(page["items"]) + cursor = page["next_cursor"] + if cursor is None: + return messages + check(cursor not in seen, "history cursor cycle") + seen.add(cursor) + path = ( + "/api/v1/chats/" + + chat["id"] + + "/messages?limit=100&cursor=" + + quote(cursor, safe="") + ) + + +def main(): + def configure(parser): + parser.add_argument("--target-id", required=True) + parser.add_argument("--action-script", required=True) + parser.add_argument("--output") + + args = arguments("Targeted API failover evidence", configure) + target = args.target_id + script = Path(args.action_script).expanduser().resolve() + check(script.is_file() and os.access(script, os.X_OK), "action script must be executable") + client = Client(args.base_url, args.ca_file, timeout=2) + alice, bob, _, chat = client.bootstrap() + observed_before = { + client.call("GET", "/api/v1/users/me", actor=alice, schema="User")["headers"][ + "X-Instance-Id" + ] + for _ in range(40) + } + check(len(observed_before) >= 2, "fewer than two API replicas observed") + check(target in observed_before, "target ID is not a currently observed API replica") + + original = None + for _ in range(20): + original = client.message( + alice, chat, "pre-failure-" + uuid.uuid4().hex, key=uuid.uuid4().hex + ) + if original["headers"]["X-Instance-Id"] == target: + break + check(original["headers"]["X-Instance-Id"] == target, "could not select requested replica") + + env = dict(os.environ) + env["TARGET_INSTANCE_ID"] = target + process = subprocess.Popen( + [str(script)], env=env, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL + ) + started = time.monotonic() + recovered_at = None + successful_writes = [] + failed_writes = 0 + failed_reads = 0 + served_after = [] + key = uuid.uuid4().hex + text = "during-failover-" + key + try: + while time.monotonic() - started < 15: + status = process.poll() + check(status is None or status == 0, "action script failed") + check(status is not None or time.monotonic() - started < 10, "action script exceeded 10s") + try: + read = client.call("GET", "/api/v1/users/me", actor=alice, schema="User") + served_after.append(read["headers"]["X-Instance-Id"]) + except Failure: + failed_reads += 1 + time.sleep(0.1) + continue + try: + response = client.message(alice, chat, text, key=key) + successful_writes.append(response["json"]) + if status == 0 and recovered_at is None: + recovered_at = time.monotonic() - started + key = uuid.uuid4().hex + text = "during-failover-" + key + except Failure: + failed_writes += 1 + time.sleep(0.1) + + check(process.poll() == 0, "action script did not finish successfully") + check(recovered_at is not None and recovered_at <= 10, "writes did not recover within 10s") + check(len(successful_writes) >= 5, "too few successful writes after the failure") + check(target not in served_after[-5:], "stopped replica still serves requests") + all_messages = history(client, bob, chat) + ids = [message["id"] for message in all_messages] + texts = [message["text"] for message in all_messages] + check(len(ids) == len(set(ids)), "duplicate message IDs in history") + check(len(texts) == len(set(texts)), "retry created duplicate message bodies") + check(original["json"] in all_messages, "confirmed pre-failure message was lost") + for message in successful_writes: + check(message in all_messages, "confirmed failover write was lost") + + elapsed = time.monotonic() - started + total_attempts = len(successful_writes) + failed_writes + result = { + "target_instance_id": target, + "observed_instances_before": len(observed_before), + "instances_serving_after": sorted(set(served_after)), + "duration_seconds": round(elapsed, 3), + "successful_writes": len(successful_writes), + "failed_write_attempts": failed_writes, + "failed_read_attempts": failed_reads, + "successful_rps": round(len(successful_writes) / elapsed, 3), + "write_error_ratio": round(failed_writes / total_attempts, 6), + "recovery_seconds": round(recovered_at, 3), + "confirmed_data_preserved": True, + "duplicate_messages": 0, + } + rendered = json.dumps(result, ensure_ascii=False, indent=2) + "\n" + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(rendered, encoding="utf-8") + print(rendered, end="") + finally: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait() + + +if __name__ == "__main__": + run(main) -- 2.54.0