diff --git a/.env.example b/.env.example index d32b9be0..495c15fb 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,28 @@ # Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Credentials live in secrets/ (see REPORT.md), only file paths are set here. APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 +SESSION_STORE=valkey IDEMPOTENCY_TTL_SECONDS=86400 +GRACEFUL_TIMEOUT=8 +MAX_INFLIGHT_REQUESTS=64 +MAX_BODY_BYTES=65536 +MAX_RSS_MB=0 +DB_POOL_MIN=1 +DB_POOL_MAX=10 +DB_POOL_TIMEOUT=3 +DB_CONNECT_TIMEOUT=3 +DB_STATEMENT_TIMEOUT_MS=5000 +DB_OP_TIMEOUT=6 +REQUEST_TIMEOUT=8 +DB_READY_TIMEOUT=1.5 +READY_CHECK_SECONDS=1 +READY_FAILURES=2 +SESSION_CACHE_TTL_SECONDS=30 +SESSION_SWEEP_SECONDS=60 +CACHE_TIMEOUT=0.2 +SESSION_REVOKED_TTL_SECONDS=60 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c3bf7779 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,31 @@ +FROM python:3.12.8-slim-bookworm@sha256:2199a62885a12290dc9c5be3ca0681d367576ab7bf037da120e564723292a2f0 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_DEFAULT_TIMEOUT=120 \ + PIP_RETRIES=10 + +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid app --home-dir /app --shell /usr/sbin/nologin app + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-deps -r requirements.txt + +COPY app/ app/ +COPY migrations/ migrations/ +COPY scripts/migrate.py scripts/migrate.py + +USER 10001:10001 + +EXPOSE 8080 + +HEALTHCHECK --interval=5s --timeout=3s --start-period=10s --retries=3 \ + CMD ["python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)"] + +STOPSIGNAL SIGTERM + +CMD ["sh", "-c", "exec uvicorn app.main:app --host \"${APP_HOST:-0.0.0.0}\" --port \"${APP_PORT:-8080}\" --timeout-graceful-shutdown \"${GRACEFUL_TIMEOUT:-8}\""] diff --git a/REPORT.md b/REPORT.md index c2679005..848f0bc7 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,203 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Курлова Алёна, БВТ2301 +- Целевая оценка; выбранный трек 5, если нужен: 5, Трек 1 — Цена масштабирования +- Commit/PR: https://gitea.privcloud.tech/mtusi2026/mtusi2026lab3/pulls/61, ветка `solution/bvt2301-kurlova`, основной commit fbb8916 +- Использованные источники, библиотеки и помощь ИИ: FastAPI, Uvicorn, psycopg 3, Valkey, Argon2id, Nginx. Помощь Claude (Anthropic) при написании кода и отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +**Версии:** Python 3.12.8, PostgreSQL 17.7, Valkey 8.1, Nginx 1.27.3, Docker Compose v2. + +**Prerequisites:** Docker, Docker Compose, Python 3.9+. + +```sh +git clone && cd mtusi2026lab3 + +# Создать секреты (значения для локального стенда; не коммитить) +mkdir -p secrets +PG_PASSWORD=$(openssl rand -hex 16) +printf '%s' "$PG_PASSWORD" > secrets/postgres_password +printf 'postgresql://messenger:%s@postgres:5432/messenger' "$PG_PASSWORD" > secrets/database_url +printf 'redis://valkey:6379/0' > secrets/cache_url + +# Создать .env (скопировать из .env.example или использовать приложенный) +cp .env.example .env # при необходимости отредактировать + +# Сборка и запуск +docker compose --env-file .env build +docker compose --env-file .env up -d + +# Дождаться готовности (обе реплики + nginx) +until curl -sf http://localhost:8080/health/ready; do sleep 1; done + +# Тесты +make test +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh + +# После failover восстановить остановленную реплику +docker compose up -d api-1 api-2 + +# Остановка (без удаления данных) +docker compose down + +# Удаление данных +docker compose down -v +``` ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +Клиент ──► Nginx (LB, :8080) + ├──► api-1 (FastAPI/Uvicorn) ──┐ + └──► api-2 (FastAPI/Uvicorn) ──┤ + ├──► PostgreSQL (источник истины) + └──► Valkey (кэш сессий, без persistence) +``` + +**Что перенесено из lab2 без изменений:** модули cursor, errors, passwords, db, sessions; миграция 0001_init.sql; Dockerfile; requirements.txt; скрипт миграций. + +**Что изменено/добавлено в lab3:** + +| Компонент | Изменение | +|---|---| +| `compose.yaml` | Два сервиса `api-1`, `api-2` за Nginx; CPU/RAM лимиты; API не публикуют порты; нет `container_name` | +| `infra/nginx.conf` | Round-robin; список серверов upstream подключается из файла, который ведёт `infra/nginx-health.sh`; пассивная защита `max_fails=1 fail_timeout=10s`; повтор на другую реплику при 502/503/ошибке соединения: для GET везде, для POST только на маршруте `/api/v1/chats/{id}/messages` (`non_idempotent`), где обязателен Idempotency-Key; `resolver 127.0.0.11` + `resolve`, чтобы nginx видел новый IP реплики после её пересоздания; keepalive-соединения к API | +| `infra/nginx-health.sh` | Активная проверка готовности внутри контейнера nginx: раз в 1 с `GET /health/ready` каждой реплики; 2 неудачи подряд → сервер помечается `down` в upstream и выполняется `nginx -s reload`; 2 успеха подряд → возвращается. Если неготовы все, никто не помечается `down` | +| `app/config.py` | Добавлен `IDEMPOTENCY_TTL_SECONDS` | +| `app/repo.py` | `create_message_idempotent()` — advisory lock, затем проверка членства в чате (и для нового запроса, и для повтора), затем INSERT message + INSERT idempotency_keys в одной транзакции | +| `app/main.py` | Валидация `Idempotency-Key`, вызов idempotent-метода, очистка просроченных ключей в sweeper; фоновая проверка связи с PostgreSQL раз в 1 с, после 2 неудач подряд бизнес-запросы получают 503 до восстановления связи | +| `migrations/0002_idempotency.sql` | Таблица `idempotency_keys` (PK: user_id, chat_id, key) | +| `scripts/stop-one.sh` | Останавливает реплику по `TARGET_INSTANCE_ID` (= имя сервиса) | +| `scripts/restart.sh` | Перезапуск всего стенда с сохранением volumes | +| `infra/scaling/`, `student-tests/` | Override-файлы и скрипты эксперимента Трека 1, rolling update, тест параллельной идемпотентности | + +**Источник истины:** PostgreSQL. Все данные (users, chats, messages, sessions, idempotency_keys) хранятся в PostgreSQL. Valkey — кэш сессий без persistence (cache-aside с NX + TTL). + +**Транзакционные границы:** Отправка сообщения с Idempotency-Key обёрнута в одну PostgreSQL-транзакцию. `pg_advisory_xact_lock(hash)` сериализует конкурентные повторы одного ключа. Вставка сообщения и запись idempotency-результата атомарны. + +**Идемпотентность:** Область ключа — (user_id, chat_id, key). Тело нормализуется: `sha256(json.dumps({"text": ...}, sort_keys=True))`. Повтор с тем же body → тот же 201 + Message. Другой body → 409. TTL по умолчанию 24 часа; sweeper удаляет просроченные записи. + +**Миграции:** Запускаются отдельным одноразовым контейнером `migrate` до старта API. Используют advisory lock для защиты от гонки. + +**Переменные .env → конфигурация:** + +| Переменная | Назначение | Значение по умолчанию | +|---|---|---| +| `INSTANCE_ID` | X-Instance-Id; задаётся в compose per-service | hostname | +| `IDEMPOTENCY_TTL_SECONDS` | Время жизни idempotency-ключа | 86400 (24 ч) | +| `SESSION_TTL_SECONDS` | TTL сессии | 3600 | +| `DB_POOL_MAX` | Макс. подключений к PostgreSQL | 10 | +| `GRACEFUL_TIMEOUT` | Время graceful shutdown Uvicorn | 8 | +| `READY_CHECK_SECONDS`, `READY_FAILURES` | Интервал фоновой проверки PostgreSQL и число неудач подряд до режима «не готов» | 1, 2 | ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +|---|---|---|---| +| Публичный smoke | `make test` | PASS: lab3 public API scenario | evidence/smoke.txt | +| ≥2 реплики, один entrypoint, общая БД и сессии | `curl -s http://localhost:8080/health/live` (несколько раз) | X-Instance-Id чередуется: api-1, api-2 | evidence/instance_ids.txt | +| Разные запросы одного пользователя — разные реплики без повторного входа | smoke test + многократные запросы с одним Bearer | X-Instance-Id меняется | evidence/instance_ids.txt | +| Остановка одной реплики — чтение/запись продолжаются ≤10 с | `make test-failover ACTION_SCRIPT=scripts/stop-one.sh` | PASS: failover preserved confirmed data | evidence/failover_both.txt | +| Повтор POST с тем же Idempotency-Key — нет дубля | smoke test (idempotency replay section) | replay == first | evidence/smoke.txt | +| Конкурентный повтор через разные реплики | `python3 student-tests/test_parallel_idempotency.py` | PASS: 8 parallel requests, 1 message | evidence/parallel_idempotency.txt | +| Health/readiness, timeouts, graceful shutdown | SIGTERM → лог "draining", 503 на новые запросы; Nginx убирает backend | docker compose logs api-1 | evidence/graceful_shutdown.txt | +| Nginx исключает упавший или неготовый backend | Активная проверка `/health/ready` (`infra/nginx-health.sh`) + `max_fails=1`; после stop — запросы идут на живую реплику | failover test; лог nginx `health: api-1 not ready, marked down` | evidence/failover_both.txt, evidence/review_scenarios.txt | +| Максимальное время обнаружения отказа | Остановленная реплика: первая же ошибка соединения повторяется на другой реплике и выключает сервер (`max_fails=1`). Неготовая реплика: ≤ 2 проверки × (1 с интервал + 1 с таймаут) ≈ 4 с, всё это время GET и POST сообщений повторяются на живой реплике | first_recovery_seconds: 0.33 с | evidence/failover_both.txt | +| Последовательная остановка каждой реплики | stop api-1 → test → restore → stop api-2 → test | Оба теста PASS | evidence/failover_both.txt | +| RPS, ошибки, время восстановления | failover output JSON | successful_writes, failed_attempts, first_recovery_seconds | evidence/failover_both.txt | +| CPU/RAM лимиты | `docker compose config \| grep -A2 limits` | api: 0.5 CPU / 256M; nginx: 0.25 CPU / 64M; pg: 1.0 CPU / 512M | evidence/limits.txt | +| API не публикуют порты мимо LB | `docker compose ps` — только nginx на 8080 | Порты: "127.0.0.1:8080->8080/tcp" только у nginx | evidence/ports.txt | +| Нет container_name | compose.yaml | Нет ни одного container_name | compose.yaml | +| Rolling update под нагрузкой | `student-tests/rolling_update.sh`: нагрузка 60 с, `up -d --no-deps --force-recreate api-1` → ждём healthy → api-2 | Оба контейнера заменены (разные ID), 910 RPS, 0 ошибок | evidence/rolling_update.txt | +| Замечания ревью PR #61 (повтор после потери членства, повтор POST /chats балансировщиком, неготовая реплика в upstream) | `python3 student-tests/test_review_scenarios.py` | До исправлений 5 FAIL (повтор → 201; 3 POST /chats ушли на две реплики; неготовая api-1 обслужила 20 из 40 GET), после — PASS | evidence/review_scenarios.txt | +| Миграции однократны | Отдельный контейнер `migrate`; advisory lock; checksum | `docker compose logs migrate` → "skip 0001 / apply 0002" | evidence/migrate.txt | +| Persistence | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | PASS: persistence preserved confirmed data | evidence/persistence.txt | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +### Трек 1. Цена масштабирования + +**Гипотеза:** при фиксированном суммарном CPU рост числа реплик сначала помогает, потому что один процесс uvicorn не использует больше ~0.5 CPU, но при очень маленьком лимите на реплику эффект пропадает. При росте ресурсов RPS упирается в общий компонент: Nginx или PostgreSQL. + +**Методика:** +- Hardware: Apple M4, 16 GB RAM, macOS, Docker Desktop. Клиент нагрузки работает на том же хосте. +- Профиль: `tests/load.py --duration 30 --concurrency 16`, closed-loop POST сообщений с Idempotency-Key. +- Прогрев 5 с, затем 3 прогона по 30 с на конфигурацию; в таблицах медиана, в скобках min..max. +- Лимиты API задаются override-файлами `infra/scaling/limits.yaml` и `infra/scaling/replicas-4.yaml`, список реплик для nginx задаётся переменной `API_UPSTREAMS` (в замерах ниже использовались отдельные файлы `nginx-{1,2,4}.conf` с тем же upstream; после ревью они заменены активной проверкой, путь запросов через nginx не изменился). Фактические лимиты проверяются через `docker inspect` (`*_limits.txt`). +- Лимиты остальных сервисов не меняются: nginx 0.25 CPU, PostgreSQL 1.0 CPU, Valkey 0.25 CPU. +- В середине каждого прогона снимается `docker stats` (`*_stats.txt`), чтобы видеть, какой контейнер упёрся в лимит. +- Скрипты: `student-tests/scaling_experiment.sh`, `student-tests/scaling_summary.py`. Сырые данные: `evidence/scaling/*.json`, сводка `evidence/scaling/summary.csv`, график `evidence/scaling/rps.svg`. + +**Эксперимент A — фиксированный бюджет API: всего 1.0 CPU / 512M:** + +| Реплики | CPU/реплику | RAM/реплику | RPS медиана [min..max] | p50 мс | p95 мс | p99 мс | Ошибки | +|---|---|---|---|---|---|---|---| +| 1 | 1.00 | 512M | 347 [140..473] | 13.5 | 205.8 | 519.1 | 0 | +| 2 | 0.50 | 256M | 784 [282..811] | 11.7 | 66.2 | 87.8 | 0 | +| 4 | 0.25 | 128M | 615 [594..620] | 9.5 | 92.9 | 167.7 | 0 | + +**Эксперимент B — растущие ресурсы, каждая реплика 0.5 CPU / 256M:** + +| Реплики | Суммарный CPU API | RPS медиана [min..max] | p50 мс | p95 мс | p99 мс | Ошибки | +|---|---|---|---|---|---|---| +| 1 | 0.5 | 414 [98..466] | 15.8 | 97.0 | 297.0 | 0 | +| 2 | 1.0 | 784 [282..811] (= A2) | 11.7 | 66.2 | 87.8 | 0 | +| 4 | 2.0 | 1014 [993..1018] | 10.8 | 47.1 | 74.1 | 0 | + +![RPS от числа реплик](evidence/scaling/rps.svg) + +**Что показали замеры:** +- Одна реплика с 1.0 CPU не быстрее одной реплики с 0.5 CPU (347 против 414, в пределах разброса). Uvicorn работает одним процессом с одним event loop, и дополнительное CPU ему не помогает. Поэтому при том же бюджете 2 × 0.5 CPU дают в 2.3 раза больше RPS. +- 4 × 0.25 CPU хуже, чем 2 × 0.5 (615 против 784, p99 168 против 88 мс): на таком маленьком лимите каждая реплика чаще упирается в CFS-квоту, а постоянные накладные расходы процесса делятся на меньший CPU. +- При росте ресурсов переход 2 → 4 реплики удваивает CPU API, но даёт только +29% RPS. +- Разброс прогонов у 1 реплики большой (98..466): одиночная реплика под квотой CPU нестабильна. Поэтому в выводах используется медиана. + +**Saturation point.** По `docker stats` в B4 (`evidence/scaling/B4_run*_stats.txt`) API используют ~40–49% из лимита 50%, PostgreSQL ~45–48% из 100%, а nginx 20–26% при лимите 25%. Упирается балансировщик. В логах API видно, что nginx ходит к upstream по `HTTP/1.0`: на каждый запрос открывается новое TCP-соединение, это и съедает его CPU. + +**Подтверждённое улучшение: keepalive от nginx к API.** В `infra/nginx.conf` добавлены `keepalive 32`, `keepalive_timeout 4s` (меньше 5 с keep-alive у uvicorn, чтобы nginx не взял соединение, которое API уже закрывает), `proxy_http_version 1.1` и пустой `Connection`. Ресурсы не менялись. Повторный замер по той же методике (`evidence/scaling/keepalive/`): + +| Конфигурация | RPS до | RPS после | p95 до → после, мс | p99 до → после, мс | Ошибки после | +|---|---|---|---|---|---| +| 2 × 0.5 CPU (основной стенд) | 784 | 1021 (+30%) | 66.2 → 44.1 | 87.8 → 73.4 | 0 | +| 4 × 0.5 CPU | 1014 | 1216 (+20%) | 47.1 → 34.1 | 74.1 → 54.3 | 0 | + +После улучшения узкое место сместилось. На 2 репликах nginx потребляет ~16%, а обе API стоят на своём лимите 50%, то есть теперь ограничивают сами реплики. На 4 репликах nginx снова на пределе 25%, PostgreSQL поднялся до ~72–75%, а API недогружены (~45%). Дальше добавлять реплики бесполезно без увеличения CPU для nginx и PostgreSQL. + +**Отклонение при замере.** В первом замере keepalive (до добавления `resolve` в upstream) один прогон B4 дал 319 ошибок из ~37 000 запросов. В трёх повторных прогонах и в итоговом замере ошибок 0, в логах nginx (`*_nginx_problems.txt`) и PostgreSQL ошибок нет. Причину установить не удалось: логи контейнеров того прогона потерялись при пересоздании. Подробности в `evidence/scaling/keepalive/B4_recheck.txt`. Скрипт теперь сохраняет ошибки nginx на каждую конфигурацию. + +### Отказы | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +|---|---|---|---| +| Остановка api-1 | Nginx направляет трафик на api-2; ≤10 с до восстановления; 0 потерь подтверждённых сообщений | 130 записей, 0 ошибок | 0.33 с / 0 потерь | +| Остановка api-2 (после восстановления api-1) | Аналогично | 130 записей, 0 ошибок | 0.33 с / 0 потерь | +| Перезапуск всего стенда | Данные и сессии сохраняются (volumes) | PASS: все данные и сессия сохранены | ~5 с перезапуск / 0 потерь | +| Rolling update api-1, затем api-2 под нагрузкой | Кратковременный рост p99; 0 потерь | 910 RPS, 0 ошибок из 54 625, p99=17.9 мс | 0 потерь | +| Пересоздание реплики с новым IP без перезапуска nginx | nginx находит новый адрес | До `resolve` nginx слал запросы на старый IP и failover-тест падал; после `resolve` обе реплики снова в балансировке | ≤5 с (valid резолвера) | +| Живая api-1 без связи с PostgreSQL (`/health/ready`=503) | Реплика исключается из бизнес-маршрутизации | Через ~2–4 с `infra/nginx-health.sh` помечает api-1 `down`; 40 из 40 GET с прогретой сессией обслужила api-2; POST сообщений 20/20 успешны; POST /chats в первые секунды до исключения получили 503 без повтора и без создания чата; после восстановления api-1 вернулась в upstream | 0 потерь, лишних записей нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +**Выдерживаемые отказы:** +- Остановка любой одной API-реплики при работающей другой +- Перезапуск всего стенда с сохранением volumes + +**Не выдерживаемые отказы:** +- Потеря PostgreSQL (единственная БД, нет репликации) — все записи и чтения недоступны +- Потеря Nginx (единственный LB) — единая точка входа +- Потеря Docker-хоста — всё на одном узле + +**Учебные упрощения:** +- Один Docker-хост, нет multi-node failover +- Nginx open-source не умеет активные health checks, поэтому они сделаны shell-скриптом в контейнере nginx (`nginx -s reload` при смене состояния). Если скрипт упадёт, upstream останется в последнем состоянии, и защитой будут только `max_fails` и проверка готовности внутри API +- В первые ~2–4 с после потери связи с БД не-идемпотентные POST (кроме сообщений), попавшие на неготовую реплику, получают 503 без повтора. Это сделано намеренно: nginx не знает, успела ли реплика что-то записать, и прозрачный повтор мог бы создать дубль +- Idempotency-ключи хранятся в PostgreSQL — при высокой нагрузке advisory lock на одном ключе может стать bottleneck +- HTTP без TLS (допустимо для локального стенда с синтетическими данными, lab7 добавит HTTPS) + +**Возможные улучшения:** +- PostgreSQL streaming replication + automatic failover (Patroni) +- HAProxy или Nginx Plus со встроенными active health checks вместо скрипта +- Idempotency через distributed lock (Valkey) + PostgreSQL для снижения contention +- Multi-node deployment (Docker Swarm / Kubernetes) diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/config.py b/app/config.py new file mode 100644 index 00000000..85eaceb0 --- /dev/null +++ b/app/config.py @@ -0,0 +1,50 @@ +import os +import socket + + +def _int(name, default): + value = os.environ.get(name) + return int(value) if value not in (None, "") else default + + +def _float(name, default): + value = os.environ.get(name) + return float(value) if value not in (None, "") else default + + +def _secret(name, default=""): + path = os.environ.get(name + "_FILE") + if path: + with open(path, encoding="utf-8") as handle: + return handle.read().strip() + return os.environ.get(name, default) + + +DATABASE_URL = _secret("DATABASE_URL") +CACHE_URL = _secret("CACHE_URL") +SESSION_STORE = os.environ.get("SESSION_STORE", "postgres") +SESSION_TTL_SECONDS = _int("SESSION_TTL_SECONDS", 3600) +SESSION_CACHE_TTL_SECONDS = _int("SESSION_CACHE_TTL_SECONDS", 30) +SESSION_SWEEP_SECONDS = _int("SESSION_SWEEP_SECONDS", 60) +DB_POOL_MIN = _int("DB_POOL_MIN", 1) +DB_POOL_MAX = _int("DB_POOL_MAX", 10) +DB_POOL_TIMEOUT = _float("DB_POOL_TIMEOUT", 3.0) +DB_CONNECT_TIMEOUT = _int("DB_CONNECT_TIMEOUT", 3) +DB_STATEMENT_TIMEOUT_MS = _int("DB_STATEMENT_TIMEOUT_MS", 5000) +DB_OP_TIMEOUT = _float("DB_OP_TIMEOUT", 6.0) +REQUEST_TIMEOUT = _float("REQUEST_TIMEOUT", 8.0) +DB_READY_TIMEOUT = _float("DB_READY_TIMEOUT", 1.5) +READY_CHECK_SECONDS = _float("READY_CHECK_SECONDS", 1.0) +READY_FAILURES = _int("READY_FAILURES", 2) +CACHE_TIMEOUT = _float("CACHE_TIMEOUT", 0.2) +MAX_INFLIGHT_REQUESTS = _int("MAX_INFLIGHT_REQUESTS", 64) +MAX_BODY_BYTES = _int("MAX_BODY_BYTES", 65536) +MAX_RSS_MB = _int("MAX_RSS_MB", 0) +PASSWORD_MIN = 12 +PASSWORD_MAX = 128 +SESSION_REVOKED_TTL_SECONDS = max( + _int("SESSION_REVOKED_TTL_SECONDS", 60), + SESSION_CACHE_TTL_SECONDS + int(REQUEST_TIMEOUT) + 5, +) +INSTANCE_ID = os.environ.get("INSTANCE_ID") or socket.gethostname() +IDEMPOTENCY_TTL_SECONDS = _int("IDEMPOTENCY_TTL_SECONDS", 86400) diff --git a/app/cursor.py b/app/cursor.py new file mode 100644 index 00000000..113aabc8 --- /dev/null +++ b/app/cursor.py @@ -0,0 +1,43 @@ +import base64 +import json +import re +import uuid +from datetime import datetime, timezone + +TS_RE = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z") +TS_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ" + + +class BadCursor(Exception): + pass + + +def fmt(ts: datetime) -> str: + return ts.astimezone(timezone.utc).strftime(TS_FORMAT) + + +def encode_cursor(scope: str, created_at: str, item_id: str) -> str: + raw = json.dumps({"s": scope, "t": created_at, "i": item_id}, separators=(",", ":")).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def decode_cursor(scope: str, cursor: str): + try: + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)) + data = json.loads(raw) + except Exception: + raise BadCursor() from None + if not isinstance(data, dict) or set(data) != {"s", "t", "i"} or data["s"] != scope: + raise BadCursor() + ts, item_id = data["t"], data["i"] + if not isinstance(ts, str) or not TS_RE.fullmatch(ts) or not isinstance(item_id, str): + raise BadCursor() + try: + parsed = datetime.strptime(ts, TS_FORMAT).replace(tzinfo=timezone.utc) + if str(uuid.UUID(item_id)) != item_id: + raise BadCursor() + except ValueError: + raise BadCursor() from None + if encode_cursor(scope, ts, item_id) != cursor: + raise BadCursor() + return parsed, item_id diff --git a/app/db.py b/app/db.py new file mode 100644 index 00000000..0bd63089 --- /dev/null +++ b/app/db.py @@ -0,0 +1,99 @@ +import asyncio +from contextlib import asynccontextmanager + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import AsyncConnectionPool, PoolTimeout + +from app import config +from app.errors import unavailable + +BROKEN = (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout) + + +async def discard(conn) -> None: + try: + await conn.close() + except Exception: + pass + + +class Guarded: + def __init__(self, conn, timeout: float): + self.conn = conn + self.timeout = timeout + + async def execute(self, *args, **kwargs): + try: + return await asyncio.wait_for(self.conn.execute(*args, **kwargs), self.timeout) + except asyncio.TimeoutError: + await discard(self.conn) + raise unavailable("Хранилище не ответило за отведённое время") from None + + def transaction(self): + return self.conn.transaction() + + +class Database: + def __init__(self) -> None: + options = "-c statement_timeout=%d -c lock_timeout=%d -c idle_in_transaction_session_timeout=%d" % ( + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS, + config.DB_STATEMENT_TIMEOUT_MS * 2, + ) + self.pool = AsyncConnectionPool( + conninfo=config.DATABASE_URL, + min_size=config.DB_POOL_MIN, + max_size=config.DB_POOL_MAX, + timeout=config.DB_POOL_TIMEOUT, + max_lifetime=600.0, + open=False, + kwargs={ + "autocommit": True, + "row_factory": dict_row, + "connect_timeout": config.DB_CONNECT_TIMEOUT, + "options": options, + }, + ) + + async def open(self) -> None: + await self.pool.open(wait=False) + + async def close(self) -> None: + await self.pool.close() + + @asynccontextmanager + async def connection(self): + try: + async with self.pool.connection(timeout=config.DB_POOL_TIMEOUT) as conn: + yield Guarded(conn, config.DB_OP_TIMEOUT) + except BROKEN: + raise unavailable("Хранилище недоступно") from None + + @asynccontextmanager + async def transaction(self): + async with self.connection() as conn: + async with conn.transaction(): + yield conn + + async def in_transaction(self, action): + async def run(): + async with self.transaction() as conn: + return await action(conn) + + try: + return await asyncio.wait_for(run(), config.DB_OP_TIMEOUT) + except asyncio.TimeoutError: + raise unavailable("Хранилище не ответило за отведённое время") from None + + async def ready(self) -> bool: + try: + async with self.pool.connection(timeout=config.DB_READY_TIMEOUT) as conn: + try: + await asyncio.wait_for(conn.execute("SELECT 1"), config.DB_READY_TIMEOUT) + except asyncio.TimeoutError: + await discard(conn) + return False + return True + except Exception: + return False diff --git a/app/errors.py b/app/errors.py new file mode 100644 index 00000000..8110d273 --- /dev/null +++ b/app/errors.py @@ -0,0 +1,37 @@ +from typing import Optional + +from fastapi.responses import JSONResponse + +BASIC_CHALLENGE = 'Basic realm="messenger", charset="UTF-8"' +BEARER_CHALLENGE = 'Bearer realm="messenger"' + + +class ApiError(Exception): + def __init__(self, status: int, code: str, message: str, headers: Optional[dict] = None): + self.status, self.code, self.message, self.headers = status, code, message, headers or {} + + +def bad(message: str) -> ApiError: + return ApiError(400, "invalid_request", message) + + +def unauthorized(scheme: str = "Bearer") -> ApiError: + challenge = BASIC_CHALLENGE if scheme == "Basic" else BEARER_CHALLENGE + message = "Нужны имя пользователя и пароль" if scheme == "Basic" else "Нужна действующая сессия" + return ApiError(401, "unauthorized", message, {"WWW-Authenticate": challenge}) + + +def not_found() -> ApiError: + return ApiError(404, "not_found", "Объект не найден") + + +def unavailable(message: str) -> ApiError: + return ApiError(503, "unavailable", message, {"Retry-After": "1"}) + + +def error_body(rid: str, instance: str, status: int, code: str, message: str, headers: Optional[dict] = None) -> JSONResponse: + return JSONResponse( + status_code=status, + content={"error": {"code": code, "message": message, "request_id": rid}}, + headers={"X-Request-Id": rid, "X-Instance-Id": instance, **(headers or {})}, + ) diff --git a/app/main.py b/app/main.py new file mode 100644 index 00000000..c847244e --- /dev/null +++ b/app/main.py @@ -0,0 +1,436 @@ +import asyncio +import base64 +import binascii +import hashlib +import json +import logging +import os +import re +import resource +import signal +import sys +import uuid +from contextlib import asynccontextmanager +from typing import List, Optional, Tuple + +from fastapi import FastAPI, Request +from fastapi.responses import JSONResponse, Response +from starlette.concurrency import run_in_threadpool + +from app import config, passwords, repo +from app.cursor import BadCursor, decode_cursor, encode_cursor +from app.db import Database +from app.errors import ApiError, bad, error_body, not_found, unauthorized, unavailable +from app.sessions import SessionStore + +logging.basicConfig(stream=sys.stdout, level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") +log = logging.getLogger("messenger") + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") +IDEMPOTENCY_KEY_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") +ZERO_WIDTH = str.maketrans({char: None for char in "​‌‍⁠"}) +EXEMPT_PATHS = ("/health/live", "/health/ready") +PAGE_SIZE = os.sysconf("SC_PAGE_SIZE") if hasattr(os, "sysconf") else 4096 + +db = Database() +sessions = SessionStore(db) +state = {"draining": False, "ready": True, "inflight": 0, "rejected": {}} + + +def rss_mb() -> float: + try: + with open("/proc/self/statm") as handle: + return int(handle.read().split()[1]) * PAGE_SIZE / 1048576 + except OSError: + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return peak / 1048576 if sys.platform == "darwin" else peak / 1024 + + +def count_rejection(code: str) -> None: + state["rejected"][code] = state["rejected"].get(code, 0) + 1 + + +async def sweeper() -> None: + while True: + await asyncio.sleep(config.SESSION_SWEEP_SECONDS) + try: + removed = await sessions.sweep() + if removed: + log.info("sessions_expired removed=%d", removed) + except Exception: + log.warning("session sweep skipped: storage unavailable") + try: + async with db.connection() as conn: + removed = await repo.cleanup_idempotency(conn, config.IDEMPOTENCY_TTL_SECONDS) + if removed: + log.info("idempotency_cleanup removed=%d", removed) + except Exception: + log.warning("idempotency cleanup skipped") + + +async def readiness_watch() -> None: + failures = 0 + while True: + if await db.ready(): + failures = 0 + if not state["ready"]: + state["ready"] = True + log.info("ready again: database reachable") + else: + failures += 1 + if failures >= config.READY_FAILURES and state["ready"]: + state["ready"] = False + log.warning("not ready: database unreachable, business requests get 503") + await asyncio.sleep(config.READY_CHECK_SECONDS) + + +@asynccontextmanager +async def lifespan(_app: FastAPI): + previous = {} + + def on_signal(signum, frame): + if not state["draining"]: + state["draining"] = True + log.info("signal %s: draining, new requests get 503", signal.Signals(signum).name) + handler = previous.get(signum) + if callable(handler): + handler(signum, frame) + + for sig in (signal.SIGTERM, signal.SIGINT): + previous[sig] = signal.getsignal(sig) + signal.signal(sig, on_signal) + await db.open() + await sessions.open() + tasks = [asyncio.create_task(sweeper()), asyncio.create_task(readiness_watch())] + log.info( + "started: instance=%s session_store=%s session_ttl=%d cache_ttl=%d pool=%d..%d max_inflight=%d max_body=%d idem_ttl=%d", + config.INSTANCE_ID, config.SESSION_STORE, config.SESSION_TTL_SECONDS, config.SESSION_CACHE_TTL_SECONDS, + config.DB_POOL_MIN, config.DB_POOL_MAX, config.MAX_INFLIGHT_REQUESTS, config.MAX_BODY_BYTES, + config.IDEMPOTENCY_TTL_SECONDS, + ) + yield + for task in tasks: + task.cancel() + await sessions.close() + await db.close() + log.info("stopped: cache=%s rejected=%s", sessions.stats, state["rejected"]) + + +app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + + +@app.middleware("http") +async def guard(request: Request, call_next): + rid = uuid.uuid4().hex + request.state.rid = rid + exempt = request.url.path in EXEMPT_PATHS + if state["draining"] and request.url.path != "/health/live": + count_rejection("draining") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Сервер завершает работу", {"Retry-After": "1", "Connection": "close"}) + if not exempt and not state["ready"]: + count_rejection("not_ready") + return error_body(rid, config.INSTANCE_ID, 503, "unavailable", "Нет связи с базой данных", {"Retry-After": "1"}) + if not exempt and state["inflight"] >= config.MAX_INFLIGHT_REQUESTS: + count_rejection("inflight") + return error_body(rid, config.INSTANCE_ID, 429, "rate_limited", "Слишком много одновременных запросов", {"Retry-After": "1"}) + state["inflight"] += 1 + try: + response = await asyncio.wait_for(call_next(request), config.REQUEST_TIMEOUT) + except asyncio.TimeoutError: + count_rejection("request_timeout") + log.warning("request timeout request_id=%s method=%s path=%s", rid, request.method, request.url.path) + response = error_body( + rid, config.INSTANCE_ID, 503, "unavailable", + "Запрос не удалось обработать за отведённое время", {"Retry-After": "1"}, + ) + except Exception: + log.exception("unhandled error request_id=%s", rid) + response = error_body(rid, config.INSTANCE_ID, 500, "internal_error", "Внутренняя ошибка сервера") + finally: + state["inflight"] -= 1 + response.headers["X-Request-Id"] = rid + response.headers["X-Instance-Id"] = config.INSTANCE_ID + return response + + +@app.exception_handler(ApiError) +async def api_error_handler(request: Request, exc: ApiError): + if exc.status in (413, 429, 503): + count_rejection(exc.code) + return error_body(request.state.rid, config.INSTANCE_ID, exc.status, exc.code, exc.message, exc.headers) + + +def parse_uuid(value: object) -> Optional[str]: + if not isinstance(value, str): + return None + try: + return str(uuid.UUID(value)) + except ValueError: + return None + + +def non_blank(value: object, max_len: int) -> bool: + if not isinstance(value, str) or not 1 <= len(value) <= max_len: + return False + return value.translate(ZERO_WIDTH).strip() != "" + + +async def read_json(request: Request, fields: Tuple[str, ...]) -> dict: + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/json": + raise ApiError(415, "unsupported_media_type", "Ожидается application/json") + too_large = ApiError(413, "payload_too_large", "Тело запроса слишком большое", {"Connection": "close"}) + declared = request.headers.get("content-length") + if declared and declared.isdigit() and int(declared) > config.MAX_BODY_BYTES: + raise too_large + raw = bytearray() + async for chunk in request.stream(): + raw += chunk + if len(raw) > config.MAX_BODY_BYTES: + raise too_large + try: + body = json.loads(bytes(raw).decode("utf-8")) + except (UnicodeDecodeError, ValueError, RecursionError): + raise bad("Некорректный JSON") from None + if not isinstance(body, dict): + raise bad("Ожидается JSON-объект") + if set(body) - set(fields): + raise bad("Лишние поля в запросе") + if set(fields) - set(body): + raise bad("Отсутствуют обязательные поля") + return body + + +def ensure_capacity() -> None: + if config.MAX_RSS_MB and rss_mb() > config.MAX_RSS_MB: + raise unavailable("Достигнут лимит памяти процесса, запись недоступна") + + +def bearer_token(request: Request) -> str: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + token = value.strip() + if scheme.lower() != "bearer" or not token: + raise unauthorized() + return token + + +async def actor(request: Request) -> dict: + user = await sessions.resolve(bearer_token(request)) + if user is None: + raise unauthorized() + return user + + +def basic_credentials(request: Request) -> Tuple[str, str]: + scheme, _, value = request.headers.get("authorization", "").partition(" ") + if scheme.lower() != "basic" or not value.strip(): + raise unauthorized("Basic") + try: + decoded = base64.b64decode(value.strip(), validate=True).decode("utf-8") + except (binascii.Error, ValueError, UnicodeDecodeError): + raise unauthorized("Basic") from None + username, separator, password = decoded.partition(":") + if not separator: + raise unauthorized("Basic") + return username, password + + +async def member_chat(user: dict, chat_id: str) -> dict: + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + async with db.connection() as conn: + chat = await repo.chat_for_member(conn, cid, user["id"]) + if chat is None: + raise not_found() + return chat + + +def page_params(request: Request) -> Tuple[int, Optional[str]]: + query = request.query_params + limit = 50 + if "limit" in query: + if not re.fullmatch(r"[0-9]{1,3}", query["limit"]) or not 1 <= int(query["limit"]) <= 100: + raise bad("limit должен быть целым 1..100") + limit = int(query["limit"]) + cursor = query.get("cursor") + if cursor is not None and cursor == "": + raise bad("Некорректный cursor") + return limit, cursor + + +def keyset(scope: str, cursor: Optional[str]): + if cursor is None: + return None + try: + return decode_cursor(scope, cursor) + except BadCursor: + raise bad("Некорректный cursor") from None + + +def page(scope: str, items: List[dict], has_more: bool) -> JSONResponse: + next_cursor = encode_cursor(scope, items[-1]["created_at"], items[-1]["id"]) if has_more and items else None + return JSONResponse({"items": items, "next_cursor": next_cursor}) + + +@app.get("/health/live") +async def live(): + return {"status": "ok"} + + +@app.get("/health/ready") +async def ready(): + if not await db.ready(): + raise unavailable("База данных недоступна") + return {"status": "ok"} + + +@app.post("/api/v1/users", status_code=201) +async def create_user(request: Request): + body = await read_json(request, ("username", "display_name", "password")) + if not isinstance(body["username"], str) or not USERNAME_RE.fullmatch(body["username"]): + raise bad("username должен соответствовать [a-z0-9_]{3,32}") + if not non_blank(body["display_name"], 100): + raise bad("display_name: 1..100 символов, не только пробелы") + password = body["password"] + if not isinstance(password, str) or not config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + raise bad("password: %d..%d символов" % (config.PASSWORD_MIN, config.PASSWORD_MAX)) + ensure_capacity() + password_hash = await run_in_threadpool(passwords.hash_password, password) + async with db.connection() as conn: + user = await repo.create_user(conn, body["username"], body["display_name"], password_hash) + if user is None: + raise ApiError(409, "conflict", "username уже занят") + log.info("user_created id=%s", user["id"]) + return JSONResponse(user, status_code=201) + + +@app.post("/api/v1/auth/sessions", status_code=201) +async def login(request: Request): + if await request.body(): + raise bad("POST /auth/sessions не принимает тело запроса") + username, password = basic_credentials(request) + row = None + if USERNAME_RE.fullmatch(username) and config.PASSWORD_MIN <= len(password) <= config.PASSWORD_MAX: + async with db.connection() as conn: + row = await repo.credentials(conn, username) + stored = row["password_hash"] if row else passwords.DUMMY_HASH + matches = await run_in_threadpool(passwords.verify_password, stored, password) + if row is None or not matches: + log.info("login_failed username_known=%s", row is not None) + raise unauthorized("Basic") + user = repo.public_user(row) + token, expires_at = await sessions.create(user) + log.info("session_created user=%s", user["id"]) + return JSONResponse({"token": token, "token_type": "Bearer", "expires_at": expires_at, "user": user}, status_code=201) + + +@app.delete("/api/v1/auth/sessions/current", status_code=204) +async def logout(request: Request): + token = bearer_token(request) + user = await sessions.resolve(token) + if user is None: + raise unauthorized() + await sessions.revoke(token) + log.info("session_revoked user=%s", user["id"]) + return Response(status_code=204) + + +@app.get("/api/v1/users/me") +async def me(request: Request): + return await actor(request) + + +@app.post("/api/v1/chats", status_code=201) +async def create_chat(request: Request): + user = await actor(request) + body = await read_json(request, ("title", "member_ids")) + if not non_blank(body["title"], 100): + raise bad("title: 1..100 символов, не только пробелы") + raw_ids = body["member_ids"] + if not isinstance(raw_ids, list) or not 1 <= len(raw_ids) <= 100: + raise bad("member_ids: массив из 1..100 UUID") + ids: List[str] = [] + for value in raw_ids: + parsed = parse_uuid(value) + if parsed is None: + raise bad("member_ids содержит некорректный UUID") + ids.append(parsed) + if len(set(ids)) != len(ids): + raise bad("member_ids содержит дубликаты") + if user["id"] not in ids: + raise bad("Создатель должен входить в member_ids") + ensure_capacity() + + async def insert(conn): + return await repo.create_chat(conn, body["title"], ids) + + try: + chat = await db.in_transaction(insert) + except repo.UnknownMember: + raise bad("member_ids содержит неизвестного пользователя") from None + log.info("chat_created id=%s members=%d", chat["id"], len(ids)) + return JSONResponse(chat, status_code=201) + + +@app.get("/api/v1/chats") +async def list_chats(request: Request): + user = await actor(request) + limit, cursor = page_params(request) + scope = "chats:" + user["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_chats(conn, user["id"], limit, after) + return page(scope, items, has_more) + + +@app.get("/api/v1/chats/{chat_id}") +async def get_chat(chat_id: str, request: Request): + return await member_chat(await actor(request), chat_id) + + +@app.post("/api/v1/chats/{chat_id}/messages", status_code=201) +async def create_message(chat_id: str, request: Request): + user = await actor(request) + cid = parse_uuid(chat_id) + if cid is None: + raise bad("chat_id должен быть UUID") + + idem_key = request.headers.get("idempotency-key", "").strip() + if not idem_key or not IDEMPOTENCY_KEY_RE.fullmatch(idem_key): + raise bad("Заголовок Idempotency-Key обязателен и должен соответствовать [A-Za-z0-9._:-]{1,128}") + + body = await read_json(request, ("text",)) + if not non_blank(body["text"], 4000): + raise bad("text: 1..4000 символов, не только пробелы") + ensure_capacity() + + body_hash = hashlib.sha256( + json.dumps({"text": body["text"]}, sort_keys=True, ensure_ascii=False).encode() + ).hexdigest() + + async def do(conn): + return await repo.create_message_idempotent(conn, cid, user["id"], body["text"], idem_key, body_hash) + + status, message = await db.in_transaction(do) + + if status == "conflict": + raise ApiError(409, "conflict", "Idempotency-Key использован с другим телом запроса") + if status == "not_member": + raise not_found() + return JSONResponse(message, status_code=201) + + +@app.get("/api/v1/chats/{chat_id}/messages") +async def list_messages(chat_id: str, request: Request): + chat = await member_chat(await actor(request), chat_id) + limit, cursor = page_params(request) + scope = "messages:" + chat["id"] + after = keyset(scope, cursor) + async with db.connection() as conn: + items, has_more = await repo.list_messages(conn, chat["id"], limit, after) + return page(scope, items, has_more) + + +@app.api_route("/{path:path}", methods=["GET", "POST", "PUT", "PATCH", "DELETE"]) +async def fallback(request: Request): + raise not_found() diff --git a/app/passwords.py b/app/passwords.py new file mode 100644 index 00000000..b509e4a7 --- /dev/null +++ b/app/passwords.py @@ -0,0 +1,16 @@ +from argon2 import PasswordHasher +from argon2.exceptions import Argon2Error, VerifyMismatchError + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash("there-is-no-such-user-password") + + +def hash_password(password: str) -> str: + return HASHER.hash(password) + + +def verify_password(stored: str, password: str) -> bool: + try: + return HASHER.verify(stored, password) + except (VerifyMismatchError, Argon2Error, ValueError): + return False diff --git a/app/repo.py b/app/repo.py new file mode 100644 index 00000000..71f1bcef --- /dev/null +++ b/app/repo.py @@ -0,0 +1,192 @@ +import hashlib +import json +from typing import List, Optional, Tuple + +import psycopg + +from app.cursor import fmt + +MESSAGE_COLUMNS = "id, chat_id, sender_id, text, created_at" +CHAT_QUERY = ( + "SELECT c.id, c.title, c.created_at," + " ARRAY(SELECT m2.user_id FROM chat_members m2 WHERE m2.chat_id = c.id ORDER BY m2.user_id) AS member_ids" +) + + +class UnknownMember(Exception): + pass + + +def decoded(value) -> str: + return bytes(value).decode("utf-8") + + +def public_user(row) -> dict: + return { + "id": str(row["id"]), + "username": row["username"], + "display_name": decoded(row["display_name"]), + "created_at": fmt(row["created_at"]), + } + + +def _chat(row) -> dict: + return { + "id": str(row["id"]), + "title": decoded(row["title"]), + "member_ids": [str(value) for value in row["member_ids"]], + "created_at": fmt(row["created_at"]), + } + + +def _message(row) -> dict: + return { + "id": str(row["id"]), + "chat_id": str(row["chat_id"]), + "sender_id": str(row["sender_id"]), + "text": decoded(row["text"]), + "created_at": fmt(row["created_at"]), + } + + +async def create_user(conn, username: str, display_name: str, password_hash: str) -> Optional[dict]: + try: + cur = await conn.execute( + "INSERT INTO users (id, username, display_name, password_hash, created_at)" + " VALUES (gen_random_uuid(), %s, %s, %s, clock_timestamp())" + " RETURNING id, username, display_name, created_at", + (username, display_name.encode("utf-8"), password_hash), + ) + except psycopg.errors.UniqueViolation: + return None + return public_user(await cur.fetchone()) + + +async def credentials(conn, username: str): + cur = await conn.execute( + "SELECT id, username, display_name, created_at, password_hash FROM users WHERE username = %s", + (username,), + ) + return await cur.fetchone() + + +async def create_chat(conn, title: str, member_ids: List[str]) -> dict: + cur = await conn.execute( + "INSERT INTO chats (id, title, created_at)" + " VALUES (gen_random_uuid(), %s, clock_timestamp())" + " RETURNING id, title, created_at", + (title.encode("utf-8"),), + ) + chat = await cur.fetchone() + cur = await conn.execute( + "INSERT INTO chat_members (chat_id, user_id) SELECT %s, u.id FROM users u WHERE u.id = ANY(%s::uuid[])", + (chat["id"], member_ids), + ) + if cur.rowcount != len(member_ids): + raise UnknownMember() + chat["member_ids"] = list(member_ids) + return _chat(chat) + + +async def chat_for_member(conn, chat_id: str, user_id: str) -> Optional[dict]: + cur = await conn.execute( + CHAT_QUERY + " FROM chats c JOIN chat_members m ON m.chat_id = c.id AND m.user_id = %s WHERE c.id = %s", + (user_id, chat_id), + ) + row = await cur.fetchone() + return _chat(row) if row else None + + +async def list_chats(conn, user_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + CHAT_QUERY + + " FROM chats c JOIN chat_members m ON m.chat_id = c.id" + " WHERE m.user_id = %s AND (%s::timestamptz IS NULL OR (c.created_at, c.id) > (%s::timestamptz, %s::uuid))" + " ORDER BY c.created_at, c.id LIMIT %s", + (user_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + return [_chat(row) for row in rows[:limit]], len(rows) > limit + + +async def create_message(conn, chat_id: str, sender_id: str, text: str) -> Optional[dict]: + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id), + ) + row = await cur.fetchone() + return _message(row) if row else None + + +async def list_messages(conn, chat_id: str, limit: int, after) -> Tuple[List[dict], bool]: + ts, last_id = after if after else (None, None) + cur = await conn.execute( + "SELECT " + MESSAGE_COLUMNS + " FROM messages" + " WHERE chat_id = %s AND (%s::timestamptz IS NULL OR (created_at, id) > (%s::timestamptz, %s::uuid))" + " ORDER BY created_at, id LIMIT %s", + (chat_id, ts, ts, last_id, limit + 1), + ) + rows = await cur.fetchall() + return [_message(row) for row in rows[:limit]], len(rows) > limit + + +def _idem_lock_id(user_id: str, chat_id: str, key: str) -> int: + raw = "%s:%s:%s" % (user_id, chat_id, key) + return int(hashlib.sha256(raw.encode()).hexdigest()[:15], 16) + + +async def create_message_idempotent(conn, chat_id: str, sender_id: str, text: str, idem_key: str, body_hash: str): + lock_id = _idem_lock_id(sender_id, chat_id, idem_key) + await conn.execute("SELECT pg_advisory_xact_lock(%s)", (lock_id,)) + + cur = await conn.execute( + "SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s", + (chat_id, sender_id), + ) + if await cur.fetchone() is None: + return "not_member", None + + cur = await conn.execute( + "SELECT request_body_hash, response_json FROM idempotency_keys" + " WHERE user_id = %s AND chat_id = %s AND key = %s", + (sender_id, chat_id, idem_key), + ) + existing = await cur.fetchone() + + if existing: + if existing["request_body_hash"] != body_hash: + return "conflict", None + return "replay", json.loads(existing["response_json"]) + + cur = await conn.execute( + "INSERT INTO messages (id, chat_id, sender_id, text, created_at)" + " SELECT gen_random_uuid(), %s, %s, %s, clock_timestamp()" + " WHERE EXISTS (SELECT 1 FROM chat_members WHERE chat_id = %s AND user_id = %s)" + " RETURNING " + MESSAGE_COLUMNS, + (chat_id, sender_id, text.encode("utf-8"), chat_id, sender_id), + ) + row = await cur.fetchone() + if row is None: + return "not_member", None + + message = _message(row) + + await conn.execute( + "INSERT INTO idempotency_keys (user_id, chat_id, key, request_body_hash, response_json, created_at)" + " VALUES (%s, %s, %s, %s, %s, clock_timestamp())", + (sender_id, chat_id, idem_key, body_hash, json.dumps(message, ensure_ascii=False)), + ) + + return "created", message + + +async def cleanup_idempotency(conn, ttl_seconds: int) -> int: + cur = await conn.execute( + "DELETE FROM idempotency_keys WHERE created_at < now() - make_interval(secs => %s)", + (ttl_seconds,), + ) + return cur.rowcount diff --git a/app/sessions.py b/app/sessions.py new file mode 100644 index 00000000..31d472f4 --- /dev/null +++ b/app/sessions.py @@ -0,0 +1,154 @@ +import hashlib +import json +import logging +import secrets +from datetime import datetime, timedelta, timezone +from typing import Optional + +import valkey.asyncio as valkey + +from app import config, repo +from app.cursor import fmt +from app.errors import unavailable + +log = logging.getLogger("messenger.sessions") + +REVOKED = "revoked" + + +class SessionStore: + def __init__(self, db) -> None: + self.db = db + self.client = None + self.stats = {"hits": 0, "misses": 0, "errors": 0} + + async def open(self) -> None: + if not config.CACHE_URL: + log.info("session cache disabled: CACHE_URL is empty") + return + self.client = valkey.Valkey.from_url( + config.CACHE_URL, + socket_timeout=config.CACHE_TIMEOUT, + socket_connect_timeout=config.CACHE_TIMEOUT, + decode_responses=True, + ) + + async def close(self) -> None: + if self.client is not None: + await self.client.aclose() + self.client = None + + @staticmethod + def digest(token: str) -> bytes: + return hashlib.sha256(token.encode("utf-8")).digest() + + @staticmethod + def key(digest: bytes) -> str: + return "session:" + digest.hex() + + async def create(self, user: dict) -> tuple: + token = secrets.token_urlsafe(32) + digest = self.digest(token) + expires_at = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS) + async with self.db.connection() as conn: + await conn.execute( + "INSERT INTO sessions (token_hash, user_id, created_at, expires_at)" + " VALUES (%s, %s, clock_timestamp(), %s)", + (digest, user["id"], expires_at), + ) + await self._cache_put(digest, user, expires_at) + return token, fmt(expires_at) + + async def resolve(self, token: str) -> Optional[dict]: + digest = self.digest(token) + cached = await self._cache_get(digest) + if cached == REVOKED: + self.stats["hits"] += 1 + return None + if cached is not None: + self.stats["hits"] += 1 + return cached + self.stats["misses"] += 1 + async with self.db.connection() as conn: + cur = await conn.execute( + "SELECT u.id, u.username, u.display_name, u.created_at, s.expires_at" + " FROM sessions s JOIN users u ON u.id = s.user_id" + " WHERE s.token_hash = %s AND s.expires_at > now()", + (digest,), + ) + row = await cur.fetchone() + if row is None: + return None + user = repo.public_user(row) + await self._cache_put(digest, user, row["expires_at"]) + return user + + async def revoke(self, token: str) -> None: + digest = self.digest(token) + await self._cache_revoke(digest) + try: + async with self.db.connection() as conn: + await conn.execute("DELETE FROM sessions WHERE token_hash = %s", (digest,)) + except Exception: + await self._cache_release(digest) + raise + + async def sweep(self) -> int: + async with self.db.connection() as conn: + cur = await conn.execute("DELETE FROM sessions WHERE expires_at <= now()") + return cur.rowcount + + def _cache_ttl(self, expires_at: datetime) -> int: + remaining = int((expires_at - datetime.now(timezone.utc)).total_seconds()) + return min(config.SESSION_CACHE_TTL_SECONDS, remaining) + + async def _cache_get(self, digest: bytes) -> Optional[dict]: + if self.client is None: + return None + try: + raw = await self.client.get(self.key(digest)) + except Exception: + self._cache_failed("get") + return None + if not raw: + return None + if raw == REVOKED: + return REVOKED + try: + return json.loads(raw) + except ValueError: + return None + + async def _cache_put(self, digest: bytes, user: dict, expires_at: datetime) -> None: + if self.client is None: + return + ttl = self._cache_ttl(expires_at) + if ttl <= 0: + return + payload = json.dumps(user) + try: + await self.client.set(self.key(digest), payload, ex=ttl, nx=True) + except Exception: + self._cache_failed("set") + + async def _cache_release(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.delete(self.key(digest)) + except Exception: + self._cache_failed("release") + + async def _cache_revoke(self, digest: bytes) -> None: + if self.client is None: + return + try: + await self.client.set(self.key(digest), REVOKED, ex=config.SESSION_REVOKED_TTL_SECONDS) + except Exception: + self._cache_failed("revoke") + raise unavailable("Кэш сессий недоступен, выход из сессии не подтверждён") from None + + def _cache_failed(self, operation: str) -> None: + self.stats["errors"] += 1 + if self.stats["errors"] % 50 == 1: + log.warning("session cache %s failed, falling back to postgres (errors=%d)", operation, self.stats["errors"]) diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..dc4d088e --- /dev/null +++ b/compose.yaml @@ -0,0 +1,167 @@ +name: mtusi-lab3 + +x-api: &api-defaults + build: + context: . + image: mtusi-messenger:lab3 + env_file: + - .env + secrets: + - database_url + - cache_url + depends_on: + postgres: + condition: service_healthy + valkey: + condition: service_healthy + migrate: + condition: service_completed_successfully + deploy: + resources: + limits: + cpus: "0.50" + memory: 256M + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2).status == 200 else 1)\""] + interval: 5s + timeout: 3s + retries: 3 + start_period: 10s + restart: unless-stopped + +services: + nginx: + image: nginx:1.27.3-alpine + command: ["/bin/sh", "/opt/nginx-health.sh"] + environment: + API_UPSTREAMS: ${API_UPSTREAMS:-api-1 api-2} + HEALTH_INTERVAL: "1" + HEALTH_FALL: "2" + HEALTH_RISE: "2" + ports: + - "127.0.0.1:8080:8080" + volumes: + - ./infra/nginx.conf:/etc/nginx/nginx.conf:ro + - ./infra/nginx-health.sh:/opt/nginx-health.sh:ro + depends_on: + api-1: + condition: service_healthy + api-2: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 64M + healthcheck: + test: ["CMD-SHELL", "wget -qO- http://127.0.0.1:8080/health/live || exit 1"] + interval: 5s + timeout: 3s + retries: 3 + start_period: 5s + restart: unless-stopped + + api-1: + <<: *api-defaults + environment: + INSTANCE_ID: api-1 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + + api-2: + <<: *api-defaults + environment: + INSTANCE_ID: api-2 + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + + postgres: + image: postgres:17.7-alpine@sha256:bb377b7239d2774ac8cc76f481596ce96c5a6b5e9d141f6d0a0ee371a6e7c0f2 + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: --data-checksums + command: + - postgres + - -c + - synchronous_commit=on + - -c + - shared_buffers=128MB + - -c + - work_mem=8MB + - -c + - max_connections=50 + - -c + - track_io_timing=on + secrets: + - postgres_password + volumes: + - pgdata:/var/lib/postgresql/data + deploy: + resources: + limits: + cpus: "1.00" + memory: 512M + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 20 + start_period: 10s + restart: unless-stopped + + valkey: + image: valkey/valkey:8.1-alpine@sha256:081c2f5cb575efc901aa80ff9cdbd1ec6a301682fd35e1ebb4b0990a4a4a8507 + command: + - valkey-server + - --save + - "" + - --appendonly + - "no" + - --maxmemory + - 64mb + - --maxmemory-policy + - allkeys-lru + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + healthcheck: + test: ["CMD", "valkey-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + restart: unless-stopped + + migrate: + build: + context: . + image: mtusi-messenger:lab3 + command: ["python", "scripts/migrate.py"] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: + - database_url + depends_on: + postgres: + condition: service_healthy + deploy: + resources: + limits: + cpus: "0.25" + memory: 128M + restart: "no" + +volumes: + pgdata: + +secrets: + postgres_password: + file: ./secrets/postgres_password + database_url: + file: ./secrets/database_url + cache_url: + file: ./secrets/cache_url diff --git a/evidence/failover_both.txt b/evidence/failover_both.txt new file mode 100644 index 00000000..81900be0 --- /dev/null +++ b/evidence/failover_both.txt @@ -0,0 +1,11 @@ +=== Failover run 1 === +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.333} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. + +=== Failover run 2 (other replica) === +python3 tests/resilience.py --mode failover --action-script "scripts/stop-one.sh" --base-url "http://localhost:8080" +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 130, "failed_attempts": 0, "first_recovery_seconds": 0.325} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/graceful_shutdown.txt b/evidence/graceful_shutdown.txt new file mode 100644 index 00000000..7742981e --- /dev/null +++ b/evidence/graceful_shutdown.txt @@ -0,0 +1,37 @@ +=== Graceful shutdown test === +Sending SIGTERM to api-1... + Container mtusi-lab3-api-1-1 Stopping + Container mtusi-lab3-api-1-1 Stopped + +=== api-1 logs (last 15 lines) === +api-1-1 | INFO: 172.19.0.6:58008 - "POST /api/v1/chats HTTP/1.0" 201 Created +api-1-1 | INFO: 172.19.0.6:58018 - "POST /api/v1/chats/0a0b21a2-0a79-4182-ac0d-b9fbac88fadc/messages HTTP/1.0" 201 Created +api-1-1 | INFO: 172.19.0.6:58022 - "POST /api/v1/chats/0a0b21a2-0a79-4182-ac0d-b9fbac88fadc/messages HTTP/1.0" 201 Created +api-1-1 | INFO: 172.19.0.6:58036 - "POST /api/v1/chats/0a0b21a2-0a79-4182-ac0d-b9fbac88fadc/messages HTTP/1.0" 409 Conflict +api-1-1 | INFO: 127.0.0.1:44656 - "GET /health/ready HTTP/1.1" 200 OK +api-1-1 | INFO: 127.0.0.1:48164 - "GET /health/ready HTTP/1.1" 200 OK +api-1-1 | INFO: 172.19.0.6:35016 - "GET /health/live HTTP/1.0" 200 OK +api-1-1 | INFO: 127.0.0.1:48392 - "GET /health/ready HTTP/1.1" 200 OK +api-1-1 | INFO: 127.0.0.1:48412 - "GET /health/ready HTTP/1.1" 200 OK +api-1-1 | 2026-10-28 12:21:22,692 INFO messenger signal SIGTERM: draining, new requests get 503 +api-1-1 | INFO: Shutting down +api-1-1 | INFO: Waiting for application shutdown. +api-1-1 | 2026-10-28 12:21:22,836 INFO messenger stopped: cache={'hits': 4, 'misses': 2, 'errors': 0} rejected={} +api-1-1 | INFO: Application shutdown complete. +api-1-1 | INFO: Finished server process [1] + +=== Restoring api-1 === + Container mtusi-lab3-postgres-1 Running + Container mtusi-lab3-valkey-1 Running + Container mtusi-lab3-postgres-1 Waiting + Container mtusi-lab3-postgres-1 Healthy + Container mtusi-lab3-migrate-1 Starting + Container mtusi-lab3-migrate-1 Started + Container mtusi-lab3-migrate-1 Waiting + Container mtusi-lab3-postgres-1 Waiting + Container mtusi-lab3-valkey-1 Waiting + Container mtusi-lab3-valkey-1 Healthy + Container mtusi-lab3-postgres-1 Healthy + Container mtusi-lab3-migrate-1 Exited + Container mtusi-lab3-api-1-1 Starting + Container mtusi-lab3-api-1-1 Started diff --git a/evidence/instance_ids.txt b/evidence/instance_ids.txt new file mode 100644 index 00000000..316df3a6 --- /dev/null +++ b/evidence/instance_ids.txt @@ -0,0 +1,6 @@ +x-instance-id: api-2 +x-instance-id: api-1 +x-instance-id: api-2 +x-instance-id: api-1 +x-instance-id: api-2 +x-instance-id: api-1 diff --git a/evidence/limits.txt b/evidence/limits.txt new file mode 100644 index 00000000..7dc1a4a0 --- /dev/null +++ b/evidence/limits.txt @@ -0,0 +1,34 @@ + limits: + cpus: 0.5 + memory: "268435456" + environment: +-- + limits: + cpus: 0.5 + memory: "268435456" + environment: +-- + limits: + cpus: 0.25 + memory: "134217728" + environment: +-- + limits: + cpus: 0.25 + memory: "67108864" + healthcheck: +-- + limits: + cpus: 1 + memory: "536870912" + environment: +-- + limits: + cpus: 0.25 + memory: "134217728" + healthcheck: +-- + limits: + cpus: "0.50" + memory: 256M + env_file: diff --git a/evidence/migrate.txt b/evidence/migrate.txt new file mode 100644 index 00000000..685a3db2 --- /dev/null +++ b/evidence/migrate.txt @@ -0,0 +1,20 @@ +migrate-1 | skip 0002_idempotency +migrate-1 | migrations complete: 2 file(s) +migrate-1 | skip 0001_init +migrate-1 | skip 0002_idempotency +migrate-1 | migrations complete: 2 file(s) +migrate-1 | skip 0001_init +migrate-1 | skip 0002_idempotency +migrate-1 | migrations complete: 2 file(s) +migrate-1 | skip 0001_init +migrate-1 | skip 0002_idempotency +migrate-1 | migrations complete: 2 file(s) +migrate-1 | skip 0001_init +migrate-1 | skip 0002_idempotency +migrate-1 | migrations complete: 2 file(s) +migrate-1 | skip 0001_init +migrate-1 | skip 0002_idempotency +migrate-1 | migrations complete: 2 file(s) +migrate-1 | skip 0001_init +migrate-1 | skip 0002_idempotency +migrate-1 | migrations complete: 2 file(s) diff --git a/evidence/parallel_idempotency.txt b/evidence/parallel_idempotency.txt new file mode 100644 index 00000000..31434d1b --- /dev/null +++ b/evidence/parallel_idempotency.txt @@ -0,0 +1,2 @@ +PASS: 8 parallel requests, 1 message created, 2 distinct replicas observed: ['api-1', 'api-2'] +PASS: same key + different body → 409 diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..793b001d --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +python3 tests/resilience.py --mode persistence --action-script "scripts/restart.sh" --base-url "http://localhost:8080" +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/ports.txt b/evidence/ports.txt new file mode 100644 index 00000000..7844713e --- /dev/null +++ b/evidence/ports.txt @@ -0,0 +1,6 @@ +NAME PORTS STATUS +mtusi-lab3-api-1-1 8080/tcp Up 12 seconds (healthy) +mtusi-lab3-api-2-1 8080/tcp Up 12 seconds (healthy) +mtusi-lab3-nginx-1 127.0.0.1:8080->8080/tcp Up 12 seconds (healthy) +mtusi-lab3-postgres-1 5432/tcp Up 12 seconds (healthy) +mtusi-lab3-valkey-1 6379/tcp Up 12 seconds (healthy) diff --git a/evidence/review_scenarios.txt b/evidence/review_scenarios.txt new file mode 100644 index 00000000..a87f5529 --- /dev/null +++ b/evidence/review_scenarios.txt @@ -0,0 +1,39 @@ +### Before fixes (commit fbb8916) +== 1. Idempotent replay after membership removal +PASS: first POST with key -> 201 (got 201) +PASS: history after removal -> 404 (got 404) +FAIL: replay with same key and body after removal -> 404 (got 201) +== 2. Live but not ready api-1 (no route to PostgreSQL) +FAIL: nginx active check marks api-1 down in upstream +PASS: Docker marks api-1 unhealthy +FAIL: 40 GET /users/me with warm session: served by not-ready api-1 = 20 +PASS: 40 GET /users/me: 200 responses = 40 +PASS: 20 POST messages during outage -> all 201 (got [201]) +== 3. LB does not transparently retry non-idempotent POST /chats +PASS: chats in DB (11) == 201 responses (11), POST /chats codes=[201, 201, 201, 201, 201, 201, 201, 201, 201, 201] +FAIL: nginx access log: 12 POST /chats, sent to two upstreams: 3 +FAIL: non_idempotent only on messages route: {'/': True} +PASS: api-1 healthy again after PostgreSQL route restored +PASS: api-1 returns to LB rotation +PASS: nginx upstream has no servers marked down +RESULT: FAIL (5) + +### After fixes +== 1. Idempotent replay after membership removal +PASS: first POST with key -> 201 (got 201) +PASS: history after removal -> 404 (got 404) +PASS: replay with same key and body after removal -> 404 (got 404) +== 2. Live but not ready api-1 (no route to PostgreSQL) +PASS: nginx active check marks api-1 down in upstream +PASS: Docker marks api-1 unhealthy +PASS: 40 GET /users/me with warm session: served by not-ready api-1 = 0 +PASS: 40 GET /users/me: 200 responses = 40 +PASS: 20 POST messages during outage -> all 201 (got [201]) +== 3. LB does not transparently retry non-idempotent POST /chats +PASS: chats in DB (9) == 201 responses (9), POST /chats codes=[503, 201, 503, 201, 201, 201, 201, 201, 201, 201] +PASS: nginx access log: 12 POST /chats, sent to two upstreams: 0 +PASS: non_idempotent only on messages route: {'~ ^/api/v1/chats/[^/]+/messages$': True, '/': False} +PASS: api-1 healthy again after PostgreSQL route restored +PASS: api-1 returns to LB rotation +PASS: nginx upstream has no servers marked down +RESULT: PASS diff --git a/evidence/rolling_load.json b/evidence/rolling_load.json new file mode 100644 index 00000000..694628a6 --- /dev/null +++ b/evidence/rolling_load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 60.0, + "elapsed_seconds": 60.012, + "concurrency": 4, + "successful_requests": 54625, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 910.231, + "successful_latency_ms": { + "p50": 3.848, + "p95": 5.7, + "p99": 17.851 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/rolling_update.txt b/evidence/rolling_update.txt new file mode 100644 index 00000000..52f2c8e4 --- /dev/null +++ b/evidence/rolling_update.txt @@ -0,0 +1,21 @@ +14:14:27 recreate api-1 (old container 572155b7facd) +14:14:33 api-1 healthy (new container a81474023dca) +14:14:43 recreate api-2 (old container 35c30c16ece5) +14:14:49 api-2 healthy (new container 1ac67f968005) +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 60.0, + "elapsed_seconds": 60.012, + "concurrency": 4, + "successful_requests": 54625, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 910.231, + "successful_latency_ms": { + "p50": 3.848, + "p95": 5.7, + "p99": 17.851 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A1_limits.txt b/evidence/scaling/A1_limits.txt new file mode 100644 index 00000000..ed28ba56 --- /dev/null +++ b/evidence/scaling/A1_limits.txt @@ -0,0 +1,2 @@ +config=A1 replicas=1 api_cpus=1.00 api_mem=512M concurrency=16 duration=30 +api-1 NanoCpus=1000000000 Memory=536870912 diff --git a/evidence/scaling/A1_run1.json b/evidence/scaling/A1_run1.json new file mode 100644 index 00000000..4abac4c8 --- /dev/null +++ b/evidence/scaling/A1_run1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.049, + "concurrency": 16, + "successful_requests": 10411, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 346.466, + "successful_latency_ms": { + "p50": 13.495, + "p95": 205.76, + "p99": 519.105 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A1_run1_stats.txt b/evidence/scaling/A1_run1_stats.txt new file mode 100644 index 00000000..51bc33cb --- /dev/null +++ b/evidence/scaling/A1_run1_stats.txt @@ -0,0 +1,4 @@ +mtusi-lab3-nginx-1 25.01% 10.68MiB / 64MiB +mtusi-lab3-api-1-1 11.14% 64.5MiB / 512MiB +mtusi-lab3-postgres-1 3.80% 59.32MiB / 512MiB +mtusi-lab3-valkey-1 0.70% 12.3MiB / 128MiB diff --git a/evidence/scaling/A1_run2.json b/evidence/scaling/A1_run2.json new file mode 100644 index 00000000..6d699ade --- /dev/null +++ b/evidence/scaling/A1_run2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.047, + "concurrency": 16, + "successful_requests": 4193, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 139.548, + "successful_latency_ms": { + "p50": 86.111, + "p95": 491.678, + "p99": 697.125 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A1_run2_stats.txt b/evidence/scaling/A1_run2_stats.txt new file mode 100644 index 00000000..6cc643c5 --- /dev/null +++ b/evidence/scaling/A1_run2_stats.txt @@ -0,0 +1,4 @@ +mtusi-lab3-nginx-1 24.54% 11.58MiB / 64MiB +mtusi-lab3-api-1-1 12.71% 64.38MiB / 512MiB +mtusi-lab3-postgres-1 7.29% 64.33MiB / 512MiB +mtusi-lab3-valkey-1 0.65% 12.26MiB / 128MiB diff --git a/evidence/scaling/A1_run3.json b/evidence/scaling/A1_run3.json new file mode 100644 index 00000000..24f0dcfb --- /dev/null +++ b/evidence/scaling/A1_run3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.166, + "concurrency": 16, + "successful_requests": 14260, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 472.712, + "successful_latency_ms": { + "p50": 13.454, + "p95": 108.372, + "p99": 498.509 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A1_run3_stats.txt b/evidence/scaling/A1_run3_stats.txt new file mode 100644 index 00000000..e4be95c1 --- /dev/null +++ b/evidence/scaling/A1_run3_stats.txt @@ -0,0 +1,4 @@ +mtusi-lab3-nginx-1 24.67% 11.53MiB / 64MiB +mtusi-lab3-api-1-1 31.22% 64.43MiB / 512MiB +mtusi-lab3-postgres-1 7.25% 77.34MiB / 512MiB +mtusi-lab3-valkey-1 0.61% 12.08MiB / 128MiB diff --git a/evidence/scaling/A2_limits.txt b/evidence/scaling/A2_limits.txt new file mode 100644 index 00000000..b9afe11e --- /dev/null +++ b/evidence/scaling/A2_limits.txt @@ -0,0 +1,3 @@ +config=A2 replicas=2 api_cpus=0.50 api_mem=256M concurrency=16 duration=30 +api-1 NanoCpus=500000000 Memory=268435456 +api-2 NanoCpus=500000000 Memory=268435456 diff --git a/evidence/scaling/A2_run1.json b/evidence/scaling/A2_run1.json new file mode 100644 index 00000000..e50073d3 --- /dev/null +++ b/evidence/scaling/A2_run1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.046, + "concurrency": 16, + "successful_requests": 24369, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 811.047, + "successful_latency_ms": { + "p50": 11.499, + "p95": 63.911, + "p99": 86.902 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A2_run1_stats.txt b/evidence/scaling/A2_run1_stats.txt new file mode 100644 index 00000000..39b53946 --- /dev/null +++ b/evidence/scaling/A2_run1_stats.txt @@ -0,0 +1,5 @@ +mtusi-lab3-nginx-1 18.84% 10.38MiB / 64MiB +mtusi-lab3-api-1-1 51.03% 63.71MiB / 256MiB +mtusi-lab3-api-2-1 43.37% 63.35MiB / 256MiB +mtusi-lab3-postgres-1 46.22% 115.9MiB / 512MiB +mtusi-lab3-valkey-1 2.79% 12.41MiB / 128MiB diff --git a/evidence/scaling/A2_run2.json b/evidence/scaling/A2_run2.json new file mode 100644 index 00000000..da8cf4a7 --- /dev/null +++ b/evidence/scaling/A2_run2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.08, + "concurrency": 16, + "successful_requests": 8481, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 281.951, + "successful_latency_ms": { + "p50": 32.217, + "p95": 112.993, + "p99": 297.73 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A2_run2_stats.txt b/evidence/scaling/A2_run2_stats.txt new file mode 100644 index 00000000..6ed88e43 --- /dev/null +++ b/evidence/scaling/A2_run2_stats.txt @@ -0,0 +1,5 @@ +mtusi-lab3-nginx-1 25.09% 11.69MiB / 64MiB +mtusi-lab3-api-1-1 27.42% 75.51MiB / 256MiB +mtusi-lab3-api-2-1 31.68% 63.59MiB / 256MiB +mtusi-lab3-postgres-1 15.93% 129.2MiB / 512MiB +mtusi-lab3-valkey-1 1.08% 12.16MiB / 128MiB diff --git a/evidence/scaling/A2_run3.json b/evidence/scaling/A2_run3.json new file mode 100644 index 00000000..cdf14e0a --- /dev/null +++ b/evidence/scaling/A2_run3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.106, + "concurrency": 16, + "successful_requests": 23597, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 783.804, + "successful_latency_ms": { + "p50": 11.682, + "p95": 66.159, + "p99": 87.785 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A2_run3_stats.txt b/evidence/scaling/A2_run3_stats.txt new file mode 100644 index 00000000..5e16630f --- /dev/null +++ b/evidence/scaling/A2_run3_stats.txt @@ -0,0 +1,5 @@ +mtusi-lab3-nginx-1 18.01% 11.25MiB / 64MiB +mtusi-lab3-api-1-1 51.18% 64.11MiB / 256MiB +mtusi-lab3-api-2-1 43.66% 63.78MiB / 256MiB +mtusi-lab3-postgres-1 43.18% 147.1MiB / 512MiB +mtusi-lab3-valkey-1 2.88% 12.25MiB / 128MiB diff --git a/evidence/scaling/A4_limits.txt b/evidence/scaling/A4_limits.txt new file mode 100644 index 00000000..4c10299e --- /dev/null +++ b/evidence/scaling/A4_limits.txt @@ -0,0 +1,5 @@ +config=A4 replicas=4 api_cpus=0.25 api_mem=128M concurrency=16 duration=30 +api-1 NanoCpus=250000000 Memory=134217728 +api-2 NanoCpus=250000000 Memory=134217728 +api-3 NanoCpus=250000000 Memory=134217728 +api-4 NanoCpus=250000000 Memory=134217728 diff --git a/evidence/scaling/A4_run1.json b/evidence/scaling/A4_run1.json new file mode 100644 index 00000000..2e9c2879 --- /dev/null +++ b/evidence/scaling/A4_run1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.048, + "concurrency": 16, + "successful_requests": 18635, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 620.172, + "successful_latency_ms": { + "p50": 10.376, + "p95": 89.425, + "p99": 105.535 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A4_run1_stats.txt b/evidence/scaling/A4_run1_stats.txt new file mode 100644 index 00000000..cf3afd36 --- /dev/null +++ b/evidence/scaling/A4_run1_stats.txt @@ -0,0 +1,7 @@ +mtusi-lab3-nginx-1 8.28% 9.543MiB / 64MiB +mtusi-lab3-api-2-1 25.56% 63.45MiB / 128MiB +mtusi-lab3-api-1-1 23.44% 63.25MiB / 128MiB +mtusi-lab3-api-3-1 24.29% 63.04MiB / 128MiB +mtusi-lab3-api-4-1 25.30% 63.8MiB / 128MiB +mtusi-lab3-postgres-1 22.11% 216.8MiB / 512MiB +mtusi-lab3-valkey-1 2.30% 12.11MiB / 128MiB diff --git a/evidence/scaling/A4_run2.json b/evidence/scaling/A4_run2.json new file mode 100644 index 00000000..208b841a --- /dev/null +++ b/evidence/scaling/A4_run2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.064, + "concurrency": 16, + "successful_requests": 17869, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 594.358, + "successful_latency_ms": { + "p50": 9.511, + "p95": 93.884, + "p99": 174.611 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A4_run2_stats.txt b/evidence/scaling/A4_run2_stats.txt new file mode 100644 index 00000000..782a2c7a --- /dev/null +++ b/evidence/scaling/A4_run2_stats.txt @@ -0,0 +1,7 @@ +mtusi-lab3-nginx-1 16.58% 10.88MiB / 64MiB +mtusi-lab3-api-2-1 24.09% 74.36MiB / 128MiB +mtusi-lab3-api-1-1 11.84% 63.49MiB / 128MiB +mtusi-lab3-api-3-1 24.96% 76.1MiB / 128MiB +mtusi-lab3-api-4-1 12.84% 63.58MiB / 128MiB +mtusi-lab3-postgres-1 21.19% 233.9MiB / 512MiB +mtusi-lab3-valkey-1 1.24% 12.81MiB / 128MiB diff --git a/evidence/scaling/A4_run3.json b/evidence/scaling/A4_run3.json new file mode 100644 index 00000000..ab2ba501 --- /dev/null +++ b/evidence/scaling/A4_run3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.131, + "concurrency": 16, + "successful_requests": 18533, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 615.075, + "successful_latency_ms": { + "p50": 9.242, + "p95": 92.912, + "p99": 167.713 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/A4_run3_stats.txt b/evidence/scaling/A4_run3_stats.txt new file mode 100644 index 00000000..a7a0d0b8 --- /dev/null +++ b/evidence/scaling/A4_run3_stats.txt @@ -0,0 +1,7 @@ +mtusi-lab3-nginx-1 16.15% 11.08MiB / 64MiB +mtusi-lab3-api-2-1 26.09% 63.18MiB / 128MiB +mtusi-lab3-api-1-1 21.89% 63.21MiB / 128MiB +mtusi-lab3-api-3-1 25.19% 72.58MiB / 128MiB +mtusi-lab3-api-4-1 24.23% 63.88MiB / 128MiB +mtusi-lab3-postgres-1 36.57% 252.6MiB / 512MiB +mtusi-lab3-valkey-1 2.19% 12.38MiB / 128MiB diff --git a/evidence/scaling/B1_limits.txt b/evidence/scaling/B1_limits.txt new file mode 100644 index 00000000..84e7107d --- /dev/null +++ b/evidence/scaling/B1_limits.txt @@ -0,0 +1,2 @@ +config=B1 replicas=1 api_cpus=0.50 api_mem=256M concurrency=16 duration=30 +api-1 NanoCpus=500000000 Memory=268435456 diff --git a/evidence/scaling/B1_run1.json b/evidence/scaling/B1_run1.json new file mode 100644 index 00000000..8be4fa4d --- /dev/null +++ b/evidence/scaling/B1_run1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.168, + "concurrency": 16, + "successful_requests": 12482, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 413.753, + "successful_latency_ms": { + "p50": 15.779, + "p95": 97.028, + "p99": 296.954 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/B1_run1_stats.txt b/evidence/scaling/B1_run1_stats.txt new file mode 100644 index 00000000..5ca100a5 --- /dev/null +++ b/evidence/scaling/B1_run1_stats.txt @@ -0,0 +1,4 @@ +mtusi-lab3-nginx-1 9.25% 10.25MiB / 64MiB +mtusi-lab3-api-1-1 50.06% 64MiB / 256MiB +mtusi-lab3-postgres-1 13.37% 177.7MiB / 512MiB +mtusi-lab3-valkey-1 2.17% 12.04MiB / 128MiB diff --git a/evidence/scaling/B1_run2.json b/evidence/scaling/B1_run2.json new file mode 100644 index 00000000..d1724bbf --- /dev/null +++ b/evidence/scaling/B1_run2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.068, + "concurrency": 16, + "successful_requests": 2946, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 97.979, + "successful_latency_ms": { + "p50": 96.015, + "p95": 601.496, + "p99": 898.354 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/B1_run2_stats.txt b/evidence/scaling/B1_run2_stats.txt new file mode 100644 index 00000000..f895bbc3 --- /dev/null +++ b/evidence/scaling/B1_run2_stats.txt @@ -0,0 +1,4 @@ +mtusi-lab3-nginx-1 24.01% 10.98MiB / 64MiB +mtusi-lab3-api-1-1 12.34% 64.64MiB / 256MiB +mtusi-lab3-postgres-1 7.14% 178.9MiB / 512MiB +mtusi-lab3-valkey-1 0.57% 12.72MiB / 128MiB diff --git a/evidence/scaling/B1_run3.json b/evidence/scaling/B1_run3.json new file mode 100644 index 00000000..d5b5d284 --- /dev/null +++ b/evidence/scaling/B1_run3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.485, + "concurrency": 16, + "successful_requests": 14214, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 466.261, + "successful_latency_ms": { + "p50": 14.117, + "p95": 94.171, + "p99": 196.651 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/B1_run3_stats.txt b/evidence/scaling/B1_run3_stats.txt new file mode 100644 index 00000000..83b094e7 --- /dev/null +++ b/evidence/scaling/B1_run3_stats.txt @@ -0,0 +1,4 @@ +mtusi-lab3-nginx-1 6.65% 10.91MiB / 64MiB +mtusi-lab3-api-1-1 50.15% 64.57MiB / 256MiB +mtusi-lab3-postgres-1 15.69% 180MiB / 512MiB +mtusi-lab3-valkey-1 1.93% 12.37MiB / 128MiB diff --git a/evidence/scaling/B4_limits.txt b/evidence/scaling/B4_limits.txt new file mode 100644 index 00000000..b5123ef8 --- /dev/null +++ b/evidence/scaling/B4_limits.txt @@ -0,0 +1,5 @@ +config=B4 replicas=4 api_cpus=0.50 api_mem=256M concurrency=16 duration=30 +api-1 NanoCpus=500000000 Memory=268435456 +api-2 NanoCpus=500000000 Memory=268435456 +api-3 NanoCpus=500000000 Memory=268435456 +api-4 NanoCpus=500000000 Memory=268435456 diff --git a/evidence/scaling/B4_run1.json b/evidence/scaling/B4_run1.json new file mode 100644 index 00000000..2e4fc6b6 --- /dev/null +++ b/evidence/scaling/B4_run1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.049, + "concurrency": 16, + "successful_requests": 30483, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1014.431, + "successful_latency_ms": { + "p50": 10.888, + "p95": 46.73, + "p99": 74.057 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/B4_run1_stats.txt b/evidence/scaling/B4_run1_stats.txt new file mode 100644 index 00000000..ba8010df --- /dev/null +++ b/evidence/scaling/B4_run1_stats.txt @@ -0,0 +1,7 @@ +mtusi-lab3-nginx-1 20.00% 10.18MiB / 64MiB +mtusi-lab3-api-1-1 41.68% 72.96MiB / 256MiB +mtusi-lab3-api-4-1 49.10% 63.3MiB / 256MiB +mtusi-lab3-api-2-1 40.04% 74.45MiB / 256MiB +mtusi-lab3-api-3-1 45.58% 73.81MiB / 256MiB +mtusi-lab3-postgres-1 48.32% 246.8MiB / 512MiB +mtusi-lab3-valkey-1 4.01% 12MiB / 128MiB diff --git a/evidence/scaling/B4_run2.json b/evidence/scaling/B4_run2.json new file mode 100644 index 00000000..ec3a1238 --- /dev/null +++ b/evidence/scaling/B4_run2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.095, + "concurrency": 16, + "successful_requests": 29869, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 992.482, + "successful_latency_ms": { + "p50": 10.804, + "p95": 48.199, + "p99": 76.479 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/B4_run2_stats.txt b/evidence/scaling/B4_run2_stats.txt new file mode 100644 index 00000000..e54bbfec --- /dev/null +++ b/evidence/scaling/B4_run2_stats.txt @@ -0,0 +1,7 @@ +mtusi-lab3-nginx-1 25.93% 11.25MiB / 64MiB +mtusi-lab3-api-1-1 37.18% 62.95MiB / 256MiB +mtusi-lab3-api-4-1 38.93% 63.41MiB / 256MiB +mtusi-lab3-api-2-1 37.11% 63.85MiB / 256MiB +mtusi-lab3-api-3-1 37.95% 62.87MiB / 256MiB +mtusi-lab3-postgres-1 59.80% 252.1MiB / 512MiB +mtusi-lab3-valkey-1 4.81% 11.68MiB / 128MiB diff --git a/evidence/scaling/B4_run3.json b/evidence/scaling/B4_run3.json new file mode 100644 index 00000000..3f5f0b21 --- /dev/null +++ b/evidence/scaling/B4_run3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.045, + "concurrency": 16, + "successful_requests": 30576, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1017.679, + "successful_latency_ms": { + "p50": 10.762, + "p95": 47.06, + "p99": 71.198 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/B4_run3_stats.txt b/evidence/scaling/B4_run3_stats.txt new file mode 100644 index 00000000..1c7a8064 --- /dev/null +++ b/evidence/scaling/B4_run3_stats.txt @@ -0,0 +1,7 @@ +mtusi-lab3-nginx-1 25.66% 11.43MiB / 64MiB +mtusi-lab3-api-1-1 39.97% 62.99MiB / 256MiB +mtusi-lab3-api-4-1 43.05% 62.9MiB / 256MiB +mtusi-lab3-api-2-1 38.31% 64.04MiB / 256MiB +mtusi-lab3-api-3-1 42.32% 63.35MiB / 256MiB +mtusi-lab3-postgres-1 44.79% 253.3MiB / 512MiB +mtusi-lab3-valkey-1 3.87% 12.4MiB / 128MiB diff --git a/evidence/scaling/experiment_log.txt b/evidence/scaling/experiment_log.txt new file mode 100644 index 00000000..a661c314 --- /dev/null +++ b/evidence/scaling/experiment_log.txt @@ -0,0 +1,26 @@ +=== A1: 1 x 1.00 CPU / 512M === + run 1: rps=346.466 err=0 p50=13.495 p95=205.76 p99=519.105 + run 2: rps=139.548 err=0 p50=86.111 p95=491.678 p99=697.125 + run 3: rps=472.712 err=0 p50=13.454 p95=108.372 p99=498.509 +=== A2: 2 x 0.50 CPU / 256M === + run 1: rps=811.047 err=0 p50=11.499 p95=63.911 p99=86.902 + run 2: rps=281.951 err=0 p50=32.217 p95=112.993 p99=297.73 + run 3: rps=783.804 err=0 p50=11.682 p95=66.159 p99=87.785 +=== A4: 4 x 0.25 CPU / 128M === + run 1: rps=620.172 err=0 p50=10.376 p95=89.425 p99=105.535 + run 2: rps=594.358 err=0 p50=9.511 p95=93.884 p99=174.611 + run 3: rps=615.075 err=0 p50=9.242 p95=92.912 p99=167.713 +=== B1: 1 x 0.50 CPU / 256M === + run 1: rps=413.753 err=0 p50=15.779 p95=97.028 p99=296.954 + run 2: rps=97.979 err=0 p50=96.015 p95=601.496 p99=898.354 + run 3: rps=466.261 err=0 p50=14.117 p95=94.171 p99=196.651 +=== B4: 4 x 0.50 CPU / 256M === + run 1: rps=1014.431 err=0 p50=10.888 p95=46.73 p99=74.057 + run 2: rps=992.482 err=0 p50=10.804 p95=48.199 p99=76.479 + run 3: rps=1017.679 err=0 p50=10.762 p95=47.06 p99=71.198 +A1 1x1.00 CPU rps=346.5 [139.5..472.7] p50=13.5 p95=205.8 p99=519.1 errors=0 +A2 2x0.50 CPU rps=783.8 [282.0..811.0] p50=11.7 p95=66.2 p99=87.8 errors=0 +A4 4x0.25 CPU rps=615.1 [594.4..620.2] p50=9.5 p95=92.9 p99=167.7 errors=0 +B1 1x0.50 CPU rps=413.8 [98.0..466.3] p50=15.8 p95=97.0 p99=297.0 errors=0 +B4 4x0.50 CPU rps=1014.4 [992.5..1017.7] p50=10.8 p95=47.1 p99=74.1 errors=0 +wrote evidence/scaling/summary.csv evidence/scaling/rps.svg diff --git a/evidence/scaling/keepalive/A2_limits.txt b/evidence/scaling/keepalive/A2_limits.txt new file mode 100644 index 00000000..b9afe11e --- /dev/null +++ b/evidence/scaling/keepalive/A2_limits.txt @@ -0,0 +1,3 @@ +config=A2 replicas=2 api_cpus=0.50 api_mem=256M concurrency=16 duration=30 +api-1 NanoCpus=500000000 Memory=268435456 +api-2 NanoCpus=500000000 Memory=268435456 diff --git a/evidence/scaling/keepalive/A2_nginx_problems.txt b/evidence/scaling/keepalive/A2_nginx_problems.txt new file mode 100644 index 00000000..e69de29b diff --git a/evidence/scaling/keepalive/A2_run1.json b/evidence/scaling/keepalive/A2_run1.json new file mode 100644 index 00000000..8fe66d4c --- /dev/null +++ b/evidence/scaling/keepalive/A2_run1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.064, + "concurrency": 16, + "successful_requests": 30683, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1020.576, + "successful_latency_ms": { + "p50": 11.476, + "p95": 44.082, + "p99": 73.405 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/keepalive/A2_run1_stats.txt b/evidence/scaling/keepalive/A2_run1_stats.txt new file mode 100644 index 00000000..8c8876ca --- /dev/null +++ b/evidence/scaling/keepalive/A2_run1_stats.txt @@ -0,0 +1,5 @@ +mtusi-lab3-nginx-1 15.65% 7.613MiB / 64MiB +mtusi-lab3-api-1-1 50.39% 63.44MiB / 256MiB +mtusi-lab3-api-2-1 48.75% 63.96MiB / 256MiB +mtusi-lab3-postgres-1 40.91% 116.4MiB / 512MiB +mtusi-lab3-valkey-1 1.95% 12.75MiB / 128MiB diff --git a/evidence/scaling/keepalive/A2_run2.json b/evidence/scaling/keepalive/A2_run2.json new file mode 100644 index 00000000..7ac492ff --- /dev/null +++ b/evidence/scaling/keepalive/A2_run2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.059, + "concurrency": 16, + "successful_requests": 28447, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 946.385, + "successful_latency_ms": { + "p50": 12.05, + "p95": 48.707, + "p99": 78.884 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/keepalive/A2_run2_stats.txt b/evidence/scaling/keepalive/A2_run2_stats.txt new file mode 100644 index 00000000..b3467b2c --- /dev/null +++ b/evidence/scaling/keepalive/A2_run2_stats.txt @@ -0,0 +1,5 @@ +mtusi-lab3-nginx-1 18.79% 7.812MiB / 64MiB +mtusi-lab3-api-1-1 47.23% 63.85MiB / 256MiB +mtusi-lab3-api-2-1 50.58% 64.79MiB / 256MiB +mtusi-lab3-postgres-1 48.47% 144MiB / 512MiB +mtusi-lab3-valkey-1 2.36% 12.38MiB / 128MiB diff --git a/evidence/scaling/keepalive/A2_run3.json b/evidence/scaling/keepalive/A2_run3.json new file mode 100644 index 00000000..6e13454e --- /dev/null +++ b/evidence/scaling/keepalive/A2_run3.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.07, + "concurrency": 16, + "successful_requests": 31667, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 1053.114, + "successful_latency_ms": { + "p50": 11.35, + "p95": 41.956, + "p99": 71.286 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scaling/keepalive/A2_run3_stats.txt b/evidence/scaling/keepalive/A2_run3_stats.txt new file mode 100644 index 00000000..d2c0c64c --- /dev/null +++ b/evidence/scaling/keepalive/A2_run3_stats.txt @@ -0,0 +1,5 @@ +mtusi-lab3-nginx-1 15.00% 8.219MiB / 64MiB +mtusi-lab3-api-1-1 51.30% 76.93MiB / 256MiB +mtusi-lab3-api-2-1 50.54% 76.83MiB / 256MiB +mtusi-lab3-postgres-1 38.70% 170.6MiB / 512MiB +mtusi-lab3-valkey-1 1.88% 13.06MiB / 128MiB diff --git a/evidence/scaling/keepalive/B4_limits.txt b/evidence/scaling/keepalive/B4_limits.txt new file mode 100644 index 00000000..b5123ef8 --- /dev/null +++ b/evidence/scaling/keepalive/B4_limits.txt @@ -0,0 +1,5 @@ +config=B4 replicas=4 api_cpus=0.50 api_mem=256M concurrency=16 duration=30 +api-1 NanoCpus=500000000 Memory=268435456 +api-2 NanoCpus=500000000 Memory=268435456 +api-3 NanoCpus=500000000 Memory=268435456 +api-4 NanoCpus=500000000 Memory=268435456 diff --git a/evidence/scaling/keepalive/B4_nginx_problems.txt b/evidence/scaling/keepalive/B4_nginx_problems.txt new file mode 100644 index 00000000..e69de29b diff --git a/evidence/scaling/keepalive/B4_recheck.txt b/evidence/scaling/keepalive/B4_recheck.txt new file mode 100644 index 00000000..1a72e904 --- /dev/null +++ b/evidence/scaling/keepalive/B4_recheck.txt @@ -0,0 +1,9 @@ +B4 (4 x 0.50 CPU / 256M, keepalive) recheck after run1 showed 319 failed requests. +3 more runs, 30 s, concurrency 16, nginx/api logs captured live: + run 1: successful_rps=1118.0 failed_requests=0 + run 2: successful_rps=1074.2 failed_requests=0 + run 3: successful_rps=1027.3 failed_requests=0 +nginx 4xx/5xx lines: 0; nginx [error]/[warn]: 0; api 4xx/5xx: 0 +postgres log (FATAL/ERROR/too many clients) for the whole session: 0 +Cause of the 319 errors in that earlier run (raw JSON superseded by the final measurement) not identified: nginx/api containers were recreated +by the script afterwards and their logs were lost. Script now saves