ЛР 3 - БПИ-2303 - Унру - на 3 #145

Closed
DeUnru wants to merge 2 commits from DeUnru/mtusi2026lab3:main into main
53 changed files with 3233 additions and 25 deletions
+2
View File
@@ -18,3 +18,5 @@ __pycache__
node_modules
evidence
.DS_Store
.superpowers
.pytest_cache
+7 -12
View File
@@ -1,13 +1,8 @@
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
# These are logical interface suggestions; map actual framework names in REPORT.md.
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
# bootstrap.sh creates a private local .env and secrets/; never commit them.
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
CACHE_URL_FILE=/run/secrets/cache_url
API_REPLICAS=2
IDEMPOTENCY_TTL_SECONDS=86400
# API config in Compose:
# DATABASE_URL_FILE=/run/secrets/database_url
# CACHE_URL_FILE=/run/secrets/cache_url
# POOL_MAX_SIZE=8 per API process
# No API host ports: only lb publishes 127.0.0.1:8080.
# Redis caches immutable User profiles; PostgreSQL owns sessions + idempotency.
+19
View File
@@ -0,0 +1,19 @@
name: Application acceptance
on: [push, pull_request]
permissions:
contents: read
jobs:
cluster:
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.13'
- run: make up
- run: make check test test-student
- run: make test-fresh
- run: make test-cluster
- if: always()
run: docker compose down
+1
View File
@@ -30,3 +30,4 @@ coverage/
.vscode/
# Keep small, sanitized evidence summaries in evidence/ under version control.
# Inspect git diff --cached: ignore rules are not a secret scanner.
.pytest_cache/
+18
View File
@@ -0,0 +1,18 @@
FROM python:3.13.5-slim
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
WORKDIR /app
COPY requirements.txt requirements-dev.txt requirements.lock ./
RUN pip install --no-cache-dir -r requirements.lock && useradd --uid 10001 --create-home messenger
COPY src/ src/
COPY migrations/ migrations/
COPY student-tests/ student-tests/
COPY scripts/ scripts/
COPY tests/ tests/
COPY contracts/ contracts/
COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./
ARG APP_RELEASE=lab3-v1
ENV APP_RELEASE=${APP_RELEASE}
LABEL org.opencontainers.image.version=${APP_RELEASE}
USER messenger
EXPOSE 8080
CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"]
+31
View File
@@ -28,3 +28,34 @@ test-persistence:
test-failover:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
.PHONY: bootstrap up down test-student test-cluster rolling
bootstrap:
./scripts/bootstrap.sh
up: bootstrap
docker compose build api
docker compose up -d --wait --wait-timeout 120
$(PYTHON) scripts/cluster.py wait
down:
docker compose down
test-student:
$(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
test-cluster:
$(PYTHON) scripts/verify_cluster.py
rolling:
$(PYTHON) scripts/cluster.py rolling
.PHONY: test-fresh load-evidence verify-persistence verify-failover
test-fresh:
./scripts/check_fresh_migrations.sh
load-evidence:
$(PYTHON) scripts/capture_load.py
verify-persistence:
$(PYTHON) scripts/verify_cluster.py persistence
verify-failover:
$(PYTHON) scripts/verify_cluster.py public-failover
$(PYTHON) scripts/verify_cluster.py failover
.PHONY: validate-container
validate-container:
$(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py
+127 -13
View File
@@ -1,37 +1,151 @@
# Отчёт — лабораторная 3
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Автор, группа: Унру Д.В. БПИ2303
- Целевая оценка; выбранный трек 5, если нужен: **3** (трек 5 не заявлен)
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, pytest; документация [Nginx load balancing](https://nginx.org/en/docs/http/load_balancing.html), [upstream/resolve](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) и [proxy_next_upstream](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/), [PostgreSQL transactions](https://www.postgresql.org/docs/17/tutorial-transactions.html).
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
```sh
git clone <fork-url> && cd <каталог-fork-лабораторной-3>
make up # secrets/.env → build API → migrate → api×2 → LB → дождаться 2 alias
curl -fsS http://localhost:8080/health/ready
docker compose ps # две API-реплики, postgres, redis, lb — healthy; migrate — Exited(0)
make check
make validate-container # полный валидатор OpenAPI без локальных pip-зависимостей
make test # публичный smoke через LB
make test-student # собственные интеграционные тесты: 14
make test-fresh # обе миграции и 14 тестов на отдельных пустых volumes
make test-cluster # все распределённые проверки, включая реальные отказы и rolling
make load-evidence # отдельные 30 с нагрузки и CPU/RAM
```
Эквивалент `make up`: `./scripts/bootstrap.sh`, `docker compose build api`, `docker compose up -d --wait --wait-timeout 120`, `python3 scripts/cluster.py wait`. Образ собирается до запуска migrate, поэтому готовый сервер из registry не требуется. Хосту доступен только LB `127.0.0.1:8080`; API и хранилища напрямую не опубликованы.
Отдельные публичные проверки:
```sh
make test-persistence ACTION_SCRIPT=scripts/restart.sh
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
# Публичный failover намеренно оставляет одну выбранную реплику остановленной.
# Восстановление:
docker compose up -d --no-recreate --wait --wait-timeout 90
python3 scripts/cluster.py wait
```
`make verify-persistence` дополнительно сохраняет снимки реальных контейнеров до/после. `make verify-failover` выполняет публичный failover и последовательные проверки каждого исходного контейнера, с восстановлением между ними. Полный `make test-cluster` последовательно запускает concurrent replay, persistence, публичный и собственные failover, SIGTERM, rolling, отказы Redis/БД и аудит инфраструктуры. Он меняет состояние стенда и собирает новый локальный образ с новым идентификатором релиза; не запускать параллельно с другими проверками.
`bootstrap.sh` создаёт случайный пароль PostgreSQL и файлы secrets один раз; существующие значения не меняет. Каталог `secrets/` — 0700 на хосте, файлы доступны uid 10001 контейнера через Compose secrets; `.env` и secrets исключены из Git и Docker build context. Для lab3 созданы собственные secrets/volumes, данные и credentials lab2 не копировались.
Безопасная остановка: `make down` — контейнеры и сети удаляются, именованные volumes сохраняются. После `make up` доступны прежние данные, действующие сессии и результаты идемпотентности. Намеренное удаление всех учебных данных: `docker compose down --volumes`. Hook `restart.sh` volumes не удаляет. `make test-fresh` удаляет только созданные им временные volumes отдельного проекта. Синтетические данные обычных тестов остаются в основной БД.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```text
┌──► API-процесс A ──┐
клиент ──HTTP──► Nginx ──┤ ├──► PostgreSQL: данные, сессии, результаты POST
127.0.0.1:8080 └──► API-процесс B ──┤
└──► Redis: кэш публичного профиля
migrate ──► PostgreSQL (один отдельный job до запуска API)
```
- **Перенос из lab2.** Перенесены исходники FastAPI, миграция 001, Argon2id/Basic/Bearer, users/chats/messages, keyset-пагинация, Redis cache-aside профиля, ограниченные пулы и собственные проверки. Отправка сообщений и тесты адаптированы к обязательному Idempotency-Key. Dockerfile/Compose изменены для отдельных миграций, нескольких API и LB. Публичные `tests/`, `contracts/`, README и COURSE сохранены из skeleton lab3.
- **Общее состояние.** PostgreSQL — источник истины для пользователей, чатов, участников, сообщений, сессий и результатов POST. API не держит в локальной памяти данных, от которых зависит корректность. Каждое защищённое обращение проверяет сессию в БД; X-User-Id игнорируется. Разные реплики обслуживают один Bearer без повторного входа. Sticky session не используется.
- **Идентификатор процесса.** `X-Instance-Id` — случайный UUID alias, стабильный для жизни конкретного процесса; добавляется ко всем его ответам, включая health/ошибки. После restart alias меняется. `X-App-Release` — дополнительный заголовок для различения сборок в rolling-эксперименте. Ошибка, сформированная самим LB при недоступных backend, может не иметь API-заголовков и JSON.
- **Общая идемпотентность** (`src/messenger/messages.py`, миграция 002). Таблица `idempotency_records`: PK(user_id,chat_id,key), request_hash, message_id FK, expires_at. Таблица обслуживает только POST сообщений, поэтому эта область соответствует user+method+chat path+key. Ключ строго ASCII `[A-Za-z0-9._:-]{1, 128}`, иначе 400. Валидированный JSON канонизируется независимо от форматирования/порядка полей, fingerprint — SHA-256. В схеме lab3 допустим только text; attachment_ids появится по контракту lab4.
- **Атомарность повтора.** После проверки сессии и членства одна транзакция выполняет INSERT ключа ON CONFLICT DO NOTHING и SELECT FOR UPDATE. Конкурирующие процессы сериализуются на одной записи. При действующем ключе тот же fingerprint возвращает тот же Message и 201, другой — 409. Для нового/истёкшего ключа INSERT сообщения и обновление результата фиксируются одним COMMIT. Исключение откатывает оба эффекта, транзиентный 5xx не сохраняется как результат. Replay снова проверяет авторизацию. Срок ключа — 24 ч; после истечения разрешена новая запись с тем же ключом, старое сообщение не удаляется.
- **Миграции.** `migrate` — отдельный одноразовый service, API зависит от `service_completed_successfully`. В API CMD мигратора нет. SQL-файлы 001/002, checksum и advisory lock сохраняют повторяемость и сериализуют возможные ручные запуски. Новая 002 только добавляет таблицу/индекс, не меняет старые поля lab2. Перед rolling один раз выполняется `docker compose run --rm --no-deps migrate`, пока старые API продолжают работу.
- **Балансировка и DNS.** Nginx OSS 1.28: `upstream messenger` с общей `zone messenger 64k` и `server api:8080 resolve`. Все A-записи одного Compose-service api участвуют в round-robin; `resolver 127.0.0.11 valid=1s ipv6=off` обновляет IP без restart/reload Nginx. Адреса не зашиты в конфигурации.
- **Пассивные проверки upstream.** `max_fails=1 fail_timeout=2s`: первая ошибка подключения/таймаут/503 исключает адрес на 2 с, затем допускается новая попытка. Активного `health_check` Nginx Plus здесь нет. Docker healthcheck каждого API раз в 2 с вызывает его readiness; healthcheck LB раз в 1 с делает GET readiness через Nginx и создаёт фоновый трафик даже без клиента. При двух backend, работающем Docker DNS и настроенных таймаутах консервативный бюджет обнаружения — до 8 с; фактические чтения/записи после отказа проверяются в пределах 10 с. В отличие от активного мониторинга, пассивное исключение происходит только после обращения к backend.
- **Таймауты и повтор.** Nginx connect 1 с, read/send 5 с; `proxy_next_upstream error timeout http_503`, не более 2 upstream-попыток и 6 с на переключение. GET может уйти на другую реплику при ошибке. Параметр `non_idempotent` не включён: уже отправленный POST автоматически не пересылается; при connect error до отправки запроса допустим безопасный выбор другого адреса. Неопределённую отправку сообщения повторяет клиент с прежним Idempotency-Key.
- **Readiness и graceful.** `/health/live` проверяет процесс; `/health/ready` требует доступную PostgreSQL и таблицу новой схемы. Несовместимая схема/ошибка зависимости →503. Redis — некритичный кэш с fallback, его отказ readiness не меняет. Uvicorn работает PID1, один worker на контейнер, graceful timeout5 с. Docker stop_grace_period10 с; hook stop-one использует 7 с, укладываясь в публичный бюджет 10 с. SIGTERM закрывает listening socket и даёт активным запросам завершиться.
- **Остановка выбранной реплики.** `scripts/stop-one.sh` — короткий wrapper Python-оркестратора `cluster.py`. Тот сопоставляет TARGET_INSTANCE_ID с реальным контейнером внутренним GET /health/live, останавливает ровно его, проверяет exit 0 и оставляет остановленным. Идентификаторы контейнеров/процессов и время записываются без credentials. `restart.sh` перезапускает все API, PostgreSQL, Redis и LB с прежними volumes.
- **Поочерёдная замена.** Compose не предоставляет здесь автоматический Swarm rolling update. Проверен эквивалент start-first: временно scale api=3 с `--no-recreate`, дождаться новой healthy-реплики и трёх alias через LB, затем stop/remove одного старого контейнера и возврат к 2. Второй старый контейнер в этом шаге не пересоздаётся. Создание до удаления старого гарантирует другой IP; оба адреса сохраняются. Шаг повторяется для второго исходного контейнера.
- **Пароли, сессии, кэш.** Argon2id (19456 KiB, time_cost2, parallelism1), opaque token256 бит, в БД только SHA-256 токена и абсолютный expiry; logout удаляет сессию. Redis хранит только User по `user:<UUID>` на 60 с после проверки сессии. Профиль неизменяем в обязательном API, инвалидация по событию не нужна. История читается из PostgreSQL и не бывает устаревшей из-за кэша.
- **Запросы/ошибки.** SQL параметризован; каждый API-пул min 1/max 8, очередь 16, ожидание 2 с, connect_timeout3 с, statement_timeout3 с, lock_timeout2 с. Redis-пул 8, connect/read по 0.2 с без retry. JSON строго UTF-8, неверные типы/поля/формат →400, неподдерживаемый media type →415; чужие чаты →404. X-Request-Id совпадает с request_id ошибки. Access-лог Uvicorn отключён, тела/токены/SQL в логи не выводятся.
Конфигурация и ресурсы:
| Переменная / service | Значение | Использование |
| --- | --- | --- |
| `SESSION_TTL_SECONDS` (.env) | 3600 по умолчанию | абсолютный срок сессии, в собственном тесте expiry — 2 с |
| `DATABASE_URL_FILE` | `/run/secrets/database_url` | API и migrate читают адрес/credentials БД |
| `CACHE_URL_FILE` | `/run/secrets/cache_url` | только API читает адрес Redis |
| `POOL_MAX_SIZE` | 8 на API-процесс | ограничение числа соединений |
| `api` | `mtusi-lab3-api:lab3`; каждая 0.75 CPU / 384 MiB | один scale-able service, штатно 2 реплики, без container_name и host ports; сети front+data |
| `lb` | `nginx:1.28.0-alpine`; 0.5 CPU / 128 MiB | единственный публичный порт 127.0.0.1:8080; только front, без DB secrets |
| `postgres` | `postgres:17.5-bookworm`; 1 CPU / 512 MiB | только internal data; volume postgres_data; pg_isready |
| `redis` | `redis:7.4.4-bookworm`; 0.25 CPU / 128 MiB | только internal data; volume redis_data; AOF; maxmemory64 MiB/allkeys-lru |
| `migrate` | тот же образ API; 0.5 CPU / 256 MiB | только data, только database_url secret; Exited(0) после применения |
Dockerfile: `python:3.13.5-slim`, полностью закреплённые зависимости в `requirements.lock`, пользователь messenger uid 10001. Установка зависимостей предшествует копированию исходников; ARG APP_RELEASE расположен после этих слоёв, поэтому смена только метки релиза не переустанавливает библиотеки. CMD — exec Uvicorn. Healthchecks задаются Compose; `.dockerignore` исключает Git, secrets/.env, сертификаты, volumes и evidence. Публичные и собственные проверки включены в образ для запуска без pip на хосте. Номера контейнеров после замен могут иметь пропуски: `cluster.py exec` выбирает реально живой контейнер, не предполагает существование api-1.
## Доказательства критериев
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Nginx, конфигурация и две реплики | `verify_cluster.py nginx` | `nginx -t` PASS, Server:nginx, два alias через LB | `evidence/nginx.json` |
| Контракт и публичный smoke (3) | `make check`, `make validate-container`, `make test` | PASS, lab3 public API scenario через LB | `evidence/check.txt`, `openapi.txt`, `smoke.txt` |
| ≥2 API, один LB, общая БД и сессии (3) | `cluster.py wait`; concurrent replay; infrastructure | один Bearer работает на разных alias; реально два API-контейнера, порты только у LB | `evidence/concurrent-idempotency.json`, `infrastructure.json` |
| Отказ любой одной реплики, чтение/запись ≤10 с (3) | публичный failover + остановка каждого исходного контейнера | оба сценария PASS; чтение известного сообщения, пока target ещё остановлен; post-stop POST в пределах 10 с | `evidence/public-failover.txt`, `sequential-failover.json` |
| Общая идемпотентность и отсутствие дублей (3) | 8 параллельных повторов через 2 alias | все 201 с одним Message, в истории 1 сообщение; другой body409 | `evidence/concurrent-idempotency.json` |
| Сохранность и прежняя сессия (3) | `make verify-persistence` | API/БД/Redis/LB действительно перезапущены; volumes те же; прежний токен и данные доступны | `evidence/persistence.txt`, `persistence.json` |
| Обязательные key/alias, TTL 24 ч, scope, rollback, повторная авторизация | `make test-student` | 14/14 PASS, включая отказ после INSERT и rollback ключа+сообщения | `evidence/student-tests.txt` |
| Миграции отдельно, с нуля и повторяемо | `make test-fresh`; migration test | применены 001/002, повтор без дублей, 14/14 PASS на новых volumes | `evidence/fresh-migrations.txt` |
| Health/readiness/timeouts/SIGTERM (4) | Nginx config; test_ready_requires_compatible_schema; SIGTERM с активным POST | схема недоступна→503; активный POST 201, новые TCP отвергнуты пока процесс ещё жив, exit 0 за 1.9171 с | `evidence/student-tests.txt`, `graceful-shutdown.json` |
| Последовательно остановлена каждая реплика, RPS/errors/recovery (4) | `verify_cluster.py failover` | два разных исходных container id; предыдущий восстановлен; подтверждения сохранены без дублей | `evidence/sequential-failover.json` |
| CPU/RAM, нет API host ports/container_name (4) | `verify_cluster.py infra`; реальное scale 2→3→2 | лимиты есть у всех сервисов, единственный public endpoint у LB, api масштабируется | `evidence/infrastructure.json`, `rolling-update.json` |
| Rolling-equivalent под нагрузкой, отдельные совместимые миграции (4) | `verify_cluster.py rolling` | два старых контейнера заменены новым образом, оба новых IP обнаружены LB; 14 975 подтверждённых сообщений сохранены | `evidence/rolling-update.json` |
| Предыдущая авторизация и отказ зависимостей сохранены | `verify_cluster.py dependencies`; собственные auth/expiry tests | Redis fallback200; PostgreSQL 503; после восстановления тот же токен работает | `evidence/dependency-failures.json`, `student-tests.txt` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
Чтобы повторно сохранить вывод: `make test > evidence/smoke.txt`, `make test-student > evidence/student-tests.txt`, `make validate-container > evidence/openapi.txt`, `make test-fresh > evidence/fresh-migrations.txt`. Распределённые сценарии сами обновляют свои JSON/text в evidence. Контракт и публичные tests не ослаблены; source lab2 переносился без них. Secrets, пароли и Bearer-токены в отчёт/evidence не записываются.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
Исследовательский трек 5 не проводился. Во всех сценариях используется реальный PostgreSQL/Redis и два API-процесса за Nginx. Генератор отказов/rolling — closed-loop4 потока; после неопределённого результата повторяет только POST сообщения, с прежним key/body. Подтверждённые ответы сверяются с полной историей, проверяются уникальность UUID и текста. Это проверка конкретных ограниченных отказов одного локального стенда.
В последовательном сценарии перед воздействиями есть 3 с базовой работы, каждое окно остановки — около 8 с, предыдущий контейнер запускается и обнаруживается через LB перед следующим. Время записи считается только по запросу, **начатому после возврата stop hook**, а общий deadline10 с отсчитывается от старта hook. Чтение известного подтверждённого сообщения также выполняется до восстановления target.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| Первый исходный API остановлен | другая реплика читает/пишет, остановленная не обслуживает | 400.056 RPS, 1 ошибочная попытка в окне; GET 200 и прежнее сообщение | POST через 1.3835 с от начала hook /0.0138 с после его завершения; GET через 1.38 с от начала |
| Второй исходный API остановлен после восстановления первого | те же гарантии | 401.166 RPS, 1 ошибочная попытка; GET 200, target оставался stopped | POST через 1.4405 с от начала hook /0.0151 с после завершения; GET через 1.4375 с от начала |
| Оба последовательных отказа | подтверждённые записи не теряются, retry не создаёт дубль | 15 915 подтверждённых сообщений =15 915 в истории | потерь/дублей не обнаружено |
| SIGTERM во время активного POST | прекратить новые соединения, завершить активный запрос | SQL-запрос наблюдался в PgSleep; POST 201, новые TCP получили ConnectionRefusedError ещё при живом процессе; exit 0 | 1.9171 с от SIGTERM / сообщение сохранено |
| Новый образ, поочерёдная start-first замена | адреса обновляются без рестарта LB, другая API продолжает работу | два шага 5.864/5.6574 с; разные IP, 3 alias при старте; во время замены 687.069 RPS, 1 ошибочная попытка (0.0126%) | 14 975 подтверждённых сообщений сохранены, дублей 0 |
| Полный restart API/PG/Redis/LB с volumes | прежний токен читает прежние данные без повторного входа | public persistence PASS, реальные StartedAt изменились | потерь подтверждённого сообщения и сессии не обнаружено |
| Redis остановлен | fallback в PostgreSQL без ослабления авторизации | GET 200 за 0.0138 с, ready 200, аноним 401 | данные/сессии в БД сохранены |
| PostgreSQL остановлен | защищённый запрос 503, процесс остаётся жив | 503; обе внутренние live 200; после старта БД та же сессия работает | ошибки не приводят к доверию по кэшу |
**Как проверялся SIGTERM:** временный BEFORE INSERT trigger задерживает ровно один синтетический текст через pg_sleep(2). После наблюдения активного запроса в pg_stat_activity процесс получает SIGTERM. Проверяется отказ новых TCP, пока процесс ещё работает, затем 201 исходного запроса, exit 0 и чтение результата через surviving API. Trigger/function удаляются в finally, контейнер восстанавливается. В коде приложения тестового endpoint/режима нет.
**Что именно обновлялось:** собран действительно другой образ с отличающейся меткой APP_RELEASE, что видно по image id и X-App-Release. Код бизнес-API и схема совместимы; новой функциональной возможности в этом тестовом релизе не добавлялось. Мигратор запускается один раз перед заменами, не каждым новым API. Проверка доказывает поочерёдную замену сборки и обновление адресов, а не безопасное выполнение произвольной destructive migration.
**Нагрузка без отказов:** `make load-evidence`, 30 с, concurrency 4, две реплики, без автоматических retry, один сохранённый прогон. Перед ним стенд прогрет функциональными сценариями; отдельного измеряемого нагрузочного прогрева нет. 24 559 успешных запросов, 0 ошибок, 814.97 successful RPS; p50/p95/p99=4.03/5.822/47.235 мс. Число сообщений в БД: до запуска — 102 032, после — 126 591. Исходный summary — `evidence/load.json`, параметры и данные — `environment.json`, замеры ресурсов — `resources.json`.
Выборочные пики CPU/RAM за нагрузочный прогон (Docker CPU 100% = одно ядро; это пики отдельных samples, не одновременная сумма):
| Контейнер | CPU peak | Memory peak |
| --- | --- | --- |
| API-процесс 1 | 68.64% | 78.41 MiB |
| API-процесс 2 | 69.43% | 66.61 MiB |
| Nginx | 10.01% | 7.895 MiB |
| PostgreSQL | 42.15% | 99.4 MiB |
| Redis | 0.99% | 11.04 MiB |
Имена конкретных контейнеров и исходные значения — `evidence/resource-summary.json`. Эти цифры не доказывают максимальную пропускную способность и не являются сравнением с lab2: ресурсы/набор данных отличаются. Closed-loop имеет coordinated omission, latency рассчитана только для успешных ответов, клиент и Docker работают на одном ноутбуке. Переходные ошибки во время отказов/замен измерены и учитывают клиентские результаты. Nginx может скрыть connect error до отправки запроса и повторить безопасный GET; уже отправленные POST не повторяются им автоматически.
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
- Один Docker-host; LB, PostgreSQL и Redis — по одному экземпляру. Проверен отказ API-процесса, не узла/диска/БД с failover. Restart с volumes не заменяет backup и не обеспечивает HA.
- Compose-замена — наш явный start-first сценарий с временной третьей репликой, не автоматический rolling-контроллер Swarm. Сам по себе обычный `docker compose up` не является доказанным безопасным rolling update. При обрыве нашего сценария может потребоваться ручное восстановление до двух healthy-реплик.
- В миграциях используются только совместимые аддитивные изменения. Breaking schema change требует expand/contract, миграции данных или отдельного окна; в этой работе не проверен. Advisory lock предотвращает одновременное применение, но сам по себе не делает любую миграцию совместимой.
- При SIGTERM/обнаружении отказа допускаются краткие HTTP/connection errors. Клиент повторяет только отправку сообщения с тем же Idempotency-Key. Создание чата и регистрация не имеют общей идемпотентности и после отправки тела автоматически не повторяются.
- TTL идемпотентности 24 ч и expiry сессии проверяются при запросе; фоновой очистки старых строк пока нет. Истёкший ключ можно использовать заново, это создаст новое сообщение.
- Redis без пароля в изолированной data-сети Compose; сессии всегда проверяются в PostgreSQL. Изменение профиля в будущем потребует инвалидации кэша. В этой работе нет TLS, rate limiting входа, PKI и secret manager.
- Нет общего snapshot между страницами; конкурентная запись с ключом до cursor может остаться позади обхода. Порядок COMMIT не обязан совпадать с порядком created_at.
- Когда БД недоступна и LB исключил обе API-реплики, даже `/health/live` через публичную точку может получить 502/503/504 от LB; внутренний `/health/live` каждого процесса остаётся 200. Это различие готовности маршрута и жизни процесса.
- UI/WebSocket, изменение участников и редактирование/удаление сообщений не входят в обязательный API. Malformed JSON без Bearer может дать 400 до выполнения зависимости аутентификации.
+89
View File
@@ -0,0 +1,89 @@
name: mtusi-lab3
services:
postgres:
image: postgres:17.5-bookworm
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
secrets: [postgres_password]
volumes: [postgres_data:/var/lib/postgresql/data]
networks: [data]
healthcheck:
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
interval: 2s
timeout: 2s
retries: 30
mem_limit: 512m
cpus: 1.0
redis:
image: redis:7.4.4-bookworm
command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru]
volumes: [redis_data:/data]
networks: [data]
healthcheck:
test: [CMD, redis-cli, ping]
interval: 2s
timeout: 2s
retries: 30
mem_limit: 128m
cpus: 0.25
migrate:
image: mtusi-lab3-api:lab3
command: [python, -m, messenger.migrate]
environment:
DATABASE_URL_FILE: /run/secrets/database_url
secrets: [database_url]
networks: [data]
depends_on:
postgres: {condition: service_healthy}
restart: 'no'
mem_limit: 256m
cpus: 0.5
api:
image: mtusi-lab3-api:lab3
build: .
scale: 2
environment:
DATABASE_URL_FILE: /run/secrets/database_url
CACHE_URL_FILE: /run/secrets/cache_url
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
POOL_MAX_SIZE: '8'
secrets: [database_url, cache_url]
networks: [front, data]
depends_on:
migrate: {condition: service_completed_successfully}
redis: {condition: service_healthy}
stop_grace_period: 10s
healthcheck:
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
interval: 2s
timeout: 5s
retries: 30
mem_limit: 384m
cpus: 0.75
lb:
image: nginx:1.28.0-alpine
volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro']
networks: [front]
ports: ['127.0.0.1:8080:8080']
depends_on:
api: {condition: service_healthy}
healthcheck:
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
interval: 1s
timeout: 8s
retries: 30
mem_limit: 128m
cpus: 0.5
networks:
front:
data:
internal: true
secrets:
postgres_password: {file: ./secrets/postgres_password}
database_url: {file: ./secrets/database_url}
cache_url: {file: ./secrets/cache_url}
volumes:
postgres_data:
redis_data:
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/check_contract.py
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
+11
View File
@@ -0,0 +1,11 @@
{
"result": "PASS",
"parallel_requests": 8,
"observed_instances": [
"a125004f5fbb4f9e91edb14bb3e64993",
"a8916b2407ed45cc87d73e88cf1639b9"
],
"responses_201": 8,
"messages_created": 1,
"different_body_status": 409
}
+20
View File
@@ -0,0 +1,20 @@
{
"result": "PASS",
"redis": {
"profile_status": 200,
"fallback_seconds": 0.0138,
"anonymous": 401,
"readiness": 200
},
"postgres": {
"authenticated_status": 503,
"response_seconds": 0.0048,
"anonymous_status": 401,
"internal_api_liveness": [
200,
200
],
"note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable."
},
"same_session_after_recovery": true
}
+39
View File
@@ -0,0 +1,39 @@
{
"captured_at_utc": "2026-11-02T08:38:18.305573+00:00",
"host_os": "macOS-27.2-arm64-arm-64bit-Mach-O",
"host_architecture": "arm64",
"docker": {
"server_version": "29.8.2",
"cpus": 12,
"memory_bytes": 8319238144,
"architecture": "aarch64"
},
"python_in_image": "Python 3.13.5",
"dataset_before": {
"users": 117,
"chats": 53,
"messages": 102032,
"sessions": 102,
"idempotency_records": 102011
},
"limits": {
"api_each": "0.75 CPU / 384 MiB",
"postgres": "1 CPU / 512 MiB",
"redis": "0.25 CPU / 128 MiB",
"lb": "0.5 CPU / 128 MiB"
},
"warmup": "API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.",
"replicas": 2,
"repetitions": 1,
"duration_seconds": 30,
"concurrency": 4,
"host_cpu": "Apple M4 Pro",
"host_memory_bytes": 25769803776,
"dataset_after": {
"users": 120,
"chats": 54,
"messages": 126591,
"sessions": 105,
"idempotency_records": 126570
}
}
+8
View File
@@ -0,0 +1,8 @@
./scripts/check_fresh_migrations.sh
Applied 001_initial.sql
Applied 002_idempotency.sql
Migrations up to date
Migrations up to date
.............. [100%]
14 passed in 3.56s
PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.
+16
View File
@@ -0,0 +1,16 @@
{
"result": "PASS",
"target_container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
"target_instance": "4ffdcca2f8644acd9b76d65f8260eaf7",
"fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards",
"in_flight_observed_in_pg_stat_activity": true,
"in_flight_response": 201,
"new_tcp_after_sigterm": {
"accepted": false,
"error_type": "ConnectionRefusedError"
},
"target_still_running_when_new_tcp_rejected": true,
"exit_code": 0,
"shutdown_seconds_from_sigterm": 1.9171,
"confirmed_message_preserved": true
}
+126
View File
@@ -0,0 +1,126 @@
{
"result": "PASS",
"containers": [
{
"service": "api",
"container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710",
"name": "mtusi-lab3-api-5",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:37:46.879119007Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741",
"memory_bytes": 402653184,
"nano_cpus": 750000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data",
"mtusi-lab3_front"
],
"http_status": 200,
"instance": "0749a2e4af8a4b849e3eae6426945546",
"release": "lab3-9d88e53e"
},
{
"service": "api",
"container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f",
"name": "mtusi-lab3-api-6",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:37:52.762454968Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741",
"memory_bytes": 402653184,
"nano_cpus": 750000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data",
"mtusi-lab3_front"
],
"http_status": 200,
"instance": "dd54c30c2e4c413ea2e28066c45e0d3c",
"release": "lab3-9d88e53e"
},
{
"service": "lb",
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
"name": "mtusi-lab3-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:32.827784792Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab3_front"
]
},
{
"service": "migrate",
"container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43",
"name": "mtusi-lab3-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T08:36:54.173486177Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data"
]
},
{
"service": "postgres",
"container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7",
"name": "mtusi-lab3-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:38:04.389392668Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab3_postgres_data"
],
"networks": [
"mtusi-lab3_data"
]
},
{
"service": "redis",
"container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4",
"name": "mtusi-lab3-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:38:03.185004292Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab3_redis_data"
],
"networks": [
"mtusi-lab3_data"
]
}
]
}
+11
View File
@@ -0,0 +1,11 @@
{
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
"name": "mtusi-lab3-api-4",
"instance": "9ab4e9373a944f05b69d934479430136",
"release": "lab3-9463abd8",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
"stopped_at_utc": "2026-11-02T08:37:19.081389+00:00",
"stop_seconds": 0.4518,
"exit_code": 0,
"left_stopped": true
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.135,
"concurrency": 4,
"successful_requests": 24559,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 814.97,
"successful_latency_ms": {
"p50": 4.03,
"p95": 5.822,
"p99": 47.235
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+17
View File
@@ -0,0 +1,17 @@
{
"lab": 3,
"workload": "closed-loop POST messages; no automatic retries",
"requested_duration_seconds": 30.0,
"elapsed_seconds": 30.135,
"concurrency": 4,
"successful_requests": 24559,
"failed_requests": 0,
"error_ratio": 0.0,
"successful_rps": 814.97,
"successful_latency_ms": {
"p50": 4.03,
"p95": 5.822,
"p99": 47.235
},
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
}
+11
View File
@@ -0,0 +1,11 @@
{
"result": "PASS",
"version": "nginx version: nginx/1.28.0",
"image": "nginx:1.28.0-alpine",
"public_server_header": "nginx",
"config_syntax": "PASS",
"observed_instances": [
"0749a2e4af8a4b849e3eae6426945546",
"dd54c30c2e4c413ea2e28066c45e0d3c"
]
}
+2
View File
@@ -0,0 +1,2 @@
python3 scripts/cluster.py exec python scripts/validate_openapi.py
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
+250
View File
@@ -0,0 +1,250 @@
{
"result": "PASS",
"before": [
{
"service": "api",
"container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
"name": "mtusi-lab3-api-3",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:24:29.880819722Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
"memory_bytes": 402653184,
"nano_cpus": 750000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data",
"mtusi-lab3_front"
],
"http_status": 200,
"instance": "a125004f5fbb4f9e91edb14bb3e64993",
"release": "lab3-9463abd8"
},
{
"service": "api",
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
"name": "mtusi-lab3-api-4",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:24:29.778877972Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
"memory_bytes": 402653184,
"nano_cpus": 750000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data",
"mtusi-lab3_front"
],
"http_status": 200,
"instance": "a8916b2407ed45cc87d73e88cf1639b9",
"release": "lab3-9463abd8"
},
{
"service": "lb",
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
"name": "mtusi-lab3-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:23.133451802Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab3_front"
]
},
{
"service": "migrate",
"container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43",
"name": "mtusi-lab3-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T08:24:32.061652Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data"
]
},
{
"service": "postgres",
"container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7",
"name": "mtusi-lab3-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:24:29.651115763Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab3_postgres_data"
],
"networks": [
"mtusi-lab3_data"
]
},
{
"service": "redis",
"container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4",
"name": "mtusi-lab3-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:24:29.682187472Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab3_redis_data"
],
"networks": [
"mtusi-lab3_data"
]
}
],
"after": [
{
"service": "api",
"container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
"name": "mtusi-lab3-api-3",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:33.199244251Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
"memory_bytes": 402653184,
"nano_cpus": 750000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data",
"mtusi-lab3_front"
],
"http_status": 200,
"instance": "f9c80459570c4c99991024bba6d2b167",
"release": "lab3-9463abd8"
},
{
"service": "api",
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
"name": "mtusi-lab3-api-4",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:33.076891376Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
"memory_bytes": 402653184,
"nano_cpus": 750000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data",
"mtusi-lab3_front"
],
"http_status": 200,
"instance": "782d112163514263a4c14c0fb6d68bfc",
"release": "lab3-9463abd8"
},
{
"service": "lb",
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
"name": "mtusi-lab3-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:32.827784792Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab3_front"
]
},
{
"service": "migrate",
"container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43",
"name": "mtusi-lab3-migrate-1",
"status": "exited",
"health": null,
"started_at": "2026-11-02T08:36:35.37548171Z",
"image": "mtusi-lab3-api:lab3",
"image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9",
"memory_bytes": 268435456,
"nano_cpus": 500000000,
"published_ports": {},
"volumes": [],
"networks": [
"mtusi-lab3_data"
]
},
{
"service": "postgres",
"container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7",
"name": "mtusi-lab3-postgres-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:32.796757626Z",
"image": "postgres:17.5-bookworm",
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
"memory_bytes": 536870912,
"nano_cpus": 1000000000,
"published_ports": {},
"volumes": [
"mtusi-lab3_postgres_data"
],
"networks": [
"mtusi-lab3_data"
]
},
{
"service": "redis",
"container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4",
"name": "mtusi-lab3-redis-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:32.773500709Z",
"image": "redis:7.4.4-bookworm",
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
"memory_bytes": 134217728,
"nano_cpus": 250000000,
"published_ports": {},
"volumes": [
"mtusi-lab3_redis_data"
],
"networks": [
"mtusi-lab3_data"
]
}
],
"public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
}
+2
View File
@@ -0,0 +1,2 @@
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
+15
View File
@@ -0,0 +1,15 @@
{
"result": "PASS",
"public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 132, \"failed_attempts\": 0, \"first_recovery_seconds\": 0.871}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
"stopped": {
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
"name": "mtusi-lab3-api-4",
"instance": "782d112163514263a4c14c0fb6d68bfc",
"release": "lab3-9463abd8",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
"stopped_at_utc": "2026-11-02T08:36:39.262177+00:00",
"stop_seconds": 0.3321,
"exit_code": 0,
"left_stopped": true
}
}
+3
View File
@@ -0,0 +1,3 @@
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 0.871}
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
+25
View File
@@ -0,0 +1,25 @@
{
"samples": 10,
"per_container_peaks": {
"mtusi-lab3-api-5": {
"cpu_percent_peak": 68.64,
"memory_mib_peak": 78.41
},
"mtusi-lab3-api-6": {
"cpu_percent_peak": 69.43,
"memory_mib_peak": 66.61
},
"mtusi-lab3-lb-1": {
"cpu_percent_peak": 10.01,
"memory_mib_peak": 7.895
},
"mtusi-lab3-postgres-1": {
"cpu_percent_peak": 42.15,
"memory_mib_peak": 99.4
},
"mtusi-lab3-redis-1": {
"cpu_percent_peak": 0.99,
"memory_mib_peak": 11.04
}
}
}
+402
View File
@@ -0,0 +1,402 @@
[
{
"at_utc": "2026-11-02T08:38:21.926644+00:00",
"containers": [
{
"CPUPerc": "50.31%",
"MemPerc": "17.21%",
"MemUsage": "66.08MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "8"
},
{
"CPUPerc": "66.02%",
"MemPerc": "17.20%",
"MemUsage": "66.05MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "9"
},
{
"CPUPerc": "9.06%",
"MemPerc": "5.98%",
"MemUsage": "7.652MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "30.22%",
"MemPerc": "15.95%",
"MemUsage": "81.65MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.14%",
"MemPerc": "8.26%",
"MemUsage": "10.58MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:24.960579+00:00",
"containers": [
{
"CPUPerc": "68.64%",
"MemPerc": "17.27%",
"MemUsage": "66.31MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "9"
},
{
"CPUPerc": "53.09%",
"MemPerc": "17.15%",
"MemUsage": "65.86MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "9.49%",
"MemPerc": "5.70%",
"MemUsage": "7.293MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "31.09%",
"MemPerc": "16.17%",
"MemUsage": "82.77MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.91%",
"MemPerc": "8.30%",
"MemUsage": "10.62MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:27.995870+00:00",
"containers": [
{
"CPUPerc": "58.43%",
"MemPerc": "20.27%",
"MemUsage": "77.83MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "10"
},
{
"CPUPerc": "56.00%",
"MemPerc": "17.20%",
"MemUsage": "66.07MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "9.79%",
"MemPerc": "5.76%",
"MemUsage": "7.379MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "33.22%",
"MemPerc": "16.73%",
"MemUsage": "85.67MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.90%",
"MemPerc": "8.13%",
"MemUsage": "10.4MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:31.030777+00:00",
"containers": [
{
"CPUPerc": "52.13%",
"MemPerc": "17.41%",
"MemUsage": "66.87MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "9"
},
{
"CPUPerc": "67.00%",
"MemPerc": "17.21%",
"MemUsage": "66.07MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "9.38%",
"MemPerc": "5.72%",
"MemUsage": "7.316MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "33.30%",
"MemPerc": "17.16%",
"MemUsage": "87.88MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.96%",
"MemPerc": "8.14%",
"MemUsage": "10.41MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:34.047170+00:00",
"containers": [
{
"CPUPerc": "65.51%",
"MemPerc": "17.21%",
"MemUsage": "66.1MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "10"
},
{
"CPUPerc": "52.34%",
"MemPerc": "17.24%",
"MemUsage": "66.21MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "9.16%",
"MemPerc": "5.85%",
"MemUsage": "7.488MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "30.55%",
"MemPerc": "17.40%",
"MemUsage": "89.08MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.93%",
"MemPerc": "8.11%",
"MemUsage": "10.38MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:37.087544+00:00",
"containers": [
{
"CPUPerc": "56.33%",
"MemPerc": "19.86%",
"MemUsage": "76.26MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "11"
},
{
"CPUPerc": "51.99%",
"MemPerc": "17.30%",
"MemUsage": "66.44MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "9.40%",
"MemPerc": "6.09%",
"MemUsage": "7.789MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "33.34%",
"MemPerc": "17.80%",
"MemUsage": "91.15MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.14%",
"MemPerc": "8.48%",
"MemUsage": "10.86MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:40.128204+00:00",
"containers": [
{
"CPUPerc": "49.82%",
"MemPerc": "17.23%",
"MemUsage": "66.16MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "10"
},
{
"CPUPerc": "64.91%",
"MemPerc": "17.32%",
"MemUsage": "66.5MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "9.44%",
"MemPerc": "5.81%",
"MemUsage": "7.438MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "30.24%",
"MemPerc": "18.14%",
"MemUsage": "92.86MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.94%",
"MemPerc": "8.63%",
"MemUsage": "11.04MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:43.146647+00:00",
"containers": [
{
"CPUPerc": "61.56%",
"MemPerc": "20.42%",
"MemUsage": "78.41MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "11"
},
{
"CPUPerc": "56.33%",
"MemPerc": "17.35%",
"MemUsage": "66.61MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "8.46%",
"MemPerc": "6.17%",
"MemUsage": "7.895MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "32.44%",
"MemPerc": "18.46%",
"MemUsage": "94.5MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.17%",
"MemPerc": "8.25%",
"MemUsage": "10.56MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:46.182717+00:00",
"containers": [
{
"CPUPerc": "64.09%",
"MemPerc": "17.39%",
"MemUsage": "66.78MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "9"
},
{
"CPUPerc": "66.93%",
"MemPerc": "17.24%",
"MemUsage": "66.19MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "8.61%",
"MemPerc": "5.89%",
"MemUsage": "7.539MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "42.15%",
"MemPerc": "19.17%",
"MemUsage": "98.16MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.14%",
"MemPerc": "8.27%",
"MemUsage": "10.58MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
},
{
"at_utc": "2026-11-02T08:38:49.213747+00:00",
"containers": [
{
"CPUPerc": "53.84%",
"MemPerc": "17.29%",
"MemUsage": "66.39MiB / 384MiB",
"Name": "mtusi-lab3-api-5",
"PIDs": "9"
},
{
"CPUPerc": "69.43%",
"MemPerc": "17.32%",
"MemUsage": "66.5MiB / 384MiB",
"Name": "mtusi-lab3-api-6",
"PIDs": "10"
},
{
"CPUPerc": "10.01%",
"MemPerc": "6.13%",
"MemUsage": "7.844MiB / 128MiB",
"Name": "mtusi-lab3-lb-1",
"PIDs": "2"
},
{
"CPUPerc": "34.69%",
"MemPerc": "19.41%",
"MemUsage": "99.4MiB / 512MiB",
"Name": "mtusi-lab3-postgres-1",
"PIDs": "16"
},
{
"CPUPerc": "0.99%",
"MemPerc": "8.46%",
"MemUsage": "10.83MiB / 128MiB",
"Name": "mtusi-lab3-redis-1",
"PIDs": "6"
}
]
}
]
+171
View File
@@ -0,0 +1,171 @@
{
"result": "PASS",
"before": [
{
"container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
"name": "mtusi-lab3-api-3",
"instance": "efae54a83fc844f4b19b1e1c2aad0385",
"release": "lab3-9463abd8",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af"
},
{
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
"name": "mtusi-lab3-api-4",
"instance": "1b7fd0264d924b3ba1b87368400b4a56",
"release": "lab3-9463abd8",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af"
}
],
"after": [
{
"container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710",
"name": "mtusi-lab3-api-5",
"instance": "0749a2e4af8a4b849e3eae6426945546",
"release": "lab3-9d88e53e",
"image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741"
},
{
"container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f",
"name": "mtusi-lab3-api-6",
"instance": "dd54c30c2e4c413ea2e28066c45e0d3c",
"release": "lab3-9d88e53e",
"image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741"
}
],
"target_release": "lab3-9d88e53e",
"lb_before": {
"service": "lb",
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
"name": "mtusi-lab3-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:32.827784792Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab3_front"
]
},
"lb_after": {
"service": "lb",
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
"name": "mtusi-lab3-lb-1",
"status": "running",
"health": "healthy",
"started_at": "2026-11-02T08:36:32.827784792Z",
"image": "nginx:1.28.0-alpine",
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
"memory_bytes": 134217728,
"nano_cpus": 500000000,
"published_ports": {
"8080/tcp": [
{
"HostIp": "127.0.0.1",
"HostPort": "8080"
}
]
},
"volumes": [],
"networks": [
"mtusi-lab3_front"
]
},
"lb_not_restarted": true,
"build_seconds": 2.2969,
"migration_output": "Migrations up to date",
"replacements": [
{
"old_container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
"new_container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710",
"old_ip": "172.23.0.4",
"new_ip": "172.23.0.5",
"dns_discovered_new_address": true,
"instances_during_start_first": [
"0749a2e4af8a4b849e3eae6426945546",
"1b7fd0264d924b3ba1b87368400b4a56",
"efae54a83fc844f4b19b1e1c2aad0385"
],
"observed_instances": [
"0749a2e4af8a4b849e3eae6426945546",
"1b7fd0264d924b3ba1b87368400b4a56"
],
"old_exit_code": 0,
"seconds": 5.864
},
{
"old_container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
"new_container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f",
"old_ip": "172.23.0.2",
"new_ip": "172.23.0.4",
"dns_discovered_new_address": true,
"instances_during_start_first": [
"0749a2e4af8a4b849e3eae6426945546",
"1b7fd0264d924b3ba1b87368400b4a56",
"dd54c30c2e4c413ea2e28066c45e0d3c"
],
"observed_instances": [
"0749a2e4af8a4b849e3eae6426945546",
"dd54c30c2e4c413ea2e28066c45e0d3c"
],
"old_exit_code": 0,
"seconds": 5.6574
}
],
"during_replacement": {
"seconds": 11.5214,
"concurrency": 4,
"successful_requests": 7916,
"failed_attempts": 1,
"successful_rps": 687.069,
"error_ratio": 0.000126,
"successful_latency_ms": {
"p50": 3.992,
"p95": 7.701,
"p99": 62.305
},
"instances": [
"0749a2e4af8a4b849e3eae6426945546",
"1b7fd0264d924b3ba1b87368400b4a56",
"dd54c30c2e4c413ea2e28066c45e0d3c",
"efae54a83fc844f4b19b1e1c2aad0385"
]
},
"total": {
"seconds": 20.7201,
"concurrency": 4,
"successful_requests": 14975,
"failed_attempts": 1,
"successful_rps": 722.729,
"error_ratio": 6.7e-05,
"successful_latency_ms": {
"p50": 4.089,
"p95": 7.12,
"p99": 59.058
},
"instances": [
"0749a2e4af8a4b849e3eae6426945546",
"1b7fd0264d924b3ba1b87368400b4a56",
"dd54c30c2e4c413ea2e28066c45e0d3c",
"efae54a83fc844f4b19b1e1c2aad0385"
]
},
"data": {
"confirmed_unique_messages": 14975,
"history_messages": 14975,
"confirmed_preserved": true,
"duplicate_ids": 0,
"duplicate_texts": 0
},
"change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate."
}
+135
View File
@@ -0,0 +1,135 @@
{
"result": "PASS",
"baseline": {
"seconds": 3.0036,
"concurrency": 4,
"successful_requests": 2095,
"failed_attempts": 0,
"successful_rps": 697.497,
"error_ratio": 0.0,
"successful_latency_ms": {
"p50": 4.402,
"p95": 6.728,
"p99": 56.167
},
"instances": [
"9ab4e9373a944f05b69d934479430136",
"f9c80459570c4c99991024bba6d2b167"
]
},
"phases": [
{
"stopped": {
"container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
"name": "mtusi-lab3-api-3",
"instance": "f9c80459570c4c99991024bba6d2b167",
"release": "lab3-9463abd8",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
"stopped_at_utc": "2026-11-02T08:37:05.018221+00:00",
"stop_seconds": 0.3922,
"exit_code": 0,
"left_stopped": true
},
"traffic": {
"seconds": 8.0014,
"concurrency": 4,
"successful_requests": 3201,
"failed_attempts": 1,
"successful_rps": 400.056,
"error_ratio": 0.000312,
"successful_latency_ms": {
"p50": 6.213,
"p95": 37.884,
"p99": 72.594
},
"instances": [
"9ab4e9373a944f05b69d934479430136",
"f9c80459570c4c99991024bba6d2b167"
]
},
"post_stop_write_seconds_from_action_start": 1.3835,
"post_stop_write_seconds_from_stop_completion": 0.0138,
"read_during_outage": {
"status": 200,
"confirmed_message_preserved": true,
"seconds_from_action_start": 1.38,
"seconds_from_stop_completion": 0.0102
},
"new_connection_after_stop": {
"accepted": false,
"error_type": "TimeoutError"
},
"stopped_remains_stopped": true
},
{
"stopped": {
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
"name": "mtusi-lab3-api-4",
"instance": "9ab4e9373a944f05b69d934479430136",
"release": "lab3-9463abd8",
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
"stopped_at_utc": "2026-11-02T08:37:19.081389+00:00",
"stop_seconds": 0.4518,
"exit_code": 0,
"left_stopped": true
},
"traffic": {
"seconds": 8.0017,
"concurrency": 4,
"successful_requests": 3210,
"failed_attempts": 1,
"successful_rps": 401.166,
"error_ratio": 0.000311,
"successful_latency_ms": {
"p50": 6.206,
"p95": 37.612,
"p99": 74.049
},
"instances": [
"4ffdcca2f8644acd9b76d65f8260eaf7",
"9ab4e9373a944f05b69d934479430136"
]
},
"post_stop_write_seconds_from_action_start": 1.4405,
"post_stop_write_seconds_from_stop_completion": 0.0151,
"read_during_outage": {
"status": 200,
"confirmed_message_preserved": true,
"seconds_from_action_start": 1.4375,
"seconds_from_stop_completion": 0.0121
},
"new_connection_after_stop": {
"accepted": false,
"error_type": "TimeoutError"
},
"stopped_remains_stopped": true
}
],
"total": {
"seconds": 31.1217,
"concurrency": 4,
"successful_requests": 15915,
"failed_attempts": 2,
"successful_rps": 511.38,
"error_ratio": 0.000126,
"successful_latency_ms": {
"p50": 5.137,
"p95": 32.066,
"p99": 65.023
},
"instances": [
"1b7fd0264d924b3ba1b87368400b4a56",
"4ffdcca2f8644acd9b76d65f8260eaf7",
"9ab4e9373a944f05b69d934479430136",
"f9c80459570c4c99991024bba6d2b167"
]
},
"data": {
"confirmed_unique_messages": 15915,
"history_messages": 15915,
"confirmed_preserved": true,
"duplicate_ids": 0,
"duplicate_texts": 0
},
"each_original_container_tested": true
}
+2
View File
@@ -0,0 +1,2 @@
python3 tests/smoke.py --base-url "http://localhost:8080"
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
+3
View File
@@ -0,0 +1,3 @@
python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
.............. [100%]
14 passed in 3.58s
+43
View File
@@ -0,0 +1,43 @@
worker_processes 1;
error_log /dev/stderr warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
access_log off;
server_tokens off;
# Docker DNS; resolve updates the upstream when replicas get new IPs.
resolver 127.0.0.11 valid=1s ipv6=off;
resolver_timeout 1s;
upstream messenger {
zone messenger 64k;
# All A records of api participate in the default round-robin balancing.
server api:8080 resolve max_fails=1 fail_timeout=2s;
}
server {
listen 8080;
location / {
proxy_pass http://messenger;
proxy_http_version 1.1;
proxy_set_header Host $http_host;
proxy_set_header Connection "";
proxy_connect_timeout 1s;
proxy_read_timeout 5s;
proxy_send_timeout 5s;
# Passive checks. Do NOT enable non_idempotent: an already-sent POST
# must not be resent by Nginx after an ambiguous response.
proxy_next_upstream error timeout http_503;
proxy_next_upstream_tries 2;
proxy_next_upstream_timeout 6s;
}
}
}
+34
View File
@@ -0,0 +1,34 @@
CREATE TABLE users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
password_hash text NOT NULL,
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chats (
id uuid PRIMARY KEY,
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
);
CREATE TABLE chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id),
PRIMARY KEY (chat_id, user_id)
);
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
CREATE INDEX chats_history_idx ON chats(created_at, id);
CREATE TABLE messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id),
sender_id uuid NOT NULL,
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
);
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
CREATE TABLE sessions (
token_hash text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id),
expires_at timestamptz NOT NULL
);
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
+11
View File
@@ -0,0 +1,11 @@
-- Additive: old lab2 tables and messages remain valid.
CREATE TABLE idempotency_records (
user_id uuid NOT NULL REFERENCES users(id),
chat_id uuid NOT NULL REFERENCES chats(id),
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
request_hash text NOT NULL,
message_id uuid REFERENCES messages(id),
expires_at timestamptz NOT NULL,
PRIMARY KEY (user_id, chat_id, key)
);
CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at);
+45
View File
@@ -0,0 +1,45 @@
annotated-types==0.8.0
anyio==4.9.0
argon2-cffi==25.1.0
argon2-cffi-bindings==26.1.0
attrs==26.1.0
certifi==2026.10.23
cffi==2.1.1
charset-normalizer==3.5.2
click==8.5.0
fastapi==0.115.12
h11==0.16.0
httpcore==1.0.9
httpx==0.28.1
idna==3.20
iniconfig==2.3.1
jsonschema==4.26.0
jsonschema-path==0.3.4
jsonschema-specifications==2025.9.1
lazy-object-proxy==1.13.0
openapi-schema-validator==0.6.3
openapi-spec-validator==0.7.2
packaging==26.3
pathable==0.4.4
pluggy==1.7.0
psycopg==3.2.9
psycopg-binary==3.2.9
psycopg-pool==3.2.6
pycparser==3.12
pydantic==2.11.5
pydantic_core==2.33.2
Pygments==2.21.0
pytest==8.4.0
PyYAML==6.0.3
redis==6.2.0
referencing==0.36.2
requests==2.34.2
rfc3339-validator==0.1.4
rpds-py==2026.9.1
six==1.17.0
sniffio==1.3.1
starlette==0.46.2
typing-inspection==0.4.4
typing_extensions==4.16.0
urllib3==2.8.0
uvicorn==0.34.3
+10
View File
@@ -0,0 +1,10 @@
fastapi==0.115.12
uvicorn==0.34.3
psycopg[binary]==3.2.9
psycopg-pool==3.2.6
redis==6.2.0
argon2-cffi==25.1.0
pytest==8.4.0
httpx==0.28.1
pydantic==2.11.5
anyio==4.9.0
+28
View File
@@ -0,0 +1,28 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
umask 077
mkdir -p secrets
python3 - <<'PY'
from pathlib import Path
import secrets
root = Path('secrets')
p = root / 'postgres_password'
if not p.exists():
p.write_text(secrets.token_hex(24))
pw = p.read_text().strip()
for name, value in {
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
'cache_url': 'redis://redis:6379/0',
}.items():
path = root / name
if not path.exists():
path.write_text(value)
# Bind-mounted Docker secrets must be readable by container uid 10001.
# Parent directory remains 0700 on the host.
for path in root.iterdir():
path.chmod(0o444)
if not Path('.env').exists():
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
print('Local secrets ready (values hidden).')
PY
+71
View File
@@ -0,0 +1,71 @@
#!/usr/bin/env python3
"""Capture repeatable short load run and sanitized environment/resource evidence."""
import datetime
import json
import platform
import re
import subprocess
import sys
import time
import cluster
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
def output(args):
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
def counts():
code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions','idempotency_records']})); c.close()"
return json.loads(output(['docker','exec',cluster.container_ids('api')[0],'python','-c',code]))
def resource_summary(samples):
peaks = {}
for sample in samples:
for container in sample['containers']:
peak = peaks.setdefault(container['Name'], {'cpu_percent_peak':0, 'memory_mib_peak':0})
peak['cpu_percent_peak'] = max(peak['cpu_percent_peak'],float(container['CPUPerc'].rstrip('%')))
value,unit = re.match(r'([0-9.]+)([A-Za-z]+)',container['MemUsage']).groups()
multiplier = {'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/1048576}[unit]
peak['memory_mib_peak'] = round(max(peak['memory_mib_peak'],float(value)*multiplier),3)
return {'samples':len(samples),'per_container_peaks':peaks}
def main():
names = cluster.container_ids()
environment = {
'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),
'host_os': platform.platform(), 'host_architecture': platform.machine(),
'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])),
'python_in_image': output(['docker','exec',cluster.container_ids('api')[0],'python','--version']),
'dataset_before': counts(),
'limits': {'api_each':'0.75 CPU / 384 MiB','postgres':'1 CPU / 512 MiB','redis':'0.25 CPU / 128 MiB','lb':'0.5 CPU / 128 MiB'},
'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.',
'replicas':len(cluster.replicas()), 'repetitions': 1, 'duration_seconds':30, 'concurrency':4,
}
if sys.platform == 'darwin':
environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string'])
environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize']))
samples = []
with (ROOT/'evidence/load-output.txt').open('w') as log:
proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT)
while proc.poll() is None:
rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names])
samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')}
for row in rows.splitlines()]})
time.sleep(1)
if proc.returncode:
raise SystemExit(proc.returncode)
environment['dataset_after'] = counts()
(ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n')
(ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n')
(ROOT/'evidence/resource-summary.json').write_text(json.dumps(resource_summary(samples),indent=2)+'\n')
print((ROOT/'evidence/load.json').read_text())
if __name__ == '__main__':
main()
+16
View File
@@ -0,0 +1,16 @@
#!/bin/sh
# A unique disposable Compose project; never touches normal project volumes.
set -eu
cd "$(dirname "$0")/.."
project="mtusi-lab3-migration-check-$$"
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
echo 'Refusing to reuse a pre-existing scratch project' >&2
exit 1
fi
cleanup() {
docker compose -p "$project" down --volumes >/dev/null 2>&1
}
trap cleanup EXIT INT TERM
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis
docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
import argparse
import datetime
import json
import os
import re
import subprocess
import sys
import time
import urllib.error
import urllib.request
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
def command(args, timeout=120):
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
def compose(*args, timeout=120):
return command(['docker', 'compose', *args], timeout=timeout)
def inspect(containers):
if not containers:
return []
return json.loads(command(['docker','inspect',*containers]))
def container_ids(service=None, all_containers=False):
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
return compose(*args).split()
def probe(container, host='127.0.0.1', path='/health/live'):
code = '''import json,sys,urllib.request,urllib.error
try:
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
except urllib.error.HTTPError as exc:
r=exc
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
'''
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
def replicas():
items = []
for data in inspect(container_ids('api')):
if data['State']['Running']:
result = probe(data['Id'])
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
return items
def snapshot():
items = []
for data in inspect(container_ids(all_containers=True)):
service = data['Config']['Labels']['com.docker.compose.service']
item = {
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
'published_ports':data['HostConfig']['PortBindings'],
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
'networks':sorted(data['NetworkSettings']['Networks']),
}
if service == 'api' and data['State']['Running']:
result = probe(data['Id'])
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
items.append(item)
return items
def wait_healthy(container, timeout=60):
deadline = time.monotonic()+timeout
while time.monotonic() < deadline:
data = inspect([container])[0]
if data['State'].get('Health',{}).get('Status') == 'healthy':
return
time.sleep(0.3)
raise RuntimeError('Container did not become healthy')
def wait_cluster(count=2, timeout=60):
deadline = time.monotonic()+timeout
seen = set()
while time.monotonic() < deadline:
try:
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
alias = response.headers.get('X-Instance-Id')
if alias:
seen.add(alias)
if len(seen) >= count:
return sorted(seen)
except (OSError, urllib.error.URLError):
pass
time.sleep(0.1)
raise RuntimeError('Load balancer did not discover required API instances')
def stop_one(target):
if not re.fullmatch(r'[a-f0-9]{32}', target):
raise ValueError('Invalid target instance alias')
matching = [item for item in replicas() if item['instance']==target]
if len(matching) != 1:
raise RuntimeError('Target must match exactly one running API process')
item = matching[0]
started = time.monotonic()
command(['docker','stop','--time','7',item['container']],timeout=9)
state = inspect([item['container']])[0]['State']
if state['Running'] or state['ExitCode'] != 0:
raise RuntimeError('API did not stop gracefully')
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
return result
def restart():
compose('restart','api','postgres','redis','lb',timeout=30)
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
return wait_cluster(2,timeout=15)
def front_address(container):
networks = inspect([container])[0]['NetworkSettings']['Networks']
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
def replace_one(container):
before = set(container_ids('api'))
if container not in before or len(before) != 2:
raise RuntimeError('Replacement expects exactly two running API containers')
old_address = front_address(container)
# Start-first ensures a genuinely different IP while the old address is still occupied.
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
expanded = set(container_ids('api'))
added = expanded-before
if len(added) != 1 or not before.issubset(expanded):
raise RuntimeError('Start-first step unexpectedly changed an existing container')
replacement = added.pop()
wait_healthy(replacement)
new_address = front_address(replacement)
if new_address == old_address:
raise RuntimeError('Replacement did not get a different IP')
expected_three = {item['instance'] for item in replicas()}
if set(wait_cluster(3)) != expected_three:
raise RuntimeError('Load balancer did not discover all three live addresses')
command(['docker','stop','--time','7',container],timeout=10)
state = inspect([container])[0]['State']
if state['ExitCode'] != 0:
raise RuntimeError('Old release did not terminate gracefully')
command(['docker','rm',container])
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
after = set(container_ids('api'))
if after != (before-{container}) | {replacement}:
raise RuntimeError('Sequential replacement changed the wrong containers')
expected = {item['instance'] for item in replicas()}
if set(wait_cluster(2)) != expected:
raise RuntimeError('Unexpected final backend aliases')
return {'old_container':container,'new_container':replacement,
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
'instances_during_start_first':sorted(expected_three),
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
def main():
parser = argparse.ArgumentParser()
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
parser.add_argument('arguments',nargs=argparse.REMAINDER)
args = parser.parse_args()
if args.action == 'wait':
print(json.dumps({'instances':wait_cluster()}))
elif args.action == 'snapshot':
print(json.dumps(snapshot(),indent=2))
elif args.action == 'stop-one':
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
elif args.action == 'exec':
ids = container_ids('api')
if not ids or not args.arguments:
raise RuntimeError('A live API container and command are required')
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
elif args.action == 'restart':
print(json.dumps({'instances':restart()}))
else:
compose('run','--rm','--no-deps','migrate')
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
if __name__ == '__main__':
main()
+4
View File
@@ -0,0 +1,4 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
exec python3 scripts/cluster.py restart
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
cd "$(dirname "$0")/.."
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
exec python3 scripts/cluster.py stop-one
+443
View File
@@ -0,0 +1,443 @@
#!/usr/bin/env python3
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
import argparse
import json
import math
import os
from pathlib import Path
import subprocess
import sys
import threading
import time
from concurrent.futures import ThreadPoolExecutor
import uuid
import cluster
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'tests'))
from client import Client, Failure, check
from resilience import history
def save(name, value):
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
print('PASS:', name)
def percentiles(values):
values = sorted(values)
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
class Traffic:
def __init__(self, concurrency=4):
self.c = Client(timeout=6)
self.alice,self.bob,_,self.chat = self.c.bootstrap()
self.concurrency = concurrency
self.done = threading.Event()
self.lock = threading.Lock()
self.attempts = []
self.confirmed = {}
self.prefix = uuid.uuid4().hex
def worker(self, number):
client = Client(timeout=6)
index = 0
while not self.done.is_set():
text = self.prefix+'-'+str(number)+'-'+str(index)
key = self.prefix+'-'+str(number)+'-'+str(index)
started = time.monotonic()
try:
response = client.message(self.alice,self.chat,text,key=key)
row = response['json']
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
with self.lock:
self.confirmed[row['id']] = row
index += 1
except Failure:
# Only idempotent message POST is retried, with its original body and key.
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
time.sleep(0.02)
with self.lock:
self.attempts.append(attempt)
def start(self):
self.started = time.monotonic()
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
return self
def stop(self):
self.done.set()
for future in self.futures:
future.result(timeout=10)
self.pool.shutdown()
self.ended = time.monotonic()
def summary(self, start=None, end=None):
start = self.started if start is None else start
end = getattr(self,'ended',time.monotonic()) if end is None else end
with self.lock:
attempts = [a for a in self.attempts if start <= a['started'] < end]
success = [a for a in attempts if a['ok']]
duration = end-start
return {'seconds':round(duration,4),'concurrency':self.concurrency,
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
'successful_rps':round(len(success)/duration,3),
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
'successful_latency_ms':percentiles([a['latency'] for a in success]),
'instances':sorted({a['instance'] for a in success})}
def verify_data(self):
rows = history(self.c,self.bob,self.chat)
actual = {row['id']:row for row in rows}
check(len(actual)==len(rows),'duplicated message ids')
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
def concurrent_replay():
cluster.wait_cluster()
c = Client()
alice,bob,_,chat = c.bootstrap()
key = uuid.uuid4().hex
barrier = threading.Barrier(8)
def send(_):
local = Client()
barrier.wait()
return local.message(alice,chat,'Один результат через разные процессы',key=key)
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(send,range(8)))
instances = {r['headers']['X-Instance-Id'] for r in results}
check(len(instances)>=2,'concurrent retry did not reach different API processes')
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
body={'text':'Другой body'},headers={'Idempotency-Key':key})
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
'responses_201':8,'messages_created':1,'different_body_status':409})
def verify_persistence():
before = cluster.snapshot()
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
after = cluster.snapshot()
roles = ['api','postgres','redis','lb']
previous = {item['container']:item for item in before}
for item in after:
if item['service'] in roles:
old = previous[item['container']]
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
check(old['volumes']==item['volumes'],'volume identities changed')
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
def public_failover():
cluster.wait_cluster()
try:
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
finally:
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
cluster.wait_cluster()
def reject_new_connection(survivor, address):
code = '''import json,socket,sys
try:
s=socket.create_connection((sys.argv[1],8080),timeout=1)
except OSError as exc:
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
else:
s.close()
print(json.dumps({'accepted':True}))
'''
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
def failover_each():
cluster.wait_cluster()
originals = cluster.replicas()
check(len(originals)==2,'two running replicas required')
traffic = Traffic().start()
phases = []
try:
time.sleep(3)
baseline_end = time.monotonic()
baseline = traffic.summary(end=baseline_end)
for original in originals:
# Alias changes on restart, so resolve the current process of this exact container.
current = next(r for r in cluster.replicas() if r['container']==original['container'])
all_before = cluster.inspect([current['container']])[0]
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
start = time.monotonic()
try:
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
stop_returned = time.monotonic()
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
read_deadline = start+10
while True:
try:
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
read_done = time.monotonic()
break
except Failure:
check(time.monotonic()<read_deadline,'reads did not recover within10s')
time.sleep(0.1)
check(read_done-start<=10,'read recovery exceeded10s')
rejected = reject_new_connection(survivor,address)
check(not rejected['accepted'],'new connection accepted by stopped API')
time.sleep(max(0,8-(time.monotonic()-start)))
end = time.monotonic()
summary = traffic.summary(start,end)
with traffic.lock:
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
recent = [a for a in successful if a['started']>stop_returned]
recovery = min(a['ended'] for a in recent)-start if recent else None
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
'seconds_from_action_start':round(read_done-start,4),
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
finally:
cluster.command(['docker','start',current['container']])
cluster.wait_healthy(current['container'])
cluster.wait_cluster()
time.sleep(2)
finally:
traffic.stop()
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
'total':traffic.summary(),'data':traffic.verify_data(),
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
def rolling_update():
cluster.wait_cluster()
before = cluster.replicas()
check(len(before)==2,'rolling demonstration expects two replicas')
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
target_release = 'lab3-'+uuid.uuid4().hex[:8]
traffic = Traffic().start()
replacements = []
try:
time.sleep(3)
build_start = time.monotonic()
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
build_end = time.monotonic()
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
swap_start = time.monotonic()
for old in before:
step_start = time.monotonic()
result = cluster.replace_one(old['container'])
result['seconds'] = round(time.monotonic()-step_start,4)
replacements.append(result)
swap_end = time.monotonic()
time.sleep(3)
finally:
traffic.stop()
after = cluster.replicas()
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
'total':traffic.summary(),'data':traffic.verify_data(),
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
def database_fixture(container, data):
code = '''import json,sys,psycopg
from psycopg import sql
from messenger.db import setting
p=json.load(sys.stdin)
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
if p['action']=='install':
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
result={'installed':True}
elif p['action']=='remove':
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
result={'removed':True}
else:
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
print(json.dumps(result))
'''
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(response.stdout)
def graceful_shutdown():
cluster.wait_cluster()
nodes = cluster.replicas()
target,survivor = nodes[0],nodes[1]
address = cluster.front_address(target['container'])
c = Client()
alice,bob,_,chat = c.bootstrap()
path = '/api/v1/chats/'+chat['id']+'/messages'
marker = 'drain_'+uuid.uuid4().hex
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
database_fixture(survivor['container'],{**fixture,'action':'install'})
code = '''import json,sys,urllib.request,urllib.error
p=json.load(sys.stdin)
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
try:
response=urllib.request.urlopen(r,timeout=6)
except urllib.error.HTTPError as exc:
response=exc
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
'''
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
def in_flight():
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
return json.loads(result.stdout)
pool = ThreadPoolExecutor(max_workers=1)
future = pool.submit(in_flight)
try:
deadline = time.monotonic()+3
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
time.sleep(0.05)
signaled = time.monotonic()
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
deadline = signaled+1.5
while True:
rejected = reject_new_connection(survivor['container'],address)
still_running = cluster.inspect([target['container']])[0]['State']['Running']
if not rejected['accepted']:
break
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
time.sleep(0.05)
check(still_running,'connection refusal observed only after process exit, not during drain')
drained = future.result(timeout=8)
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
check(drained['message']['text']==marker,'in-flight body changed')
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
elapsed = time.monotonic()-signaled
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
'confirmed_message_preserved':True}
finally:
pool.shutdown(wait=True)
database_fixture(survivor['container'],{**fixture,'action':'remove'})
cluster.command(['docker','start',target['container']])
cluster.wait_healthy(target['container'])
cluster.wait_cluster()
save('graceful-shutdown.json',result)
def infrastructure():
state = cluster.snapshot()
for item in state:
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
if item['service']!='lb':
check(not item['published_ports'],'service exposed a port bypassing LB')
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
save('infrastructure.json',{'result':'PASS','containers':state})
def dependency_failures():
c = Client(timeout=6)
actor = c.user('dependencies')
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result = {'result':'PASS'}
try:
cluster.compose('stop','-t','2','redis')
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
c.call('GET','/api/v1/users/me',401)
c.call('GET','/health/ready',schema='Health')
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
finally:
cluster.compose('start','redis')
import urllib.request
import urllib.error
def status(path, token=None):
headers = {'Authorization':'Bearer '+token} if token else {}
request = urllib.request.Request(c.base+path,headers=headers)
started = time.monotonic()
try:
response = urllib.request.urlopen(request,timeout=6)
except urllib.error.HTTPError as error:
response = error
with response:
return response.code, round(time.monotonic()-started,4)
try:
cluster.compose('stop','-t','2','postgres')
code,elapsed = status('/api/v1/users/me',actor['token'])
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
anonymous,_ = status('/api/v1/users/me')
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
check(all(code==200 for code in live),'API process liveness depends on DB')
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
finally:
cluster.compose('start','postgres')
cluster.wait_cluster()
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
result['same_session_after_recovery'] = True
save('dependency-failures.json',result)
def verify_nginx():
state = cluster.snapshot()
lb = next(item for item in state if item['service']=='lb')
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
c = Client()
response = c.call('GET','/health/ready',schema='Health')
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
args = parser.parse_args()
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
if args.mode=='all':
for function in modes.values():
function()
else:
modes[args.mode]()
if __name__ == '__main__':
main()
View File
+252
View File
@@ -0,0 +1,252 @@
"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads."""
import json
import logging
import os
import re
from contextlib import asynccontextmanager
from uuid import uuid4
import psycopg
from fastapi import Depends, FastAPI, Request, Response
from fastapi.exceptions import RequestValidationError
from fastapi.responses import JSONResponse
from psycopg_pool import PoolTimeout, TooManyRequests
from redis.exceptions import RedisError
from starlette.exceptions import HTTPException
from starlette.concurrency import run_in_threadpool
from .db import Database
from .errors import ApiError, invalid, not_found, unauthorized
from .models import CreateChat, CreateMessage, CreateUser, uuid_value
from .messages import store_message, validate_key
from .pagination import decode_cursor, page, page_limit, public, stamp
from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session
LOG = logging.getLogger('messenger')
USER_FIELDS = 'id, username, display_name, created_at'
def error_response(request, error):
return JSONResponse({'error': {'code': error.code, 'message': error.message,
'request_id': request.state.request_id}},
status_code=error.status, headers=error.headers)
def authenticate(request: Request):
token = bearer(request.headers.get('Authorization', ''))
token_hash = digest(token)
with request.app.state.db.pool.connection() as conn:
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()',
(token_hash,)).fetchone()
if not row:
raise unauthorized()
request.state.token_hash = token_hash
return row['user_id']
def require_member(conn, chat_id, user_id):
found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone()
if not found:
raise not_found()
def chat_public(conn, row):
item = public(row)
item['member_ids'] = [str(member['user_id']) for member in conn.execute(
'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()]
return item
def create_app():
@asynccontextmanager
async def lifespan(app):
app.state.db = Database()
app.state.db.open()
try:
yield
finally:
app.state.db.close()
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
app.state.instance_id = uuid4().hex
app.state.release = os.environ.get('APP_RELEASE', 'lab3-v1')
@app.middleware('http')
async def request_context(request: Request, call_next):
request.state.request_id = str(uuid4())
try:
path = request.url.path
json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path)
if request.method == 'POST' and json_endpoint:
media = request.headers.get('Content-Type', '').split(';')
unsupported = media[0].strip().lower() != 'application/json'
for parameter in media[1:]:
name, _, value = parameter.strip().partition('=')
if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'):
unsupported = True
if unsupported:
if path != '/api/v1/users':
await run_in_threadpool(authenticate, request)
raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8')
try:
json.loads((await request.body()).decode('utf-8'))
except (UnicodeError, json.JSONDecodeError):
raise invalid() from None
response = await call_next(request)
except ApiError as exc:
response = error_response(request, exc)
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests):
response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно'))
except Exception as exc:
# Log exception class only: credentials, SQL and message content stay out of logs.
LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__)
response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка'))
response.headers['X-Request-Id'] = request.state.request_id
response.headers['X-Instance-Id'] = app.state.instance_id
response.headers['X-App-Release'] = app.state.release
return response
@app.exception_handler(ApiError)
async def api_error(request, exc):
return error_response(request, exc)
@app.exception_handler(RequestValidationError)
async def validation_error(request, exc):
return error_response(request, invalid())
@app.exception_handler(HTTPException)
async def http_error(request, exc):
return error_response(request, not_found() if exc.status_code == 404 else invalid())
@app.get('/health/live')
def live():
return {'status': 'ok'}
@app.get('/health/ready')
def ready(request: Request):
try:
with request.app.state.db.pool.connection() as conn:
conn.execute('SELECT 1 FROM idempotency_records LIMIT 0')
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None
return {'status': 'ok'}
@app.post('/api/v1/users', status_code=201)
def register(body: CreateUser, request: Request):
password_hash = HASHER.hash(body.password)
try:
with request.app.state.db.pool.connection() as conn:
row = conn.execute(
f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}',
(uuid4(), body.username, body.display_name, password_hash)).fetchone()
except psycopg.errors.UniqueViolation:
raise ApiError(409, 'conflict', 'Имя пользователя занято') from None
return public(row)
@app.post('/api/v1/auth/sessions', status_code=201)
async def create_session(request: Request):
# Run hashing and synchronous database calls outside the event loop.
username, password = basic_credentials(request.headers.get('Authorization', ''))
if await request.body():
raise invalid()
return await run_in_threadpool(login, request, username, password)
def login(request, username, password):
db = request.app.state.db
with db.pool.connection() as conn:
row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone()
valid = check_password(row['password_hash'] if row else DUMMY_HASH, password)
if not row or not valid:
raise unauthorized(basic=True)
row.pop('password_hash')
token, expires_at = new_session(db.session_ttl)
with db.pool.connection() as conn:
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
(digest(token), row['id'], expires_at))
return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)}
@app.get('/api/v1/users/me')
def me(request: Request, user_id=Depends(authenticate)):
db = request.app.state.db
key = 'user:' + str(user_id)
try:
cached = db.cache.get(key)
if cached:
return json.loads(cached)
except (RedisError, ValueError):
pass
with db.pool.connection() as conn:
row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone()
if not row:
raise unauthorized()
result = public(row)
try:
db.cache.set(key, json.dumps(result), ex=60)
except RedisError:
pass
return result
@app.delete('/api/v1/auth/sessions/current', status_code=204)
def logout(request: Request, user_id=Depends(authenticate)):
with request.app.state.db.pool.connection() as conn:
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,))
return Response(status_code=204)
@app.post('/api/v1/chats', status_code=201)
def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)):
if str(user_id) not in body.member_ids:
raise invalid()
try:
with request.app.state.db.pool.connection() as conn:
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
(uuid4(), body.title)).fetchone()
# An unknown user raises AFTER the chat INSERT; transaction context rolls it all back.
with conn.cursor() as cur:
cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',
[(row['id'], uuid_value(member)) for member in body.member_ids])
result = chat_public(conn, row)
except psycopg.errors.ForeignKeyViolation:
raise invalid() from None
return result
@app.get('/api/v1/chats')
def list_chats(request: Request, user_id=Depends(authenticate)):
limit = page_limit(request.query_params.get('limit', '50'))
scope = 'chats:' + str(user_id)
after = decode_cursor(scope, request.query_params.get('cursor'))
where = ' AND (c.created_at,c.id) > (%s,%s)' if after else ''
params = (user_id, *after, limit+1) if after else (user_id, limit+1)
with request.app.state.db.pool.connection() as conn:
rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id '
'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall()
result = page(scope, rows, limit)
result['items'] = [chat_public(conn, row) for row in rows[:limit]]
return result
@app.get('/api/v1/chats/{chat_id}')
def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone()
return chat_public(conn, row)
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
key = validate_key(request.headers.get('Idempotency-Key', ''))
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
row = store_message(conn, user_id, cid, body.text, key)
return public(row)
@app.get('/api/v1/chats/{chat_id}/messages')
def history(chat_id: str, request: Request, user_id=Depends(authenticate)):
cid = uuid_value(chat_id)
limit = page_limit(request.query_params.get('limit', '50'))
scope = 'messages:' + str(cid)
with request.app.state.db.pool.connection() as conn:
require_member(conn, cid, user_id)
after = decode_cursor(scope, request.query_params.get('cursor'))
where = ' AND (created_at,id) > (%s,%s)' if after else ''
params = (cid, *after, limit+1) if after else (cid, limit+1)
rows = conn.execute('SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s'+where+
' ORDER BY created_at,id LIMIT %s', params).fetchall()
return page(scope, rows, limit)
return app
+37
View File
@@ -0,0 +1,37 @@
"""Bounded dependencies; PostgreSQL is always the source of truth."""
import os
from pathlib import Path
from psycopg.rows import dict_row
from psycopg_pool import ConnectionPool
from redis import Redis
from redis.backoff import NoBackoff
from redis.retry import Retry
def setting(name, default=None):
filename = os.environ.get(name + '_FILE')
value = Path(filename).read_text().strip() if filename else os.environ.get(name, default)
if value is None:
raise RuntimeError(f'Missing configuration: {name}')
return value
class Database:
def __init__(self):
self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600'))
if self.session_ttl < 1:
raise ValueError('SESSION_TTL_SECONDS must be positive')
self.pool = ConnectionPool(
setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')),
timeout=2, max_waiting=16, open=False,
kwargs={'row_factory': dict_row, 'connect_timeout': 3,
'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'},
)
self.cache = Redis.from_url(
setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True,
socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8,
retry=Retry(NoBackoff(), 0),
)
def open(self):
self.pool.open(wait=True, timeout=30)
def close(self):
self.pool.close()
self.cache.close()
+16
View File
@@ -0,0 +1,16 @@
class ApiError(Exception):
def __init__(self, status, code, message, headers=None):
self.status = status
self.code = code
self.message = message
self.headers = headers or {}
def invalid():
return ApiError(400, 'invalid_request', 'Неверный запрос')
def unauthorized(basic=False):
scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme})
def not_found():
return ApiError(404, 'not_found', 'Объект не найден')
+33
View File
@@ -0,0 +1,33 @@
"""One PostgreSQL transaction owns both a message and its retry result."""
import hashlib
import json
import re
from uuid import uuid4
from .errors import ApiError, invalid
def validate_key(key):
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
raise invalid()
return key
def store_message(conn, user_id, chat_id, text, key):
canonical = json.dumps({'text': text}, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
params = (user_id, chat_id, key)
# A competing INSERT waits for the other transaction, then the next statement sees its result.
conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) "
"VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING",
(*params, fingerprint))
record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active '
'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone()
if record['message_id'] is not None and record['active']:
if record['request_hash'] != fingerprint:
raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением')
return conn.execute('SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE id=%s',
(record['message_id'],)).fetchone()
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone()
conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' "
'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params))
return row
+27
View File
@@ -0,0 +1,27 @@
"""Versioned, atomic, repeatable migrations, serialized with an advisory lock."""
import hashlib
from pathlib import Path
import psycopg
from .db import setting
def main():
directory = Path(__file__).resolve().parents[2] / 'migrations'
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
conn.execute('SELECT pg_advisory_xact_lock(20261102)')
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())')
for path in sorted(directory.glob('*.sql')):
sql = path.read_text()
checksum = hashlib.sha256(sql.encode()).hexdigest()
found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone()
if found:
if found[0] != checksum:
raise RuntimeError('Applied migration checksum changed: ' + path.name)
continue
conn.execute(sql)
conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum))
print('Applied', path.name)
print('Migrations up to date')
if __name__ == '__main__':
main()
+55
View File
@@ -0,0 +1,55 @@
import re
from uuid import UUID
from pydantic import BaseModel, ConfigDict, Field, field_validator
from .errors import invalid
def uuid_value(value):
try:
# Require the UUID string representation, not integer/coercible values.
if not isinstance(value, str) or len(value) != 36:
raise ValueError
return UUID(value)
except (ValueError, TypeError, AttributeError):
raise invalid() from None
class StrictModel(BaseModel):
model_config = ConfigDict(extra='forbid', strict=True)
@field_validator('*')
@classmethod
def valid_strings(cls, value):
if isinstance(value, str):
if not value.strip():
raise ValueError('Blank string')
# PostgreSQL cannot store NUL or unpaired Unicode surrogates.
if '\x00' in value:
raise ValueError('NUL is not supported')
value.encode('utf-8')
return value
class CreateUser(StrictModel):
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
display_name: str = Field(min_length=1, max_length=100)
password: str = Field(min_length=12, max_length=128)
@field_validator('username')
@classmethod
def username_ascii(cls, value):
if not re.fullmatch(r'[a-z0-9_]{3,32}', value):
raise ValueError('Invalid username')
return value
class CreateChat(StrictModel):
title: str = Field(min_length=1, max_length=100)
member_ids: list[str] = Field(min_length=1, max_length=100)
@field_validator('member_ids')
@classmethod
def members(cls, value):
ids = [uuid_value(item) for item in value]
if len(ids) != len(set(ids)):
raise ValueError('Duplicate member')
return [str(item) for item in ids]
class CreateMessage(StrictModel):
text: str = Field(min_length=1, max_length=4000)
+42
View File
@@ -0,0 +1,42 @@
import base64
import json
import re
from datetime import datetime, timezone
from .errors import invalid
from .models import uuid_value
def stamp(value):
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
def encode_cursor(scope, row):
raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode()
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
def decode_cursor(scope, value):
if value is None:
return None
try:
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value):
raise ValueError
data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True))
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
raise ValueError
ts = datetime.fromisoformat(data[1].replace('Z', '+00:00'))
if ts.tzinfo is None:
raise ValueError
return ts, uuid_value(data[2])
except (ValueError, TypeError, AttributeError, OverflowError):
raise invalid() from None
def page_limit(value):
if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100:
raise invalid()
return int(value)
def public(row):
return {key: stamp(value) if isinstance(value, datetime) else str(value) if hasattr(value, 'hex') else value
for key, value in row.items()}
def page(scope, rows, limit):
return {'items': [public(row) for row in rows[:limit]],
'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None}
+41
View File
@@ -0,0 +1,41 @@
import base64
import hashlib
import re
import secrets
from datetime import datetime, timedelta, timezone
from argon2 import PasswordHasher
from argon2.exceptions import VerificationError, InvalidHashError
from .errors import unauthorized
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
DUMMY_HASH = HASHER.hash('synthetic-dummy-password')
def digest(token):
return hashlib.sha256(token.encode()).hexdigest()
def basic_credentials(header):
try:
scheme, encoded = header.split(' ', 1)
if scheme.lower() != 'basic':
raise ValueError
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128:
raise ValueError
return username, password
except (ValueError, UnicodeError):
raise unauthorized(basic=True) from None
def check_password(encoded, password):
try:
return HASHER.verify(encoded, password)
except (VerificationError, InvalidHashError):
return False
def bearer(header):
try:
scheme, token = header.split(' ', 1)
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
raise ValueError
return token
except ValueError:
raise unauthorized() from None
def new_session(ttl):
return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl)
+131
View File
@@ -0,0 +1,131 @@
"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities."""
import base64
import json
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import pytest
from fastapi.testclient import TestClient
from messenger.app import create_app
PASSWORD = 'Тестовый:пароль🔐' * 8
@pytest.fixture
def client():
with TestClient(create_app()) as c:
yield c
def user(c):
body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD)
r = c.post('/api/v1/users', json=body)
assert r.status_code == 201, r.text
return r.json(), body
def login(c, body):
credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode()
r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials})
assert r.status_code == 201, r.text
return {'Authorization': 'Bearer ' + r.json()['token']}, r.json()
def test_auth_unicode_logout_and_cache(client):
u, body = user(client)
h, session = login(client, body)
assert session['user'] == u
assert len(session['token']) >= 43
key = 'user:' + u['id']
client.app.state.db.cache.delete(key)
for _ in range(2):
assert client.get('/api/v1/users/me', headers=h).json() == u
cached = json.loads(client.app.state.db.cache.get(key))
assert cached == u
assert 0 < client.app.state.db.cache.ttl(key) <= 60
assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204
r = client.get('/api/v1/users/me', headers=h)
assert r.status_code == 401
assert r.headers['www-authenticate'] == 'Bearer realm="messenger"'
assert r.json()['error']['request_id'] == r.headers['x-request-id']
def test_session_expiry(monkeypatch):
monkeypatch.setenv('SESSION_TTL_SECONDS', '2')
with TestClient(create_app()) as client:
_, body = user(client)
h, _ = login(client, body)
assert client.get('/api/v1/users/me', headers=h).status_code == 200
time.sleep(2.1)
assert client.get('/api/v1/users/me', headers=h).status_code == 401
def test_registration_race(client):
body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD)
with ThreadPoolExecutor(max_workers=6) as executor:
codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6)))
assert sorted(codes) == [201, 409, 409, 409, 409, 409]
def test_chat_rollback_after_insert(client):
u, body = user(client)
h, _ = login(client, body)
title = 'rollback_' + uuid.uuid4().hex
r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]})
assert r.status_code == 400
with client.app.state.db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0
assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0
def test_cursor_access_and_strict_validation(client):
a, ab = user(client)
b, bb = user(client)
ah, _ = login(client, ab)
bh, _ = login(client, bb)
chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)]
path = '/api/v1/chats/' + chats[0]['id']
for text in ['Один 👋', 'Два', 'Три']:
assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201
p = client.get(path+'/messages?limit=2', headers=ah).json()
assert [m['text'] for m in p['items']] == ['Один 👋','Два']
tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json()
assert [m['text'] for m in tail['items']] == ['Три']
assert tail['next_cursor'] is None
r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']})
assert r.status_code == 400
assert client.get(path, headers=bh).status_code == 404
for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]:
assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400
for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']:
assert client.get(path+'/messages?'+query,headers=ah).status_code == 400
for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]:
assert client.post('/api/v1/users',json=payload).status_code == 400
assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400
def test_session_storage_and_migrations(client):
from messenger.migrate import main as migrate
u, body = user(client)
_, session = login(client, body)
db = client.app.state.db
def snapshot():
with db.pool.connection() as conn:
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n']
for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']}
before = snapshot()
migrate()
migrate()
assert snapshot() == before
with db.pool.connection() as conn:
row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone()
assert row['token_hash'] != session['token']
assert len(row['token_hash']) == 64
stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash']
assert stored.startswith('$argon2id$')
assert stored != body['password']
def test_json_media_type_and_utf8(client):
body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD)
raw = json.dumps(body).encode()
for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]:
r = client.post('/api/v1/users',content=raw,headers=headers)
assert r.status_code == 415
assert r.json()['error']['code'] == 'unsupported_media_type'
for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']:
r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'})
assert r.status_code == 400, encoding
r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'})
assert r.status_code == 401
+111
View File
@@ -0,0 +1,111 @@
"""Regressions for lab3 shared message results, real PostgreSQL (no mocks)."""
from concurrent.futures import ThreadPoolExecutor
import uuid
from psycopg import sql
from test_api import client, user, login
def setup_chat(c, extra=None):
u, body = user(c)
headers, _ = login(c, body)
members = [u['id']] + (extra or [])
chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json()
return u, headers, '/api/v1/chats/'+chat['id']+'/messages'
def test_instance_alias_on_success_and_errors(client):
responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'),
client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})]
ids = [r.headers.get('X-Instance-Id') for r in responses]
assert ids[0] and len(set(ids)) == 1
assert all(r.headers.get('X-Request-Id') for r in responses)
def test_key_required_replay_and_conflict(client):
_, headers, path = setup_chat(client)
assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400
for key in ['', 'a b', 'bad/', 'x'*129]:
assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400
h = {**headers, 'Idempotency-Key':'valid:._-123'}
first = client.post(path, headers=h, json={'text':'same'})
assert first.status_code == 201
replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ')
assert replay.status_code == 201 and replay.json() == first.json()
assert client.post(path, headers=h, json={'text':'other'}).status_code == 409
assert len(client.get(path,headers=headers).json()['items']) == 1
def test_concurrent_key_produces_one_message(client):
_, headers, path = setup_chat(client)
h = {**headers, 'Idempotency-Key':uuid.uuid4().hex}
with ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16)))
assert [r.status_code for r in results] == [201]*16
assert all(r.json() == results[0].json() for r in results)
assert client.get(path,headers=headers).json()['items'] == [results[0].json()]
def test_scope_and_replay_rechecks_authorization(client):
b, bb = user(client)
bh, _ = login(client, bb)
a, ah, path = setup_chat(client, [b['id']])
key = 'same-in-independent-scopes'
first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json()
second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'})
assert second.status_code == 201 and second.json()['id'] != first['id']
chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json()
other_path = '/api/v1/chats/'+chat['id']+'/messages'
third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'})
assert third.status_code == 201 and third.json()['id'] != first['id']
outsider, ob = user(client)
oh, _ = login(client, ob)
assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404
assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204
assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401
def test_expired_key_can_be_reused(client):
u, h, path = setup_chat(client)
key = uuid.uuid4().hex
first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json()
with client.app.state.db.pool.connection() as conn:
ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds']
assert 86390 <= ttl <= 86400
conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key))
second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'})
assert second.status_code == 201 and second.json()['id'] != first['id']
assert len(client.get(path,headers=h).json()['items']) == 2
def test_rollback_side_effect_and_key_on_database_error(client):
u, h, path = setup_chat(client)
key = uuid.uuid4().hex
text = 'fault_' + uuid.uuid4().hex
function = 'fault_' + uuid.uuid4().hex
trigger = 'trigger_' + uuid.uuid4().hex
db = client.app.state.db
# AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook.
with db.pool.connection() as conn:
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text)))
conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
try:
failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
assert failed.status_code == 500
assert failed.headers['X-Instance-Id']
with db.pool.connection() as conn:
assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0
assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0
finally:
with db.pool.connection() as conn:
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger)))
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
assert retried.status_code == 201
assert client.get(path,headers=h).json()['items'] == [retried.json()]
def test_ready_requires_compatible_schema(client):
temporary = 'temporarily_unavailable_' + uuid.uuid4().hex
with client.app.state.db.pool.connection() as conn:
conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary)))
try:
r = client.get('/health/ready')
assert r.status_code == 503
assert r.json()['error']['code'] == 'unavailable'
assert client.get('/health/live').status_code == 200
finally:
with client.app.state.db.pool.connection() as conn:
conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))