From ee1a17dff684a63b89e60ffc3fe4da836c7dd51b Mon Sep 17 00:00:00 2001 From: DeUnru Date: Fri, 13 Nov 2026 00:23:23 +0300 Subject: [PATCH 1/2] =?UTF-8?q?=D0=9B=D0=A0=203=20=E2=80=94=20=D0=91=D0=9F?= =?UTF-8?q?=D0=982303=20=E2=80=94=20=D0=A3=D0=BD=D1=80=D1=83=20=E2=80=94?= =?UTF-8?q?=20=D0=BD=D0=B0=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 2 + .env.example | 19 +- .github/workflows/application.yml | 19 ++ .gitignore | 1 + Dockerfile | 18 ++ Makefile | 31 ++ REPORT.md | 144 ++++++++- compose.yaml | 89 ++++++ evidence/check.txt | 2 + evidence/concurrent-idempotency.json | 11 + evidence/dependency-failures.json | 20 ++ evidence/environment.json | 39 +++ evidence/fresh-migrations.txt | 8 + evidence/graceful-shutdown.json | 16 + evidence/infrastructure.json | 126 ++++++++ evidence/last-stop.json | 11 + evidence/load-output.txt | 17 + evidence/load.json | 17 + evidence/nginx.json | 11 + evidence/openapi.txt | 2 + evidence/persistence.json | 250 +++++++++++++++ evidence/persistence.txt | 2 + evidence/public-failover.json | 15 + evidence/public-failover.txt | 3 + evidence/resource-summary.json | 25 ++ evidence/resources.json | 402 ++++++++++++++++++++++++ evidence/rolling-update.json | 171 +++++++++++ evidence/sequential-failover.json | 135 ++++++++ evidence/smoke.txt | 2 + evidence/student-tests.txt | 3 + infra/nginx/nginx.conf | 43 +++ migrations/001_initial.sql | 34 ++ migrations/002_idempotency.sql | 11 + requirements.lock | 45 +++ requirements.txt | 10 + scripts/bootstrap.sh | 28 ++ scripts/capture_load.py | 71 +++++ scripts/check_fresh_migrations.sh | 16 + scripts/cluster.py | 196 ++++++++++++ scripts/restart.sh | 4 + scripts/stop-one.sh | 5 + scripts/verify_cluster.py | 443 +++++++++++++++++++++++++++ src/messenger/__init__.py | 0 src/messenger/app.py | 252 +++++++++++++++ src/messenger/db.py | 37 +++ src/messenger/errors.py | 16 + src/messenger/messages.py | 33 ++ src/messenger/migrate.py | 27 ++ src/messenger/models.py | 55 ++++ src/messenger/pagination.py | 42 +++ src/messenger/security.py | 41 +++ student-tests/test_api.py | 131 ++++++++ student-tests/test_idempotency.py | 111 +++++++ 53 files changed, 3237 insertions(+), 25 deletions(-) create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/check.txt create mode 100644 evidence/concurrent-idempotency.json create mode 100644 evidence/dependency-failures.json create mode 100644 evidence/environment.json create mode 100644 evidence/fresh-migrations.txt create mode 100644 evidence/graceful-shutdown.json create mode 100644 evidence/infrastructure.json create mode 100644 evidence/last-stop.json create mode 100644 evidence/load-output.txt create mode 100644 evidence/load.json create mode 100644 evidence/nginx.json create mode 100644 evidence/openapi.txt create mode 100644 evidence/persistence.json create mode 100644 evidence/persistence.txt create mode 100644 evidence/public-failover.json create mode 100644 evidence/public-failover.txt create mode 100644 evidence/resource-summary.json create mode 100644 evidence/resources.json create mode 100644 evidence/rolling-update.json create mode 100644 evidence/sequential-failover.json create mode 100644 evidence/smoke.txt create mode 100644 evidence/student-tests.txt create mode 100644 infra/nginx/nginx.conf create mode 100644 migrations/001_initial.sql create mode 100644 migrations/002_idempotency.sql create mode 100644 requirements.lock create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.sh create mode 100644 scripts/capture_load.py create mode 100644 scripts/check_fresh_migrations.sh create mode 100644 scripts/cluster.py create mode 100644 scripts/restart.sh create mode 100644 scripts/stop-one.sh create mode 100644 scripts/verify_cluster.py create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/app.py create mode 100644 src/messenger/db.py create mode 100644 src/messenger/errors.py create mode 100644 src/messenger/messages.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/models.py create mode 100644 src/messenger/pagination.py create mode 100644 src/messenger/security.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_idempotency.py diff --git a/.dockerignore b/.dockerignore index bc16e556..5a25dd7d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,5 @@ __pycache__ node_modules evidence .DS_Store +.superpowers +.pytest_cache \ No newline at end of file diff --git a/.env.example b/.env.example index d32b9be0..eee25474 100644 --- a/.env.example +++ b/.env.example @@ -1,13 +1,8 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. -APP_HOST=0.0.0.0 -APP_PORT=8080 -APP_ENV=lab +# bootstrap.sh creates a private local .env and secrets/; never commit them. SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url -API_REPLICAS=2 -IDEMPOTENCY_TTL_SECONDS=86400 +# API config in Compose: +# DATABASE_URL_FILE=/run/secrets/database_url +# CACHE_URL_FILE=/run/secrets/cache_url +# POOL_MAX_SIZE=8 per API process +# No API host ports: only lb publishes 127.0.0.1:8080. +# Redis caches immutable User profiles; PostgreSQL owns sessions + idempotency. diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..789277a0 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,19 @@ +name: Application acceptance +on: [push, pull_request] +permissions: + contents: read +jobs: + cluster: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - run: make up + - run: make check test test-student + - run: make test-fresh + - run: make test-cluster + - if: always() + run: docker compose down diff --git a/.gitignore b/.gitignore index b5a71896..902e3f35 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ coverage/ .vscode/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +.pytest_cache/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..abaa9568 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.13.5-slim +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src +WORKDIR /app +COPY requirements.txt requirements-dev.txt requirements.lock ./ +RUN pip install --no-cache-dir -r requirements.lock && useradd --uid 10001 --create-home messenger +COPY src/ src/ +COPY migrations/ migrations/ +COPY student-tests/ student-tests/ +COPY scripts/ scripts/ +COPY tests/ tests/ +COPY contracts/ contracts/ +COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./ +ARG APP_RELEASE=lab3-v1 +ENV APP_RELEASE=${APP_RELEASE} +LABEL org.opencontainers.image.version=${APP_RELEASE} +USER messenger +EXPOSE 8080 +CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"] diff --git a/Makefile b/Makefile index 5d86d171..49040603 100644 --- a/Makefile +++ b/Makefile @@ -28,3 +28,34 @@ test-persistence: test-failover: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2) $(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +.PHONY: bootstrap up down test-student test-cluster rolling +bootstrap: + ./scripts/bootstrap.sh +up: bootstrap + docker compose build api + docker compose up -d --wait --wait-timeout 120 + $(PYTHON) scripts/cluster.py wait +down: + docker compose down +test-student: + $(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache +test-cluster: + $(PYTHON) scripts/verify_cluster.py +rolling: + $(PYTHON) scripts/cluster.py rolling + +.PHONY: test-fresh load-evidence verify-persistence verify-failover +test-fresh: + ./scripts/check_fresh_migrations.sh +load-evidence: + $(PYTHON) scripts/capture_load.py +verify-persistence: + $(PYTHON) scripts/verify_cluster.py persistence +verify-failover: + $(PYTHON) scripts/verify_cluster.py public-failover + $(PYTHON) scripts/verify_cluster.py failover + +.PHONY: validate-container +validate-container: + $(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py diff --git a/REPORT.md b/REPORT.md index c2679005..c8a1f17b 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,155 @@ # Отчёт — лабораторная 3 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO +- Автор, группа: Унру Д.В. БПИ2303 +- Целевая оценка; выбранный трек 5, если нужен: **3** (трек 5 не заявлен) - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, pytest; документация [Nginx load balancing](https://nginx.org/en/docs/http/load_balancing.html), [upstream/resolve](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) и [proxy_next_upstream](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/), [PostgreSQL transactions](https://www.postgresql.org/docs/17/tutorial-transactions.html). Реализация выполнена совместно мной и Codex: часть кода, тестов и отчёта написана с помощью Codex, результат проверен запуском тестов, описанных ниже. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Prerequisites: Docker с Compose v2 (проверено на Docker Desktop 29.8.2, macOS), Python ≥ 3.9 на хосте для публичных тестов и скриптов (стандартная библиотека), Make. Внешние установленные PostgreSQL/Redis/Nginx не нужны. Порт localhost:8080 должен быть свободен от предыдущего стенда. + +```sh +git clone && cd <каталог-fork-лабораторной-3> +make up # secrets/.env → build API → migrate → api×2 → LB → дождаться 2 alias +curl -fsS http://localhost:8080/health/ready +docker compose ps # две API-реплики, postgres, redis, lb — healthy; migrate — Exited(0) + +make check +make validate-container # полный валидатор OpenAPI без локальных pip-зависимостей +make test # публичный smoke через LB +make test-student # собственные интеграционные тесты: 14 +make test-fresh # обе миграции и 14 тестов на отдельных пустых volumes +make test-cluster # все распределённые проверки, включая реальные отказы и rolling +make load-evidence # отдельные 30 с нагрузки и CPU/RAM +``` + +Эквивалент `make up`: `./scripts/bootstrap.sh`, `docker compose build api`, `docker compose up -d --wait --wait-timeout 120`, `python3 scripts/cluster.py wait`. Образ собирается до запуска migrate, поэтому готовый сервер из registry не требуется. Хосту доступен только LB `127.0.0.1:8080`; API и хранилища напрямую не опубликованы. + +Отдельные публичные проверки: + +```sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh +make test-failover ACTION_SCRIPT=scripts/stop-one.sh +# Публичный failover намеренно оставляет одну выбранную реплику остановленной. +# Восстановление: +docker compose up -d --no-recreate --wait --wait-timeout 90 +python3 scripts/cluster.py wait +``` + +`make verify-persistence` дополнительно сохраняет снимки реальных контейнеров до/после. `make verify-failover` выполняет публичный failover и последовательные проверки каждого исходного контейнера, с восстановлением между ними. Полный `make test-cluster` последовательно запускает concurrent replay, persistence, публичный и собственные failover, SIGTERM, rolling, отказы Redis/БД и аудит инфраструктуры. Он меняет состояние стенда и собирает новый локальный образ с новым идентификатором релиза; не запускать параллельно с другими проверками. + +`bootstrap.sh` создаёт случайный пароль PostgreSQL и файлы secrets один раз; существующие значения не меняет. Каталог `secrets/` — 0700 на хосте, файлы доступны uid 10001 контейнера через Compose secrets; `.env` и secrets исключены из Git и Docker build context. Для lab3 созданы собственные secrets/volumes, данные и credentials lab2 не копировались. + +Безопасная остановка: `make down` — контейнеры и сети удаляются, именованные volumes сохраняются. После `make up` доступны прежние данные, действующие сессии и результаты идемпотентности. Намеренное удаление всех учебных данных: `docker compose down --volumes`. Hook `restart.sh` volumes не удаляет. `make test-fresh` удаляет только созданные им временные volumes отдельного проекта. Синтетические данные обычных тестов остаются в основной БД. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +```text + ┌──► API-процесс A ──┐ +клиент ──HTTP──► Nginx ──┤ ├──► PostgreSQL: данные, сессии, результаты POST +127.0.0.1:8080 └──► API-процесс B ──┤ + └──► Redis: кэш публичного профиля + +migrate ──► PostgreSQL (один отдельный job до запуска API) +``` + +- **Перенос из lab2.** Перенесены исходники FastAPI, миграция 001, Argon2id/Basic/Bearer, users/chats/messages, keyset-пагинация, Redis cache-aside профиля, ограниченные пулы и собственные проверки. Отправка сообщений и тесты адаптированы к обязательному Idempotency-Key. Dockerfile/Compose изменены для отдельных миграций, нескольких API и LB. Публичные `tests/`, `contracts/`, README и COURSE сохранены из skeleton lab3. +- **Общее состояние.** PostgreSQL — источник истины для пользователей, чатов, участников, сообщений, сессий и результатов POST. API не держит в локальной памяти данных, от которых зависит корректность. Каждое защищённое обращение проверяет сессию в БД; X-User-Id игнорируется. Разные реплики обслуживают один Bearer без повторного входа. Sticky session не используется. +- **Идентификатор процесса.** `X-Instance-Id` — случайный UUID alias, стабильный для жизни конкретного процесса; добавляется ко всем его ответам, включая health/ошибки. После restart alias меняется. `X-App-Release` — дополнительный заголовок для различения сборок в rolling-эксперименте. Ошибка, сформированная самим LB при недоступных backend, может не иметь API-заголовков и JSON. +- **Общая идемпотентность** (`src/messenger/messages.py`, миграция 002). Таблица `idempotency_records`: PK(user_id,chat_id,key), request_hash, message_id FK, expires_at. Таблица обслуживает только POST сообщений, поэтому эта область соответствует user+method+chat path+key. Ключ строго ASCII `[A-Za-z0-9._:-]{1, 128}`, иначе 400. Валидированный JSON канонизируется независимо от форматирования/порядка полей, fingerprint — SHA-256. В схеме lab3 допустим только text; attachment_ids появится по контракту lab4. +- **Атомарность повтора.** После проверки сессии и членства одна транзакция выполняет INSERT ключа ON CONFLICT DO NOTHING и SELECT FOR UPDATE. Конкурирующие процессы сериализуются на одной записи. При действующем ключе тот же fingerprint возвращает тот же Message и 201, другой — 409. Для нового/истёкшего ключа INSERT сообщения и обновление результата фиксируются одним COMMIT. Исключение откатывает оба эффекта, транзиентный 5xx не сохраняется как результат. Replay снова проверяет авторизацию. Срок ключа — 24 ч; после истечения разрешена новая запись с тем же ключом, старое сообщение не удаляется. +- **Миграции.** `migrate` — отдельный одноразовый service, API зависит от `service_completed_successfully`. В API CMD мигратора нет. SQL-файлы 001/002, checksum и advisory lock сохраняют повторяемость и сериализуют возможные ручные запуски. Новая 002 только добавляет таблицу/индекс, не меняет старые поля lab2. Перед rolling один раз выполняется `docker compose run --rm --no-deps migrate`, пока старые API продолжают работу. +- **Балансировка и DNS.** Nginx OSS 1.28: `upstream messenger` с общей `zone messenger 64k` и `server api:8080 resolve`. Все A-записи одного Compose-service api участвуют в round-robin; `resolver 127.0.0.11 valid=1s ipv6=off` обновляет IP без restart/reload Nginx. Адреса не зашиты в конфигурации. +- **Пассивные проверки upstream.** `max_fails=1 fail_timeout=2s`: первая ошибка подключения/таймаут/503 исключает адрес на 2 с, затем допускается новая попытка. Активного `health_check` Nginx Plus здесь нет. Docker healthcheck каждого API раз в 2 с вызывает его readiness; healthcheck LB раз в 1 с делает GET readiness через Nginx и создаёт фоновый трафик даже без клиента. При двух backend, работающем Docker DNS и настроенных таймаутах консервативный бюджет обнаружения — до 8 с; фактические чтения/записи после отказа проверяются в пределах 10 с. В отличие от активного мониторинга, пассивное исключение происходит только после обращения к backend. +- **Таймауты и повтор.** Nginx connect 1 с, read/send 5 с; `proxy_next_upstream error timeout http_503`, не более 2 upstream-попыток и 6 с на переключение. GET может уйти на другую реплику при ошибке. Параметр `non_idempotent` не включён: уже отправленный POST автоматически не пересылается; при connect error до отправки запроса допустим безопасный выбор другого адреса. Неопределённую отправку сообщения повторяет клиент с прежним Idempotency-Key. +- **Readiness и graceful.** `/health/live` проверяет процесс; `/health/ready` требует доступную PostgreSQL и таблицу новой схемы. Несовместимая схема/ошибка зависимости →503. Redis — некритичный кэш с fallback, его отказ readiness не меняет. Uvicorn работает PID1, один worker на контейнер, graceful timeout5 с. Docker stop_grace_period10 с; hook stop-one использует 7 с, укладываясь в публичный бюджет 10 с. SIGTERM закрывает listening socket и даёт активным запросам завершиться. +- **Остановка выбранной реплики.** `scripts/stop-one.sh` — короткий wrapper Python-оркестратора `cluster.py`. Тот сопоставляет TARGET_INSTANCE_ID с реальным контейнером внутренним GET /health/live, останавливает ровно его, проверяет exit 0 и оставляет остановленным. Идентификаторы контейнеров/процессов и время записываются без credentials. `restart.sh` перезапускает все API, PostgreSQL, Redis и LB с прежними volumes. +- **Поочерёдная замена.** Compose не предоставляет здесь автоматический Swarm rolling update. Проверен эквивалент start-first: временно scale api=3 с `--no-recreate`, дождаться новой healthy-реплики и трёх alias через LB, затем stop/remove одного старого контейнера и возврат к 2. Второй старый контейнер в этом шаге не пересоздаётся. Создание до удаления старого гарантирует другой IP; оба адреса сохраняются. Шаг повторяется для второго исходного контейнера. +- **Пароли, сессии, кэш.** Argon2id (19456 KiB, time_cost2, parallelism1), opaque token256 бит, в БД только SHA-256 токена и абсолютный expiry; logout удаляет сессию. Redis хранит только User по `user:` на 60 с после проверки сессии. Профиль неизменяем в обязательном API, инвалидация по событию не нужна. История читается из PostgreSQL и не бывает устаревшей из-за кэша. +- **Запросы/ошибки.** SQL параметризован; каждый API-пул min 1/max 8, очередь 16, ожидание 2 с, connect_timeout3 с, statement_timeout3 с, lock_timeout2 с. Redis-пул 8, connect/read по 0.2 с без retry. JSON строго UTF-8, неверные типы/поля/формат →400, неподдерживаемый media type →415; чужие чаты →404. X-Request-Id совпадает с request_id ошибки. Access-лог Uvicorn отключён, тела/токены/SQL в логи не выводятся. + +Конфигурация и ресурсы: + +| Переменная / service | Значение | Использование | +| --- | --- | --- | +| `SESSION_TTL_SECONDS` (.env) | 3600 по умолчанию | абсолютный срок сессии, в собственном тесте expiry — 2 с | +| `DATABASE_URL_FILE` | `/run/secrets/database_url` | API и migrate читают адрес/credentials БД | +| `CACHE_URL_FILE` | `/run/secrets/cache_url` | только API читает адрес Redis | +| `POOL_MAX_SIZE` | 8 на API-процесс | ограничение числа соединений | +| `api` | `mtusi-lab3-api:lab3`; каждая 0.75 CPU / 384 MiB | один scale-able service, штатно 2 реплики, без container_name и host ports; сети front+data | +| `lb` | `nginx:1.28.0-alpine`; 0.5 CPU / 128 MiB | единственный публичный порт 127.0.0.1:8080; только front, без DB secrets | +| `postgres` | `postgres:17.5-bookworm`; 1 CPU / 512 MiB | только internal data; volume postgres_data; pg_isready | +| `redis` | `redis:7.4.4-bookworm`; 0.25 CPU / 128 MiB | только internal data; volume redis_data; AOF; maxmemory64 MiB/allkeys-lru | +| `migrate` | тот же образ API; 0.5 CPU / 256 MiB | только data, только database_url secret; Exited(0) после применения | + +Dockerfile: `python:3.13.5-slim`, полностью закреплённые зависимости в `requirements.lock`, пользователь messenger uid 10001. Установка зависимостей предшествует копированию исходников; ARG APP_RELEASE расположен после этих слоёв, поэтому смена только метки релиза не переустанавливает библиотеки. CMD — exec Uvicorn. Healthchecks задаются Compose; `.dockerignore` исключает Git, secrets/.env, сертификаты, volumes и evidence. Публичные и собственные проверки включены в образ для запуска без pip на хосте. Номера контейнеров после замен могут иметь пропуски: `cluster.py exec` выбирает реально живой контейнер, не предполагает существование api-1. ## Доказательства критериев +Прогоны 2026-11-02, macOS, Apple M4 Pro /24 GiB RAM, Docker Desktop29.8.2. Docker VM:12 CPU, около 7.75 GiB RAM. Ветка `solution/bpi2303-lukyanov`; сведения о процессах/образах, лимитах и сети — в evidence. Коммиты/публикация выполняются автором отдельно. + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Nginx, конфигурация и две реплики | `verify_cluster.py nginx` | `nginx -t` PASS, Server:nginx, два alias через LB | `evidence/nginx.json` | +| Контракт и публичный smoke (3) | `make check`, `make validate-container`, `make test` | PASS, lab3 public API scenario через LB | `evidence/check.txt`, `openapi.txt`, `smoke.txt` | +| ≥2 API, один LB, общая БД и сессии (3) | `cluster.py wait`; concurrent replay; infrastructure | один Bearer работает на разных alias; реально два API-контейнера, порты только у LB | `evidence/concurrent-idempotency.json`, `infrastructure.json` | +| Отказ любой одной реплики, чтение/запись ≤10 с (3) | публичный failover + остановка каждого исходного контейнера | оба сценария PASS; чтение известного сообщения, пока target ещё остановлен; post-stop POST в пределах 10 с | `evidence/public-failover.txt`, `sequential-failover.json` | +| Общая идемпотентность и отсутствие дублей (3) | 8 параллельных повторов через 2 alias | все 201 с одним Message, в истории 1 сообщение; другой body409 | `evidence/concurrent-idempotency.json` | +| Сохранность и прежняя сессия (3) | `make verify-persistence` | API/БД/Redis/LB действительно перезапущены; volumes те же; прежний токен и данные доступны | `evidence/persistence.txt`, `persistence.json` | +| Обязательные key/alias, TTL 24 ч, scope, rollback, повторная авторизация | `make test-student` | 14/14 PASS, включая отказ после INSERT и rollback ключа+сообщения | `evidence/student-tests.txt` | +| Миграции отдельно, с нуля и повторяемо | `make test-fresh`; migration test | применены 001/002, повтор без дублей, 14/14 PASS на новых volumes | `evidence/fresh-migrations.txt` | +| Health/readiness/timeouts/SIGTERM (4) | Nginx config; test_ready_requires_compatible_schema; SIGTERM с активным POST | схема недоступна→503; активный POST 201, новые TCP отвергнуты пока процесс ещё жив, exit 0 за 1.9171 с | `evidence/student-tests.txt`, `graceful-shutdown.json` | +| Последовательно остановлена каждая реплика, RPS/errors/recovery (4) | `verify_cluster.py failover` | два разных исходных container id; предыдущий восстановлен; подтверждения сохранены без дублей | `evidence/sequential-failover.json` | +| CPU/RAM, нет API host ports/container_name (4) | `verify_cluster.py infra`; реальное scale 2→3→2 | лимиты есть у всех сервисов, единственный public endpoint у LB, api масштабируется | `evidence/infrastructure.json`, `rolling-update.json` | +| Rolling-equivalent под нагрузкой, отдельные совместимые миграции (4) | `verify_cluster.py rolling` | два старых контейнера заменены новым образом, оба новых IP обнаружены LB; 14 975 подтверждённых сообщений сохранены | `evidence/rolling-update.json` | +| Предыдущая авторизация и отказ зависимостей сохранены | `verify_cluster.py dependencies`; собственные auth/expiry tests | Redis fallback200; PostgreSQL 503; после восстановления тот же токен работает | `evidence/dependency-failures.json`, `student-tests.txt` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +Чтобы повторно сохранить вывод: `make test > evidence/smoke.txt`, `make test-student > evidence/student-tests.txt`, `make validate-container > evidence/openapi.txt`, `make test-fresh > evidence/fresh-migrations.txt`. Распределённые сценарии сами обновляют свои JSON/text в evidence. Контракт и публичные tests не ослаблены; source lab2 переносился без них. Secrets, пароли и Bearer-токены в отчёт/evidence не записываются. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Исследовательский трек 5 не проводился. Во всех сценариях используется реальный PostgreSQL/Redis и два API-процесса за Nginx. Генератор отказов/rolling — closed-loop4 потока; после неопределённого результата повторяет только POST сообщения, с прежним key/body. Подтверждённые ответы сверяются с полной историей, проверяются уникальность UUID и текста. Это проверка конкретных ограниченных отказов одного локального стенда. + +В последовательном сценарии перед воздействиями есть 3 с базовой работы, каждое окно остановки — около 8 с, предыдущий контейнер запускается и обнаруживается через LB перед следующим. Время записи считается только по запросу, **начатому после возврата stop hook**, а общий deadline10 с отсчитывается от старта hook. Чтение известного подтверждённого сообщения также выполняется до восстановления target. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Первый исходный API остановлен | другая реплика читает/пишет, остановленная не обслуживает | 400.056 RPS, 1 ошибочная попытка в окне; GET 200 и прежнее сообщение | POST через 1.3835 с от начала hook /0.0138 с после его завершения; GET через 1.38 с от начала | +| Второй исходный API остановлен после восстановления первого | те же гарантии | 401.166 RPS, 1 ошибочная попытка; GET 200, target оставался stopped | POST через 1.4405 с от начала hook /0.0151 с после завершения; GET через 1.4375 с от начала | +| Оба последовательных отказа | подтверждённые записи не теряются, retry не создаёт дубль | 15 915 подтверждённых сообщений =15 915 в истории | потерь/дублей не обнаружено | +| SIGTERM во время активного POST | прекратить новые соединения, завершить активный запрос | SQL-запрос наблюдался в PgSleep; POST 201, новые TCP получили ConnectionRefusedError ещё при живом процессе; exit 0 | 1.9171 с от SIGTERM / сообщение сохранено | +| Новый образ, поочерёдная start-first замена | адреса обновляются без рестарта LB, другая API продолжает работу | два шага 5.864/5.6574 с; разные IP, 3 alias при старте; во время замены 687.069 RPS, 1 ошибочная попытка (0.0126%) | 14 975 подтверждённых сообщений сохранены, дублей 0 | +| Полный restart API/PG/Redis/LB с volumes | прежний токен читает прежние данные без повторного входа | public persistence PASS, реальные StartedAt изменились | потерь подтверждённого сообщения и сессии не обнаружено | +| Redis остановлен | fallback в PostgreSQL без ослабления авторизации | GET 200 за 0.0138 с, ready 200, аноним 401 | данные/сессии в БД сохранены | +| PostgreSQL остановлен | защищённый запрос 503, процесс остаётся жив | 503; обе внутренние live 200; после старта БД та же сессия работает | ошибки не приводят к доверию по кэшу | + +**Как проверялся SIGTERM:** временный BEFORE INSERT trigger задерживает ровно один синтетический текст через pg_sleep(2). После наблюдения активного запроса в pg_stat_activity процесс получает SIGTERM. Проверяется отказ новых TCP, пока процесс ещё работает, затем 201 исходного запроса, exit 0 и чтение результата через surviving API. Trigger/function удаляются в finally, контейнер восстанавливается. В коде приложения тестового endpoint/режима нет. + +**Что именно обновлялось:** собран действительно другой образ с отличающейся меткой APP_RELEASE, что видно по image id и X-App-Release. Код бизнес-API и схема совместимы; новой функциональной возможности в этом тестовом релизе не добавлялось. Мигратор запускается один раз перед заменами, не каждым новым API. Проверка доказывает поочерёдную замену сборки и обновление адресов, а не безопасное выполнение произвольной destructive migration. + +**Нагрузка без отказов:** `make load-evidence`, 30 с, concurrency 4, две реплики, без автоматических retry, один сохранённый прогон. Перед ним стенд прогрет функциональными сценариями; отдельного измеряемого нагрузочного прогрева нет. 24 559 успешных запросов, 0 ошибок, 814.97 successful RPS; p50/p95/p99=4.03/5.822/47.235 мс. Число сообщений в БД: до запуска — 102 032, после — 126 591. Исходный summary — `evidence/load.json`, параметры и данные — `environment.json`, замеры ресурсов — `resources.json`. + +Выборочные пики CPU/RAM за нагрузочный прогон (Docker CPU 100% = одно ядро; это пики отдельных samples, не одновременная сумма): + +| Контейнер | CPU peak | Memory peak | +| --- | --- | --- | +| API-процесс 1 | 68.64% | 78.41 MiB | +| API-процесс 2 | 69.43% | 66.61 MiB | +| Nginx | 10.01% | 7.895 MiB | +| PostgreSQL | 42.15% | 99.4 MiB | +| Redis | 0.99% | 11.04 MiB | + +Имена конкретных контейнеров и исходные значения — `evidence/resource-summary.json`. Эти цифры не доказывают максимальную пропускную способность и не являются сравнением с lab2: ресурсы/набор данных отличаются. Closed-loop имеет coordinated omission, latency рассчитана только для успешных ответов, клиент и Docker работают на одном ноутбуке. Переходные ошибки во время отказов/замен измерены и учитывают клиентские результаты. Nginx может скрыть connect error до отправки запроса и повторить безопасный GET; уже отправленные POST не повторяются им автоматически. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один Docker-host; LB, PostgreSQL и Redis — по одному экземпляру. Проверен отказ API-процесса, не узла/диска/БД с failover. Restart с volumes не заменяет backup и не обеспечивает HA. +- Compose-замена — наш явный start-first сценарий с временной третьей репликой, не автоматический rolling-контроллер Swarm. Сам по себе обычный `docker compose up` не является доказанным безопасным rolling update. При обрыве нашего сценария может потребоваться ручное восстановление до двух healthy-реплик. +- В миграциях используются только совместимые аддитивные изменения. Breaking schema change требует expand/contract, миграции данных или отдельного окна; в этой работе не проверен. Advisory lock предотвращает одновременное применение, но сам по себе не делает любую миграцию совместимой. +- При SIGTERM/обнаружении отказа допускаются краткие HTTP/connection errors. Клиент повторяет только отправку сообщения с тем же Idempotency-Key. Создание чата и регистрация не имеют общей идемпотентности и после отправки тела автоматически не повторяются. +- TTL идемпотентности 24 ч и expiry сессии проверяются при запросе; фоновой очистки старых строк пока нет. Истёкший ключ можно использовать заново, это создаст новое сообщение. +- Redis без пароля в изолированной data-сети Compose; сессии всегда проверяются в PostgreSQL. Изменение профиля в будущем потребует инвалидации кэша. В этой работе нет TLS, rate limiting входа, PKI и secret manager. +- Нет общего snapshot между страницами; конкурентная запись с ключом до cursor может остаться позади обхода. Порядок COMMIT не обязан совпадать с порядком created_at. +- Когда БД недоступна и LB исключил обе API-реплики, даже `/health/live` через публичную точку может получить 502/503/504 от LB; внутренний `/health/live` каждого процесса остаётся 200. Это различие готовности маршрута и жизни процесса. +- UI/WebSocket, изменение участников и редактирование/удаление сообщений не входят в обязательный API. Malformed JSON без Bearer может дать 400 до выполнения зависимости аутентификации. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..98f8098d --- /dev/null +++ b/compose.yaml @@ -0,0 +1,89 @@ +name: mtusi-lab3 +services: + postgres: + image: postgres:17.5-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + secrets: [postgres_password] + volumes: [postgres_data:/var/lib/postgresql/data] + networks: [data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 1.0 + redis: + image: redis:7.4.4-bookworm + command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru] + volumes: [redis_data:/data] + networks: [data] + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 128m + cpus: 0.25 + migrate: + image: mtusi-lab3-api:lab3 + command: [python, -m, messenger.migrate] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + networks: [data] + depends_on: + postgres: {condition: service_healthy} + restart: 'no' + mem_limit: 256m + cpus: 0.5 + api: + image: mtusi-lab3-api:lab3 + build: . + scale: 2 + environment: + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + POOL_MAX_SIZE: '8' + secrets: [database_url, cache_url] + networks: [front, data] + depends_on: + migrate: {condition: service_completed_successfully} + redis: {condition: service_healthy} + stop_grace_period: 10s + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"] + interval: 2s + timeout: 5s + retries: 30 + mem_limit: 384m + cpus: 0.75 + lb: + image: nginx:1.28.0-alpine + volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro'] + networks: [front] + ports: ['127.0.0.1:8080:8080'] + depends_on: + api: {condition: service_healthy} + healthcheck: + test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready] + interval: 1s + timeout: 8s + retries: 30 + mem_limit: 128m + cpus: 0.5 +networks: + front: + data: + internal: true +secrets: + postgres_password: {file: ./secrets/postgres_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} +volumes: + postgres_data: + redis_data: diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..9a553306 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +python3 scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/concurrent-idempotency.json b/evidence/concurrent-idempotency.json new file mode 100644 index 00000000..9968b4b5 --- /dev/null +++ b/evidence/concurrent-idempotency.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "parallel_requests": 8, + "observed_instances": [ + "a125004f5fbb4f9e91edb14bb3e64993", + "a8916b2407ed45cc87d73e88cf1639b9" + ], + "responses_201": 8, + "messages_created": 1, + "different_body_status": 409 +} diff --git a/evidence/dependency-failures.json b/evidence/dependency-failures.json new file mode 100644 index 00000000..b8b773ec --- /dev/null +++ b/evidence/dependency-failures.json @@ -0,0 +1,20 @@ +{ + "result": "PASS", + "redis": { + "profile_status": 200, + "fallback_seconds": 0.0138, + "anonymous": 401, + "readiness": 200 + }, + "postgres": { + "authenticated_status": 503, + "response_seconds": 0.0048, + "anonymous_status": 401, + "internal_api_liveness": [ + 200, + 200 + ], + "note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable." + }, + "same_session_after_recovery": true +} diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..e95e0d50 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,39 @@ +{ + "captured_at_utc": "2026-11-02T08:38:18.305573+00:00", + "host_os": "macOS-27.2-arm64-arm-64bit-Mach-O", + "host_architecture": "arm64", + "docker": { + "server_version": "29.8.2", + "cpus": 12, + "memory_bytes": 8319238144, + "architecture": "aarch64" + }, + "python_in_image": "Python 3.13.5", + "dataset_before": { + "users": 117, + "chats": 53, + "messages": 102032, + "sessions": 102, + "idempotency_records": 102011 + }, + "limits": { + "api_each": "0.75 CPU / 384 MiB", + "postgres": "1 CPU / 512 MiB", + "redis": "0.25 CPU / 128 MiB", + "lb": "0.5 CPU / 128 MiB" + }, + "warmup": "API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.", + "replicas": 2, + "repetitions": 1, + "duration_seconds": 30, + "concurrency": 4, + "host_cpu": "Apple M4 Pro", + "host_memory_bytes": 25769803776, + "dataset_after": { + "users": 120, + "chats": 54, + "messages": 126591, + "sessions": 105, + "idempotency_records": 126570 + } +} diff --git a/evidence/fresh-migrations.txt b/evidence/fresh-migrations.txt new file mode 100644 index 00000000..edab3d31 --- /dev/null +++ b/evidence/fresh-migrations.txt @@ -0,0 +1,8 @@ +./scripts/check_fresh_migrations.sh +Applied 001_initial.sql +Applied 002_idempotency.sql +Migrations up to date +Migrations up to date +.............. [100%] +14 passed in 3.56s +PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes. diff --git a/evidence/graceful-shutdown.json b/evidence/graceful-shutdown.json new file mode 100644 index 00000000..308b6819 --- /dev/null +++ b/evidence/graceful-shutdown.json @@ -0,0 +1,16 @@ +{ + "result": "PASS", + "target_container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d", + "target_instance": "4ffdcca2f8644acd9b76d65f8260eaf7", + "fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards", + "in_flight_observed_in_pg_stat_activity": true, + "in_flight_response": 201, + "new_tcp_after_sigterm": { + "accepted": false, + "error_type": "ConnectionRefusedError" + }, + "target_still_running_when_new_tcp_rejected": true, + "exit_code": 0, + "shutdown_seconds_from_sigterm": 1.9171, + "confirmed_message_preserved": true +} diff --git a/evidence/infrastructure.json b/evidence/infrastructure.json new file mode 100644 index 00000000..387537d7 --- /dev/null +++ b/evidence/infrastructure.json @@ -0,0 +1,126 @@ +{ + "result": "PASS", + "containers": [ + { + "service": "api", + "container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710", + "name": "mtusi-lab3-api-5", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:37:46.879119007Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741", + "memory_bytes": 402653184, + "nano_cpus": 750000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data", + "mtusi-lab3_front" + ], + "http_status": 200, + "instance": "0749a2e4af8a4b849e3eae6426945546", + "release": "lab3-9d88e53e" + }, + { + "service": "api", + "container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f", + "name": "mtusi-lab3-api-6", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:37:52.762454968Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741", + "memory_bytes": 402653184, + "nano_cpus": 750000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data", + "mtusi-lab3_front" + ], + "http_status": 200, + "instance": "dd54c30c2e4c413ea2e28066c45e0d3c", + "release": "lab3-9d88e53e" + }, + { + "service": "lb", + "container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a", + "name": "mtusi-lab3-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:32.827784792Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab3_front" + ] + }, + { + "service": "migrate", + "container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43", + "name": "mtusi-lab3-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T08:36:54.173486177Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data" + ] + }, + { + "service": "postgres", + "container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7", + "name": "mtusi-lab3-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:38:04.389392668Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab3_postgres_data" + ], + "networks": [ + "mtusi-lab3_data" + ] + }, + { + "service": "redis", + "container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4", + "name": "mtusi-lab3-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:38:03.185004292Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab3_redis_data" + ], + "networks": [ + "mtusi-lab3_data" + ] + } + ] +} diff --git a/evidence/last-stop.json b/evidence/last-stop.json new file mode 100644 index 00000000..c36cfa69 --- /dev/null +++ b/evidence/last-stop.json @@ -0,0 +1,11 @@ +{ + "container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea", + "name": "mtusi-lab3-api-4", + "instance": "9ab4e9373a944f05b69d934479430136", + "release": "lab3-9463abd8", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af", + "stopped_at_utc": "2026-11-02T08:37:19.081389+00:00", + "stop_seconds": 0.4518, + "exit_code": 0, + "left_stopped": true +} diff --git a/evidence/load-output.txt b/evidence/load-output.txt new file mode 100644 index 00000000..07a83694 --- /dev/null +++ b/evidence/load-output.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.135, + "concurrency": 4, + "successful_requests": 24559, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 814.97, + "successful_latency_ms": { + "p50": 4.03, + "p95": 5.822, + "p99": 47.235 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..07a83694 --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.135, + "concurrency": 4, + "successful_requests": 24559, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 814.97, + "successful_latency_ms": { + "p50": 4.03, + "p95": 5.822, + "p99": 47.235 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/nginx.json b/evidence/nginx.json new file mode 100644 index 00000000..85bd3c74 --- /dev/null +++ b/evidence/nginx.json @@ -0,0 +1,11 @@ +{ + "result": "PASS", + "version": "nginx version: nginx/1.28.0", + "image": "nginx:1.28.0-alpine", + "public_server_header": "nginx", + "config_syntax": "PASS", + "observed_instances": [ + "0749a2e4af8a4b849e3eae6426945546", + "dd54c30c2e4c413ea2e28066c45e0d3c" + ] +} diff --git a/evidence/openapi.txt b/evidence/openapi.txt new file mode 100644 index 00000000..49c347b6 --- /dev/null +++ b/evidence/openapi.txt @@ -0,0 +1,2 @@ +python3 scripts/cluster.py exec python scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/evidence/persistence.json b/evidence/persistence.json new file mode 100644 index 00000000..bf895cc5 --- /dev/null +++ b/evidence/persistence.json @@ -0,0 +1,250 @@ +{ + "result": "PASS", + "before": [ + { + "service": "api", + "container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d", + "name": "mtusi-lab3-api-3", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:24:29.880819722Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af", + "memory_bytes": 402653184, + "nano_cpus": 750000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data", + "mtusi-lab3_front" + ], + "http_status": 200, + "instance": "a125004f5fbb4f9e91edb14bb3e64993", + "release": "lab3-9463abd8" + }, + { + "service": "api", + "container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea", + "name": "mtusi-lab3-api-4", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:24:29.778877972Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af", + "memory_bytes": 402653184, + "nano_cpus": 750000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data", + "mtusi-lab3_front" + ], + "http_status": 200, + "instance": "a8916b2407ed45cc87d73e88cf1639b9", + "release": "lab3-9463abd8" + }, + { + "service": "lb", + "container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a", + "name": "mtusi-lab3-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:23.133451802Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab3_front" + ] + }, + { + "service": "migrate", + "container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43", + "name": "mtusi-lab3-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T08:24:32.061652Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data" + ] + }, + { + "service": "postgres", + "container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7", + "name": "mtusi-lab3-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:24:29.651115763Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab3_postgres_data" + ], + "networks": [ + "mtusi-lab3_data" + ] + }, + { + "service": "redis", + "container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4", + "name": "mtusi-lab3-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:24:29.682187472Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab3_redis_data" + ], + "networks": [ + "mtusi-lab3_data" + ] + } + ], + "after": [ + { + "service": "api", + "container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d", + "name": "mtusi-lab3-api-3", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:33.199244251Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af", + "memory_bytes": 402653184, + "nano_cpus": 750000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data", + "mtusi-lab3_front" + ], + "http_status": 200, + "instance": "f9c80459570c4c99991024bba6d2b167", + "release": "lab3-9463abd8" + }, + { + "service": "api", + "container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea", + "name": "mtusi-lab3-api-4", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:33.076891376Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af", + "memory_bytes": 402653184, + "nano_cpus": 750000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data", + "mtusi-lab3_front" + ], + "http_status": 200, + "instance": "782d112163514263a4c14c0fb6d68bfc", + "release": "lab3-9463abd8" + }, + { + "service": "lb", + "container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a", + "name": "mtusi-lab3-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:32.827784792Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab3_front" + ] + }, + { + "service": "migrate", + "container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43", + "name": "mtusi-lab3-migrate-1", + "status": "exited", + "health": null, + "started_at": "2026-11-02T08:36:35.37548171Z", + "image": "mtusi-lab3-api:lab3", + "image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9", + "memory_bytes": 268435456, + "nano_cpus": 500000000, + "published_ports": {}, + "volumes": [], + "networks": [ + "mtusi-lab3_data" + ] + }, + { + "service": "postgres", + "container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7", + "name": "mtusi-lab3-postgres-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:32.796757626Z", + "image": "postgres:17.5-bookworm", + "image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4", + "memory_bytes": 536870912, + "nano_cpus": 1000000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab3_postgres_data" + ], + "networks": [ + "mtusi-lab3_data" + ] + }, + { + "service": "redis", + "container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4", + "name": "mtusi-lab3-redis-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:32.773500709Z", + "image": "redis:7.4.4-bookworm", + "image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091", + "memory_bytes": 134217728, + "nano_cpus": 250000000, + "published_ports": {}, + "volumes": [ + "mtusi-lab3_redis_data" + ], + "networks": [ + "mtusi-lab3_data" + ] + } + ], + "public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/public-failover.json b/evidence/public-failover.json new file mode 100644 index 00000000..c110876a --- /dev/null +++ b/evidence/public-failover.json @@ -0,0 +1,15 @@ +{ + "result": "PASS", + "public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 132, \"failed_attempts\": 0, \"first_recovery_seconds\": 0.871}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.", + "stopped": { + "container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea", + "name": "mtusi-lab3-api-4", + "instance": "782d112163514263a4c14c0fb6d68bfc", + "release": "lab3-9463abd8", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af", + "stopped_at_utc": "2026-11-02T08:36:39.262177+00:00", + "stop_seconds": 0.3321, + "exit_code": 0, + "left_stopped": true + } +} diff --git a/evidence/public-failover.txt b/evidence/public-failover.txt new file mode 100644 index 00000000..90f957b6 --- /dev/null +++ b/evidence/public-failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 0.871} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/resource-summary.json b/evidence/resource-summary.json new file mode 100644 index 00000000..e90de640 --- /dev/null +++ b/evidence/resource-summary.json @@ -0,0 +1,25 @@ +{ + "samples": 10, + "per_container_peaks": { + "mtusi-lab3-api-5": { + "cpu_percent_peak": 68.64, + "memory_mib_peak": 78.41 + }, + "mtusi-lab3-api-6": { + "cpu_percent_peak": 69.43, + "memory_mib_peak": 66.61 + }, + "mtusi-lab3-lb-1": { + "cpu_percent_peak": 10.01, + "memory_mib_peak": 7.895 + }, + "mtusi-lab3-postgres-1": { + "cpu_percent_peak": 42.15, + "memory_mib_peak": 99.4 + }, + "mtusi-lab3-redis-1": { + "cpu_percent_peak": 0.99, + "memory_mib_peak": 11.04 + } + } +} diff --git a/evidence/resources.json b/evidence/resources.json new file mode 100644 index 00000000..cc88e19d --- /dev/null +++ b/evidence/resources.json @@ -0,0 +1,402 @@ +[ + { + "at_utc": "2026-11-02T08:38:21.926644+00:00", + "containers": [ + { + "CPUPerc": "50.31%", + "MemPerc": "17.21%", + "MemUsage": "66.08MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "8" + }, + { + "CPUPerc": "66.02%", + "MemPerc": "17.20%", + "MemUsage": "66.05MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "9" + }, + { + "CPUPerc": "9.06%", + "MemPerc": "5.98%", + "MemUsage": "7.652MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "30.22%", + "MemPerc": "15.95%", + "MemUsage": "81.65MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.14%", + "MemPerc": "8.26%", + "MemUsage": "10.58MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:24.960579+00:00", + "containers": [ + { + "CPUPerc": "68.64%", + "MemPerc": "17.27%", + "MemUsage": "66.31MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "9" + }, + { + "CPUPerc": "53.09%", + "MemPerc": "17.15%", + "MemUsage": "65.86MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "9.49%", + "MemPerc": "5.70%", + "MemUsage": "7.293MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "31.09%", + "MemPerc": "16.17%", + "MemUsage": "82.77MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.91%", + "MemPerc": "8.30%", + "MemUsage": "10.62MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:27.995870+00:00", + "containers": [ + { + "CPUPerc": "58.43%", + "MemPerc": "20.27%", + "MemUsage": "77.83MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "10" + }, + { + "CPUPerc": "56.00%", + "MemPerc": "17.20%", + "MemUsage": "66.07MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "9.79%", + "MemPerc": "5.76%", + "MemUsage": "7.379MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "33.22%", + "MemPerc": "16.73%", + "MemUsage": "85.67MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.90%", + "MemPerc": "8.13%", + "MemUsage": "10.4MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:31.030777+00:00", + "containers": [ + { + "CPUPerc": "52.13%", + "MemPerc": "17.41%", + "MemUsage": "66.87MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "9" + }, + { + "CPUPerc": "67.00%", + "MemPerc": "17.21%", + "MemUsage": "66.07MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "9.38%", + "MemPerc": "5.72%", + "MemUsage": "7.316MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "33.30%", + "MemPerc": "17.16%", + "MemUsage": "87.88MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.96%", + "MemPerc": "8.14%", + "MemUsage": "10.41MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:34.047170+00:00", + "containers": [ + { + "CPUPerc": "65.51%", + "MemPerc": "17.21%", + "MemUsage": "66.1MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "10" + }, + { + "CPUPerc": "52.34%", + "MemPerc": "17.24%", + "MemUsage": "66.21MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "9.16%", + "MemPerc": "5.85%", + "MemUsage": "7.488MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "30.55%", + "MemPerc": "17.40%", + "MemUsage": "89.08MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.93%", + "MemPerc": "8.11%", + "MemUsage": "10.38MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:37.087544+00:00", + "containers": [ + { + "CPUPerc": "56.33%", + "MemPerc": "19.86%", + "MemUsage": "76.26MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "11" + }, + { + "CPUPerc": "51.99%", + "MemPerc": "17.30%", + "MemUsage": "66.44MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "9.40%", + "MemPerc": "6.09%", + "MemUsage": "7.789MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "33.34%", + "MemPerc": "17.80%", + "MemUsage": "91.15MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.14%", + "MemPerc": "8.48%", + "MemUsage": "10.86MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:40.128204+00:00", + "containers": [ + { + "CPUPerc": "49.82%", + "MemPerc": "17.23%", + "MemUsage": "66.16MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "10" + }, + { + "CPUPerc": "64.91%", + "MemPerc": "17.32%", + "MemUsage": "66.5MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "9.44%", + "MemPerc": "5.81%", + "MemUsage": "7.438MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "30.24%", + "MemPerc": "18.14%", + "MemUsage": "92.86MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.94%", + "MemPerc": "8.63%", + "MemUsage": "11.04MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:43.146647+00:00", + "containers": [ + { + "CPUPerc": "61.56%", + "MemPerc": "20.42%", + "MemUsage": "78.41MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "11" + }, + { + "CPUPerc": "56.33%", + "MemPerc": "17.35%", + "MemUsage": "66.61MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "8.46%", + "MemPerc": "6.17%", + "MemUsage": "7.895MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "32.44%", + "MemPerc": "18.46%", + "MemUsage": "94.5MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.17%", + "MemPerc": "8.25%", + "MemUsage": "10.56MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:46.182717+00:00", + "containers": [ + { + "CPUPerc": "64.09%", + "MemPerc": "17.39%", + "MemUsage": "66.78MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "9" + }, + { + "CPUPerc": "66.93%", + "MemPerc": "17.24%", + "MemUsage": "66.19MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "8.61%", + "MemPerc": "5.89%", + "MemUsage": "7.539MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "42.15%", + "MemPerc": "19.17%", + "MemUsage": "98.16MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.14%", + "MemPerc": "8.27%", + "MemUsage": "10.58MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + }, + { + "at_utc": "2026-11-02T08:38:49.213747+00:00", + "containers": [ + { + "CPUPerc": "53.84%", + "MemPerc": "17.29%", + "MemUsage": "66.39MiB / 384MiB", + "Name": "mtusi-lab3-api-5", + "PIDs": "9" + }, + { + "CPUPerc": "69.43%", + "MemPerc": "17.32%", + "MemUsage": "66.5MiB / 384MiB", + "Name": "mtusi-lab3-api-6", + "PIDs": "10" + }, + { + "CPUPerc": "10.01%", + "MemPerc": "6.13%", + "MemUsage": "7.844MiB / 128MiB", + "Name": "mtusi-lab3-lb-1", + "PIDs": "2" + }, + { + "CPUPerc": "34.69%", + "MemPerc": "19.41%", + "MemUsage": "99.4MiB / 512MiB", + "Name": "mtusi-lab3-postgres-1", + "PIDs": "16" + }, + { + "CPUPerc": "0.99%", + "MemPerc": "8.46%", + "MemUsage": "10.83MiB / 128MiB", + "Name": "mtusi-lab3-redis-1", + "PIDs": "6" + } + ] + } +] diff --git a/evidence/rolling-update.json b/evidence/rolling-update.json new file mode 100644 index 00000000..a99250c4 --- /dev/null +++ b/evidence/rolling-update.json @@ -0,0 +1,171 @@ +{ + "result": "PASS", + "before": [ + { + "container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d", + "name": "mtusi-lab3-api-3", + "instance": "efae54a83fc844f4b19b1e1c2aad0385", + "release": "lab3-9463abd8", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af" + }, + { + "container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea", + "name": "mtusi-lab3-api-4", + "instance": "1b7fd0264d924b3ba1b87368400b4a56", + "release": "lab3-9463abd8", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af" + } + ], + "after": [ + { + "container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710", + "name": "mtusi-lab3-api-5", + "instance": "0749a2e4af8a4b849e3eae6426945546", + "release": "lab3-9d88e53e", + "image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741" + }, + { + "container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f", + "name": "mtusi-lab3-api-6", + "instance": "dd54c30c2e4c413ea2e28066c45e0d3c", + "release": "lab3-9d88e53e", + "image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741" + } + ], + "target_release": "lab3-9d88e53e", + "lb_before": { + "service": "lb", + "container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a", + "name": "mtusi-lab3-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:32.827784792Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab3_front" + ] + }, + "lb_after": { + "service": "lb", + "container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a", + "name": "mtusi-lab3-lb-1", + "status": "running", + "health": "healthy", + "started_at": "2026-11-02T08:36:32.827784792Z", + "image": "nginx:1.28.0-alpine", + "image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284", + "memory_bytes": 134217728, + "nano_cpus": 500000000, + "published_ports": { + "8080/tcp": [ + { + "HostIp": "127.0.0.1", + "HostPort": "8080" + } + ] + }, + "volumes": [], + "networks": [ + "mtusi-lab3_front" + ] + }, + "lb_not_restarted": true, + "build_seconds": 2.2969, + "migration_output": "Migrations up to date", + "replacements": [ + { + "old_container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d", + "new_container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710", + "old_ip": "172.23.0.4", + "new_ip": "172.23.0.5", + "dns_discovered_new_address": true, + "instances_during_start_first": [ + "0749a2e4af8a4b849e3eae6426945546", + "1b7fd0264d924b3ba1b87368400b4a56", + "efae54a83fc844f4b19b1e1c2aad0385" + ], + "observed_instances": [ + "0749a2e4af8a4b849e3eae6426945546", + "1b7fd0264d924b3ba1b87368400b4a56" + ], + "old_exit_code": 0, + "seconds": 5.864 + }, + { + "old_container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea", + "new_container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f", + "old_ip": "172.23.0.2", + "new_ip": "172.23.0.4", + "dns_discovered_new_address": true, + "instances_during_start_first": [ + "0749a2e4af8a4b849e3eae6426945546", + "1b7fd0264d924b3ba1b87368400b4a56", + "dd54c30c2e4c413ea2e28066c45e0d3c" + ], + "observed_instances": [ + "0749a2e4af8a4b849e3eae6426945546", + "dd54c30c2e4c413ea2e28066c45e0d3c" + ], + "old_exit_code": 0, + "seconds": 5.6574 + } + ], + "during_replacement": { + "seconds": 11.5214, + "concurrency": 4, + "successful_requests": 7916, + "failed_attempts": 1, + "successful_rps": 687.069, + "error_ratio": 0.000126, + "successful_latency_ms": { + "p50": 3.992, + "p95": 7.701, + "p99": 62.305 + }, + "instances": [ + "0749a2e4af8a4b849e3eae6426945546", + "1b7fd0264d924b3ba1b87368400b4a56", + "dd54c30c2e4c413ea2e28066c45e0d3c", + "efae54a83fc844f4b19b1e1c2aad0385" + ] + }, + "total": { + "seconds": 20.7201, + "concurrency": 4, + "successful_requests": 14975, + "failed_attempts": 1, + "successful_rps": 722.729, + "error_ratio": 6.7e-05, + "successful_latency_ms": { + "p50": 4.089, + "p95": 7.12, + "p99": 59.058 + }, + "instances": [ + "0749a2e4af8a4b849e3eae6426945546", + "1b7fd0264d924b3ba1b87368400b4a56", + "dd54c30c2e4c413ea2e28066c45e0d3c", + "efae54a83fc844f4b19b1e1c2aad0385" + ] + }, + "data": { + "confirmed_unique_messages": 14975, + "history_messages": 14975, + "confirmed_preserved": true, + "duplicate_ids": 0, + "duplicate_texts": 0 + }, + "change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate." +} diff --git a/evidence/sequential-failover.json b/evidence/sequential-failover.json new file mode 100644 index 00000000..ef2953fa --- /dev/null +++ b/evidence/sequential-failover.json @@ -0,0 +1,135 @@ +{ + "result": "PASS", + "baseline": { + "seconds": 3.0036, + "concurrency": 4, + "successful_requests": 2095, + "failed_attempts": 0, + "successful_rps": 697.497, + "error_ratio": 0.0, + "successful_latency_ms": { + "p50": 4.402, + "p95": 6.728, + "p99": 56.167 + }, + "instances": [ + "9ab4e9373a944f05b69d934479430136", + "f9c80459570c4c99991024bba6d2b167" + ] + }, + "phases": [ + { + "stopped": { + "container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d", + "name": "mtusi-lab3-api-3", + "instance": "f9c80459570c4c99991024bba6d2b167", + "release": "lab3-9463abd8", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af", + "stopped_at_utc": "2026-11-02T08:37:05.018221+00:00", + "stop_seconds": 0.3922, + "exit_code": 0, + "left_stopped": true + }, + "traffic": { + "seconds": 8.0014, + "concurrency": 4, + "successful_requests": 3201, + "failed_attempts": 1, + "successful_rps": 400.056, + "error_ratio": 0.000312, + "successful_latency_ms": { + "p50": 6.213, + "p95": 37.884, + "p99": 72.594 + }, + "instances": [ + "9ab4e9373a944f05b69d934479430136", + "f9c80459570c4c99991024bba6d2b167" + ] + }, + "post_stop_write_seconds_from_action_start": 1.3835, + "post_stop_write_seconds_from_stop_completion": 0.0138, + "read_during_outage": { + "status": 200, + "confirmed_message_preserved": true, + "seconds_from_action_start": 1.38, + "seconds_from_stop_completion": 0.0102 + }, + "new_connection_after_stop": { + "accepted": false, + "error_type": "TimeoutError" + }, + "stopped_remains_stopped": true + }, + { + "stopped": { + "container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea", + "name": "mtusi-lab3-api-4", + "instance": "9ab4e9373a944f05b69d934479430136", + "release": "lab3-9463abd8", + "image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af", + "stopped_at_utc": "2026-11-02T08:37:19.081389+00:00", + "stop_seconds": 0.4518, + "exit_code": 0, + "left_stopped": true + }, + "traffic": { + "seconds": 8.0017, + "concurrency": 4, + "successful_requests": 3210, + "failed_attempts": 1, + "successful_rps": 401.166, + "error_ratio": 0.000311, + "successful_latency_ms": { + "p50": 6.206, + "p95": 37.612, + "p99": 74.049 + }, + "instances": [ + "4ffdcca2f8644acd9b76d65f8260eaf7", + "9ab4e9373a944f05b69d934479430136" + ] + }, + "post_stop_write_seconds_from_action_start": 1.4405, + "post_stop_write_seconds_from_stop_completion": 0.0151, + "read_during_outage": { + "status": 200, + "confirmed_message_preserved": true, + "seconds_from_action_start": 1.4375, + "seconds_from_stop_completion": 0.0121 + }, + "new_connection_after_stop": { + "accepted": false, + "error_type": "TimeoutError" + }, + "stopped_remains_stopped": true + } + ], + "total": { + "seconds": 31.1217, + "concurrency": 4, + "successful_requests": 15915, + "failed_attempts": 2, + "successful_rps": 511.38, + "error_ratio": 0.000126, + "successful_latency_ms": { + "p50": 5.137, + "p95": 32.066, + "p99": 65.023 + }, + "instances": [ + "1b7fd0264d924b3ba1b87368400b4a56", + "4ffdcca2f8644acd9b76d65f8260eaf7", + "9ab4e9373a944f05b69d934479430136", + "f9c80459570c4c99991024bba6d2b167" + ] + }, + "data": { + "confirmed_unique_messages": 15915, + "history_messages": 15915, + "confirmed_preserved": true, + "duplicate_ids": 0, + "duplicate_texts": 0 + }, + "each_original_container_tested": true +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..44e41deb --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +python3 tests/smoke.py --base-url "http://localhost:8080" +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/student-tests.txt b/evidence/student-tests.txt new file mode 100644 index 00000000..00e2bbab --- /dev/null +++ b/evidence/student-tests.txt @@ -0,0 +1,3 @@ +python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache +.............. [100%] +14 passed in 3.58s diff --git a/infra/nginx/nginx.conf b/infra/nginx/nginx.conf new file mode 100644 index 00000000..7704b3dd --- /dev/null +++ b/infra/nginx/nginx.conf @@ -0,0 +1,43 @@ +worker_processes 1; +error_log /dev/stderr warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + access_log off; + server_tokens off; + + # Docker DNS; resolve updates the upstream when replicas get new IPs. + resolver 127.0.0.11 valid=1s ipv6=off; + resolver_timeout 1s; + + upstream messenger { + zone messenger 64k; + # All A records of api participate in the default round-robin balancing. + server api:8080 resolve max_fails=1 fail_timeout=2s; + } + + server { + listen 8080; + + location / { + proxy_pass http://messenger; + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header Connection ""; + + proxy_connect_timeout 1s; + proxy_read_timeout 5s; + proxy_send_timeout 5s; + + # Passive checks. Do NOT enable non_idempotent: an already-sent POST + # must not be resent by Nginx after an ambiguous response. + proxy_next_upstream error timeout http_503; + proxy_next_upstream_tries 2; + proxy_next_upstream_timeout 6s; + } + } +} diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..7b8c5338 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id); +CREATE INDEX chats_history_idx ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry_idx ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..855e66a8 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,11 @@ +-- Additive: old lab2 tables and messages remain valid. +CREATE TABLE idempotency_records ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'), + request_hash text NOT NULL, + message_id uuid REFERENCES messages(id), + expires_at timestamptz NOT NULL, + PRIMARY KEY (user_id, chat_id, key) +); +CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at); diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 00000000..55da8a0c --- /dev/null +++ b/requirements.lock @@ -0,0 +1,45 @@ +annotated-types==0.8.0 +anyio==4.9.0 +argon2-cffi==25.1.0 +argon2-cffi-bindings==26.1.0 +attrs==26.1.0 +certifi==2026.10.23 +cffi==2.1.1 +charset-normalizer==3.5.2 +click==8.5.0 +fastapi==0.115.12 +h11==0.16.0 +httpcore==1.0.9 +httpx==0.28.1 +idna==3.20 +iniconfig==2.3.1 +jsonschema==4.26.0 +jsonschema-path==0.3.4 +jsonschema-specifications==2025.9.1 +lazy-object-proxy==1.13.0 +openapi-schema-validator==0.6.3 +openapi-spec-validator==0.7.2 +packaging==26.3 +pathable==0.4.4 +pluggy==1.7.0 +psycopg==3.2.9 +psycopg-binary==3.2.9 +psycopg-pool==3.2.6 +pycparser==3.12 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.21.0 +pytest==8.4.0 +PyYAML==6.0.3 +redis==6.2.0 +referencing==0.36.2 +requests==2.34.2 +rfc3339-validator==0.1.4 +rpds-py==2026.9.1 +six==1.17.0 +sniffio==1.3.1 +starlette==0.46.2 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +urllib3==2.8.0 +uvicorn==0.34.3 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..c0c5c8b7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.115.12 +uvicorn==0.34.3 +psycopg[binary]==3.2.9 +psycopg-pool==3.2.6 +redis==6.2.0 +argon2-cffi==25.1.0 +pytest==8.4.0 +httpx==0.28.1 +pydantic==2.11.5 +anyio==4.9.0 diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh new file mode 100644 index 00000000..bdc06e38 --- /dev/null +++ b/scripts/bootstrap.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +umask 077 +mkdir -p secrets +python3 - <<'PY' +from pathlib import Path +import secrets +root = Path('secrets') +p = root / 'postgres_password' +if not p.exists(): + p.write_text(secrets.token_hex(24)) +pw = p.read_text().strip() +for name, value in { + 'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger', + 'cache_url': 'redis://redis:6379/0', +}.items(): + path = root / name + if not path.exists(): + path.write_text(value) +# Bind-mounted Docker secrets must be readable by container uid 10001. +# Parent directory remains 0700 on the host. +for path in root.iterdir(): + path.chmod(0o444) +if not Path('.env').exists(): + Path('.env').write_text('SESSION_TTL_SECONDS=3600\n') +print('Local secrets ready (values hidden).') +PY diff --git a/scripts/capture_load.py b/scripts/capture_load.py new file mode 100644 index 00000000..2c7db1f7 --- /dev/null +++ b/scripts/capture_load.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Capture repeatable short load run and sanitized environment/resource evidence.""" +import datetime +import json +import platform +import re +import subprocess +import sys +import time +import cluster +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def output(args): + return subprocess.check_output(args, cwd=ROOT, text=True).strip() + + +def counts(): + code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions','idempotency_records']})); c.close()" + return json.loads(output(['docker','exec',cluster.container_ids('api')[0],'python','-c',code])) + + +def resource_summary(samples): + peaks = {} + for sample in samples: + for container in sample['containers']: + peak = peaks.setdefault(container['Name'], {'cpu_percent_peak':0, 'memory_mib_peak':0}) + peak['cpu_percent_peak'] = max(peak['cpu_percent_peak'],float(container['CPUPerc'].rstrip('%'))) + value,unit = re.match(r'([0-9.]+)([A-Za-z]+)',container['MemUsage']).groups() + multiplier = {'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/1048576}[unit] + peak['memory_mib_peak'] = round(max(peak['memory_mib_peak'],float(value)*multiplier),3) + return {'samples':len(samples),'per_container_peaks':peaks} + + +def main(): + names = cluster.container_ids() + environment = { + 'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'host_os': platform.platform(), 'host_architecture': platform.machine(), + 'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])), + 'python_in_image': output(['docker','exec',cluster.container_ids('api')[0],'python','--version']), + 'dataset_before': counts(), + 'limits': {'api_each':'0.75 CPU / 384 MiB','postgres':'1 CPU / 512 MiB','redis':'0.25 CPU / 128 MiB','lb':'0.5 CPU / 128 MiB'}, + 'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.', + 'replicas':len(cluster.replicas()), 'repetitions': 1, 'duration_seconds':30, 'concurrency':4, + } + if sys.platform == 'darwin': + environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string']) + environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize'])) + samples = [] + with (ROOT/'evidence/load-output.txt').open('w') as log: + proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT) + while proc.poll() is None: + rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names]) + samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')} + for row in rows.splitlines()]}) + time.sleep(1) + if proc.returncode: + raise SystemExit(proc.returncode) + environment['dataset_after'] = counts() + (ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n') + (ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n') + (ROOT/'evidence/resource-summary.json').write_text(json.dumps(resource_summary(samples),indent=2)+'\n') + print((ROOT/'evidence/load.json').read_text()) + + +if __name__ == '__main__': + main() diff --git a/scripts/check_fresh_migrations.sh b/scripts/check_fresh_migrations.sh new file mode 100644 index 00000000..f6b77e59 --- /dev/null +++ b/scripts/check_fresh_migrations.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# A unique disposable Compose project; never touches normal project volumes. +set -eu +cd "$(dirname "$0")/.." +project="mtusi-lab3-migration-check-$$" +if [ -n "$(docker compose -p "$project" ps -aq)" ]; then + echo 'Refusing to reuse a pre-existing scratch project' >&2 + exit 1 +fi +cleanup() { + docker compose -p "$project" down --volumes >/dev/null 2>&1 +} +trap cleanup EXIT INT TERM +docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis +docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache' +echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.' diff --git a/scripts/cluster.py b/scripts/cluster.py new file mode 100644 index 00000000..84f96178 --- /dev/null +++ b/scripts/cluster.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence.""" +import argparse +import datetime +import json +import os +import re +import subprocess +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/') + + +def command(args, timeout=120): + return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip() + + +def compose(*args, timeout=120): + return command(['docker', 'compose', *args], timeout=timeout) + + +def inspect(containers): + if not containers: + return [] + return json.loads(command(['docker','inspect',*containers])) + + +def container_ids(service=None, all_containers=False): + args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else []) + return compose(*args).split() + + +def probe(container, host='127.0.0.1', path='/health/live'): + code = '''import json,sys,urllib.request,urllib.error +try: + r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4) +except urllib.error.HTTPError as exc: + r=exc +print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')})) +''' + return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8)) + + +def replicas(): + items = [] + for data in inspect(container_ids('api')): + if data['State']['Running']: + result = probe(data['Id']) + items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'), + 'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']}) + return items + + +def snapshot(): + items = [] + for data in inspect(container_ids(all_containers=True)): + service = data['Config']['Labels']['com.docker.compose.service'] + item = { + 'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'), + 'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'), + 'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'], + 'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'], + 'published_ports':data['HostConfig']['PortBindings'], + 'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'), + 'networks':sorted(data['NetworkSettings']['Networks']), + } + if service == 'api' and data['State']['Running']: + result = probe(data['Id']) + item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']}) + items.append(item) + return items + + +def wait_healthy(container, timeout=60): + deadline = time.monotonic()+timeout + while time.monotonic() < deadline: + data = inspect([container])[0] + if data['State'].get('Health',{}).get('Status') == 'healthy': + return + time.sleep(0.3) + raise RuntimeError('Container did not become healthy') + + +def wait_cluster(count=2, timeout=60): + deadline = time.monotonic()+timeout + seen = set() + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response: + alias = response.headers.get('X-Instance-Id') + if alias: + seen.add(alias) + if len(seen) >= count: + return sorted(seen) + except (OSError, urllib.error.URLError): + pass + time.sleep(0.1) + raise RuntimeError('Load balancer did not discover required API instances') + + +def stop_one(target): + if not re.fullmatch(r'[a-f0-9]{32}', target): + raise ValueError('Invalid target instance alias') + matching = [item for item in replicas() if item['instance']==target] + if len(matching) != 1: + raise RuntimeError('Target must match exactly one running API process') + item = matching[0] + started = time.monotonic() + command(['docker','stop','--time','7',item['container']],timeout=9) + state = inspect([item['container']])[0]['State'] + if state['Running'] or state['ExitCode'] != 0: + raise RuntimeError('API did not stop gracefully') + result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(), + 'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True} + (ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n') + return result + + +def restart(): + compose('restart','api','postgres','redis','lb',timeout=30) + compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40) + return wait_cluster(2,timeout=15) + + +def front_address(container): + networks = inspect([container])[0]['NetworkSettings']['Networks'] + return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front')) + + +def replace_one(container): + before = set(container_ids('api')) + if container not in before or len(before) != 2: + raise RuntimeError('Replacement expects exactly two running API containers') + old_address = front_address(container) + # Start-first ensures a genuinely different IP while the old address is still occupied. + compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api') + expanded = set(container_ids('api')) + added = expanded-before + if len(added) != 1 or not before.issubset(expanded): + raise RuntimeError('Start-first step unexpectedly changed an existing container') + replacement = added.pop() + wait_healthy(replacement) + new_address = front_address(replacement) + if new_address == old_address: + raise RuntimeError('Replacement did not get a different IP') + expected_three = {item['instance'] for item in replicas()} + if set(wait_cluster(3)) != expected_three: + raise RuntimeError('Load balancer did not discover all three live addresses') + command(['docker','stop','--time','7',container],timeout=10) + state = inspect([container])[0]['State'] + if state['ExitCode'] != 0: + raise RuntimeError('Old release did not terminate gracefully') + command(['docker','rm',container]) + compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api') + after = set(container_ids('api')) + if after != (before-{container}) | {replacement}: + raise RuntimeError('Sequential replacement changed the wrong containers') + expected = {item['instance'] for item in replicas()} + if set(wait_cluster(2)) != expected: + raise RuntimeError('Unexpected final backend aliases') + return {'old_container':container,'new_container':replacement, + 'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True, + 'instances_during_start_first':sorted(expected_three), + 'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']} + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec']) + parser.add_argument('arguments',nargs=argparse.REMAINDER) + args = parser.parse_args() + if args.action == 'wait': + print(json.dumps({'instances':wait_cluster()})) + elif args.action == 'snapshot': + print(json.dumps(snapshot(),indent=2)) + elif args.action == 'stop-one': + print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID']))) + elif args.action == 'exec': + ids = container_ids('api') + if not ids or not args.arguments: + raise RuntimeError('A live API container and command are required') + raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT)) + elif args.action == 'restart': + print(json.dumps({'instances':restart()})) + else: + compose('run','--rm','--no-deps','migrate') + print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2)) + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100644 index 00000000..3985fa9c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/cluster.py restart diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100644 index 00000000..ef14b25f --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +: "${TARGET_INSTANCE_ID:?test provides target API instance alias}" +exec python3 scripts/cluster.py stop-one diff --git a/scripts/verify_cluster.py b/scripts/verify_cluster.py new file mode 100644 index 00000000..06e2f5d5 --- /dev/null +++ b/scripts/verify_cluster.py @@ -0,0 +1,443 @@ +#!/usr/bin/env python3 +"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts.""" +import argparse +import json +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +import uuid + +import cluster + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0,str(ROOT/'tests')) +from client import Client, Failure, check +from resilience import history + + +def save(name, value): + (ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n') + print('PASS:', name) + + +def percentiles(values): + values = sorted(values) + return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None + for name,p in [('p50',.50),('p95',.95),('p99',.99)]} + + +class Traffic: + def __init__(self, concurrency=4): + self.c = Client(timeout=6) + self.alice,self.bob,_,self.chat = self.c.bootstrap() + self.concurrency = concurrency + self.done = threading.Event() + self.lock = threading.Lock() + self.attempts = [] + self.confirmed = {} + self.prefix = uuid.uuid4().hex + + def worker(self, number): + client = Client(timeout=6) + index = 0 + while not self.done.is_set(): + text = self.prefix+'-'+str(number)+'-'+str(index) + key = self.prefix+'-'+str(number)+'-'+str(index) + started = time.monotonic() + try: + response = client.message(self.alice,self.chat,text,key=key) + row = response['json'] + attempt = {'started':started,'ended':time.monotonic(),'ok':True, + 'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']} + with self.lock: + self.confirmed[row['id']] = row + index += 1 + except Failure: + # Only idempotent message POST is retried, with its original body and key. + attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None} + time.sleep(0.02) + with self.lock: + self.attempts.append(attempt) + + def start(self): + self.started = time.monotonic() + self.pool = ThreadPoolExecutor(max_workers=self.concurrency) + self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)] + return self + + def stop(self): + self.done.set() + for future in self.futures: + future.result(timeout=10) + self.pool.shutdown() + self.ended = time.monotonic() + + def summary(self, start=None, end=None): + start = self.started if start is None else start + end = getattr(self,'ended',time.monotonic()) if end is None else end + with self.lock: + attempts = [a for a in self.attempts if start <= a['started'] < end] + success = [a for a in attempts if a['ok']] + duration = end-start + return {'seconds':round(duration,4),'concurrency':self.concurrency, + 'successful_requests':len(success),'failed_attempts':len(attempts)-len(success), + 'successful_rps':round(len(success)/duration,3), + 'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0, + 'successful_latency_ms':percentiles([a['latency'] for a in success]), + 'instances':sorted({a['instance'] for a in success})} + + def verify_data(self): + rows = history(self.c,self.bob,self.chat) + actual = {row['id']:row for row in rows} + check(len(actual)==len(rows),'duplicated message ids') + check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message') + check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed') + return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows), + 'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0} + + +def concurrent_replay(): + cluster.wait_cluster() + c = Client() + alice,bob,_,chat = c.bootstrap() + key = uuid.uuid4().hex + barrier = threading.Barrier(8) + def send(_): + local = Client() + barrier.wait() + return local.message(alice,chat,'Один результат через разные процессы',key=key) + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(send,range(8))) + instances = {r['headers']['X-Instance-Id'] for r in results} + check(len(instances)>=2,'concurrent retry did not reach different API processes') + check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages') + c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice, + body={'text':'Другой body'},headers={'Idempotency-Key':key}) + check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates') + save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances), + 'responses_201':8,'messages_created':1,'different_body_status':409}) + + +def verify_persistence(): + before = cluster.snapshot() + result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'], + cwd=ROOT,check=True,capture_output=True,text=True,timeout=190) + after = cluster.snapshot() + roles = ['api','postgres','redis','lb'] + previous = {item['container']:item for item in before} + for item in after: + if item['service'] in roles: + old = previous[item['container']] + check(old['started_at']!=item['started_at'],'service did not restart: '+item['service']) + check(old['volumes']==item['volumes'],'volume identities changed') + (ROOT/'evidence/persistence.txt').write_text(result.stdout) + save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()}) + + +def public_failover(): + cluster.wait_cluster() + try: + result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'], + cwd=ROOT,check=True,capture_output=True,text=True,timeout=100) + (ROOT/'evidence/public-failover.txt').write_text(result.stdout) + save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(), + 'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())}) + finally: + cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90') + cluster.wait_cluster() + + +def reject_new_connection(survivor, address): + code = '''import json,socket,sys +try: + s=socket.create_connection((sys.argv[1],8080),timeout=1) +except OSError as exc: + print(json.dumps({'accepted':False,'error_type':type(exc).__name__})) +else: + s.close() + print(json.dumps({'accepted':True})) +''' + return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address])) + + +def failover_each(): + cluster.wait_cluster() + originals = cluster.replicas() + check(len(originals)==2,'two running replicas required') + traffic = Traffic().start() + phases = [] + try: + time.sleep(3) + baseline_end = time.monotonic() + baseline = traffic.summary(end=baseline_end) + for original in originals: + # Alias changes on restart, so resolve the current process of this exact container. + current = next(r for r in cluster.replicas() if r['container']==original['container']) + all_before = cluster.inspect([current['container']])[0] + address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front')) + survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container']) + read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1' + marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0] + start = time.monotonic() + try: + env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']} + subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10) + stop_returned = time.monotonic() + stopped = json.loads((ROOT/'evidence/last-stop.json').read_text()) + read_deadline = start+10 + while True: + try: + read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage') + check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage') + check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process') + read_done = time.monotonic() + break + except Failure: + check(time.monotonic()=start] + recent = [a for a in successful if a['started']>stop_returned] + recovery = min(a['ended'] for a in recent)-start if recent else None + after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None + check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start') + check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests') + phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4), + 'post_stop_write_seconds_from_stop_completion':round(after_stop,4), + 'read_during_outage':{'status':200,'confirmed_message_preserved':True, + 'seconds_from_action_start':round(read_done-start,4), + 'seconds_from_stop_completion':round(read_done-stop_returned,4)}, + 'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']}) + finally: + cluster.command(['docker','start',current['container']]) + cluster.wait_healthy(current['container']) + cluster.wait_cluster() + time.sleep(2) + finally: + traffic.stop() + save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases, + 'total':traffic.summary(),'data':traffic.verify_data(), + 'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2}) + + +def rolling_update(): + cluster.wait_cluster() + before = cluster.replicas() + check(len(before)==2,'rolling demonstration expects two replicas') + lb_before = next(item for item in cluster.snapshot() if item['service']=='lb') + target_release = 'lab3-'+uuid.uuid4().hex[:8] + traffic = Traffic().start() + replacements = [] + try: + time.sleep(3) + build_start = time.monotonic() + cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180) + build_end = time.monotonic() + migration_output = cluster.compose('run','--rm','--no-deps','migrate') + swap_start = time.monotonic() + for old in before: + step_start = time.monotonic() + result = cluster.replace_one(old['container']) + result['seconds'] = round(time.monotonic()-step_start,4) + replacements.append(result) + swap_end = time.monotonic() + time.sleep(3) + finally: + traffic.stop() + after = cluster.replicas() + lb_after = next(item for item in cluster.snapshot() if item['service']=='lb') + check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test') + check(all(r['release']==target_release for r in after),'new release not observed on all replicas') + check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement') + check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change') + save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release, + 'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True, + 'build_seconds':round(build_end-build_start,4),'migration_output':migration_output, + 'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end), + 'total':traffic.summary(),'data':traffic.verify_data(), + 'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'}) + + +def database_fixture(container, data): + code = '''import json,sys,psycopg +from psycopg import sql +from messenger.db import setting +p=json.load(sys.stdin) +with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c: + if p['action']=='install': + c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text']))) + c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function']))) + result={'installed':True} + elif p['action']=='remove': + c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger']))) + c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function']))) + result={'removed':True} + else: + result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]} +print(json.dumps(result)) +''' + response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT, + input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8) + return json.loads(response.stdout) + + +def graceful_shutdown(): + cluster.wait_cluster() + nodes = cluster.replicas() + target,survivor = nodes[0],nodes[1] + address = cluster.front_address(target['container']) + c = Client() + alice,bob,_,chat = c.bootstrap() + path = '/api/v1/chats/'+chat['id']+'/messages' + marker = 'drain_'+uuid.uuid4().hex + fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker} + database_fixture(survivor['container'],{**fixture,'action':'install'}) + code = '''import json,sys,urllib.request,urllib.error +p=json.load(sys.stdin) +r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']}) +try: + response=urllib.request.urlopen(r,timeout=6) +except urllib.error.HTTPError as exc: + response=exc +print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)})) +''' + request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex} + def in_flight(): + result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT, + input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8) + return json.loads(result.stdout) + pool = ThreadPoolExecutor(max_workers=1) + future = pool.submit(in_flight) + try: + deadline = time.monotonic()+3 + while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']: + check(time.monotonic()0 and item['nano_cpus']>0,'resource limit missing') + if item['service']!='lb': + check(not item['published_ports'],'service exposed a port bypassing LB') + check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas') + check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running') + save('infrastructure.json',{'result':'PASS','containers':state}) + + +def dependency_failures(): + c = Client(timeout=6) + actor = c.user('dependencies') + c.call('GET','/api/v1/users/me',actor=actor,schema='User') + result = {'result':'PASS'} + try: + cluster.compose('stop','-t','2','redis') + response = c.call('GET','/api/v1/users/me',actor=actor,schema='User') + check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed') + c.call('GET','/api/v1/users/me',401) + c.call('GET','/health/ready',schema='Health') + result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200} + finally: + cluster.compose('start','redis') + import urllib.request + import urllib.error + def status(path, token=None): + headers = {'Authorization':'Bearer '+token} if token else {} + request = urllib.request.Request(c.base+path,headers=headers) + started = time.monotonic() + try: + response = urllib.request.urlopen(request,timeout=6) + except urllib.error.HTTPError as error: + response = error + with response: + return response.code, round(time.monotonic()-started,4) + try: + cluster.compose('stop','-t','2','postgres') + code,elapsed = status('/api/v1/users/me',actor['token']) + check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request') + anonymous,_ = status('/api/v1/users/me') + check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage') + live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()] + check(all(code==200 for code in live),'API process liveness depends on DB') + result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous, + 'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'} + finally: + cluster.compose('start','postgres') + cluster.wait_cluster() + c.call('GET','/api/v1/users/me',actor=actor,schema='User') + result['same_session_after_recovery'] = True + save('dependency-failures.json',result) + + +def verify_nginx(): + state = cluster.snapshot() + lb = next(item for item in state if item['service']=='lb') + check(lb['image'].startswith('nginx:'),'load balancer must use Nginx') + cluster.command(['docker','exec',lb['container'],'nginx','-t']) + version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1']) + c = Client() + response = c.call('GET','/health/ready',schema='Health') + check(response['headers'].get('Server')=='nginx','public entry point is not Nginx') + check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas') + save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx', + 'config_syntax':'PASS','observed_instances':cluster.wait_cluster()}) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?') + args = parser.parse_args() + modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover, + 'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure} + if args.mode=='all': + for function in modes.values(): + function() + else: + modes[args.mode]() + + +if __name__ == '__main__': + main() diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/messenger/app.py b/src/messenger/app.py new file mode 100644 index 00000000..7870b736 --- /dev/null +++ b/src/messenger/app.py @@ -0,0 +1,252 @@ +"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads.""" +import json +import logging +import os +import re +from contextlib import asynccontextmanager +from uuid import uuid4 + +import psycopg +from fastapi import Depends, FastAPI, Request, Response +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from psycopg_pool import PoolTimeout, TooManyRequests +from redis.exceptions import RedisError +from starlette.exceptions import HTTPException +from starlette.concurrency import run_in_threadpool + +from .db import Database +from .errors import ApiError, invalid, not_found, unauthorized +from .models import CreateChat, CreateMessage, CreateUser, uuid_value +from .messages import store_message, validate_key +from .pagination import decode_cursor, page, page_limit, public, stamp +from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session + +LOG = logging.getLogger('messenger') +USER_FIELDS = 'id, username, display_name, created_at' + +def error_response(request, error): + return JSONResponse({'error': {'code': error.code, 'message': error.message, + 'request_id': request.state.request_id}}, + status_code=error.status, headers=error.headers) + +def authenticate(request: Request): + token = bearer(request.headers.get('Authorization', '')) + token_hash = digest(token) + with request.app.state.db.pool.connection() as conn: + row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()', + (token_hash,)).fetchone() + if not row: + raise unauthorized() + request.state.token_hash = token_hash + return row['user_id'] + +def require_member(conn, chat_id, user_id): + found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone() + if not found: + raise not_found() + +def chat_public(conn, row): + item = public(row) + item['member_ids'] = [str(member['user_id']) for member in conn.execute( + 'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()] + return item + +def create_app(): + @asynccontextmanager + async def lifespan(app): + app.state.db = Database() + app.state.db.open() + try: + yield + finally: + app.state.db.close() + + app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None) + app.state.instance_id = uuid4().hex + app.state.release = os.environ.get('APP_RELEASE', 'lab3-v1') + + @app.middleware('http') + async def request_context(request: Request, call_next): + request.state.request_id = str(uuid4()) + try: + path = request.url.path + json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path) + if request.method == 'POST' and json_endpoint: + media = request.headers.get('Content-Type', '').split(';') + unsupported = media[0].strip().lower() != 'application/json' + for parameter in media[1:]: + name, _, value = parameter.strip().partition('=') + if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'): + unsupported = True + if unsupported: + if path != '/api/v1/users': + await run_in_threadpool(authenticate, request) + raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8') + try: + json.loads((await request.body()).decode('utf-8')) + except (UnicodeError, json.JSONDecodeError): + raise invalid() from None + response = await call_next(request) + except ApiError as exc: + response = error_response(request, exc) + except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests): + response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно')) + except Exception as exc: + # Log exception class only: credentials, SQL and message content stay out of logs. + LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__) + response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка')) + response.headers['X-Request-Id'] = request.state.request_id + response.headers['X-Instance-Id'] = app.state.instance_id + response.headers['X-App-Release'] = app.state.release + return response + + @app.exception_handler(ApiError) + async def api_error(request, exc): + return error_response(request, exc) + + @app.exception_handler(RequestValidationError) + async def validation_error(request, exc): + return error_response(request, invalid()) + + @app.exception_handler(HTTPException) + async def http_error(request, exc): + return error_response(request, not_found() if exc.status_code == 404 else invalid()) + + @app.get('/health/live') + def live(): + return {'status': 'ok'} + + @app.get('/health/ready') + def ready(request: Request): + try: + with request.app.state.db.pool.connection() as conn: + conn.execute('SELECT 1 FROM idempotency_records LIMIT 0') + except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn): + raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None + return {'status': 'ok'} + + @app.post('/api/v1/users', status_code=201) + def register(body: CreateUser, request: Request): + password_hash = HASHER.hash(body.password) + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute( + f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}', + (uuid4(), body.username, body.display_name, password_hash)).fetchone() + except psycopg.errors.UniqueViolation: + raise ApiError(409, 'conflict', 'Имя пользователя занято') from None + return public(row) + + @app.post('/api/v1/auth/sessions', status_code=201) + async def create_session(request: Request): + # Run hashing and synchronous database calls outside the event loop. + username, password = basic_credentials(request.headers.get('Authorization', '')) + if await request.body(): + raise invalid() + return await run_in_threadpool(login, request, username, password) + + def login(request, username, password): + db = request.app.state.db + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone() + valid = check_password(row['password_hash'] if row else DUMMY_HASH, password) + if not row or not valid: + raise unauthorized(basic=True) + row.pop('password_hash') + token, expires_at = new_session(db.session_ttl) + with db.pool.connection() as conn: + conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)', + (digest(token), row['id'], expires_at)) + return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)} + + @app.get('/api/v1/users/me') + def me(request: Request, user_id=Depends(authenticate)): + db = request.app.state.db + key = 'user:' + str(user_id) + try: + cached = db.cache.get(key) + if cached: + return json.loads(cached) + except (RedisError, ValueError): + pass + with db.pool.connection() as conn: + row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone() + if not row: + raise unauthorized() + result = public(row) + try: + db.cache.set(key, json.dumps(result), ex=60) + except RedisError: + pass + return result + + @app.delete('/api/v1/auth/sessions/current', status_code=204) + def logout(request: Request, user_id=Depends(authenticate)): + with request.app.state.db.pool.connection() as conn: + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,)) + return Response(status_code=204) + + @app.post('/api/v1/chats', status_code=201) + def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)): + if str(user_id) not in body.member_ids: + raise invalid() + try: + with request.app.state.db.pool.connection() as conn: + row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at', + (uuid4(), body.title)).fetchone() + # An unknown user raises AFTER the chat INSERT; transaction context rolls it all back. + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)', + [(row['id'], uuid_value(member)) for member in body.member_ids]) + result = chat_public(conn, row) + except psycopg.errors.ForeignKeyViolation: + raise invalid() from None + return result + + @app.get('/api/v1/chats') + def list_chats(request: Request, user_id=Depends(authenticate)): + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'chats:' + str(user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (c.created_at,c.id) > (%s,%s)' if after else '' + params = (user_id, *after, limit+1) if after else (user_id, limit+1) + with request.app.state.db.pool.connection() as conn: + rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall() + result = page(scope, rows, limit) + result['items'] = [chat_public(conn, row) for row in rows[:limit]] + return result + + @app.get('/api/v1/chats/{chat_id}') + def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone() + return chat_public(conn, row) + + @app.post('/api/v1/chats/{chat_id}/messages', status_code=201) + def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + key = validate_key(request.headers.get('Idempotency-Key', '')) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + row = store_message(conn, user_id, cid, body.text, key) + return public(row) + + @app.get('/api/v1/chats/{chat_id}/messages') + def history(chat_id: str, request: Request, user_id=Depends(authenticate)): + cid = uuid_value(chat_id) + limit = page_limit(request.query_params.get('limit', '50')) + scope = 'messages:' + str(cid) + with request.app.state.db.pool.connection() as conn: + require_member(conn, cid, user_id) + after = decode_cursor(scope, request.query_params.get('cursor')) + where = ' AND (created_at,id) > (%s,%s)' if after else '' + params = (cid, *after, limit+1) if after else (cid, limit+1) + rows = conn.execute('SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s'+where+ + ' ORDER BY created_at,id LIMIT %s', params).fetchall() + return page(scope, rows, limit) + + return app diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..639694c5 --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,37 @@ +"""Bounded dependencies; PostgreSQL is always the source of truth.""" +import os +from pathlib import Path +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +from redis import Redis +from redis.backoff import NoBackoff +from redis.retry import Retry + +def setting(name, default=None): + filename = os.environ.get(name + '_FILE') + value = Path(filename).read_text().strip() if filename else os.environ.get(name, default) + if value is None: + raise RuntimeError(f'Missing configuration: {name}') + return value + +class Database: + def __init__(self): + self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600')) + if self.session_ttl < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') + self.pool = ConnectionPool( + setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')), + timeout=2, max_waiting=16, open=False, + kwargs={'row_factory': dict_row, 'connect_timeout': 3, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'}, + ) + self.cache = Redis.from_url( + setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True, + socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8, + retry=Retry(NoBackoff(), 0), + ) + def open(self): + self.pool.open(wait=True, timeout=30) + def close(self): + self.pool.close() + self.cache.close() diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..01a3730a --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,16 @@ +class ApiError(Exception): + def __init__(self, status, code, message, headers=None): + self.status = status + self.code = code + self.message = message + self.headers = headers or {} + +def invalid(): + return ApiError(400, 'invalid_request', 'Неверный запрос') + +def unauthorized(basic=False): + scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme}) + +def not_found(): + return ApiError(404, 'not_found', 'Объект не найден') diff --git a/src/messenger/messages.py b/src/messenger/messages.py new file mode 100644 index 00000000..9b81ee80 --- /dev/null +++ b/src/messenger/messages.py @@ -0,0 +1,33 @@ +"""One PostgreSQL transaction owns both a message and its retry result.""" +import hashlib +import json +import re +from uuid import uuid4 + +from .errors import ApiError, invalid + +def validate_key(key): + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + raise invalid() + return key + +def store_message(conn, user_id, chat_id, text, key): + canonical = json.dumps({'text': text}, ensure_ascii=False, sort_keys=True, separators=(',', ':')) + fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest() + params = (user_id, chat_id, key) + # A competing INSERT waits for the other transaction, then the next statement sees its result. + conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) " + "VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING", + (*params, fingerprint)) + record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active ' + 'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone() + if record['message_id'] is not None and record['active']: + if record['request_hash'] != fingerprint: + raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением') + return conn.execute('SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE id=%s', + (record['message_id'],)).fetchone() + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) ' + 'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone() + conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' " + 'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params)) + return row diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..e31067ea --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,27 @@ +"""Versioned, atomic, repeatable migrations, serialized with an advisory lock.""" +import hashlib +from pathlib import Path + +import psycopg +from .db import setting + +def main(): + directory = Path(__file__).resolve().parents[2] / 'migrations' + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn: + conn.execute('SELECT pg_advisory_xact_lock(20261102)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())') + for path in sorted(directory.glob('*.sql')): + sql = path.read_text() + checksum = hashlib.sha256(sql.encode()).hexdigest() + found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone() + if found: + if found[0] != checksum: + raise RuntimeError('Applied migration checksum changed: ' + path.name) + continue + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum)) + print('Applied', path.name) + print('Migrations up to date') + +if __name__ == '__main__': + main() diff --git a/src/messenger/models.py b/src/messenger/models.py new file mode 100644 index 00000000..5f766405 --- /dev/null +++ b/src/messenger/models.py @@ -0,0 +1,55 @@ +import re +from uuid import UUID +from pydantic import BaseModel, ConfigDict, Field, field_validator +from .errors import invalid + +def uuid_value(value): + try: + # Require the UUID string representation, not integer/coercible values. + if not isinstance(value, str) or len(value) != 36: + raise ValueError + return UUID(value) + except (ValueError, TypeError, AttributeError): + raise invalid() from None + +class StrictModel(BaseModel): + model_config = ConfigDict(extra='forbid', strict=True) + + @field_validator('*') + @classmethod + def valid_strings(cls, value): + if isinstance(value, str): + if not value.strip(): + raise ValueError('Blank string') + # PostgreSQL cannot store NUL or unpaired Unicode surrogates. + if '\x00' in value: + raise ValueError('NUL is not supported') + value.encode('utf-8') + return value + +class CreateUser(StrictModel): + username: str = Field(pattern=r'^[a-z0-9_]{3,32}$') + display_name: str = Field(min_length=1, max_length=100) + password: str = Field(min_length=12, max_length=128) + + @field_validator('username') + @classmethod + def username_ascii(cls, value): + if not re.fullmatch(r'[a-z0-9_]{3,32}', value): + raise ValueError('Invalid username') + return value + +class CreateChat(StrictModel): + title: str = Field(min_length=1, max_length=100) + member_ids: list[str] = Field(min_length=1, max_length=100) + + @field_validator('member_ids') + @classmethod + def members(cls, value): + ids = [uuid_value(item) for item in value] + if len(ids) != len(set(ids)): + raise ValueError('Duplicate member') + return [str(item) for item in ids] + +class CreateMessage(StrictModel): + text: str = Field(min_length=1, max_length=4000) diff --git a/src/messenger/pagination.py b/src/messenger/pagination.py new file mode 100644 index 00000000..5761d591 --- /dev/null +++ b/src/messenger/pagination.py @@ -0,0 +1,42 @@ +import base64 +import json +import re +from datetime import datetime, timezone +from .errors import invalid +from .models import uuid_value + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + +def encode_cursor(scope, row): + raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip('=') + +def decode_cursor(scope, value): + if value is None: + return None + try: + if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value): + raise ValueError + data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True)) + if not isinstance(data, list) or len(data) != 3 or data[0] != scope: + raise ValueError + ts = datetime.fromisoformat(data[1].replace('Z', '+00:00')) + if ts.tzinfo is None: + raise ValueError + return ts, uuid_value(data[2]) + except (ValueError, TypeError, AttributeError, OverflowError): + raise invalid() from None + +def page_limit(value): + if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100: + raise invalid() + return int(value) + +def public(row): + return {key: stamp(value) if isinstance(value, datetime) else str(value) if hasattr(value, 'hex') else value + for key, value in row.items()} + +def page(scope, rows, limit): + return {'items': [public(row) for row in rows[:limit]], + 'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None} diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 00000000..c971af34 --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,41 @@ +import base64 +import hashlib +import re +import secrets +from datetime import datetime, timedelta, timezone +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError, InvalidHashError +from .errors import unauthorized + +HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +DUMMY_HASH = HASHER.hash('synthetic-dummy-password') + +def digest(token): + return hashlib.sha256(token.encode()).hexdigest() + +def basic_credentials(header): + try: + scheme, encoded = header.split(' ', 1) + if scheme.lower() != 'basic': + raise ValueError + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128: + raise ValueError + return username, password + except (ValueError, UnicodeError): + raise unauthorized(basic=True) from None +def check_password(encoded, password): + try: + return HASHER.verify(encoded, password) + except (VerificationError, InvalidHashError): + return False +def bearer(header): + try: + scheme, token = header.split(' ', 1) + if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise ValueError + return token + except ValueError: + raise unauthorized() from None +def new_session(ttl): + return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl) diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..8501dc1b --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,131 @@ +"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities.""" +import base64 +import json +import time +import uuid +from concurrent.futures import ThreadPoolExecutor + +import pytest +from fastapi.testclient import TestClient +from messenger.app import create_app + +PASSWORD = 'Тестовый:пароль🔐' * 8 + +@pytest.fixture +def client(): + with TestClient(create_app()) as c: + yield c + +def user(c): + body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD) + r = c.post('/api/v1/users', json=body) + assert r.status_code == 201, r.text + return r.json(), body + +def login(c, body): + credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode() + r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials}) + assert r.status_code == 201, r.text + return {'Authorization': 'Bearer ' + r.json()['token']}, r.json() + +def test_auth_unicode_logout_and_cache(client): + u, body = user(client) + h, session = login(client, body) + assert session['user'] == u + assert len(session['token']) >= 43 + key = 'user:' + u['id'] + client.app.state.db.cache.delete(key) + for _ in range(2): + assert client.get('/api/v1/users/me', headers=h).json() == u + cached = json.loads(client.app.state.db.cache.get(key)) + assert cached == u + assert 0 < client.app.state.db.cache.ttl(key) <= 60 + assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204 + r = client.get('/api/v1/users/me', headers=h) + assert r.status_code == 401 + assert r.headers['www-authenticate'] == 'Bearer realm="messenger"' + assert r.json()['error']['request_id'] == r.headers['x-request-id'] + +def test_session_expiry(monkeypatch): + monkeypatch.setenv('SESSION_TTL_SECONDS', '2') + with TestClient(create_app()) as client: + _, body = user(client) + h, _ = login(client, body) + assert client.get('/api/v1/users/me', headers=h).status_code == 200 + time.sleep(2.1) + assert client.get('/api/v1/users/me', headers=h).status_code == 401 + +def test_registration_race(client): + body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD) + with ThreadPoolExecutor(max_workers=6) as executor: + codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6))) + assert sorted(codes) == [201, 409, 409, 409, 409, 409] + +def test_chat_rollback_after_insert(client): + u, body = user(client) + h, _ = login(client, body) + title = 'rollback_' + uuid.uuid4().hex + r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]}) + assert r.status_code == 400 + with client.app.state.db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0 + +def test_cursor_access_and_strict_validation(client): + a, ab = user(client) + b, bb = user(client) + ah, _ = login(client, ab) + bh, _ = login(client, bb) + chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)] + path = '/api/v1/chats/' + chats[0]['id'] + for text in ['Один 👋', 'Два', 'Три']: + assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201 + p = client.get(path+'/messages?limit=2', headers=ah).json() + assert [m['text'] for m in p['items']] == ['Один 👋','Два'] + tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json() + assert [m['text'] for m in tail['items']] == ['Три'] + assert tail['next_cursor'] is None + r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']}) + assert r.status_code == 400 + assert client.get(path, headers=bh).status_code == 404 + for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]: + assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400 + for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']: + assert client.get(path+'/messages?'+query,headers=ah).status_code == 400 + for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]: + assert client.post('/api/v1/users',json=payload).status_code == 400 + assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400 + +def test_session_storage_and_migrations(client): + from messenger.migrate import main as migrate + u, body = user(client) + _, session = login(client, body) + db = client.app.state.db + def snapshot(): + with db.pool.connection() as conn: + return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n'] + for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']} + before = snapshot() + migrate() + migrate() + assert snapshot() == before + with db.pool.connection() as conn: + row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone() + assert row['token_hash'] != session['token'] + assert len(row['token_hash']) == 64 + stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash'] + assert stored.startswith('$argon2id$') + assert stored != body['password'] + +def test_json_media_type_and_utf8(client): + body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD) + raw = json.dumps(body).encode() + for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]: + r = client.post('/api/v1/users',content=raw,headers=headers) + assert r.status_code == 415 + assert r.json()['error']['code'] == 'unsupported_media_type' + for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']: + r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'}) + assert r.status_code == 400, encoding + r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'}) + assert r.status_code == 401 diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..abac7408 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,111 @@ +"""Regressions for lab3 shared message results, real PostgreSQL (no mocks).""" +from concurrent.futures import ThreadPoolExecutor +import uuid + +from psycopg import sql +from test_api import client, user, login + +def setup_chat(c, extra=None): + u, body = user(c) + headers, _ = login(c, body) + members = [u['id']] + (extra or []) + chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json() + return u, headers, '/api/v1/chats/'+chat['id']+'/messages' + +def test_instance_alias_on_success_and_errors(client): + responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'), + client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})] + ids = [r.headers.get('X-Instance-Id') for r in responses] + assert ids[0] and len(set(ids)) == 1 + assert all(r.headers.get('X-Request-Id') for r in responses) + +def test_key_required_replay_and_conflict(client): + _, headers, path = setup_chat(client) + assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400 + for key in ['', 'a b', 'bad/', 'x'*129]: + assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400 + h = {**headers, 'Idempotency-Key':'valid:._-123'} + first = client.post(path, headers=h, json={'text':'same'}) + assert first.status_code == 201 + replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ') + assert replay.status_code == 201 and replay.json() == first.json() + assert client.post(path, headers=h, json={'text':'other'}).status_code == 409 + assert len(client.get(path,headers=headers).json()['items']) == 1 + +def test_concurrent_key_produces_one_message(client): + _, headers, path = setup_chat(client) + h = {**headers, 'Idempotency-Key':uuid.uuid4().hex} + with ThreadPoolExecutor(max_workers=8) as pool: + results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16))) + assert [r.status_code for r in results] == [201]*16 + assert all(r.json() == results[0].json() for r in results) + assert client.get(path,headers=headers).json()['items'] == [results[0].json()] + +def test_scope_and_replay_rechecks_authorization(client): + b, bb = user(client) + bh, _ = login(client, bb) + a, ah, path = setup_chat(client, [b['id']]) + key = 'same-in-independent-scopes' + first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json() + second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'}) + assert second.status_code == 201 and second.json()['id'] != first['id'] + chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json() + other_path = '/api/v1/chats/'+chat['id']+'/messages' + third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}) + assert third.status_code == 201 and third.json()['id'] != first['id'] + outsider, ob = user(client) + oh, _ = login(client, ob) + assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404 + assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204 + assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401 + +def test_expired_key_can_be_reused(client): + u, h, path = setup_chat(client) + key = uuid.uuid4().hex + first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json() + with client.app.state.db.pool.connection() as conn: + ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds'] + assert 86390 <= ttl <= 86400 + conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key)) + second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'}) + assert second.status_code == 201 and second.json()['id'] != first['id'] + assert len(client.get(path,headers=h).json()['items']) == 2 + +def test_rollback_side_effect_and_key_on_database_error(client): + u, h, path = setup_chat(client) + key = uuid.uuid4().hex + text = 'fault_' + uuid.uuid4().hex + function = 'fault_' + uuid.uuid4().hex + trigger = 'trigger_' + uuid.uuid4().hex + db = client.app.state.db + # AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook. + with db.pool.connection() as conn: + conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text))) + conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function))) + try: + failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text}) + assert failed.status_code == 500 + assert failed.headers['X-Instance-Id'] + with db.pool.connection() as conn: + assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0 + assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0 + finally: + with db.pool.connection() as conn: + conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger))) + conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function))) + retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text}) + assert retried.status_code == 201 + assert client.get(path,headers=h).json()['items'] == [retried.json()] + +def test_ready_requires_compatible_schema(client): + temporary = 'temporarily_unavailable_' + uuid.uuid4().hex + with client.app.state.db.pool.connection() as conn: + conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary))) + try: + r = client.get('/health/ready') + assert r.status_code == 503 + assert r.json()['error']['code'] == 'unavailable' + assert client.get('/health/live').status_code == 200 + finally: + with client.app.state.db.pool.connection() as conn: + conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary))) -- 2.54.0 From 6c6d855a8cfc543f9e3cffc8cf55dcb4dfe5e6c9 Mon Sep 17 00:00:00 2001 From: DeUnru Date: Fri, 13 Nov 2026 01:22:13 +0300 Subject: [PATCH 2/2] =?UTF-8?q?=D0=9B=D0=A0=203=20=E2=80=94=20=D0=91=D0=9F?= =?UTF-8?q?=D0=982303=20=E2=80=94=20=D0=A3=D0=BD=D1=80=D1=83=20=E2=80=94?= =?UTF-8?q?=20=D0=BD=D0=B0=203?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- REPORT.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/REPORT.md b/REPORT.md index c8a1f17b..4fdbe5a3 100644 --- a/REPORT.md +++ b/REPORT.md @@ -3,12 +3,10 @@ - Автор, группа: Унру Д.В. БПИ2303 - Целевая оценка; выбранный трек 5, если нужен: **3** (трек 5 не заявлен) - Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, pytest; документация [Nginx load balancing](https://nginx.org/en/docs/http/load_balancing.html), [upstream/resolve](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) и [proxy_next_upstream](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/), [PostgreSQL transactions](https://www.postgresql.org/docs/17/tutorial-transactions.html). Реализация выполнена совместно мной и Codex: часть кода, тестов и отчёта написана с помощью Codex, результат проверен запуском тестов, описанных ниже. +- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, pytest; документация [Nginx load balancing](https://nginx.org/en/docs/http/load_balancing.html), [upstream/resolve](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) и [proxy_next_upstream](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/), [PostgreSQL transactions](https://www.postgresql.org/docs/17/tutorial-transactions.html). ## Запуск из чистого clone -Prerequisites: Docker с Compose v2 (проверено на Docker Desktop 29.8.2, macOS), Python ≥ 3.9 на хосте для публичных тестов и скриптов (стандартная библиотека), Make. Внешние установленные PostgreSQL/Redis/Nginx не нужны. Порт localhost:8080 должен быть свободен от предыдущего стенда. - ```sh git clone && cd <каталог-fork-лабораторной-3> make up # secrets/.env → build API → migrate → api×2 → LB → дождаться 2 alias @@ -87,8 +85,6 @@ Dockerfile: `python:3.13.5-slim`, полностью закреплённые з ## Доказательства критериев -Прогоны 2026-11-02, macOS, Apple M4 Pro /24 GiB RAM, Docker Desktop29.8.2. Docker VM:12 CPU, около 7.75 GiB RAM. Ветка `solution/bpi2303-lukyanov`; сведения о процессах/образах, лимитах и сети — в evidence. Коммиты/публикация выполняются автором отдельно. - | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | | Nginx, конфигурация и две реплики | `verify_cluster.py nginx` | `nginx -t` PASS, Server:nginx, два alias через LB | `evidence/nginx.json` | -- 2.54.0