ЛР 3 - БПИ-2303 - Унру - на 3 #145
@@ -18,3 +18,5 @@ __pycache__
|
||||
node_modules
|
||||
evidence
|
||||
.DS_Store
|
||||
.superpowers
|
||||
.pytest_cache
|
||||
+7
-12
@@ -1,13 +1,8 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
# bootstrap.sh creates a private local .env and secrets/; never commit them.
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
# API config in Compose:
|
||||
# DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# POOL_MAX_SIZE=8 per API process
|
||||
# No API host ports: only lb publishes 127.0.0.1:8080.
|
||||
# Redis caches immutable User profiles; PostgreSQL owns sessions + idempotency.
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
name: Application acceptance
|
||||
on: [push, pull_request]
|
||||
permissions:
|
||||
contents: read
|
||||
jobs:
|
||||
cluster:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
- run: make up
|
||||
- run: make check test test-student
|
||||
- run: make test-fresh
|
||||
- run: make test-cluster
|
||||
- if: always()
|
||||
run: docker compose down
|
||||
@@ -30,3 +30,4 @@ coverage/
|
||||
.vscode/
|
||||
# Keep small, sanitized evidence summaries in evidence/ under version control.
|
||||
# Inspect git diff --cached: ignore rules are not a secret scanner.
|
||||
.pytest_cache/
|
||||
+18
@@ -0,0 +1,18 @@
|
||||
FROM python:3.13.5-slim
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src
|
||||
WORKDIR /app
|
||||
COPY requirements.txt requirements-dev.txt requirements.lock ./
|
||||
RUN pip install --no-cache-dir -r requirements.lock && useradd --uid 10001 --create-home messenger
|
||||
COPY src/ src/
|
||||
COPY migrations/ migrations/
|
||||
COPY student-tests/ student-tests/
|
||||
COPY scripts/ scripts/
|
||||
COPY tests/ tests/
|
||||
COPY contracts/ contracts/
|
||||
COPY lab.json Makefile README.md COURSE.md CONTRIBUTING.md REPORT.md ./
|
||||
ARG APP_RELEASE=lab3-v1
|
||||
ENV APP_RELEASE=${APP_RELEASE}
|
||||
LABEL org.opencontainers.image.version=${APP_RELEASE}
|
||||
USER messenger
|
||||
EXPOSE 8080
|
||||
CMD ["uvicorn", "messenger.app:create_app", "--factory", "--host", "0.0.0.0", "--port", "8080", "--timeout-graceful-shutdown", "5", "--no-access-log"]
|
||||
@@ -28,3 +28,34 @@ test-persistence:
|
||||
test-failover:
|
||||
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable stop-one hook"; exit 2)
|
||||
$(PYTHON) tests/resilience.py --mode failover --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
|
||||
|
||||
.PHONY: bootstrap up down test-student test-cluster rolling
|
||||
bootstrap:
|
||||
./scripts/bootstrap.sh
|
||||
up: bootstrap
|
||||
docker compose build api
|
||||
docker compose up -d --wait --wait-timeout 120
|
||||
$(PYTHON) scripts/cluster.py wait
|
||||
down:
|
||||
docker compose down
|
||||
test-student:
|
||||
$(PYTHON) scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
|
||||
test-cluster:
|
||||
$(PYTHON) scripts/verify_cluster.py
|
||||
rolling:
|
||||
$(PYTHON) scripts/cluster.py rolling
|
||||
|
||||
.PHONY: test-fresh load-evidence verify-persistence verify-failover
|
||||
test-fresh:
|
||||
./scripts/check_fresh_migrations.sh
|
||||
load-evidence:
|
||||
$(PYTHON) scripts/capture_load.py
|
||||
verify-persistence:
|
||||
$(PYTHON) scripts/verify_cluster.py persistence
|
||||
verify-failover:
|
||||
$(PYTHON) scripts/verify_cluster.py public-failover
|
||||
$(PYTHON) scripts/verify_cluster.py failover
|
||||
|
||||
.PHONY: validate-container
|
||||
validate-container:
|
||||
$(PYTHON) scripts/cluster.py exec python scripts/validate_openapi.py
|
||||
|
||||
@@ -1,37 +1,151 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Автор, группа: Унру Д.В. БПИ2303
|
||||
- Целевая оценка; выбранный трек 5, если нужен: **3** (трек 5 не заявлен)
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Использованные источники, библиотеки и помощь ИИ: FastAPI, Pydantic v2, Uvicorn, Psycopg 3, redis-py, argon2-cffi, pytest; документация [Nginx load balancing](https://nginx.org/en/docs/http/load_balancing.html), [upstream/resolve](https://nginx.org/en/docs/http/ngx_http_upstream_module.html) и [proxy_next_upstream](https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_next_upstream), [Docker Compose services](https://docs.docker.com/reference/compose-file/services/), [PostgreSQL transactions](https://www.postgresql.org/docs/17/tutorial-transactions.html).
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
```sh
|
||||
git clone <fork-url> && cd <каталог-fork-лабораторной-3>
|
||||
make up # secrets/.env → build API → migrate → api×2 → LB → дождаться 2 alias
|
||||
curl -fsS http://localhost:8080/health/ready
|
||||
docker compose ps # две API-реплики, postgres, redis, lb — healthy; migrate — Exited(0)
|
||||
|
||||
make check
|
||||
make validate-container # полный валидатор OpenAPI без локальных pip-зависимостей
|
||||
make test # публичный smoke через LB
|
||||
make test-student # собственные интеграционные тесты: 14
|
||||
make test-fresh # обе миграции и 14 тестов на отдельных пустых volumes
|
||||
make test-cluster # все распределённые проверки, включая реальные отказы и rolling
|
||||
make load-evidence # отдельные 30 с нагрузки и CPU/RAM
|
||||
```
|
||||
|
||||
Эквивалент `make up`: `./scripts/bootstrap.sh`, `docker compose build api`, `docker compose up -d --wait --wait-timeout 120`, `python3 scripts/cluster.py wait`. Образ собирается до запуска migrate, поэтому готовый сервер из registry не требуется. Хосту доступен только LB `127.0.0.1:8080`; API и хранилища напрямую не опубликованы.
|
||||
|
||||
Отдельные публичные проверки:
|
||||
|
||||
```sh
|
||||
make test-persistence ACTION_SCRIPT=scripts/restart.sh
|
||||
make test-failover ACTION_SCRIPT=scripts/stop-one.sh
|
||||
# Публичный failover намеренно оставляет одну выбранную реплику остановленной.
|
||||
# Восстановление:
|
||||
docker compose up -d --no-recreate --wait --wait-timeout 90
|
||||
python3 scripts/cluster.py wait
|
||||
```
|
||||
|
||||
`make verify-persistence` дополнительно сохраняет снимки реальных контейнеров до/после. `make verify-failover` выполняет публичный failover и последовательные проверки каждого исходного контейнера, с восстановлением между ними. Полный `make test-cluster` последовательно запускает concurrent replay, persistence, публичный и собственные failover, SIGTERM, rolling, отказы Redis/БД и аудит инфраструктуры. Он меняет состояние стенда и собирает новый локальный образ с новым идентификатором релиза; не запускать параллельно с другими проверками.
|
||||
|
||||
`bootstrap.sh` создаёт случайный пароль PostgreSQL и файлы secrets один раз; существующие значения не меняет. Каталог `secrets/` — 0700 на хосте, файлы доступны uid 10001 контейнера через Compose secrets; `.env` и secrets исключены из Git и Docker build context. Для lab3 созданы собственные secrets/volumes, данные и credentials lab2 не копировались.
|
||||
|
||||
Безопасная остановка: `make down` — контейнеры и сети удаляются, именованные volumes сохраняются. После `make up` доступны прежние данные, действующие сессии и результаты идемпотентности. Намеренное удаление всех учебных данных: `docker compose down --volumes`. Hook `restart.sh` volumes не удаляет. `make test-fresh` удаляет только созданные им временные volumes отдельного проекта. Синтетические данные обычных тестов остаются в основной БД.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```text
|
||||
┌──► API-процесс A ──┐
|
||||
клиент ──HTTP──► Nginx ──┤ ├──► PostgreSQL: данные, сессии, результаты POST
|
||||
127.0.0.1:8080 └──► API-процесс B ──┤
|
||||
└──► Redis: кэш публичного профиля
|
||||
|
||||
migrate ──► PostgreSQL (один отдельный job до запуска API)
|
||||
```
|
||||
|
||||
- **Перенос из lab2.** Перенесены исходники FastAPI, миграция 001, Argon2id/Basic/Bearer, users/chats/messages, keyset-пагинация, Redis cache-aside профиля, ограниченные пулы и собственные проверки. Отправка сообщений и тесты адаптированы к обязательному Idempotency-Key. Dockerfile/Compose изменены для отдельных миграций, нескольких API и LB. Публичные `tests/`, `contracts/`, README и COURSE сохранены из skeleton lab3.
|
||||
- **Общее состояние.** PostgreSQL — источник истины для пользователей, чатов, участников, сообщений, сессий и результатов POST. API не держит в локальной памяти данных, от которых зависит корректность. Каждое защищённое обращение проверяет сессию в БД; X-User-Id игнорируется. Разные реплики обслуживают один Bearer без повторного входа. Sticky session не используется.
|
||||
- **Идентификатор процесса.** `X-Instance-Id` — случайный UUID alias, стабильный для жизни конкретного процесса; добавляется ко всем его ответам, включая health/ошибки. После restart alias меняется. `X-App-Release` — дополнительный заголовок для различения сборок в rolling-эксперименте. Ошибка, сформированная самим LB при недоступных backend, может не иметь API-заголовков и JSON.
|
||||
- **Общая идемпотентность** (`src/messenger/messages.py`, миграция 002). Таблица `idempotency_records`: PK(user_id,chat_id,key), request_hash, message_id FK, expires_at. Таблица обслуживает только POST сообщений, поэтому эта область соответствует user+method+chat path+key. Ключ строго ASCII `[A-Za-z0-9._:-]{1, 128}`, иначе 400. Валидированный JSON канонизируется независимо от форматирования/порядка полей, fingerprint — SHA-256. В схеме lab3 допустим только text; attachment_ids появится по контракту lab4.
|
||||
- **Атомарность повтора.** После проверки сессии и членства одна транзакция выполняет INSERT ключа ON CONFLICT DO NOTHING и SELECT FOR UPDATE. Конкурирующие процессы сериализуются на одной записи. При действующем ключе тот же fingerprint возвращает тот же Message и 201, другой — 409. Для нового/истёкшего ключа INSERT сообщения и обновление результата фиксируются одним COMMIT. Исключение откатывает оба эффекта, транзиентный 5xx не сохраняется как результат. Replay снова проверяет авторизацию. Срок ключа — 24 ч; после истечения разрешена новая запись с тем же ключом, старое сообщение не удаляется.
|
||||
- **Миграции.** `migrate` — отдельный одноразовый service, API зависит от `service_completed_successfully`. В API CMD мигратора нет. SQL-файлы 001/002, checksum и advisory lock сохраняют повторяемость и сериализуют возможные ручные запуски. Новая 002 только добавляет таблицу/индекс, не меняет старые поля lab2. Перед rolling один раз выполняется `docker compose run --rm --no-deps migrate`, пока старые API продолжают работу.
|
||||
- **Балансировка и DNS.** Nginx OSS 1.28: `upstream messenger` с общей `zone messenger 64k` и `server api:8080 resolve`. Все A-записи одного Compose-service api участвуют в round-robin; `resolver 127.0.0.11 valid=1s ipv6=off` обновляет IP без restart/reload Nginx. Адреса не зашиты в конфигурации.
|
||||
- **Пассивные проверки upstream.** `max_fails=1 fail_timeout=2s`: первая ошибка подключения/таймаут/503 исключает адрес на 2 с, затем допускается новая попытка. Активного `health_check` Nginx Plus здесь нет. Docker healthcheck каждого API раз в 2 с вызывает его readiness; healthcheck LB раз в 1 с делает GET readiness через Nginx и создаёт фоновый трафик даже без клиента. При двух backend, работающем Docker DNS и настроенных таймаутах консервативный бюджет обнаружения — до 8 с; фактические чтения/записи после отказа проверяются в пределах 10 с. В отличие от активного мониторинга, пассивное исключение происходит только после обращения к backend.
|
||||
- **Таймауты и повтор.** Nginx connect 1 с, read/send 5 с; `proxy_next_upstream error timeout http_503`, не более 2 upstream-попыток и 6 с на переключение. GET может уйти на другую реплику при ошибке. Параметр `non_idempotent` не включён: уже отправленный POST автоматически не пересылается; при connect error до отправки запроса допустим безопасный выбор другого адреса. Неопределённую отправку сообщения повторяет клиент с прежним Idempotency-Key.
|
||||
- **Readiness и graceful.** `/health/live` проверяет процесс; `/health/ready` требует доступную PostgreSQL и таблицу новой схемы. Несовместимая схема/ошибка зависимости →503. Redis — некритичный кэш с fallback, его отказ readiness не меняет. Uvicorn работает PID1, один worker на контейнер, graceful timeout5 с. Docker stop_grace_period10 с; hook stop-one использует 7 с, укладываясь в публичный бюджет 10 с. SIGTERM закрывает listening socket и даёт активным запросам завершиться.
|
||||
- **Остановка выбранной реплики.** `scripts/stop-one.sh` — короткий wrapper Python-оркестратора `cluster.py`. Тот сопоставляет TARGET_INSTANCE_ID с реальным контейнером внутренним GET /health/live, останавливает ровно его, проверяет exit 0 и оставляет остановленным. Идентификаторы контейнеров/процессов и время записываются без credentials. `restart.sh` перезапускает все API, PostgreSQL, Redis и LB с прежними volumes.
|
||||
- **Поочерёдная замена.** Compose не предоставляет здесь автоматический Swarm rolling update. Проверен эквивалент start-first: временно scale api=3 с `--no-recreate`, дождаться новой healthy-реплики и трёх alias через LB, затем stop/remove одного старого контейнера и возврат к 2. Второй старый контейнер в этом шаге не пересоздаётся. Создание до удаления старого гарантирует другой IP; оба адреса сохраняются. Шаг повторяется для второго исходного контейнера.
|
||||
- **Пароли, сессии, кэш.** Argon2id (19456 KiB, time_cost2, parallelism1), opaque token256 бит, в БД только SHA-256 токена и абсолютный expiry; logout удаляет сессию. Redis хранит только User по `user:<UUID>` на 60 с после проверки сессии. Профиль неизменяем в обязательном API, инвалидация по событию не нужна. История читается из PostgreSQL и не бывает устаревшей из-за кэша.
|
||||
- **Запросы/ошибки.** SQL параметризован; каждый API-пул min 1/max 8, очередь 16, ожидание 2 с, connect_timeout3 с, statement_timeout3 с, lock_timeout2 с. Redis-пул 8, connect/read по 0.2 с без retry. JSON строго UTF-8, неверные типы/поля/формат →400, неподдерживаемый media type →415; чужие чаты →404. X-Request-Id совпадает с request_id ошибки. Access-лог Uvicorn отключён, тела/токены/SQL в логи не выводятся.
|
||||
|
||||
Конфигурация и ресурсы:
|
||||
|
||||
| Переменная / service | Значение | Использование |
|
||||
| --- | --- | --- |
|
||||
| `SESSION_TTL_SECONDS` (.env) | 3600 по умолчанию | абсолютный срок сессии, в собственном тесте expiry — 2 с |
|
||||
| `DATABASE_URL_FILE` | `/run/secrets/database_url` | API и migrate читают адрес/credentials БД |
|
||||
| `CACHE_URL_FILE` | `/run/secrets/cache_url` | только API читает адрес Redis |
|
||||
| `POOL_MAX_SIZE` | 8 на API-процесс | ограничение числа соединений |
|
||||
| `api` | `mtusi-lab3-api:lab3`; каждая 0.75 CPU / 384 MiB | один scale-able service, штатно 2 реплики, без container_name и host ports; сети front+data |
|
||||
| `lb` | `nginx:1.28.0-alpine`; 0.5 CPU / 128 MiB | единственный публичный порт 127.0.0.1:8080; только front, без DB secrets |
|
||||
| `postgres` | `postgres:17.5-bookworm`; 1 CPU / 512 MiB | только internal data; volume postgres_data; pg_isready |
|
||||
| `redis` | `redis:7.4.4-bookworm`; 0.25 CPU / 128 MiB | только internal data; volume redis_data; AOF; maxmemory64 MiB/allkeys-lru |
|
||||
| `migrate` | тот же образ API; 0.5 CPU / 256 MiB | только data, только database_url secret; Exited(0) после применения |
|
||||
|
||||
Dockerfile: `python:3.13.5-slim`, полностью закреплённые зависимости в `requirements.lock`, пользователь messenger uid 10001. Установка зависимостей предшествует копированию исходников; ARG APP_RELEASE расположен после этих слоёв, поэтому смена только метки релиза не переустанавливает библиотеки. CMD — exec Uvicorn. Healthchecks задаются Compose; `.dockerignore` исключает Git, secrets/.env, сертификаты, volumes и evidence. Публичные и собственные проверки включены в образ для запуска без pip на хосте. Номера контейнеров после замен могут иметь пропуски: `cluster.py exec` выбирает реально живой контейнер, не предполагает существование api-1.
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Nginx, конфигурация и две реплики | `verify_cluster.py nginx` | `nginx -t` PASS, Server:nginx, два alias через LB | `evidence/nginx.json` |
|
||||
| Контракт и публичный smoke (3) | `make check`, `make validate-container`, `make test` | PASS, lab3 public API scenario через LB | `evidence/check.txt`, `openapi.txt`, `smoke.txt` |
|
||||
| ≥2 API, один LB, общая БД и сессии (3) | `cluster.py wait`; concurrent replay; infrastructure | один Bearer работает на разных alias; реально два API-контейнера, порты только у LB | `evidence/concurrent-idempotency.json`, `infrastructure.json` |
|
||||
| Отказ любой одной реплики, чтение/запись ≤10 с (3) | публичный failover + остановка каждого исходного контейнера | оба сценария PASS; чтение известного сообщения, пока target ещё остановлен; post-stop POST в пределах 10 с | `evidence/public-failover.txt`, `sequential-failover.json` |
|
||||
| Общая идемпотентность и отсутствие дублей (3) | 8 параллельных повторов через 2 alias | все 201 с одним Message, в истории 1 сообщение; другой body409 | `evidence/concurrent-idempotency.json` |
|
||||
| Сохранность и прежняя сессия (3) | `make verify-persistence` | API/БД/Redis/LB действительно перезапущены; volumes те же; прежний токен и данные доступны | `evidence/persistence.txt`, `persistence.json` |
|
||||
| Обязательные key/alias, TTL 24 ч, scope, rollback, повторная авторизация | `make test-student` | 14/14 PASS, включая отказ после INSERT и rollback ключа+сообщения | `evidence/student-tests.txt` |
|
||||
| Миграции отдельно, с нуля и повторяемо | `make test-fresh`; migration test | применены 001/002, повтор без дублей, 14/14 PASS на новых volumes | `evidence/fresh-migrations.txt` |
|
||||
| Health/readiness/timeouts/SIGTERM (4) | Nginx config; test_ready_requires_compatible_schema; SIGTERM с активным POST | схема недоступна→503; активный POST 201, новые TCP отвергнуты пока процесс ещё жив, exit 0 за 1.9171 с | `evidence/student-tests.txt`, `graceful-shutdown.json` |
|
||||
| Последовательно остановлена каждая реплика, RPS/errors/recovery (4) | `verify_cluster.py failover` | два разных исходных container id; предыдущий восстановлен; подтверждения сохранены без дублей | `evidence/sequential-failover.json` |
|
||||
| CPU/RAM, нет API host ports/container_name (4) | `verify_cluster.py infra`; реальное scale 2→3→2 | лимиты есть у всех сервисов, единственный public endpoint у LB, api масштабируется | `evidence/infrastructure.json`, `rolling-update.json` |
|
||||
| Rolling-equivalent под нагрузкой, отдельные совместимые миграции (4) | `verify_cluster.py rolling` | два старых контейнера заменены новым образом, оба новых IP обнаружены LB; 14 975 подтверждённых сообщений сохранены | `evidence/rolling-update.json` |
|
||||
| Предыдущая авторизация и отказ зависимостей сохранены | `verify_cluster.py dependencies`; собственные auth/expiry tests | Redis fallback200; PostgreSQL 503; после восстановления тот же токен работает | `evidence/dependency-failures.json`, `student-tests.txt` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Чтобы повторно сохранить вывод: `make test > evidence/smoke.txt`, `make test-student > evidence/student-tests.txt`, `make validate-container > evidence/openapi.txt`, `make test-fresh > evidence/fresh-migrations.txt`. Распределённые сценарии сами обновляют свои JSON/text в evidence. Контракт и публичные tests не ослаблены; source lab2 переносился без них. Secrets, пароли и Bearer-токены в отчёт/evidence не записываются.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
Исследовательский трек 5 не проводился. Во всех сценариях используется реальный PostgreSQL/Redis и два API-процесса за Nginx. Генератор отказов/rolling — closed-loop4 потока; после неопределённого результата повторяет только POST сообщения, с прежним key/body. Подтверждённые ответы сверяются с полной историей, проверяются уникальность UUID и текста. Это проверка конкретных ограниченных отказов одного локального стенда.
|
||||
|
||||
В последовательном сценарии перед воздействиями есть 3 с базовой работы, каждое окно остановки — около 8 с, предыдущий контейнер запускается и обнаруживается через LB перед следующим. Время записи считается только по запросу, **начатому после возврата stop hook**, а общий deadline10 с отсчитывается от старта hook. Чтение известного подтверждённого сообщения также выполняется до восстановления target.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| Первый исходный API остановлен | другая реплика читает/пишет, остановленная не обслуживает | 400.056 RPS, 1 ошибочная попытка в окне; GET 200 и прежнее сообщение | POST через 1.3835 с от начала hook /0.0138 с после его завершения; GET через 1.38 с от начала |
|
||||
| Второй исходный API остановлен после восстановления первого | те же гарантии | 401.166 RPS, 1 ошибочная попытка; GET 200, target оставался stopped | POST через 1.4405 с от начала hook /0.0151 с после завершения; GET через 1.4375 с от начала |
|
||||
| Оба последовательных отказа | подтверждённые записи не теряются, retry не создаёт дубль | 15 915 подтверждённых сообщений =15 915 в истории | потерь/дублей не обнаружено |
|
||||
| SIGTERM во время активного POST | прекратить новые соединения, завершить активный запрос | SQL-запрос наблюдался в PgSleep; POST 201, новые TCP получили ConnectionRefusedError ещё при живом процессе; exit 0 | 1.9171 с от SIGTERM / сообщение сохранено |
|
||||
| Новый образ, поочерёдная start-first замена | адреса обновляются без рестарта LB, другая API продолжает работу | два шага 5.864/5.6574 с; разные IP, 3 alias при старте; во время замены 687.069 RPS, 1 ошибочная попытка (0.0126%) | 14 975 подтверждённых сообщений сохранены, дублей 0 |
|
||||
| Полный restart API/PG/Redis/LB с volumes | прежний токен читает прежние данные без повторного входа | public persistence PASS, реальные StartedAt изменились | потерь подтверждённого сообщения и сессии не обнаружено |
|
||||
| Redis остановлен | fallback в PostgreSQL без ослабления авторизации | GET 200 за 0.0138 с, ready 200, аноним 401 | данные/сессии в БД сохранены |
|
||||
| PostgreSQL остановлен | защищённый запрос 503, процесс остаётся жив | 503; обе внутренние live 200; после старта БД та же сессия работает | ошибки не приводят к доверию по кэшу |
|
||||
|
||||
**Как проверялся SIGTERM:** временный BEFORE INSERT trigger задерживает ровно один синтетический текст через pg_sleep(2). После наблюдения активного запроса в pg_stat_activity процесс получает SIGTERM. Проверяется отказ новых TCP, пока процесс ещё работает, затем 201 исходного запроса, exit 0 и чтение результата через surviving API. Trigger/function удаляются в finally, контейнер восстанавливается. В коде приложения тестового endpoint/режима нет.
|
||||
|
||||
**Что именно обновлялось:** собран действительно другой образ с отличающейся меткой APP_RELEASE, что видно по image id и X-App-Release. Код бизнес-API и схема совместимы; новой функциональной возможности в этом тестовом релизе не добавлялось. Мигратор запускается один раз перед заменами, не каждым новым API. Проверка доказывает поочерёдную замену сборки и обновление адресов, а не безопасное выполнение произвольной destructive migration.
|
||||
|
||||
**Нагрузка без отказов:** `make load-evidence`, 30 с, concurrency 4, две реплики, без автоматических retry, один сохранённый прогон. Перед ним стенд прогрет функциональными сценариями; отдельного измеряемого нагрузочного прогрева нет. 24 559 успешных запросов, 0 ошибок, 814.97 successful RPS; p50/p95/p99=4.03/5.822/47.235 мс. Число сообщений в БД: до запуска — 102 032, после — 126 591. Исходный summary — `evidence/load.json`, параметры и данные — `environment.json`, замеры ресурсов — `resources.json`.
|
||||
|
||||
Выборочные пики CPU/RAM за нагрузочный прогон (Docker CPU 100% = одно ядро; это пики отдельных samples, не одновременная сумма):
|
||||
|
||||
| Контейнер | CPU peak | Memory peak |
|
||||
| --- | --- | --- |
|
||||
| API-процесс 1 | 68.64% | 78.41 MiB |
|
||||
| API-процесс 2 | 69.43% | 66.61 MiB |
|
||||
| Nginx | 10.01% | 7.895 MiB |
|
||||
| PostgreSQL | 42.15% | 99.4 MiB |
|
||||
| Redis | 0.99% | 11.04 MiB |
|
||||
|
||||
Имена конкретных контейнеров и исходные значения — `evidence/resource-summary.json`. Эти цифры не доказывают максимальную пропускную способность и не являются сравнением с lab2: ресурсы/набор данных отличаются. Closed-loop имеет coordinated omission, latency рассчитана только для успешных ответов, клиент и Docker работают на одном ноутбуке. Переходные ошибки во время отказов/замен измерены и учитывают клиентские результаты. Nginx может скрыть connect error до отправки запроса и повторить безопасный GET; уже отправленные POST не повторяются им автоматически.
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker-host; LB, PostgreSQL и Redis — по одному экземпляру. Проверен отказ API-процесса, не узла/диска/БД с failover. Restart с volumes не заменяет backup и не обеспечивает HA.
|
||||
- Compose-замена — наш явный start-first сценарий с временной третьей репликой, не автоматический rolling-контроллер Swarm. Сам по себе обычный `docker compose up` не является доказанным безопасным rolling update. При обрыве нашего сценария может потребоваться ручное восстановление до двух healthy-реплик.
|
||||
- В миграциях используются только совместимые аддитивные изменения. Breaking schema change требует expand/contract, миграции данных или отдельного окна; в этой работе не проверен. Advisory lock предотвращает одновременное применение, но сам по себе не делает любую миграцию совместимой.
|
||||
- При SIGTERM/обнаружении отказа допускаются краткие HTTP/connection errors. Клиент повторяет только отправку сообщения с тем же Idempotency-Key. Создание чата и регистрация не имеют общей идемпотентности и после отправки тела автоматически не повторяются.
|
||||
- TTL идемпотентности 24 ч и expiry сессии проверяются при запросе; фоновой очистки старых строк пока нет. Истёкший ключ можно использовать заново, это создаст новое сообщение.
|
||||
- Redis без пароля в изолированной data-сети Compose; сессии всегда проверяются в PostgreSQL. Изменение профиля в будущем потребует инвалидации кэша. В этой работе нет TLS, rate limiting входа, PKI и secret manager.
|
||||
- Нет общего snapshot между страницами; конкурентная запись с ключом до cursor может остаться позади обхода. Порядок COMMIT не обязан совпадать с порядком created_at.
|
||||
- Когда БД недоступна и LB исключил обе API-реплики, даже `/health/live` через публичную точку может получить 502/503/504 от LB; внутренний `/health/live` каждого процесса остаётся 200. Это различие готовности маршрута и жизни процесса.
|
||||
- UI/WebSocket, изменение участников и редактирование/удаление сообщений не входят в обязательный API. Malformed JSON без Bearer может дать 400 до выполнения зависимости аутентификации.
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
name: mtusi-lab3
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.5-bookworm
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets: [postgres_password]
|
||||
volumes: [postgres_data:/var/lib/postgresql/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD-SHELL, 'pg_isready -U messenger -d messenger']
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 512m
|
||||
cpus: 1.0
|
||||
redis:
|
||||
image: redis:7.4.4-bookworm
|
||||
command: [redis-server, --appendonly, 'yes', --maxmemory, 64mb, --maxmemory-policy, allkeys-lru]
|
||||
volumes: [redis_data:/data]
|
||||
networks: [data]
|
||||
healthcheck:
|
||||
test: [CMD, redis-cli, ping]
|
||||
interval: 2s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.25
|
||||
migrate:
|
||||
image: mtusi-lab3-api:lab3
|
||||
command: [python, -m, messenger.migrate]
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
secrets: [database_url]
|
||||
networks: [data]
|
||||
depends_on:
|
||||
postgres: {condition: service_healthy}
|
||||
restart: 'no'
|
||||
mem_limit: 256m
|
||||
cpus: 0.5
|
||||
api:
|
||||
image: mtusi-lab3-api:lab3
|
||||
build: .
|
||||
scale: 2
|
||||
environment:
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
CACHE_URL_FILE: /run/secrets/cache_url
|
||||
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600}
|
||||
POOL_MAX_SIZE: '8'
|
||||
secrets: [database_url, cache_url]
|
||||
networks: [front, data]
|
||||
depends_on:
|
||||
migrate: {condition: service_completed_successfully}
|
||||
redis: {condition: service_healthy}
|
||||
stop_grace_period: 10s
|
||||
healthcheck:
|
||||
test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=4)"]
|
||||
interval: 2s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
mem_limit: 384m
|
||||
cpus: 0.75
|
||||
lb:
|
||||
image: nginx:1.28.0-alpine
|
||||
volumes: ['./infra/nginx/nginx.conf:/etc/nginx/nginx.conf:ro']
|
||||
networks: [front]
|
||||
ports: ['127.0.0.1:8080:8080']
|
||||
depends_on:
|
||||
api: {condition: service_healthy}
|
||||
healthcheck:
|
||||
test: [CMD, wget, -q, -O, /dev/null, http://127.0.0.1:8080/health/ready]
|
||||
interval: 1s
|
||||
timeout: 8s
|
||||
retries: 30
|
||||
mem_limit: 128m
|
||||
cpus: 0.5
|
||||
networks:
|
||||
front:
|
||||
data:
|
||||
internal: true
|
||||
secrets:
|
||||
postgres_password: {file: ./secrets/postgres_password}
|
||||
database_url: {file: ./secrets/database_url}
|
||||
cache_url: {file: ./secrets/cache_url}
|
||||
volumes:
|
||||
postgres_data:
|
||||
redis_data:
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"parallel_requests": 8,
|
||||
"observed_instances": [
|
||||
"a125004f5fbb4f9e91edb14bb3e64993",
|
||||
"a8916b2407ed45cc87d73e88cf1639b9"
|
||||
],
|
||||
"responses_201": 8,
|
||||
"messages_created": 1,
|
||||
"different_body_status": 409
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"redis": {
|
||||
"profile_status": 200,
|
||||
"fallback_seconds": 0.0138,
|
||||
"anonymous": 401,
|
||||
"readiness": 200
|
||||
},
|
||||
"postgres": {
|
||||
"authenticated_status": 503,
|
||||
"response_seconds": 0.0048,
|
||||
"anonymous_status": 401,
|
||||
"internal_api_liveness": [
|
||||
200,
|
||||
200
|
||||
],
|
||||
"note": "Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable."
|
||||
},
|
||||
"same_session_after_recovery": true
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"captured_at_utc": "2026-11-02T08:38:18.305573+00:00",
|
||||
"host_os": "macOS-27.2-arm64-arm-64bit-Mach-O",
|
||||
"host_architecture": "arm64",
|
||||
"docker": {
|
||||
"server_version": "29.8.2",
|
||||
"cpus": 12,
|
||||
"memory_bytes": 8319238144,
|
||||
"architecture": "aarch64"
|
||||
},
|
||||
"python_in_image": "Python 3.13.5",
|
||||
"dataset_before": {
|
||||
"users": 117,
|
||||
"chats": 53,
|
||||
"messages": 102032,
|
||||
"sessions": 102,
|
||||
"idempotency_records": 102011
|
||||
},
|
||||
"limits": {
|
||||
"api_each": "0.75 CPU / 384 MiB",
|
||||
"postgres": "1 CPU / 512 MiB",
|
||||
"redis": "0.25 CPU / 128 MiB",
|
||||
"lb": "0.5 CPU / 128 MiB"
|
||||
},
|
||||
"warmup": "API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.",
|
||||
"replicas": 2,
|
||||
"repetitions": 1,
|
||||
"duration_seconds": 30,
|
||||
"concurrency": 4,
|
||||
"host_cpu": "Apple M4 Pro",
|
||||
"host_memory_bytes": 25769803776,
|
||||
"dataset_after": {
|
||||
"users": 120,
|
||||
"chats": 54,
|
||||
"messages": 126591,
|
||||
"sessions": 105,
|
||||
"idempotency_records": 126570
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
./scripts/check_fresh_migrations.sh
|
||||
Applied 001_initial.sql
|
||||
Applied 002_idempotency.sql
|
||||
Migrations up to date
|
||||
Migrations up to date
|
||||
.............. [100%]
|
||||
14 passed in 3.56s
|
||||
PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.
|
||||
@@ -0,0 +1,16 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"target_container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
|
||||
"target_instance": "4ffdcca2f8644acd9b76d65f8260eaf7",
|
||||
"fixture": "temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards",
|
||||
"in_flight_observed_in_pg_stat_activity": true,
|
||||
"in_flight_response": 201,
|
||||
"new_tcp_after_sigterm": {
|
||||
"accepted": false,
|
||||
"error_type": "ConnectionRefusedError"
|
||||
},
|
||||
"target_still_running_when_new_tcp_rejected": true,
|
||||
"exit_code": 0,
|
||||
"shutdown_seconds_from_sigterm": 1.9171,
|
||||
"confirmed_message_preserved": true
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"containers": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710",
|
||||
"name": "mtusi-lab3-api-5",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:37:46.879119007Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 750000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data",
|
||||
"mtusi-lab3_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "0749a2e4af8a4b849e3eae6426945546",
|
||||
"release": "lab3-9d88e53e"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f",
|
||||
"name": "mtusi-lab3-api-6",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:37:52.762454968Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 750000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data",
|
||||
"mtusi-lab3_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "dd54c30c2e4c413ea2e28066c45e0d3c",
|
||||
"release": "lab3-9d88e53e"
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
|
||||
"name": "mtusi-lab3-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:32.827784792Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43",
|
||||
"name": "mtusi-lab3-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T08:36:54.173486177Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7",
|
||||
"name": "mtusi-lab3-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:38:04.389392668Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab3_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4",
|
||||
"name": "mtusi-lab3-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:38:03.185004292Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab3_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
|
||||
"name": "mtusi-lab3-api-4",
|
||||
"instance": "9ab4e9373a944f05b69d934479430136",
|
||||
"release": "lab3-9463abd8",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
|
||||
"stopped_at_utc": "2026-11-02T08:37:19.081389+00:00",
|
||||
"stop_seconds": 0.4518,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.135,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 24559,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 814.97,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.03,
|
||||
"p95": 5.822,
|
||||
"p99": 47.235
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"lab": 3,
|
||||
"workload": "closed-loop POST messages; no automatic retries",
|
||||
"requested_duration_seconds": 30.0,
|
||||
"elapsed_seconds": 30.135,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 24559,
|
||||
"failed_requests": 0,
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 814.97,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.03,
|
||||
"p95": 5.822,
|
||||
"p99": 47.235
|
||||
},
|
||||
"note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead."
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"version": "nginx version: nginx/1.28.0",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"public_server_header": "nginx",
|
||||
"config_syntax": "PASS",
|
||||
"observed_instances": [
|
||||
"0749a2e4af8a4b849e3eae6426945546",
|
||||
"dd54c30c2e4c413ea2e28066c45e0d3c"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 scripts/cluster.py exec python scripts/validate_openapi.py
|
||||
PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested.
|
||||
@@ -0,0 +1,250 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"before": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
|
||||
"name": "mtusi-lab3-api-3",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:24:29.880819722Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 750000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data",
|
||||
"mtusi-lab3_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "a125004f5fbb4f9e91edb14bb3e64993",
|
||||
"release": "lab3-9463abd8"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
|
||||
"name": "mtusi-lab3-api-4",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:24:29.778877972Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 750000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data",
|
||||
"mtusi-lab3_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "a8916b2407ed45cc87d73e88cf1639b9",
|
||||
"release": "lab3-9463abd8"
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
|
||||
"name": "mtusi-lab3-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:23.133451802Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43",
|
||||
"name": "mtusi-lab3-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T08:24:32.061652Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7",
|
||||
"name": "mtusi-lab3-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:24:29.651115763Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab3_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4",
|
||||
"name": "mtusi-lab3-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:24:29.682187472Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab3_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"service": "api",
|
||||
"container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
|
||||
"name": "mtusi-lab3-api-3",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:33.199244251Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 750000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data",
|
||||
"mtusi-lab3_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "f9c80459570c4c99991024bba6d2b167",
|
||||
"release": "lab3-9463abd8"
|
||||
},
|
||||
{
|
||||
"service": "api",
|
||||
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
|
||||
"name": "mtusi-lab3-api-4",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:33.076891376Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
|
||||
"memory_bytes": 402653184,
|
||||
"nano_cpus": 750000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data",
|
||||
"mtusi-lab3_front"
|
||||
],
|
||||
"http_status": 200,
|
||||
"instance": "782d112163514263a4c14c0fb6d68bfc",
|
||||
"release": "lab3-9463abd8"
|
||||
},
|
||||
{
|
||||
"service": "lb",
|
||||
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
|
||||
"name": "mtusi-lab3-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:32.827784792Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_front"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "migrate",
|
||||
"container": "f28386721cb033e6cbc187963a0d606cac9d5cf04cb4c24b9bb87dd392942e43",
|
||||
"name": "mtusi-lab3-migrate-1",
|
||||
"status": "exited",
|
||||
"health": null,
|
||||
"started_at": "2026-11-02T08:36:35.37548171Z",
|
||||
"image": "mtusi-lab3-api:lab3",
|
||||
"image_id": "sha256:1eb73e648436c8e2da04f5f1003b4804a91cdcef3cae9410c0f79d48ecd94ae9",
|
||||
"memory_bytes": 268435456,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "postgres",
|
||||
"container": "bfad6f9df6b7586ed9138173fc0414a86e67b4c1922a37ef3e645c461980deb7",
|
||||
"name": "mtusi-lab3-postgres-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:32.796757626Z",
|
||||
"image": "postgres:17.5-bookworm",
|
||||
"image_id": "sha256:fbcea1bd13b6a882cd6caa6b58db3ae5c102efe50ec625b3e2a5cbc50db5bfe4",
|
||||
"memory_bytes": 536870912,
|
||||
"nano_cpus": 1000000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab3_postgres_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"service": "redis",
|
||||
"container": "f8f6db5d380f2beacbd3c2af1d5b098a74da04f30b889ea3befafd0677ff08e4",
|
||||
"name": "mtusi-lab3-redis-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:32.773500709Z",
|
||||
"image": "redis:7.4.4-bookworm",
|
||||
"image_id": "sha256:7b5ee51c14cf5febd4adde3a4b6cced72efdf154df7e9a0383163b13e16e9091",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 250000000,
|
||||
"published_ports": {},
|
||||
"volumes": [
|
||||
"mtusi-lab3_redis_data"
|
||||
],
|
||||
"networks": [
|
||||
"mtusi-lab3_data"
|
||||
]
|
||||
}
|
||||
],
|
||||
"public_test": "Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.\nPASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence."
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"public_test": "Executing explicit failover hook; hook output suppressed to avoid leaking secrets.\n{\"observed_instances_before\": 2, \"successful_writes\": 132, \"failed_attempts\": 0, \"first_recovery_seconds\": 0.871}\nPASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.",
|
||||
"stopped": {
|
||||
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
|
||||
"name": "mtusi-lab3-api-4",
|
||||
"instance": "782d112163514263a4c14c0fb6d68bfc",
|
||||
"release": "lab3-9463abd8",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
|
||||
"stopped_at_utc": "2026-11-02T08:36:39.262177+00:00",
|
||||
"stop_seconds": 0.3321,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 132, "failed_attempts": 0, "first_recovery_seconds": 0.871}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"samples": 10,
|
||||
"per_container_peaks": {
|
||||
"mtusi-lab3-api-5": {
|
||||
"cpu_percent_peak": 68.64,
|
||||
"memory_mib_peak": 78.41
|
||||
},
|
||||
"mtusi-lab3-api-6": {
|
||||
"cpu_percent_peak": 69.43,
|
||||
"memory_mib_peak": 66.61
|
||||
},
|
||||
"mtusi-lab3-lb-1": {
|
||||
"cpu_percent_peak": 10.01,
|
||||
"memory_mib_peak": 7.895
|
||||
},
|
||||
"mtusi-lab3-postgres-1": {
|
||||
"cpu_percent_peak": 42.15,
|
||||
"memory_mib_peak": 99.4
|
||||
},
|
||||
"mtusi-lab3-redis-1": {
|
||||
"cpu_percent_peak": 0.99,
|
||||
"memory_mib_peak": 11.04
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,402 @@
|
||||
[
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:21.926644+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "50.31%",
|
||||
"MemPerc": "17.21%",
|
||||
"MemUsage": "66.08MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "8"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "66.02%",
|
||||
"MemPerc": "17.20%",
|
||||
"MemUsage": "66.05MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "9"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "9.06%",
|
||||
"MemPerc": "5.98%",
|
||||
"MemUsage": "7.652MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "30.22%",
|
||||
"MemPerc": "15.95%",
|
||||
"MemUsage": "81.65MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.14%",
|
||||
"MemPerc": "8.26%",
|
||||
"MemUsage": "10.58MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:24.960579+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "68.64%",
|
||||
"MemPerc": "17.27%",
|
||||
"MemUsage": "66.31MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "9"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "53.09%",
|
||||
"MemPerc": "17.15%",
|
||||
"MemUsage": "65.86MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "9.49%",
|
||||
"MemPerc": "5.70%",
|
||||
"MemUsage": "7.293MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "31.09%",
|
||||
"MemPerc": "16.17%",
|
||||
"MemUsage": "82.77MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.91%",
|
||||
"MemPerc": "8.30%",
|
||||
"MemUsage": "10.62MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:27.995870+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "58.43%",
|
||||
"MemPerc": "20.27%",
|
||||
"MemUsage": "77.83MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "56.00%",
|
||||
"MemPerc": "17.20%",
|
||||
"MemUsage": "66.07MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "9.79%",
|
||||
"MemPerc": "5.76%",
|
||||
"MemUsage": "7.379MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "33.22%",
|
||||
"MemPerc": "16.73%",
|
||||
"MemUsage": "85.67MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.90%",
|
||||
"MemPerc": "8.13%",
|
||||
"MemUsage": "10.4MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:31.030777+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "52.13%",
|
||||
"MemPerc": "17.41%",
|
||||
"MemUsage": "66.87MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "9"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "67.00%",
|
||||
"MemPerc": "17.21%",
|
||||
"MemUsage": "66.07MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "9.38%",
|
||||
"MemPerc": "5.72%",
|
||||
"MemUsage": "7.316MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "33.30%",
|
||||
"MemPerc": "17.16%",
|
||||
"MemUsage": "87.88MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.96%",
|
||||
"MemPerc": "8.14%",
|
||||
"MemUsage": "10.41MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:34.047170+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "65.51%",
|
||||
"MemPerc": "17.21%",
|
||||
"MemUsage": "66.1MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "52.34%",
|
||||
"MemPerc": "17.24%",
|
||||
"MemUsage": "66.21MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "9.16%",
|
||||
"MemPerc": "5.85%",
|
||||
"MemUsage": "7.488MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "30.55%",
|
||||
"MemPerc": "17.40%",
|
||||
"MemUsage": "89.08MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.93%",
|
||||
"MemPerc": "8.11%",
|
||||
"MemUsage": "10.38MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:37.087544+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "56.33%",
|
||||
"MemPerc": "19.86%",
|
||||
"MemUsage": "76.26MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "11"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "51.99%",
|
||||
"MemPerc": "17.30%",
|
||||
"MemUsage": "66.44MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "9.40%",
|
||||
"MemPerc": "6.09%",
|
||||
"MemUsage": "7.789MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "33.34%",
|
||||
"MemPerc": "17.80%",
|
||||
"MemUsage": "91.15MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.14%",
|
||||
"MemPerc": "8.48%",
|
||||
"MemUsage": "10.86MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:40.128204+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "49.82%",
|
||||
"MemPerc": "17.23%",
|
||||
"MemUsage": "66.16MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "64.91%",
|
||||
"MemPerc": "17.32%",
|
||||
"MemUsage": "66.5MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "9.44%",
|
||||
"MemPerc": "5.81%",
|
||||
"MemUsage": "7.438MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "30.24%",
|
||||
"MemPerc": "18.14%",
|
||||
"MemUsage": "92.86MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.94%",
|
||||
"MemPerc": "8.63%",
|
||||
"MemUsage": "11.04MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:43.146647+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "61.56%",
|
||||
"MemPerc": "20.42%",
|
||||
"MemUsage": "78.41MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "11"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "56.33%",
|
||||
"MemPerc": "17.35%",
|
||||
"MemUsage": "66.61MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "8.46%",
|
||||
"MemPerc": "6.17%",
|
||||
"MemUsage": "7.895MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "32.44%",
|
||||
"MemPerc": "18.46%",
|
||||
"MemUsage": "94.5MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.17%",
|
||||
"MemPerc": "8.25%",
|
||||
"MemUsage": "10.56MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:46.182717+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "64.09%",
|
||||
"MemPerc": "17.39%",
|
||||
"MemUsage": "66.78MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "9"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "66.93%",
|
||||
"MemPerc": "17.24%",
|
||||
"MemUsage": "66.19MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "8.61%",
|
||||
"MemPerc": "5.89%",
|
||||
"MemUsage": "7.539MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "42.15%",
|
||||
"MemPerc": "19.17%",
|
||||
"MemUsage": "98.16MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.14%",
|
||||
"MemPerc": "8.27%",
|
||||
"MemUsage": "10.58MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"at_utc": "2026-11-02T08:38:49.213747+00:00",
|
||||
"containers": [
|
||||
{
|
||||
"CPUPerc": "53.84%",
|
||||
"MemPerc": "17.29%",
|
||||
"MemUsage": "66.39MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-5",
|
||||
"PIDs": "9"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "69.43%",
|
||||
"MemPerc": "17.32%",
|
||||
"MemUsage": "66.5MiB / 384MiB",
|
||||
"Name": "mtusi-lab3-api-6",
|
||||
"PIDs": "10"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "10.01%",
|
||||
"MemPerc": "6.13%",
|
||||
"MemUsage": "7.844MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-lb-1",
|
||||
"PIDs": "2"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "34.69%",
|
||||
"MemPerc": "19.41%",
|
||||
"MemUsage": "99.4MiB / 512MiB",
|
||||
"Name": "mtusi-lab3-postgres-1",
|
||||
"PIDs": "16"
|
||||
},
|
||||
{
|
||||
"CPUPerc": "0.99%",
|
||||
"MemPerc": "8.46%",
|
||||
"MemUsage": "10.83MiB / 128MiB",
|
||||
"Name": "mtusi-lab3-redis-1",
|
||||
"PIDs": "6"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,171 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"before": [
|
||||
{
|
||||
"container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
|
||||
"name": "mtusi-lab3-api-3",
|
||||
"instance": "efae54a83fc844f4b19b1e1c2aad0385",
|
||||
"release": "lab3-9463abd8",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af"
|
||||
},
|
||||
{
|
||||
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
|
||||
"name": "mtusi-lab3-api-4",
|
||||
"instance": "1b7fd0264d924b3ba1b87368400b4a56",
|
||||
"release": "lab3-9463abd8",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af"
|
||||
}
|
||||
],
|
||||
"after": [
|
||||
{
|
||||
"container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710",
|
||||
"name": "mtusi-lab3-api-5",
|
||||
"instance": "0749a2e4af8a4b849e3eae6426945546",
|
||||
"release": "lab3-9d88e53e",
|
||||
"image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741"
|
||||
},
|
||||
{
|
||||
"container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f",
|
||||
"name": "mtusi-lab3-api-6",
|
||||
"instance": "dd54c30c2e4c413ea2e28066c45e0d3c",
|
||||
"release": "lab3-9d88e53e",
|
||||
"image_id": "sha256:86bd4cd854e6ccea91ee6f75e259b5808646c82e0946952fff8703ef205bb741"
|
||||
}
|
||||
],
|
||||
"target_release": "lab3-9d88e53e",
|
||||
"lb_before": {
|
||||
"service": "lb",
|
||||
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
|
||||
"name": "mtusi-lab3-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:32.827784792Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_front"
|
||||
]
|
||||
},
|
||||
"lb_after": {
|
||||
"service": "lb",
|
||||
"container": "8e7f7169b22230a255252691b145b2bd14394b16f9feb1a1f2fc4163fe03d59a",
|
||||
"name": "mtusi-lab3-lb-1",
|
||||
"status": "running",
|
||||
"health": "healthy",
|
||||
"started_at": "2026-11-02T08:36:32.827784792Z",
|
||||
"image": "nginx:1.28.0-alpine",
|
||||
"image_id": "sha256:30f1c0d78e0ad60901648be663a710bdadf19e4c10ac6782c235200619158284",
|
||||
"memory_bytes": 134217728,
|
||||
"nano_cpus": 500000000,
|
||||
"published_ports": {
|
||||
"8080/tcp": [
|
||||
{
|
||||
"HostIp": "127.0.0.1",
|
||||
"HostPort": "8080"
|
||||
}
|
||||
]
|
||||
},
|
||||
"volumes": [],
|
||||
"networks": [
|
||||
"mtusi-lab3_front"
|
||||
]
|
||||
},
|
||||
"lb_not_restarted": true,
|
||||
"build_seconds": 2.2969,
|
||||
"migration_output": "Migrations up to date",
|
||||
"replacements": [
|
||||
{
|
||||
"old_container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
|
||||
"new_container": "8a1922748bf43fe98645bc9a03b0971445616c48bdf2df0690a386c293ebb710",
|
||||
"old_ip": "172.23.0.4",
|
||||
"new_ip": "172.23.0.5",
|
||||
"dns_discovered_new_address": true,
|
||||
"instances_during_start_first": [
|
||||
"0749a2e4af8a4b849e3eae6426945546",
|
||||
"1b7fd0264d924b3ba1b87368400b4a56",
|
||||
"efae54a83fc844f4b19b1e1c2aad0385"
|
||||
],
|
||||
"observed_instances": [
|
||||
"0749a2e4af8a4b849e3eae6426945546",
|
||||
"1b7fd0264d924b3ba1b87368400b4a56"
|
||||
],
|
||||
"old_exit_code": 0,
|
||||
"seconds": 5.864
|
||||
},
|
||||
{
|
||||
"old_container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
|
||||
"new_container": "d63828d8ad840ddaa3e63b1dfe9716b2ace17e9eb9f05ec5bd5df73641fbf85f",
|
||||
"old_ip": "172.23.0.2",
|
||||
"new_ip": "172.23.0.4",
|
||||
"dns_discovered_new_address": true,
|
||||
"instances_during_start_first": [
|
||||
"0749a2e4af8a4b849e3eae6426945546",
|
||||
"1b7fd0264d924b3ba1b87368400b4a56",
|
||||
"dd54c30c2e4c413ea2e28066c45e0d3c"
|
||||
],
|
||||
"observed_instances": [
|
||||
"0749a2e4af8a4b849e3eae6426945546",
|
||||
"dd54c30c2e4c413ea2e28066c45e0d3c"
|
||||
],
|
||||
"old_exit_code": 0,
|
||||
"seconds": 5.6574
|
||||
}
|
||||
],
|
||||
"during_replacement": {
|
||||
"seconds": 11.5214,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 7916,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 687.069,
|
||||
"error_ratio": 0.000126,
|
||||
"successful_latency_ms": {
|
||||
"p50": 3.992,
|
||||
"p95": 7.701,
|
||||
"p99": 62.305
|
||||
},
|
||||
"instances": [
|
||||
"0749a2e4af8a4b849e3eae6426945546",
|
||||
"1b7fd0264d924b3ba1b87368400b4a56",
|
||||
"dd54c30c2e4c413ea2e28066c45e0d3c",
|
||||
"efae54a83fc844f4b19b1e1c2aad0385"
|
||||
]
|
||||
},
|
||||
"total": {
|
||||
"seconds": 20.7201,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 14975,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 722.729,
|
||||
"error_ratio": 6.7e-05,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.089,
|
||||
"p95": 7.12,
|
||||
"p99": 59.058
|
||||
},
|
||||
"instances": [
|
||||
"0749a2e4af8a4b849e3eae6426945546",
|
||||
"1b7fd0264d924b3ba1b87368400b4a56",
|
||||
"dd54c30c2e4c413ea2e28066c45e0d3c",
|
||||
"efae54a83fc844f4b19b1e1c2aad0385"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"confirmed_unique_messages": 14975,
|
||||
"history_messages": 14975,
|
||||
"confirmed_preserved": true,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0
|
||||
},
|
||||
"change": "New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate."
|
||||
}
|
||||
@@ -0,0 +1,135 @@
|
||||
{
|
||||
"result": "PASS",
|
||||
"baseline": {
|
||||
"seconds": 3.0036,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 2095,
|
||||
"failed_attempts": 0,
|
||||
"successful_rps": 697.497,
|
||||
"error_ratio": 0.0,
|
||||
"successful_latency_ms": {
|
||||
"p50": 4.402,
|
||||
"p95": 6.728,
|
||||
"p99": 56.167
|
||||
},
|
||||
"instances": [
|
||||
"9ab4e9373a944f05b69d934479430136",
|
||||
"f9c80459570c4c99991024bba6d2b167"
|
||||
]
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"stopped": {
|
||||
"container": "e8c5e3cb84a8918b5603e202cd1d66334d12a4943321e6db6d26716cc361b19d",
|
||||
"name": "mtusi-lab3-api-3",
|
||||
"instance": "f9c80459570c4c99991024bba6d2b167",
|
||||
"release": "lab3-9463abd8",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
|
||||
"stopped_at_utc": "2026-11-02T08:37:05.018221+00:00",
|
||||
"stop_seconds": 0.3922,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"traffic": {
|
||||
"seconds": 8.0014,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 3201,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 400.056,
|
||||
"error_ratio": 0.000312,
|
||||
"successful_latency_ms": {
|
||||
"p50": 6.213,
|
||||
"p95": 37.884,
|
||||
"p99": 72.594
|
||||
},
|
||||
"instances": [
|
||||
"9ab4e9373a944f05b69d934479430136",
|
||||
"f9c80459570c4c99991024bba6d2b167"
|
||||
]
|
||||
},
|
||||
"post_stop_write_seconds_from_action_start": 1.3835,
|
||||
"post_stop_write_seconds_from_stop_completion": 0.0138,
|
||||
"read_during_outage": {
|
||||
"status": 200,
|
||||
"confirmed_message_preserved": true,
|
||||
"seconds_from_action_start": 1.38,
|
||||
"seconds_from_stop_completion": 0.0102
|
||||
},
|
||||
"new_connection_after_stop": {
|
||||
"accepted": false,
|
||||
"error_type": "TimeoutError"
|
||||
},
|
||||
"stopped_remains_stopped": true
|
||||
},
|
||||
{
|
||||
"stopped": {
|
||||
"container": "9ad409d735547c65b7a73a7bf803909e61256faec3266341934bf89608d848ea",
|
||||
"name": "mtusi-lab3-api-4",
|
||||
"instance": "9ab4e9373a944f05b69d934479430136",
|
||||
"release": "lab3-9463abd8",
|
||||
"image_id": "sha256:953592eec2a47c9875c4d1c6331455e1a105c66f9320e2d086c2dcb0a02d51af",
|
||||
"stopped_at_utc": "2026-11-02T08:37:19.081389+00:00",
|
||||
"stop_seconds": 0.4518,
|
||||
"exit_code": 0,
|
||||
"left_stopped": true
|
||||
},
|
||||
"traffic": {
|
||||
"seconds": 8.0017,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 3210,
|
||||
"failed_attempts": 1,
|
||||
"successful_rps": 401.166,
|
||||
"error_ratio": 0.000311,
|
||||
"successful_latency_ms": {
|
||||
"p50": 6.206,
|
||||
"p95": 37.612,
|
||||
"p99": 74.049
|
||||
},
|
||||
"instances": [
|
||||
"4ffdcca2f8644acd9b76d65f8260eaf7",
|
||||
"9ab4e9373a944f05b69d934479430136"
|
||||
]
|
||||
},
|
||||
"post_stop_write_seconds_from_action_start": 1.4405,
|
||||
"post_stop_write_seconds_from_stop_completion": 0.0151,
|
||||
"read_during_outage": {
|
||||
"status": 200,
|
||||
"confirmed_message_preserved": true,
|
||||
"seconds_from_action_start": 1.4375,
|
||||
"seconds_from_stop_completion": 0.0121
|
||||
},
|
||||
"new_connection_after_stop": {
|
||||
"accepted": false,
|
||||
"error_type": "TimeoutError"
|
||||
},
|
||||
"stopped_remains_stopped": true
|
||||
}
|
||||
],
|
||||
"total": {
|
||||
"seconds": 31.1217,
|
||||
"concurrency": 4,
|
||||
"successful_requests": 15915,
|
||||
"failed_attempts": 2,
|
||||
"successful_rps": 511.38,
|
||||
"error_ratio": 0.000126,
|
||||
"successful_latency_ms": {
|
||||
"p50": 5.137,
|
||||
"p95": 32.066,
|
||||
"p99": 65.023
|
||||
},
|
||||
"instances": [
|
||||
"1b7fd0264d924b3ba1b87368400b4a56",
|
||||
"4ffdcca2f8644acd9b76d65f8260eaf7",
|
||||
"9ab4e9373a944f05b69d934479430136",
|
||||
"f9c80459570c4c99991024bba6d2b167"
|
||||
]
|
||||
},
|
||||
"data": {
|
||||
"confirmed_unique_messages": 15915,
|
||||
"history_messages": 15915,
|
||||
"confirmed_preserved": true,
|
||||
"duplicate_ids": 0,
|
||||
"duplicate_texts": 0
|
||||
},
|
||||
"each_original_container_tested": true
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
python3 tests/smoke.py --base-url "http://localhost:8080"
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
@@ -0,0 +1,3 @@
|
||||
python3 scripts/cluster.py exec python -m pytest student-tests -q --tb=short -o cache_dir=/tmp/pytest-cache
|
||||
.............. [100%]
|
||||
14 passed in 3.58s
|
||||
@@ -0,0 +1,43 @@
|
||||
worker_processes 1;
|
||||
error_log /dev/stderr warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
http {
|
||||
access_log off;
|
||||
server_tokens off;
|
||||
|
||||
# Docker DNS; resolve updates the upstream when replicas get new IPs.
|
||||
resolver 127.0.0.11 valid=1s ipv6=off;
|
||||
resolver_timeout 1s;
|
||||
|
||||
upstream messenger {
|
||||
zone messenger 64k;
|
||||
# All A records of api participate in the default round-robin balancing.
|
||||
server api:8080 resolve max_fails=1 fail_timeout=2s;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
|
||||
location / {
|
||||
proxy_pass http://messenger;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header Connection "";
|
||||
|
||||
proxy_connect_timeout 1s;
|
||||
proxy_read_timeout 5s;
|
||||
proxy_send_timeout 5s;
|
||||
|
||||
# Passive checks. Do NOT enable non_idempotent: an already-sent POST
|
||||
# must not be resent by Nginx after an ambiguous response.
|
||||
proxy_next_upstream error timeout http_503;
|
||||
proxy_next_upstream_tries 2;
|
||||
proxy_next_upstream_timeout 6s;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
CREATE TABLE users (
|
||||
id uuid PRIMARY KEY,
|
||||
username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'),
|
||||
display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100),
|
||||
password_hash text NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chats (
|
||||
id uuid PRIMARY KEY,
|
||||
title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp()
|
||||
);
|
||||
CREATE TABLE chat_members (
|
||||
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
PRIMARY KEY (chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX chat_members_user_idx ON chat_members(user_id, chat_id);
|
||||
CREATE INDEX chats_history_idx ON chats(created_at, id);
|
||||
CREATE TABLE messages (
|
||||
id uuid PRIMARY KEY,
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
sender_id uuid NOT NULL,
|
||||
text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000),
|
||||
created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
|
||||
FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id)
|
||||
);
|
||||
CREATE INDEX messages_history_idx ON messages(chat_id, created_at, id);
|
||||
CREATE TABLE sessions (
|
||||
token_hash text PRIMARY KEY,
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
expires_at timestamptz NOT NULL
|
||||
);
|
||||
CREATE INDEX sessions_expiry_idx ON sessions(expires_at);
|
||||
@@ -0,0 +1,11 @@
|
||||
-- Additive: old lab2 tables and messages remain valid.
|
||||
CREATE TABLE idempotency_records (
|
||||
user_id uuid NOT NULL REFERENCES users(id),
|
||||
chat_id uuid NOT NULL REFERENCES chats(id),
|
||||
key text NOT NULL CHECK (key ~ '^[A-Za-z0-9._:-]{1,128}$'),
|
||||
request_hash text NOT NULL,
|
||||
message_id uuid REFERENCES messages(id),
|
||||
expires_at timestamptz NOT NULL,
|
||||
PRIMARY KEY (user_id, chat_id, key)
|
||||
);
|
||||
CREATE INDEX idempotency_expiry_idx ON idempotency_records(expires_at);
|
||||
@@ -0,0 +1,45 @@
|
||||
annotated-types==0.8.0
|
||||
anyio==4.9.0
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
attrs==26.1.0
|
||||
certifi==2026.10.23
|
||||
cffi==2.1.1
|
||||
charset-normalizer==3.5.2
|
||||
click==8.5.0
|
||||
fastapi==0.115.12
|
||||
h11==0.16.0
|
||||
httpcore==1.0.9
|
||||
httpx==0.28.1
|
||||
idna==3.20
|
||||
iniconfig==2.3.1
|
||||
jsonschema==4.26.0
|
||||
jsonschema-path==0.3.4
|
||||
jsonschema-specifications==2025.9.1
|
||||
lazy-object-proxy==1.13.0
|
||||
openapi-schema-validator==0.6.3
|
||||
openapi-spec-validator==0.7.2
|
||||
packaging==26.3
|
||||
pathable==0.4.4
|
||||
pluggy==1.7.0
|
||||
psycopg==3.2.9
|
||||
psycopg-binary==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
pycparser==3.12
|
||||
pydantic==2.11.5
|
||||
pydantic_core==2.33.2
|
||||
Pygments==2.21.0
|
||||
pytest==8.4.0
|
||||
PyYAML==6.0.3
|
||||
redis==6.2.0
|
||||
referencing==0.36.2
|
||||
requests==2.34.2
|
||||
rfc3339-validator==0.1.4
|
||||
rpds-py==2026.9.1
|
||||
six==1.17.0
|
||||
sniffio==1.3.1
|
||||
starlette==0.46.2
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
urllib3==2.8.0
|
||||
uvicorn==0.34.3
|
||||
@@ -0,0 +1,10 @@
|
||||
fastapi==0.115.12
|
||||
uvicorn==0.34.3
|
||||
psycopg[binary]==3.2.9
|
||||
psycopg-pool==3.2.6
|
||||
redis==6.2.0
|
||||
argon2-cffi==25.1.0
|
||||
pytest==8.4.0
|
||||
httpx==0.28.1
|
||||
pydantic==2.11.5
|
||||
anyio==4.9.0
|
||||
@@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
umask 077
|
||||
mkdir -p secrets
|
||||
python3 - <<'PY'
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
root = Path('secrets')
|
||||
p = root / 'postgres_password'
|
||||
if not p.exists():
|
||||
p.write_text(secrets.token_hex(24))
|
||||
pw = p.read_text().strip()
|
||||
for name, value in {
|
||||
'database_url': f'postgresql://messenger:{pw}@postgres:5432/messenger',
|
||||
'cache_url': 'redis://redis:6379/0',
|
||||
}.items():
|
||||
path = root / name
|
||||
if not path.exists():
|
||||
path.write_text(value)
|
||||
# Bind-mounted Docker secrets must be readable by container uid 10001.
|
||||
# Parent directory remains 0700 on the host.
|
||||
for path in root.iterdir():
|
||||
path.chmod(0o444)
|
||||
if not Path('.env').exists():
|
||||
Path('.env').write_text('SESSION_TTL_SECONDS=3600\n')
|
||||
print('Local secrets ready (values hidden).')
|
||||
PY
|
||||
@@ -0,0 +1,71 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Capture repeatable short load run and sanitized environment/resource evidence."""
|
||||
import datetime
|
||||
import json
|
||||
import platform
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import cluster
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
|
||||
def output(args):
|
||||
return subprocess.check_output(args, cwd=ROOT, text=True).strip()
|
||||
|
||||
|
||||
def counts():
|
||||
code = "import json,psycopg; from messenger.db import setting; c=psycopg.connect(setting('DATABASE_URL')); print(json.dumps({t:c.execute('SELECT count(*) FROM '+t).fetchone()[0] for t in ['users','chats','messages','sessions','idempotency_records']})); c.close()"
|
||||
return json.loads(output(['docker','exec',cluster.container_ids('api')[0],'python','-c',code]))
|
||||
|
||||
|
||||
def resource_summary(samples):
|
||||
peaks = {}
|
||||
for sample in samples:
|
||||
for container in sample['containers']:
|
||||
peak = peaks.setdefault(container['Name'], {'cpu_percent_peak':0, 'memory_mib_peak':0})
|
||||
peak['cpu_percent_peak'] = max(peak['cpu_percent_peak'],float(container['CPUPerc'].rstrip('%')))
|
||||
value,unit = re.match(r'([0-9.]+)([A-Za-z]+)',container['MemUsage']).groups()
|
||||
multiplier = {'GiB':1024,'MiB':1,'KiB':1/1024,'B':1/1048576}[unit]
|
||||
peak['memory_mib_peak'] = round(max(peak['memory_mib_peak'],float(value)*multiplier),3)
|
||||
return {'samples':len(samples),'per_container_peaks':peaks}
|
||||
|
||||
|
||||
def main():
|
||||
names = cluster.container_ids()
|
||||
environment = {
|
||||
'captured_at_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'host_os': platform.platform(), 'host_architecture': platform.machine(),
|
||||
'docker': json.loads(output(['docker','info','--format','{"server_version":"{{.ServerVersion}}","cpus":{{.NCPU}},"memory_bytes":{{.MemTotal}},"architecture":"{{.Architecture}}"}'])),
|
||||
'python_in_image': output(['docker','exec',cluster.container_ids('api')[0],'python','--version']),
|
||||
'dataset_before': counts(),
|
||||
'limits': {'api_each':'0.75 CPU / 384 MiB','postgres':'1 CPU / 512 MiB','redis':'0.25 CPU / 128 MiB','lb':'0.5 CPU / 128 MiB'},
|
||||
'warmup': 'API already healthy; acceptance, persistence and dependency checks performed before benchmark. No separate timed load warmup.',
|
||||
'replicas':len(cluster.replicas()), 'repetitions': 1, 'duration_seconds':30, 'concurrency':4,
|
||||
}
|
||||
if sys.platform == 'darwin':
|
||||
environment['host_cpu'] = output(['sysctl','-n','machdep.cpu.brand_string'])
|
||||
environment['host_memory_bytes'] = int(output(['sysctl','-n','hw.memsize']))
|
||||
samples = []
|
||||
with (ROOT/'evidence/load-output.txt').open('w') as log:
|
||||
proc = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4','--output','evidence/load.json'],cwd=ROOT,stdout=log,stderr=subprocess.STDOUT)
|
||||
while proc.poll() is None:
|
||||
rows = output(['docker','stats','--no-stream','--format','{{json .}}',*names])
|
||||
samples.append({'at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'containers':[{k:v for k,v in json.loads(row).items() if k in ('Name','CPUPerc','MemUsage','MemPerc','PIDs')}
|
||||
for row in rows.splitlines()]})
|
||||
time.sleep(1)
|
||||
if proc.returncode:
|
||||
raise SystemExit(proc.returncode)
|
||||
environment['dataset_after'] = counts()
|
||||
(ROOT/'evidence/environment.json').write_text(json.dumps(environment,ensure_ascii=False,indent=2)+'\n')
|
||||
(ROOT/'evidence/resources.json').write_text(json.dumps(samples,indent=2)+'\n')
|
||||
(ROOT/'evidence/resource-summary.json').write_text(json.dumps(resource_summary(samples),indent=2)+'\n')
|
||||
print((ROOT/'evidence/load.json').read_text())
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# A unique disposable Compose project; never touches normal project volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
project="mtusi-lab3-migration-check-$$"
|
||||
if [ -n "$(docker compose -p "$project" ps -aq)" ]; then
|
||||
echo 'Refusing to reuse a pre-existing scratch project' >&2
|
||||
exit 1
|
||||
fi
|
||||
cleanup() {
|
||||
docker compose -p "$project" down --volumes >/dev/null 2>&1
|
||||
}
|
||||
trap cleanup EXIT INT TERM
|
||||
docker compose -p "$project" up -d --wait --wait-timeout 90 postgres redis
|
||||
docker compose -p "$project" run --rm --no-deps api sh -c 'python -m messenger.migrate && python -m messenger.migrate && python -m pytest student-tests -q -o cache_dir=/tmp/pytest-cache'
|
||||
echo 'PASS: migrations and application tests on fresh disposable PostgreSQL and Redis volumes.'
|
||||
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local Compose orchestration. No credentials, tokens or environment dumps in evidence."""
|
||||
import argparse
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
BASE_URL = os.environ.get('BASE_URL', 'http://localhost:8080').rstrip('/')
|
||||
|
||||
|
||||
def command(args, timeout=120):
|
||||
return subprocess.run(args, cwd=ROOT, check=True, capture_output=True, text=True, timeout=timeout).stdout.strip()
|
||||
|
||||
|
||||
def compose(*args, timeout=120):
|
||||
return command(['docker', 'compose', *args], timeout=timeout)
|
||||
|
||||
|
||||
def inspect(containers):
|
||||
if not containers:
|
||||
return []
|
||||
return json.loads(command(['docker','inspect',*containers]))
|
||||
|
||||
|
||||
def container_ids(service=None, all_containers=False):
|
||||
args = ['ps'] + (['--all'] if all_containers else []) + ['-q'] + ([service] if service else [])
|
||||
return compose(*args).split()
|
||||
|
||||
|
||||
def probe(container, host='127.0.0.1', path='/health/live'):
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
try:
|
||||
r=urllib.request.urlopen('http://'+sys.argv[1]+':8080'+sys.argv[2],timeout=4)
|
||||
except urllib.error.HTTPError as exc:
|
||||
r=exc
|
||||
print(json.dumps({'status':r.code,'instance':r.headers.get('X-Instance-Id'),'release':r.headers.get('X-App-Release')}))
|
||||
'''
|
||||
return json.loads(command(['docker','exec',container,'python','-c',code,host,path], timeout=8))
|
||||
|
||||
|
||||
def replicas():
|
||||
items = []
|
||||
for data in inspect(container_ids('api')):
|
||||
if data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
items.append({'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'instance':result['instance'], 'release':result['release'], 'image_id':data['Image']})
|
||||
return items
|
||||
|
||||
|
||||
def snapshot():
|
||||
items = []
|
||||
for data in inspect(container_ids(all_containers=True)):
|
||||
service = data['Config']['Labels']['com.docker.compose.service']
|
||||
item = {
|
||||
'service':service, 'container':data['Id'], 'name':data['Name'].lstrip('/'),
|
||||
'status':data['State']['Status'], 'health':data['State'].get('Health',{}).get('Status'),
|
||||
'started_at':data['State']['StartedAt'], 'image':data['Config']['Image'], 'image_id':data['Image'],
|
||||
'memory_bytes':data['HostConfig']['Memory'], 'nano_cpus':data['HostConfig']['NanoCpus'],
|
||||
'published_ports':data['HostConfig']['PortBindings'],
|
||||
'volumes':sorted(m['Name'] for m in data['Mounts'] if m['Type']=='volume'),
|
||||
'networks':sorted(data['NetworkSettings']['Networks']),
|
||||
}
|
||||
if service == 'api' and data['State']['Running']:
|
||||
result = probe(data['Id'])
|
||||
item.update({'http_status':result['status'], 'instance':result['instance'], 'release':result['release']})
|
||||
items.append(item)
|
||||
return items
|
||||
|
||||
|
||||
def wait_healthy(container, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
while time.monotonic() < deadline:
|
||||
data = inspect([container])[0]
|
||||
if data['State'].get('Health',{}).get('Status') == 'healthy':
|
||||
return
|
||||
time.sleep(0.3)
|
||||
raise RuntimeError('Container did not become healthy')
|
||||
|
||||
|
||||
def wait_cluster(count=2, timeout=60):
|
||||
deadline = time.monotonic()+timeout
|
||||
seen = set()
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with urllib.request.urlopen(BASE_URL+'/health/ready',timeout=5) as response:
|
||||
alias = response.headers.get('X-Instance-Id')
|
||||
if alias:
|
||||
seen.add(alias)
|
||||
if len(seen) >= count:
|
||||
return sorted(seen)
|
||||
except (OSError, urllib.error.URLError):
|
||||
pass
|
||||
time.sleep(0.1)
|
||||
raise RuntimeError('Load balancer did not discover required API instances')
|
||||
|
||||
|
||||
def stop_one(target):
|
||||
if not re.fullmatch(r'[a-f0-9]{32}', target):
|
||||
raise ValueError('Invalid target instance alias')
|
||||
matching = [item for item in replicas() if item['instance']==target]
|
||||
if len(matching) != 1:
|
||||
raise RuntimeError('Target must match exactly one running API process')
|
||||
item = matching[0]
|
||||
started = time.monotonic()
|
||||
command(['docker','stop','--time','7',item['container']],timeout=9)
|
||||
state = inspect([item['container']])[0]['State']
|
||||
if state['Running'] or state['ExitCode'] != 0:
|
||||
raise RuntimeError('API did not stop gracefully')
|
||||
result = {**item, 'stopped_at_utc':datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
||||
'stop_seconds':round(time.monotonic()-started,4), 'exit_code':state['ExitCode'], 'left_stopped':True}
|
||||
(ROOT/'evidence/last-stop.json').write_text(json.dumps(result,indent=2)+'\n')
|
||||
return result
|
||||
|
||||
|
||||
def restart():
|
||||
compose('restart','api','postgres','redis','lb',timeout=30)
|
||||
compose('up','-d','--no-recreate','--wait','--wait-timeout','35',timeout=40)
|
||||
return wait_cluster(2,timeout=15)
|
||||
|
||||
|
||||
def front_address(container):
|
||||
networks = inspect([container])[0]['NetworkSettings']['Networks']
|
||||
return next(value['IPAddress'] for name,value in networks.items() if name.endswith('_front'))
|
||||
|
||||
|
||||
def replace_one(container):
|
||||
before = set(container_ids('api'))
|
||||
if container not in before or len(before) != 2:
|
||||
raise RuntimeError('Replacement expects exactly two running API containers')
|
||||
old_address = front_address(container)
|
||||
# Start-first ensures a genuinely different IP while the old address is still occupied.
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=3','api')
|
||||
expanded = set(container_ids('api'))
|
||||
added = expanded-before
|
||||
if len(added) != 1 or not before.issubset(expanded):
|
||||
raise RuntimeError('Start-first step unexpectedly changed an existing container')
|
||||
replacement = added.pop()
|
||||
wait_healthy(replacement)
|
||||
new_address = front_address(replacement)
|
||||
if new_address == old_address:
|
||||
raise RuntimeError('Replacement did not get a different IP')
|
||||
expected_three = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(3)) != expected_three:
|
||||
raise RuntimeError('Load balancer did not discover all three live addresses')
|
||||
command(['docker','stop','--time','7',container],timeout=10)
|
||||
state = inspect([container])[0]['State']
|
||||
if state['ExitCode'] != 0:
|
||||
raise RuntimeError('Old release did not terminate gracefully')
|
||||
command(['docker','rm',container])
|
||||
compose('up','-d','--no-deps','--no-recreate','--scale','api=2','api')
|
||||
after = set(container_ids('api'))
|
||||
if after != (before-{container}) | {replacement}:
|
||||
raise RuntimeError('Sequential replacement changed the wrong containers')
|
||||
expected = {item['instance'] for item in replicas()}
|
||||
if set(wait_cluster(2)) != expected:
|
||||
raise RuntimeError('Unexpected final backend aliases')
|
||||
return {'old_container':container,'new_container':replacement,
|
||||
'old_ip':old_address,'new_ip':new_address,'dns_discovered_new_address':True,
|
||||
'instances_during_start_first':sorted(expected_three),
|
||||
'observed_instances':sorted(expected),'old_exit_code':state['ExitCode']}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('action',choices=['wait','snapshot','stop-one','restart','rolling','exec'])
|
||||
parser.add_argument('arguments',nargs=argparse.REMAINDER)
|
||||
args = parser.parse_args()
|
||||
if args.action == 'wait':
|
||||
print(json.dumps({'instances':wait_cluster()}))
|
||||
elif args.action == 'snapshot':
|
||||
print(json.dumps(snapshot(),indent=2))
|
||||
elif args.action == 'stop-one':
|
||||
print(json.dumps(stop_one(os.environ['TARGET_INSTANCE_ID'])))
|
||||
elif args.action == 'exec':
|
||||
ids = container_ids('api')
|
||||
if not ids or not args.arguments:
|
||||
raise RuntimeError('A live API container and command are required')
|
||||
raise SystemExit(subprocess.call(['docker','exec',ids[0],*args.arguments],cwd=ROOT))
|
||||
elif args.action == 'restart':
|
||||
print(json.dumps({'instances':restart()}))
|
||||
else:
|
||||
compose('run','--rm','--no-deps','migrate')
|
||||
print(json.dumps([replace_one(item['container']) for item in replicas()],indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
exec python3 scripts/cluster.py restart
|
||||
@@ -0,0 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
exec python3 scripts/cluster.py stop-one
|
||||
@@ -0,0 +1,443 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Grade4 distributed acceptance through the only public entry point; no tokens in artifacts."""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
import cluster
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'tests'))
|
||||
from client import Client, Failure, check
|
||||
from resilience import history
|
||||
|
||||
|
||||
def save(name, value):
|
||||
(ROOT/'evidence'/name).write_text(json.dumps(value,ensure_ascii=False,indent=2)+'\n')
|
||||
print('PASS:', name)
|
||||
|
||||
|
||||
def percentiles(values):
|
||||
values = sorted(values)
|
||||
return {name:round(1000*values[max(0,math.ceil(p*len(values))-1)],3) if values else None
|
||||
for name,p in [('p50',.50),('p95',.95),('p99',.99)]}
|
||||
|
||||
|
||||
class Traffic:
|
||||
def __init__(self, concurrency=4):
|
||||
self.c = Client(timeout=6)
|
||||
self.alice,self.bob,_,self.chat = self.c.bootstrap()
|
||||
self.concurrency = concurrency
|
||||
self.done = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.attempts = []
|
||||
self.confirmed = {}
|
||||
self.prefix = uuid.uuid4().hex
|
||||
|
||||
def worker(self, number):
|
||||
client = Client(timeout=6)
|
||||
index = 0
|
||||
while not self.done.is_set():
|
||||
text = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
key = self.prefix+'-'+str(number)+'-'+str(index)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = client.message(self.alice,self.chat,text,key=key)
|
||||
row = response['json']
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':True,
|
||||
'latency':response['elapsed'],'instance':response['headers']['X-Instance-Id']}
|
||||
with self.lock:
|
||||
self.confirmed[row['id']] = row
|
||||
index += 1
|
||||
except Failure:
|
||||
# Only idempotent message POST is retried, with its original body and key.
|
||||
attempt = {'started':started,'ended':time.monotonic(),'ok':False,'instance':None}
|
||||
time.sleep(0.02)
|
||||
with self.lock:
|
||||
self.attempts.append(attempt)
|
||||
|
||||
def start(self):
|
||||
self.started = time.monotonic()
|
||||
self.pool = ThreadPoolExecutor(max_workers=self.concurrency)
|
||||
self.futures = [self.pool.submit(self.worker,n) for n in range(self.concurrency)]
|
||||
return self
|
||||
|
||||
def stop(self):
|
||||
self.done.set()
|
||||
for future in self.futures:
|
||||
future.result(timeout=10)
|
||||
self.pool.shutdown()
|
||||
self.ended = time.monotonic()
|
||||
|
||||
def summary(self, start=None, end=None):
|
||||
start = self.started if start is None else start
|
||||
end = getattr(self,'ended',time.monotonic()) if end is None else end
|
||||
with self.lock:
|
||||
attempts = [a for a in self.attempts if start <= a['started'] < end]
|
||||
success = [a for a in attempts if a['ok']]
|
||||
duration = end-start
|
||||
return {'seconds':round(duration,4),'concurrency':self.concurrency,
|
||||
'successful_requests':len(success),'failed_attempts':len(attempts)-len(success),
|
||||
'successful_rps':round(len(success)/duration,3),
|
||||
'error_ratio':round((len(attempts)-len(success))/len(attempts),6) if attempts else 0,
|
||||
'successful_latency_ms':percentiles([a['latency'] for a in success]),
|
||||
'instances':sorted({a['instance'] for a in success})}
|
||||
|
||||
def verify_data(self):
|
||||
rows = history(self.c,self.bob,self.chat)
|
||||
actual = {row['id']:row for row in rows}
|
||||
check(len(actual)==len(rows),'duplicated message ids')
|
||||
check(len({row['text'] for row in rows})==len(rows),'retry duplicated a message')
|
||||
check(all(actual.get(mid)==row for mid,row in self.confirmed.items()),'confirmed message lost or changed')
|
||||
return {'confirmed_unique_messages':len(self.confirmed), 'history_messages':len(rows),
|
||||
'confirmed_preserved':True, 'duplicate_ids':0, 'duplicate_texts':0}
|
||||
|
||||
|
||||
def concurrent_replay():
|
||||
cluster.wait_cluster()
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
key = uuid.uuid4().hex
|
||||
barrier = threading.Barrier(8)
|
||||
def send(_):
|
||||
local = Client()
|
||||
barrier.wait()
|
||||
return local.message(alice,chat,'Один результат через разные процессы',key=key)
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(send,range(8)))
|
||||
instances = {r['headers']['X-Instance-Id'] for r in results}
|
||||
check(len(instances)>=2,'concurrent retry did not reach different API processes')
|
||||
check(all(r['json']==results[0]['json'] for r in results),'concurrent key returned different messages')
|
||||
c.call('POST','/api/v1/chats/'+chat['id']+'/messages',409,actor=alice,
|
||||
body={'text':'Другой body'},headers={'Idempotency-Key':key})
|
||||
check(history(c,bob,chat)==[results[0]['json']],'repeated key created duplicates')
|
||||
save('concurrent-idempotency.json',{'result':'PASS','parallel_requests':8,'observed_instances':sorted(instances),
|
||||
'responses_201':8,'messages_created':1,'different_body_status':409})
|
||||
|
||||
|
||||
def verify_persistence():
|
||||
before = cluster.snapshot()
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','persistence','--action-script','scripts/restart.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=190)
|
||||
after = cluster.snapshot()
|
||||
roles = ['api','postgres','redis','lb']
|
||||
previous = {item['container']:item for item in before}
|
||||
for item in after:
|
||||
if item['service'] in roles:
|
||||
old = previous[item['container']]
|
||||
check(old['started_at']!=item['started_at'],'service did not restart: '+item['service'])
|
||||
check(old['volumes']==item['volumes'],'volume identities changed')
|
||||
(ROOT/'evidence/persistence.txt').write_text(result.stdout)
|
||||
save('persistence.json',{'result':'PASS','before':before,'after':after,'public_test':result.stdout.strip()})
|
||||
|
||||
|
||||
def public_failover():
|
||||
cluster.wait_cluster()
|
||||
try:
|
||||
result = subprocess.run([sys.executable,'tests/resilience.py','--mode','failover','--action-script','scripts/stop-one.sh'],
|
||||
cwd=ROOT,check=True,capture_output=True,text=True,timeout=100)
|
||||
(ROOT/'evidence/public-failover.txt').write_text(result.stdout)
|
||||
save('public-failover.json',{'result':'PASS','public_test':result.stdout.strip(),
|
||||
'stopped':json.loads((ROOT/'evidence/last-stop.json').read_text())})
|
||||
finally:
|
||||
cluster.compose('up','-d','--no-recreate','--wait','--wait-timeout','90')
|
||||
cluster.wait_cluster()
|
||||
|
||||
|
||||
def reject_new_connection(survivor, address):
|
||||
code = '''import json,socket,sys
|
||||
try:
|
||||
s=socket.create_connection((sys.argv[1],8080),timeout=1)
|
||||
except OSError as exc:
|
||||
print(json.dumps({'accepted':False,'error_type':type(exc).__name__}))
|
||||
else:
|
||||
s.close()
|
||||
print(json.dumps({'accepted':True}))
|
||||
'''
|
||||
return json.loads(cluster.command(['docker','exec',survivor,'python','-c',code,address]))
|
||||
|
||||
|
||||
def failover_each():
|
||||
cluster.wait_cluster()
|
||||
originals = cluster.replicas()
|
||||
check(len(originals)==2,'two running replicas required')
|
||||
traffic = Traffic().start()
|
||||
phases = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
baseline_end = time.monotonic()
|
||||
baseline = traffic.summary(end=baseline_end)
|
||||
for original in originals:
|
||||
# Alias changes on restart, so resolve the current process of this exact container.
|
||||
current = next(r for r in cluster.replicas() if r['container']==original['container'])
|
||||
all_before = cluster.inspect([current['container']])[0]
|
||||
address = next(v['IPAddress'] for k,v in all_before['NetworkSettings']['Networks'].items() if k.endswith('_front'))
|
||||
survivor = next(r['container'] for r in cluster.replicas() if r['container']!=current['container'])
|
||||
read_path = '/api/v1/chats/'+traffic.chat['id']+'/messages?limit=1'
|
||||
marker = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')['json']['items'][0]
|
||||
start = time.monotonic()
|
||||
try:
|
||||
env = {**os.environ,'TARGET_INSTANCE_ID':current['instance']}
|
||||
subprocess.run(['scripts/stop-one.sh'],cwd=ROOT,env=env,check=True,capture_output=True,text=True,timeout=10)
|
||||
stop_returned = time.monotonic()
|
||||
stopped = json.loads((ROOT/'evidence/last-stop.json').read_text())
|
||||
read_deadline = start+10
|
||||
while True:
|
||||
try:
|
||||
read = traffic.c.call('GET',read_path,actor=traffic.bob,schema='MessagePage')
|
||||
check(read['json']['items'][0]==marker,'pre-failure message unavailable during outage')
|
||||
check(read['headers']['X-Instance-Id']!=current['instance'],'read served by stopped process')
|
||||
read_done = time.monotonic()
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<read_deadline,'reads did not recover within10s')
|
||||
time.sleep(0.1)
|
||||
check(read_done-start<=10,'read recovery exceeded10s')
|
||||
rejected = reject_new_connection(survivor,address)
|
||||
check(not rejected['accepted'],'new connection accepted by stopped API')
|
||||
time.sleep(max(0,8-(time.monotonic()-start)))
|
||||
end = time.monotonic()
|
||||
summary = traffic.summary(start,end)
|
||||
with traffic.lock:
|
||||
successful = [a for a in traffic.attempts if a['ok'] and a['started']>=start]
|
||||
recent = [a for a in successful if a['started']>stop_returned]
|
||||
recovery = min(a['ended'] for a in recent)-start if recent else None
|
||||
after_stop = min(a['ended'] for a in recent)-stop_returned if recent else None
|
||||
check(recovery is not None and recovery<=10,'post-stop writes did not recover within10s of action start')
|
||||
check(len(recent)>=5 and all(a['instance']!=current['instance'] for a in recent[-5:]),'stopped alias still served requests')
|
||||
phases.append({'stopped':stopped, 'traffic':summary,'post_stop_write_seconds_from_action_start':round(recovery,4),
|
||||
'post_stop_write_seconds_from_stop_completion':round(after_stop,4),
|
||||
'read_during_outage':{'status':200,'confirmed_message_preserved':True,
|
||||
'seconds_from_action_start':round(read_done-start,4),
|
||||
'seconds_from_stop_completion':round(read_done-stop_returned,4)},
|
||||
'new_connection_after_stop':rejected,'stopped_remains_stopped':not cluster.inspect([current['container']])[0]['State']['Running']})
|
||||
finally:
|
||||
cluster.command(['docker','start',current['container']])
|
||||
cluster.wait_healthy(current['container'])
|
||||
cluster.wait_cluster()
|
||||
time.sleep(2)
|
||||
finally:
|
||||
traffic.stop()
|
||||
save('sequential-failover.json',{'result':'PASS','baseline':baseline,'phases':phases,
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'each_original_container_tested':len({p['stopped']['container'] for p in phases})==2})
|
||||
|
||||
|
||||
def rolling_update():
|
||||
cluster.wait_cluster()
|
||||
before = cluster.replicas()
|
||||
check(len(before)==2,'rolling demonstration expects two replicas')
|
||||
lb_before = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
target_release = 'lab3-'+uuid.uuid4().hex[:8]
|
||||
traffic = Traffic().start()
|
||||
replacements = []
|
||||
try:
|
||||
time.sleep(3)
|
||||
build_start = time.monotonic()
|
||||
cluster.compose('build','--build-arg','APP_RELEASE='+target_release,'api',timeout=180)
|
||||
build_end = time.monotonic()
|
||||
migration_output = cluster.compose('run','--rm','--no-deps','migrate')
|
||||
swap_start = time.monotonic()
|
||||
for old in before:
|
||||
step_start = time.monotonic()
|
||||
result = cluster.replace_one(old['container'])
|
||||
result['seconds'] = round(time.monotonic()-step_start,4)
|
||||
replacements.append(result)
|
||||
swap_end = time.monotonic()
|
||||
time.sleep(3)
|
||||
finally:
|
||||
traffic.stop()
|
||||
after = cluster.replicas()
|
||||
lb_after = next(item for item in cluster.snapshot() if item['service']=='lb')
|
||||
check(lb_before['container']==lb_after['container'] and lb_before['started_at']==lb_after['started_at'],'LB restarted during DNS test')
|
||||
check(all(r['release']==target_release for r in after),'new release not observed on all replicas')
|
||||
check(not {r['container'] for r in before}&{r['container'] for r in after},'old containers remain after replacement')
|
||||
check(not {r['image_id'] for r in before}&{r['image_id'] for r in after},'image did not actually change')
|
||||
save('rolling-update.json',{'result':'PASS','before':before,'after':after,'target_release':target_release,
|
||||
'lb_before':lb_before,'lb_after':lb_after,'lb_not_restarted':True,
|
||||
'build_seconds':round(build_end-build_start,4),'migration_output':migration_output,
|
||||
'replacements':replacements,'during_replacement':traffic.summary(swap_start,swap_end),
|
||||
'total':traffic.summary(),'data':traffic.verify_data(),
|
||||
'change':'New image with a distinct release header; API code and schema remain compatible. Start-first: scale to3, prove new IP in DNS/LB, stop/remove one old container, return to2 with --no-recreate.'})
|
||||
|
||||
|
||||
def database_fixture(container, data):
|
||||
code = '''import json,sys,psycopg
|
||||
from psycopg import sql
|
||||
from messenger.db import setting
|
||||
p=json.load(sys.stdin)
|
||||
with psycopg.connect(setting('DATABASE_URL'),connect_timeout=3) as c:
|
||||
if p['action']=='install':
|
||||
c.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN PERFORM pg_sleep(2); END IF; RETURN NEW; END $$").format(sql.Identifier(p['function']),sql.Literal(p['text'])))
|
||||
c.execute(sql.SQL('CREATE TRIGGER {} BEFORE INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(p['trigger']),sql.Identifier(p['function'])))
|
||||
result={'installed':True}
|
||||
elif p['action']=='remove':
|
||||
c.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(p['trigger'])))
|
||||
c.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(p['function'])))
|
||||
result={'removed':True}
|
||||
else:
|
||||
result={'waiting':c.execute("SELECT count(*) FROM pg_stat_activity WHERE datname=current_database() AND wait_event='PgSleep' AND query LIKE 'INSERT INTO messages%%'").fetchone()[0]}
|
||||
print(json.dumps(result))
|
||||
'''
|
||||
response = subprocess.run(['docker','exec','-i',container,'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(response.stdout)
|
||||
|
||||
|
||||
def graceful_shutdown():
|
||||
cluster.wait_cluster()
|
||||
nodes = cluster.replicas()
|
||||
target,survivor = nodes[0],nodes[1]
|
||||
address = cluster.front_address(target['container'])
|
||||
c = Client()
|
||||
alice,bob,_,chat = c.bootstrap()
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
marker = 'drain_'+uuid.uuid4().hex
|
||||
fixture = {'function':'drain_'+uuid.uuid4().hex,'trigger':'drain_'+uuid.uuid4().hex,'text':marker}
|
||||
database_fixture(survivor['container'],{**fixture,'action':'install'})
|
||||
code = '''import json,sys,urllib.request,urllib.error
|
||||
p=json.load(sys.stdin)
|
||||
r=urllib.request.Request('http://'+p['address']+':8080'+p['path'],data=json.dumps({'text':p['text']}).encode(),method='POST',headers={'Content-Type':'application/json','Authorization':'Bearer '+p['token'],'Idempotency-Key':p['key']})
|
||||
try:
|
||||
response=urllib.request.urlopen(r,timeout=6)
|
||||
except urllib.error.HTTPError as exc:
|
||||
response=exc
|
||||
print(json.dumps({'status':response.code,'instance':response.headers.get('X-Instance-Id'),'message':json.load(response)}))
|
||||
'''
|
||||
request_data = {'address':address,'path':path,'text':marker,'token':alice['token'],'key':uuid.uuid4().hex}
|
||||
def in_flight():
|
||||
result = subprocess.run(['docker','exec','-i',survivor['container'],'python','-c',code],cwd=ROOT,
|
||||
input=json.dumps(request_data),capture_output=True,text=True,check=True,timeout=8)
|
||||
return json.loads(result.stdout)
|
||||
pool = ThreadPoolExecutor(max_workers=1)
|
||||
future = pool.submit(in_flight)
|
||||
try:
|
||||
deadline = time.monotonic()+3
|
||||
while not database_fixture(survivor['container'],{'action':'waiting'})['waiting']:
|
||||
check(time.monotonic()<deadline and not future.done(),'did not observe an in-flight SQL request')
|
||||
time.sleep(0.05)
|
||||
signaled = time.monotonic()
|
||||
cluster.command(['docker','kill','--signal','SIGTERM',target['container']])
|
||||
deadline = signaled+1.5
|
||||
while True:
|
||||
rejected = reject_new_connection(survivor['container'],address)
|
||||
still_running = cluster.inspect([target['container']])[0]['State']['Running']
|
||||
if not rejected['accepted']:
|
||||
break
|
||||
check(time.monotonic()<deadline,'new TCP connections still accepted after SIGTERM')
|
||||
time.sleep(0.05)
|
||||
check(still_running,'connection refusal observed only after process exit, not during drain')
|
||||
drained = future.result(timeout=8)
|
||||
check(drained['status']==201 and drained['instance']==target['instance'],'in-flight request did not finish on signaled process')
|
||||
check(drained['message']['text']==marker,'in-flight body changed')
|
||||
exit_code = cluster.command(['docker','wait',target['container']],timeout=8)
|
||||
elapsed = time.monotonic()-signaled
|
||||
check(exit_code=='0' and elapsed<5,'SIGTERM exceeded graceful budget or exited with error')
|
||||
check(history(c,bob,chat)==[drained['message']],'drained message missing or duplicated')
|
||||
result = {'result':'PASS','target_container':target['container'],'target_instance':target['instance'],
|
||||
'fixture':'temporary BEFORE INSERT trigger sleeps2s for one synthetic text; removed afterwards',
|
||||
'in_flight_observed_in_pg_stat_activity':True,'in_flight_response':201,
|
||||
'new_tcp_after_sigterm':rejected,'target_still_running_when_new_tcp_rejected':still_running,
|
||||
'exit_code':int(exit_code),'shutdown_seconds_from_sigterm':round(elapsed,4),
|
||||
'confirmed_message_preserved':True}
|
||||
finally:
|
||||
pool.shutdown(wait=True)
|
||||
database_fixture(survivor['container'],{**fixture,'action':'remove'})
|
||||
cluster.command(['docker','start',target['container']])
|
||||
cluster.wait_healthy(target['container'])
|
||||
cluster.wait_cluster()
|
||||
save('graceful-shutdown.json',result)
|
||||
|
||||
|
||||
def infrastructure():
|
||||
state = cluster.snapshot()
|
||||
for item in state:
|
||||
check(item['memory_bytes']>0 and item['nano_cpus']>0,'resource limit missing')
|
||||
if item['service']!='lb':
|
||||
check(not item['published_ports'],'service exposed a port bypassing LB')
|
||||
check(len([r for r in state if r['service']=='api' and r['status']=='running'])==2,'expected2live replicas')
|
||||
check(all(r['service']!='migrate' or r['status']=='exited' for r in state),'migration job still running')
|
||||
save('infrastructure.json',{'result':'PASS','containers':state})
|
||||
|
||||
|
||||
def dependency_failures():
|
||||
c = Client(timeout=6)
|
||||
actor = c.user('dependencies')
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result = {'result':'PASS'}
|
||||
try:
|
||||
cluster.compose('stop','-t','2','redis')
|
||||
response = c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
check(response['json']==actor['user'] and response['elapsed']<3,'Redis fallback failed')
|
||||
c.call('GET','/api/v1/users/me',401)
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
result['redis'] = {'profile_status':200,'fallback_seconds':round(response['elapsed'],4),'anonymous':401,'readiness':200}
|
||||
finally:
|
||||
cluster.compose('start','redis')
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
def status(path, token=None):
|
||||
headers = {'Authorization':'Bearer '+token} if token else {}
|
||||
request = urllib.request.Request(c.base+path,headers=headers)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
response = urllib.request.urlopen(request,timeout=6)
|
||||
except urllib.error.HTTPError as error:
|
||||
response = error
|
||||
with response:
|
||||
return response.code, round(time.monotonic()-started,4)
|
||||
try:
|
||||
cluster.compose('stop','-t','2','postgres')
|
||||
code,elapsed = status('/api/v1/users/me',actor['token'])
|
||||
check(code in (502,503,504) and elapsed<6,'PostgreSQL outage did not safely reject authenticated request')
|
||||
anonymous,_ = status('/api/v1/users/me')
|
||||
check(anonymous in (401,502,503,504),'anonymous user trusted during DB outage')
|
||||
live = [cluster.probe(node['container'])['status'] for node in cluster.replicas()]
|
||||
check(all(code==200 for code in live),'API process liveness depends on DB')
|
||||
result['postgres'] = {'authenticated_status':code,'response_seconds':elapsed,'anonymous_status':anonymous,
|
||||
'internal_api_liveness':live,'note':'Nginx-generated502/503/504 may have no API JSON/instance headers when all backends become unavailable.'}
|
||||
finally:
|
||||
cluster.compose('start','postgres')
|
||||
cluster.wait_cluster()
|
||||
c.call('GET','/api/v1/users/me',actor=actor,schema='User')
|
||||
result['same_session_after_recovery'] = True
|
||||
save('dependency-failures.json',result)
|
||||
|
||||
|
||||
def verify_nginx():
|
||||
state = cluster.snapshot()
|
||||
lb = next(item for item in state if item['service']=='lb')
|
||||
check(lb['image'].startswith('nginx:'),'load balancer must use Nginx')
|
||||
cluster.command(['docker','exec',lb['container'],'nginx','-t'])
|
||||
version = cluster.command(['docker','exec',lb['container'],'sh','-c','nginx -v 2>&1'])
|
||||
c = Client()
|
||||
response = c.call('GET','/health/ready',schema='Health')
|
||||
check(response['headers'].get('Server')=='nginx','public entry point is not Nginx')
|
||||
check(len(cluster.wait_cluster())==2,'Nginx must discover two replicas')
|
||||
save('nginx.json',{'result':'PASS','version':version,'image':lb['image'],'public_server_header':'nginx',
|
||||
'config_syntax':'PASS','observed_instances':cluster.wait_cluster()})
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('mode',choices=['idempotency','persistence','public-failover','failover','rolling','sigterm','dependencies','infra','nginx','all'],default='all',nargs='?')
|
||||
args = parser.parse_args()
|
||||
modes = {'nginx':verify_nginx,'idempotency':concurrent_replay,'persistence':verify_persistence,'public-failover':public_failover,
|
||||
'failover':failover_each,'sigterm':graceful_shutdown,'rolling':rolling_update,'dependencies':dependency_failures,'infra':infrastructure}
|
||||
if args.mode=='all':
|
||||
for function in modes.values():
|
||||
function()
|
||||
else:
|
||||
modes[args.mode]()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Contract-compatible synchronous REST API, with short bounded I/O in worker threads."""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from contextlib import asynccontextmanager
|
||||
from uuid import uuid4
|
||||
|
||||
import psycopg
|
||||
from fastapi import Depends, FastAPI, Request, Response
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from psycopg_pool import PoolTimeout, TooManyRequests
|
||||
from redis.exceptions import RedisError
|
||||
from starlette.exceptions import HTTPException
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
|
||||
from .db import Database
|
||||
from .errors import ApiError, invalid, not_found, unauthorized
|
||||
from .models import CreateChat, CreateMessage, CreateUser, uuid_value
|
||||
from .messages import store_message, validate_key
|
||||
from .pagination import decode_cursor, page, page_limit, public, stamp
|
||||
from .security import HASHER, DUMMY_HASH, basic_credentials, bearer, check_password, digest, new_session
|
||||
|
||||
LOG = logging.getLogger('messenger')
|
||||
USER_FIELDS = 'id, username, display_name, created_at'
|
||||
|
||||
def error_response(request, error):
|
||||
return JSONResponse({'error': {'code': error.code, 'message': error.message,
|
||||
'request_id': request.state.request_id}},
|
||||
status_code=error.status, headers=error.headers)
|
||||
|
||||
def authenticate(request: Request):
|
||||
token = bearer(request.headers.get('Authorization', ''))
|
||||
token_hash = digest(token)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s AND expires_at > clock_timestamp()',
|
||||
(token_hash,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
request.state.token_hash = token_hash
|
||||
return row['user_id']
|
||||
|
||||
def require_member(conn, chat_id, user_id):
|
||||
found = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', (chat_id, user_id)).fetchone()
|
||||
if not found:
|
||||
raise not_found()
|
||||
|
||||
def chat_public(conn, row):
|
||||
item = public(row)
|
||||
item['member_ids'] = [str(member['user_id']) for member in conn.execute(
|
||||
'SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY user_id', (row['id'],)).fetchall()]
|
||||
return item
|
||||
|
||||
def create_app():
|
||||
@asynccontextmanager
|
||||
async def lifespan(app):
|
||||
app.state.db = Database()
|
||||
app.state.db.open()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
app.state.db.close()
|
||||
|
||||
app = FastAPI(lifespan=lifespan, docs_url=None, redoc_url=None, openapi_url=None)
|
||||
app.state.instance_id = uuid4().hex
|
||||
app.state.release = os.environ.get('APP_RELEASE', 'lab3-v1')
|
||||
|
||||
@app.middleware('http')
|
||||
async def request_context(request: Request, call_next):
|
||||
request.state.request_id = str(uuid4())
|
||||
try:
|
||||
path = request.url.path
|
||||
json_endpoint = path in ('/api/v1/users', '/api/v1/chats') or re.fullmatch(r'/api/v1/chats/[^/]+/messages', path)
|
||||
if request.method == 'POST' and json_endpoint:
|
||||
media = request.headers.get('Content-Type', '').split(';')
|
||||
unsupported = media[0].strip().lower() != 'application/json'
|
||||
for parameter in media[1:]:
|
||||
name, _, value = parameter.strip().partition('=')
|
||||
if name.lower() == 'charset' and value.strip('"').lower() not in ('utf-8', 'utf8'):
|
||||
unsupported = True
|
||||
if unsupported:
|
||||
if path != '/api/v1/users':
|
||||
await run_in_threadpool(authenticate, request)
|
||||
raise ApiError(415, 'unsupported_media_type', 'Требуется application/json в UTF-8')
|
||||
try:
|
||||
json.loads((await request.body()).decode('utf-8'))
|
||||
except (UnicodeError, json.JSONDecodeError):
|
||||
raise invalid() from None
|
||||
response = await call_next(request)
|
||||
except ApiError as exc:
|
||||
response = error_response(request, exc)
|
||||
except (psycopg.OperationalError, psycopg.InterfaceError, PoolTimeout, TooManyRequests):
|
||||
response = error_response(request, ApiError(503, 'unavailable', 'Хранилище временно недоступно'))
|
||||
except Exception as exc:
|
||||
# Log exception class only: credentials, SQL and message content stay out of logs.
|
||||
LOG.error('request_failed request_id=%s type=%s', request.state.request_id, type(exc).__name__)
|
||||
response = error_response(request, ApiError(500, 'internal_error', 'Внутренняя ошибка'))
|
||||
response.headers['X-Request-Id'] = request.state.request_id
|
||||
response.headers['X-Instance-Id'] = app.state.instance_id
|
||||
response.headers['X-App-Release'] = app.state.release
|
||||
return response
|
||||
|
||||
@app.exception_handler(ApiError)
|
||||
async def api_error(request, exc):
|
||||
return error_response(request, exc)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error(request, exc):
|
||||
return error_response(request, invalid())
|
||||
|
||||
@app.exception_handler(HTTPException)
|
||||
async def http_error(request, exc):
|
||||
return error_response(request, not_found() if exc.status_code == 404 else invalid())
|
||||
|
||||
@app.get('/health/live')
|
||||
def live():
|
||||
return {'status': 'ok'}
|
||||
|
||||
@app.get('/health/ready')
|
||||
def ready(request: Request):
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
conn.execute('SELECT 1 FROM idempotency_records LIMIT 0')
|
||||
except (psycopg.errors.UndefinedTable, psycopg.errors.UndefinedColumn):
|
||||
raise ApiError(503, 'unavailable', 'Схема хранилища ещё не готова') from None
|
||||
return {'status': 'ok'}
|
||||
|
||||
@app.post('/api/v1/users', status_code=201)
|
||||
def register(body: CreateUser, request: Request):
|
||||
password_hash = HASHER.hash(body.password)
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute(
|
||||
f'INSERT INTO users(id,username,display_name,password_hash) VALUES (%s,%s,%s,%s) RETURNING {USER_FIELDS}',
|
||||
(uuid4(), body.username, body.display_name, password_hash)).fetchone()
|
||||
except psycopg.errors.UniqueViolation:
|
||||
raise ApiError(409, 'conflict', 'Имя пользователя занято') from None
|
||||
return public(row)
|
||||
|
||||
@app.post('/api/v1/auth/sessions', status_code=201)
|
||||
async def create_session(request: Request):
|
||||
# Run hashing and synchronous database calls outside the event loop.
|
||||
username, password = basic_credentials(request.headers.get('Authorization', ''))
|
||||
if await request.body():
|
||||
raise invalid()
|
||||
return await run_in_threadpool(login, request, username, password)
|
||||
|
||||
def login(request, username, password):
|
||||
db = request.app.state.db
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT '+USER_FIELDS+', password_hash FROM users WHERE username=%s', (username,)).fetchone()
|
||||
valid = check_password(row['password_hash'] if row else DUMMY_HASH, password)
|
||||
if not row or not valid:
|
||||
raise unauthorized(basic=True)
|
||||
row.pop('password_hash')
|
||||
token, expires_at = new_session(db.session_ttl)
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute('INSERT INTO sessions(token_hash,user_id,expires_at) VALUES (%s,%s,%s)',
|
||||
(digest(token), row['id'], expires_at))
|
||||
return {'token': token, 'token_type': 'Bearer', 'expires_at': stamp(expires_at), 'user': public(row)}
|
||||
|
||||
@app.get('/api/v1/users/me')
|
||||
def me(request: Request, user_id=Depends(authenticate)):
|
||||
db = request.app.state.db
|
||||
key = 'user:' + str(user_id)
|
||||
try:
|
||||
cached = db.cache.get(key)
|
||||
if cached:
|
||||
return json.loads(cached)
|
||||
except (RedisError, ValueError):
|
||||
pass
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT '+USER_FIELDS+' FROM users WHERE id=%s', (user_id,)).fetchone()
|
||||
if not row:
|
||||
raise unauthorized()
|
||||
result = public(row)
|
||||
try:
|
||||
db.cache.set(key, json.dumps(result), ex=60)
|
||||
except RedisError:
|
||||
pass
|
||||
return result
|
||||
|
||||
@app.delete('/api/v1/auth/sessions/current', status_code=204)
|
||||
def logout(request: Request, user_id=Depends(authenticate)):
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
conn.execute('DELETE FROM sessions WHERE token_hash=%s', (request.state.token_hash,))
|
||||
return Response(status_code=204)
|
||||
|
||||
@app.post('/api/v1/chats', status_code=201)
|
||||
def create_chat(body: CreateChat, request: Request, user_id=Depends(authenticate)):
|
||||
if str(user_id) not in body.member_ids:
|
||||
raise invalid()
|
||||
try:
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING id,title,created_at',
|
||||
(uuid4(), body.title)).fetchone()
|
||||
# An unknown user raises AFTER the chat INSERT; transaction context rolls it all back.
|
||||
with conn.cursor() as cur:
|
||||
cur.executemany('INSERT INTO chat_members(chat_id,user_id) VALUES (%s,%s)',
|
||||
[(row['id'], uuid_value(member)) for member in body.member_ids])
|
||||
result = chat_public(conn, row)
|
||||
except psycopg.errors.ForeignKeyViolation:
|
||||
raise invalid() from None
|
||||
return result
|
||||
|
||||
@app.get('/api/v1/chats')
|
||||
def list_chats(request: Request, user_id=Depends(authenticate)):
|
||||
limit = page_limit(request.query_params.get('limit', '50'))
|
||||
scope = 'chats:' + str(user_id)
|
||||
after = decode_cursor(scope, request.query_params.get('cursor'))
|
||||
where = ' AND (c.created_at,c.id) > (%s,%s)' if after else ''
|
||||
params = (user_id, *after, limit+1) if after else (user_id, limit+1)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
rows = conn.execute('SELECT c.id,c.title,c.created_at FROM chats c JOIN chat_members m ON m.chat_id=c.id '
|
||||
'WHERE m.user_id=%s'+where+' ORDER BY c.created_at,c.id LIMIT %s', params).fetchall()
|
||||
result = page(scope, rows, limit)
|
||||
result['items'] = [chat_public(conn, row) for row in rows[:limit]]
|
||||
return result
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}')
|
||||
def get_chat(chat_id: str, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
row = conn.execute('SELECT id,title,created_at FROM chats WHERE id=%s', (cid,)).fetchone()
|
||||
return chat_public(conn, row)
|
||||
|
||||
@app.post('/api/v1/chats/{chat_id}/messages', status_code=201)
|
||||
def send_message(chat_id: str, body: CreateMessage, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
key = validate_key(request.headers.get('Idempotency-Key', ''))
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
row = store_message(conn, user_id, cid, body.text, key)
|
||||
return public(row)
|
||||
|
||||
@app.get('/api/v1/chats/{chat_id}/messages')
|
||||
def history(chat_id: str, request: Request, user_id=Depends(authenticate)):
|
||||
cid = uuid_value(chat_id)
|
||||
limit = page_limit(request.query_params.get('limit', '50'))
|
||||
scope = 'messages:' + str(cid)
|
||||
with request.app.state.db.pool.connection() as conn:
|
||||
require_member(conn, cid, user_id)
|
||||
after = decode_cursor(scope, request.query_params.get('cursor'))
|
||||
where = ' AND (created_at,id) > (%s,%s)' if after else ''
|
||||
params = (cid, *after, limit+1) if after else (cid, limit+1)
|
||||
rows = conn.execute('SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE chat_id=%s'+where+
|
||||
' ORDER BY created_at,id LIMIT %s', params).fetchall()
|
||||
return page(scope, rows, limit)
|
||||
|
||||
return app
|
||||
@@ -0,0 +1,37 @@
|
||||
"""Bounded dependencies; PostgreSQL is always the source of truth."""
|
||||
import os
|
||||
from pathlib import Path
|
||||
from psycopg.rows import dict_row
|
||||
from psycopg_pool import ConnectionPool
|
||||
from redis import Redis
|
||||
from redis.backoff import NoBackoff
|
||||
from redis.retry import Retry
|
||||
|
||||
def setting(name, default=None):
|
||||
filename = os.environ.get(name + '_FILE')
|
||||
value = Path(filename).read_text().strip() if filename else os.environ.get(name, default)
|
||||
if value is None:
|
||||
raise RuntimeError(f'Missing configuration: {name}')
|
||||
return value
|
||||
|
||||
class Database:
|
||||
def __init__(self):
|
||||
self.session_ttl = int(setting('SESSION_TTL_SECONDS', '3600'))
|
||||
if self.session_ttl < 1:
|
||||
raise ValueError('SESSION_TTL_SECONDS must be positive')
|
||||
self.pool = ConnectionPool(
|
||||
setting('DATABASE_URL'), min_size=1, max_size=int(setting('POOL_MAX_SIZE', '8')),
|
||||
timeout=2, max_waiting=16, open=False,
|
||||
kwargs={'row_factory': dict_row, 'connect_timeout': 3,
|
||||
'options': '-c statement_timeout=3000 -c lock_timeout=2000 -c idle_in_transaction_session_timeout=5000'},
|
||||
)
|
||||
self.cache = Redis.from_url(
|
||||
setting('CACHE_URL', 'redis://redis:6379/0'), decode_responses=True,
|
||||
socket_connect_timeout=0.2, socket_timeout=0.2, max_connections=8,
|
||||
retry=Retry(NoBackoff(), 0),
|
||||
)
|
||||
def open(self):
|
||||
self.pool.open(wait=True, timeout=30)
|
||||
def close(self):
|
||||
self.pool.close()
|
||||
self.cache.close()
|
||||
@@ -0,0 +1,16 @@
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status, code, message, headers=None):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.headers = headers or {}
|
||||
|
||||
def invalid():
|
||||
return ApiError(400, 'invalid_request', 'Неверный запрос')
|
||||
|
||||
def unauthorized(basic=False):
|
||||
scheme = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"'
|
||||
return ApiError(401, 'unauthorized', 'Требуется аутентификация', {'WWW-Authenticate': scheme})
|
||||
|
||||
def not_found():
|
||||
return ApiError(404, 'not_found', 'Объект не найден')
|
||||
@@ -0,0 +1,33 @@
|
||||
"""One PostgreSQL transaction owns both a message and its retry result."""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
from uuid import uuid4
|
||||
|
||||
from .errors import ApiError, invalid
|
||||
|
||||
def validate_key(key):
|
||||
if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key):
|
||||
raise invalid()
|
||||
return key
|
||||
|
||||
def store_message(conn, user_id, chat_id, text, key):
|
||||
canonical = json.dumps({'text': text}, ensure_ascii=False, sort_keys=True, separators=(',', ':'))
|
||||
fingerprint = hashlib.sha256(canonical.encode('utf-8')).hexdigest()
|
||||
params = (user_id, chat_id, key)
|
||||
# A competing INSERT waits for the other transaction, then the next statement sees its result.
|
||||
conn.execute("INSERT INTO idempotency_records(user_id,chat_id,key,request_hash,expires_at) "
|
||||
"VALUES (%s,%s,%s,%s,clock_timestamp()+interval '24 hours') ON CONFLICT DO NOTHING",
|
||||
(*params, fingerprint))
|
||||
record = conn.execute('SELECT request_hash,message_id,expires_at > clock_timestamp() AS active '
|
||||
'FROM idempotency_records WHERE user_id=%s AND chat_id=%s AND key=%s FOR UPDATE', params).fetchone()
|
||||
if record['message_id'] is not None and record['active']:
|
||||
if record['request_hash'] != fingerprint:
|
||||
raise ApiError(409, 'conflict', 'Ключ уже использован с другим сообщением')
|
||||
return conn.execute('SELECT id,chat_id,sender_id,text,created_at FROM messages WHERE id=%s',
|
||||
(record['message_id'],)).fetchone()
|
||||
row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) '
|
||||
'RETURNING id,chat_id,sender_id,text,created_at', (uuid4(), chat_id, user_id, text)).fetchone()
|
||||
conn.execute("UPDATE idempotency_records SET request_hash=%s,message_id=%s,expires_at=clock_timestamp()+interval '24 hours' "
|
||||
'WHERE user_id=%s AND chat_id=%s AND key=%s', (fingerprint, row['id'], *params))
|
||||
return row
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Versioned, atomic, repeatable migrations, serialized with an advisory lock."""
|
||||
import hashlib
|
||||
from pathlib import Path
|
||||
|
||||
import psycopg
|
||||
from .db import setting
|
||||
|
||||
def main():
|
||||
directory = Path(__file__).resolve().parents[2] / 'migrations'
|
||||
with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn:
|
||||
conn.execute('SELECT pg_advisory_xact_lock(20261102)')
|
||||
conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations (version text PRIMARY KEY, sha256 text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())')
|
||||
for path in sorted(directory.glob('*.sql')):
|
||||
sql = path.read_text()
|
||||
checksum = hashlib.sha256(sql.encode()).hexdigest()
|
||||
found = conn.execute('SELECT sha256 FROM schema_migrations WHERE version=%s', (path.name,)).fetchone()
|
||||
if found:
|
||||
if found[0] != checksum:
|
||||
raise RuntimeError('Applied migration checksum changed: ' + path.name)
|
||||
continue
|
||||
conn.execute(sql)
|
||||
conn.execute('INSERT INTO schema_migrations(version,sha256) VALUES (%s,%s)', (path.name, checksum))
|
||||
print('Applied', path.name)
|
||||
print('Migrations up to date')
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,55 @@
|
||||
import re
|
||||
from uuid import UUID
|
||||
from pydantic import BaseModel, ConfigDict, Field, field_validator
|
||||
from .errors import invalid
|
||||
|
||||
def uuid_value(value):
|
||||
try:
|
||||
# Require the UUID string representation, not integer/coercible values.
|
||||
if not isinstance(value, str) or len(value) != 36:
|
||||
raise ValueError
|
||||
return UUID(value)
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
raise invalid() from None
|
||||
|
||||
class StrictModel(BaseModel):
|
||||
model_config = ConfigDict(extra='forbid', strict=True)
|
||||
|
||||
@field_validator('*')
|
||||
@classmethod
|
||||
def valid_strings(cls, value):
|
||||
if isinstance(value, str):
|
||||
if not value.strip():
|
||||
raise ValueError('Blank string')
|
||||
# PostgreSQL cannot store NUL or unpaired Unicode surrogates.
|
||||
if '\x00' in value:
|
||||
raise ValueError('NUL is not supported')
|
||||
value.encode('utf-8')
|
||||
return value
|
||||
|
||||
class CreateUser(StrictModel):
|
||||
username: str = Field(pattern=r'^[a-z0-9_]{3,32}$')
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator('username')
|
||||
@classmethod
|
||||
def username_ascii(cls, value):
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', value):
|
||||
raise ValueError('Invalid username')
|
||||
return value
|
||||
|
||||
class CreateChat(StrictModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[str] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator('member_ids')
|
||||
@classmethod
|
||||
def members(cls, value):
|
||||
ids = [uuid_value(item) for item in value]
|
||||
if len(ids) != len(set(ids)):
|
||||
raise ValueError('Duplicate member')
|
||||
return [str(item) for item in ids]
|
||||
|
||||
class CreateMessage(StrictModel):
|
||||
text: str = Field(min_length=1, max_length=4000)
|
||||
@@ -0,0 +1,42 @@
|
||||
import base64
|
||||
import json
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from .errors import invalid
|
||||
from .models import uuid_value
|
||||
|
||||
def stamp(value):
|
||||
return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
|
||||
|
||||
def encode_cursor(scope, row):
|
||||
raw = json.dumps([scope, stamp(row['created_at']), str(row['id'])], separators=(',', ':')).encode()
|
||||
return base64.urlsafe_b64encode(raw).decode().rstrip('=')
|
||||
|
||||
def decode_cursor(scope, value):
|
||||
if value is None:
|
||||
return None
|
||||
try:
|
||||
if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', value):
|
||||
raise ValueError
|
||||
data = json.loads(base64.b64decode(value + '=' * (-len(value) % 4), altchars=b'-_', validate=True))
|
||||
if not isinstance(data, list) or len(data) != 3 or data[0] != scope:
|
||||
raise ValueError
|
||||
ts = datetime.fromisoformat(data[1].replace('Z', '+00:00'))
|
||||
if ts.tzinfo is None:
|
||||
raise ValueError
|
||||
return ts, uuid_value(data[2])
|
||||
except (ValueError, TypeError, AttributeError, OverflowError):
|
||||
raise invalid() from None
|
||||
|
||||
def page_limit(value):
|
||||
if not re.fullmatch(r'[0-9]{1,3}', value) or not 1 <= int(value) <= 100:
|
||||
raise invalid()
|
||||
return int(value)
|
||||
|
||||
def public(row):
|
||||
return {key: stamp(value) if isinstance(value, datetime) else str(value) if hasattr(value, 'hex') else value
|
||||
for key, value in row.items()}
|
||||
|
||||
def page(scope, rows, limit):
|
||||
return {'items': [public(row) for row in rows[:limit]],
|
||||
'next_cursor': encode_cursor(scope, rows[limit-1]) if len(rows) > limit else None}
|
||||
@@ -0,0 +1,41 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import re
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerificationError, InvalidHashError
|
||||
from .errors import unauthorized
|
||||
|
||||
HASHER = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1)
|
||||
DUMMY_HASH = HASHER.hash('synthetic-dummy-password')
|
||||
|
||||
def digest(token):
|
||||
return hashlib.sha256(token.encode()).hexdigest()
|
||||
|
||||
def basic_credentials(header):
|
||||
try:
|
||||
scheme, encoded = header.split(' ', 1)
|
||||
if scheme.lower() != 'basic':
|
||||
raise ValueError
|
||||
username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1)
|
||||
if not re.fullmatch(r'[a-z0-9_]{3,32}', username) or not 12 <= len(password) <= 128:
|
||||
raise ValueError
|
||||
return username, password
|
||||
except (ValueError, UnicodeError):
|
||||
raise unauthorized(basic=True) from None
|
||||
def check_password(encoded, password):
|
||||
try:
|
||||
return HASHER.verify(encoded, password)
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
def bearer(header):
|
||||
try:
|
||||
scheme, token = header.split(' ', 1)
|
||||
if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token):
|
||||
raise ValueError
|
||||
return token
|
||||
except ValueError:
|
||||
raise unauthorized() from None
|
||||
def new_session(ttl):
|
||||
return secrets.token_urlsafe(32), datetime.now(timezone.utc) + timedelta(seconds=ttl)
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Integration tests: real PostgreSQL and Redis, isolated synthetic identities."""
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from messenger.app import create_app
|
||||
|
||||
PASSWORD = 'Тестовый:пароль🔐' * 8
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
with TestClient(create_app()) as c:
|
||||
yield c
|
||||
|
||||
def user(c):
|
||||
body = dict(username='u_' + uuid.uuid4().hex[:24], display_name='Алиса 👋', password=PASSWORD)
|
||||
r = c.post('/api/v1/users', json=body)
|
||||
assert r.status_code == 201, r.text
|
||||
return r.json(), body
|
||||
|
||||
def login(c, body):
|
||||
credentials = base64.b64encode((body['username'] + ':' + body['password']).encode()).decode()
|
||||
r = c.post('/api/v1/auth/sessions', headers={'Authorization': 'Basic ' + credentials})
|
||||
assert r.status_code == 201, r.text
|
||||
return {'Authorization': 'Bearer ' + r.json()['token']}, r.json()
|
||||
|
||||
def test_auth_unicode_logout_and_cache(client):
|
||||
u, body = user(client)
|
||||
h, session = login(client, body)
|
||||
assert session['user'] == u
|
||||
assert len(session['token']) >= 43
|
||||
key = 'user:' + u['id']
|
||||
client.app.state.db.cache.delete(key)
|
||||
for _ in range(2):
|
||||
assert client.get('/api/v1/users/me', headers=h).json() == u
|
||||
cached = json.loads(client.app.state.db.cache.get(key))
|
||||
assert cached == u
|
||||
assert 0 < client.app.state.db.cache.ttl(key) <= 60
|
||||
assert client.delete('/api/v1/auth/sessions/current', headers=h).status_code == 204
|
||||
r = client.get('/api/v1/users/me', headers=h)
|
||||
assert r.status_code == 401
|
||||
assert r.headers['www-authenticate'] == 'Bearer realm="messenger"'
|
||||
assert r.json()['error']['request_id'] == r.headers['x-request-id']
|
||||
|
||||
def test_session_expiry(monkeypatch):
|
||||
monkeypatch.setenv('SESSION_TTL_SECONDS', '2')
|
||||
with TestClient(create_app()) as client:
|
||||
_, body = user(client)
|
||||
h, _ = login(client, body)
|
||||
assert client.get('/api/v1/users/me', headers=h).status_code == 200
|
||||
time.sleep(2.1)
|
||||
assert client.get('/api/v1/users/me', headers=h).status_code == 401
|
||||
|
||||
def test_registration_race(client):
|
||||
body = dict(username='race_' + uuid.uuid4().hex[:22], display_name='Race', password=PASSWORD)
|
||||
with ThreadPoolExecutor(max_workers=6) as executor:
|
||||
codes = list(executor.map(lambda _: client.post('/api/v1/users', json=body).status_code, range(6)))
|
||||
assert sorted(codes) == [201, 409, 409, 409, 409, 409]
|
||||
|
||||
def test_chat_rollback_after_insert(client):
|
||||
u, body = user(client)
|
||||
h, _ = login(client, body)
|
||||
title = 'rollback_' + uuid.uuid4().hex
|
||||
r = client.post('/api/v1/chats', headers=h, json={'title': title, 'member_ids': [u['id'], str(uuid.uuid4())]})
|
||||
assert r.status_code == 400
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM chats WHERE title=%s', (title,)).fetchone()['n'] == 0
|
||||
assert conn.execute('SELECT count(*) AS n FROM chat_members m LEFT JOIN chats c ON c.id=m.chat_id WHERE c.id IS NULL').fetchone()['n'] == 0
|
||||
|
||||
def test_cursor_access_and_strict_validation(client):
|
||||
a, ab = user(client)
|
||||
b, bb = user(client)
|
||||
ah, _ = login(client, ab)
|
||||
bh, _ = login(client, bb)
|
||||
chats = [client.post('/api/v1/chats', headers=ah, json={'title':'Чат','member_ids':[a['id']]}).json() for _ in range(2)]
|
||||
path = '/api/v1/chats/' + chats[0]['id']
|
||||
for text in ['Один 👋', 'Два', 'Три']:
|
||||
assert client.post(path+'/messages', headers={**ah,'Idempotency-Key':uuid.uuid4().hex}, json={'text':text}).status_code == 201
|
||||
p = client.get(path+'/messages?limit=2', headers=ah).json()
|
||||
assert [m['text'] for m in p['items']] == ['Один 👋','Два']
|
||||
tail = client.get(path+'/messages', headers=ah, params={'cursor':p['next_cursor'],'limit':1}).json()
|
||||
assert [m['text'] for m in tail['items']] == ['Три']
|
||||
assert tail['next_cursor'] is None
|
||||
r = client.get('/api/v1/chats/'+chats[1]['id']+'/messages',headers=ah, params={'cursor':p['next_cursor']})
|
||||
assert r.status_code == 400
|
||||
assert client.get(path, headers=bh).status_code == 404
|
||||
for payload in [{'text':123}, {'text':' '}, {'text':'ok','sender_id':b['id']}, {'text':'x'*4001}]:
|
||||
assert client.post(path+'/messages',headers={**ah,'Idempotency-Key':uuid.uuid4().hex},json=payload).status_code == 400
|
||||
for query in ['limit=0','limit=101','limit=1.0','cursor=bad','cursor=']:
|
||||
assert client.get(path+'/messages?'+query,headers=ah).status_code == 400
|
||||
for payload in [{'username':123,'display_name':'a','password':PASSWORD}, {'username':'valid','display_name':' ','password':PASSWORD}]:
|
||||
assert client.post('/api/v1/users',json=payload).status_code == 400
|
||||
assert client.post('/api/v1/users',content='{',headers={'Content-Type':'application/json'}).status_code == 400
|
||||
|
||||
def test_session_storage_and_migrations(client):
|
||||
from messenger.migrate import main as migrate
|
||||
u, body = user(client)
|
||||
_, session = login(client, body)
|
||||
db = client.app.state.db
|
||||
def snapshot():
|
||||
with db.pool.connection() as conn:
|
||||
return {table: conn.execute('SELECT count(*) AS n FROM '+table).fetchone()['n']
|
||||
for table in ['users','chats','chat_members','messages','sessions','schema_migrations','idempotency_records']}
|
||||
before = snapshot()
|
||||
migrate()
|
||||
migrate()
|
||||
assert snapshot() == before
|
||||
with db.pool.connection() as conn:
|
||||
row = conn.execute('SELECT token_hash,expires_at FROM sessions WHERE user_id=%s', (u['id'],)).fetchone()
|
||||
assert row['token_hash'] != session['token']
|
||||
assert len(row['token_hash']) == 64
|
||||
stored = conn.execute('SELECT password_hash FROM users WHERE id=%s', (u['id'],)).fetchone()['password_hash']
|
||||
assert stored.startswith('$argon2id$')
|
||||
assert stored != body['password']
|
||||
|
||||
def test_json_media_type_and_utf8(client):
|
||||
body = dict(username='media_' + uuid.uuid4().hex[:24], display_name='UTF8', password=PASSWORD)
|
||||
raw = json.dumps(body).encode()
|
||||
for headers in [{}, {'Content-Type':'text/plain'}, {'Content-Type':'application/xml'}]:
|
||||
r = client.post('/api/v1/users',content=raw,headers=headers)
|
||||
assert r.status_code == 415
|
||||
assert r.json()['error']['code'] == 'unsupported_media_type'
|
||||
for encoding in ['utf-16', 'utf-16-le', 'utf-16-be', 'utf-32-le', 'utf-32-be']:
|
||||
r = client.post('/api/v1/users',content=json.dumps(body).encode(encoding),headers={'Content-Type':'application/json'})
|
||||
assert r.status_code == 400, encoding
|
||||
r = client.post('/api/v1/chats',content=b'{}',headers={'Content-Type':'text/plain'})
|
||||
assert r.status_code == 401
|
||||
@@ -0,0 +1,111 @@
|
||||
"""Regressions for lab3 shared message results, real PostgreSQL (no mocks)."""
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import uuid
|
||||
|
||||
from psycopg import sql
|
||||
from test_api import client, user, login
|
||||
|
||||
def setup_chat(c, extra=None):
|
||||
u, body = user(c)
|
||||
headers, _ = login(c, body)
|
||||
members = [u['id']] + (extra or [])
|
||||
chat = c.post('/api/v1/chats', headers=headers, json={'title':'Идемпотентность', 'member_ids':members}).json()
|
||||
return u, headers, '/api/v1/chats/'+chat['id']+'/messages'
|
||||
|
||||
def test_instance_alias_on_success_and_errors(client):
|
||||
responses = [client.get('/health/live'), client.get('/health/ready'), client.get('/api/v1/users/me'),
|
||||
client.get('/unknown'), client.post('/api/v1/users', content='{', headers={'Content-Type':'application/json'})]
|
||||
ids = [r.headers.get('X-Instance-Id') for r in responses]
|
||||
assert ids[0] and len(set(ids)) == 1
|
||||
assert all(r.headers.get('X-Request-Id') for r in responses)
|
||||
|
||||
def test_key_required_replay_and_conflict(client):
|
||||
_, headers, path = setup_chat(client)
|
||||
assert client.post(path, headers=headers, json={'text':'same'}).status_code == 400
|
||||
for key in ['', 'a b', 'bad/', 'x'*129]:
|
||||
assert client.post(path, headers={**headers,'Idempotency-Key':key},json={'text':'same'}).status_code == 400
|
||||
h = {**headers, 'Idempotency-Key':'valid:._-123'}
|
||||
first = client.post(path, headers=h, json={'text':'same'})
|
||||
assert first.status_code == 201
|
||||
replay = client.post(path, headers={**h,'Content-Type':'application/json'},content=' { "text" : "same" } ')
|
||||
assert replay.status_code == 201 and replay.json() == first.json()
|
||||
assert client.post(path, headers=h, json={'text':'other'}).status_code == 409
|
||||
assert len(client.get(path,headers=headers).json()['items']) == 1
|
||||
|
||||
def test_concurrent_key_produces_one_message(client):
|
||||
_, headers, path = setup_chat(client)
|
||||
h = {**headers, 'Idempotency-Key':uuid.uuid4().hex}
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(lambda _: client.post(path,headers=h,json={'text':'concurrent'}), range(16)))
|
||||
assert [r.status_code for r in results] == [201]*16
|
||||
assert all(r.json() == results[0].json() for r in results)
|
||||
assert client.get(path,headers=headers).json()['items'] == [results[0].json()]
|
||||
|
||||
def test_scope_and_replay_rechecks_authorization(client):
|
||||
b, bb = user(client)
|
||||
bh, _ = login(client, bb)
|
||||
a, ah, path = setup_chat(client, [b['id']])
|
||||
key = 'same-in-independent-scopes'
|
||||
first = client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).json()
|
||||
second = client.post(path,headers={**bh,'Idempotency-Key':key},json={'text':'b'})
|
||||
assert second.status_code == 201 and second.json()['id'] != first['id']
|
||||
chat = client.post('/api/v1/chats',headers=ah,json={'title':'Другая область','member_ids':[a['id']]}).json()
|
||||
other_path = '/api/v1/chats/'+chat['id']+'/messages'
|
||||
third = client.post(other_path,headers={**ah,'Idempotency-Key':key},json={'text':'a'})
|
||||
assert third.status_code == 201 and third.json()['id'] != first['id']
|
||||
outsider, ob = user(client)
|
||||
oh, _ = login(client, ob)
|
||||
assert client.post(path,headers={**oh,'Idempotency-Key':key},json={'text':'a'}).status_code == 404
|
||||
assert client.delete('/api/v1/auth/sessions/current',headers=ah).status_code == 204
|
||||
assert client.post(path,headers={**ah,'Idempotency-Key':key},json={'text':'a'}).status_code == 401
|
||||
|
||||
def test_expired_key_can_be_reused(client):
|
||||
u, h, path = setup_chat(client)
|
||||
key = uuid.uuid4().hex
|
||||
first = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'first'}).json()
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
ttl = conn.execute('SELECT extract(epoch FROM expires_at-clock_timestamp()) AS seconds FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['seconds']
|
||||
assert 86390 <= ttl <= 86400
|
||||
conn.execute("UPDATE idempotency_records SET expires_at=clock_timestamp()-interval '1 second' WHERE user_id=%s AND key=%s", (u['id'],key))
|
||||
second = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':'second'})
|
||||
assert second.status_code == 201 and second.json()['id'] != first['id']
|
||||
assert len(client.get(path,headers=h).json()['items']) == 2
|
||||
|
||||
def test_rollback_side_effect_and_key_on_database_error(client):
|
||||
u, h, path = setup_chat(client)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'fault_' + uuid.uuid4().hex
|
||||
function = 'fault_' + uuid.uuid4().hex
|
||||
trigger = 'trigger_' + uuid.uuid4().hex
|
||||
db = client.app.state.db
|
||||
# AFTER INSERT fails after the side effect. This is a DB-only test fixture, not a production hook.
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL("CREATE FUNCTION {}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN IF NEW.text = {} THEN RAISE EXCEPTION 'synthetic insert fault'; END IF; RETURN NEW; END $$").format(sql.Identifier(function), sql.Literal(text)))
|
||||
conn.execute(sql.SQL('CREATE TRIGGER {} AFTER INSERT ON messages FOR EACH ROW EXECUTE FUNCTION {}()').format(sql.Identifier(trigger),sql.Identifier(function)))
|
||||
try:
|
||||
failed = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
|
||||
assert failed.status_code == 500
|
||||
assert failed.headers['X-Instance-Id']
|
||||
with db.pool.connection() as conn:
|
||||
assert conn.execute('SELECT count(*) AS n FROM messages WHERE text=%s',(text,)).fetchone()['n'] == 0
|
||||
assert conn.execute('SELECT count(*) AS n FROM idempotency_records WHERE user_id=%s AND key=%s',(u['id'],key)).fetchone()['n'] == 0
|
||||
finally:
|
||||
with db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('DROP TRIGGER IF EXISTS {} ON messages').format(sql.Identifier(trigger)))
|
||||
conn.execute(sql.SQL('DROP FUNCTION IF EXISTS {}()').format(sql.Identifier(function)))
|
||||
retried = client.post(path,headers={**h,'Idempotency-Key':key},json={'text':text})
|
||||
assert retried.status_code == 201
|
||||
assert client.get(path,headers=h).json()['items'] == [retried.json()]
|
||||
|
||||
def test_ready_requires_compatible_schema(client):
|
||||
temporary = 'temporarily_unavailable_' + uuid.uuid4().hex
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('ALTER TABLE idempotency_records RENAME TO {}').format(sql.Identifier(temporary)))
|
||||
try:
|
||||
r = client.get('/health/ready')
|
||||
assert r.status_code == 503
|
||||
assert r.json()['error']['code'] == 'unavailable'
|
||||
assert client.get('/health/live').status_code == 200
|
||||
finally:
|
||||
with client.app.state.db.pool.connection() as conn:
|
||||
conn.execute(sql.SQL('ALTER TABLE {} RENAME TO idempotency_records').format(sql.Identifier(temporary)))
|
||||
Reference in New Issue
Block a user