ЛР 3 — БПИ2302 — Мякишев — на 4 #63
+35
-11
@@ -1,13 +1,37 @@
|
||||
# Copy to .env for your LOCAL synthetic test environment; never commit .env.
|
||||
# These are logical interface suggestions; map actual framework names in REPORT.md.
|
||||
APP_HOST=0.0.0.0
|
||||
APP_PORT=8080
|
||||
APP_ENV=lab
|
||||
SESSION_TTL_SECONDS=3600
|
||||
# Choose credentials locally. File paths contain no secret values.
|
||||
DATABASE_URL_FILE=/run/secrets/database_url
|
||||
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
|
||||
SESSION_STORE=postgres
|
||||
CACHE_URL_FILE=/run/secrets/cache_url
|
||||
# Names follow pydantic-settings: prefix APP_CONFIG__, nesting via "__" (see app/core/config.py).
|
||||
# Mapping to the skeleton's logical names: APP_HOST -> APP_CONFIG__RUN__HOST, APP_PORT -> APP_CONFIG__RUN__PORT,
|
||||
# DATABASE_URL_FILE -> APP_CONFIG__DB__* (host/port/name/user + password file),
|
||||
# SESSION_TTL_SECONDS -> APP_CONFIG__JWT__SESSION_TTL_SECONDS, SESSION_STORE=postgres (sessions table),
|
||||
# IDEMPOTENCY_TTL_SECONDS -> APP_CONFIG__IDEMPOTENCY__TTL_SECONDS, API_REPLICAS -> API_REPLICAS (compose.yaml).
|
||||
APP_CONFIG__RUN__HOST=0.0.0.0
|
||||
APP_CONFIG__RUN__PORT=8080
|
||||
APP_CONFIG__LOGGING__LEVEL=INFO
|
||||
|
||||
# PostgreSQL. Values below are for running the API from the venv on the host;
|
||||
# inside Compose the defaults (host=postgres, /run/secrets/postgres_password) are used.
|
||||
APP_CONFIG__DB__HOST=127.0.0.1
|
||||
APP_CONFIG__DB__PORT=5432
|
||||
APP_CONFIG__DB__NAME=messenger
|
||||
APP_CONFIG__DB__USER=messenger
|
||||
# Path only, never the password itself. Relative paths are resolved from the project root.
|
||||
APP_CONFIG__DB__PASSWORD_FILE=secrets/postgres_password
|
||||
|
||||
# Sessions: JWT signed with a key from a secret file + server-side session row in PostgreSQL
|
||||
# (shared by all API replicas, so no sticky sessions are needed).
|
||||
APP_CONFIG__JWT__SECRET_FILE=secrets/jwt_secret
|
||||
APP_CONFIG__JWT__SESSION_TTL_SECONDS=3600
|
||||
|
||||
# Idempotency-Key: a repeat within this window returns the original message (contract: 24 h).
|
||||
APP_CONFIG__IDEMPOTENCY__TTL_SECONDS=86400
|
||||
|
||||
# Graceful shutdown: after SIGTERM /health/ready answers 503 for DRAIN seconds, then the port closes
|
||||
# and in-flight requests get GRACEFUL seconds. DRAIN must exceed the LB detection time (fall*inter + check timeout = 2.5 s).
|
||||
APP_CONFIG__RUN__DRAIN_SECONDS=3
|
||||
APP_CONFIG__RUN__GRACEFUL_TIMEOUT=5
|
||||
|
||||
# Used by compose.yaml.
|
||||
# Host port for PostgreSQL (bound to 127.0.0.1 only); API replicas publish no ports at all.
|
||||
POSTGRES_HOST_PORT=5432
|
||||
# Number of API replicas behind the load balancer.
|
||||
API_REPLICAS=2
|
||||
IDEMPOTENCY_TTL_SECONDS=86400
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
*.sh text eol=lf
|
||||
*.cmd text eol=crlf
|
||||
# Читаются внутри Linux-контейнеров: всегда LF, даже при checkout на Windows.
|
||||
*.cfg text eol=lf
|
||||
Dockerfile text eol=lf
|
||||
@@ -1,3 +1,5 @@
|
||||
# Personal study notes, not part of the submission
|
||||
report_help.md
|
||||
# Local/private state
|
||||
.env
|
||||
.env.*
|
||||
@@ -16,6 +18,7 @@ backups/
|
||||
# Runtime/build artifacts
|
||||
.venv/
|
||||
__pycache__/
|
||||
.pytest_cache/
|
||||
*.py[cod]
|
||||
node_modules/
|
||||
dist/
|
||||
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
FROM python:3.14.3-slim@sha256:5e59aae31ff0e87511226be8e2b94d78c58f05216efda3b07dbbed938ec8583b
|
||||
|
||||
ARG APP_VERSION=dev
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
APP_CONFIG__RUN__VERSION=${APP_VERSION}
|
||||
|
||||
LABEL app.version=${APP_VERSION}
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app/ .
|
||||
|
||||
RUN useradd --no-create-home --shell /usr/sbin/nologin appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
# Liveness не зависит от БД; готовность (/health/ready) проверяют тесты и restart-скрипт.
|
||||
HEALTHCHECK --interval=5s --timeout=3s --start-period=10s \
|
||||
CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/live')"
|
||||
|
||||
CMD ["python", "main.py"]
|
||||
@@ -1,37 +1,164 @@
|
||||
# Отчёт — лабораторная 3
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- Автор, группа: Мякишев Владислав, БПИ2302
|
||||
- Целевая оценка; выбранный трек 4
|
||||
- Commit/PR: ветка `solution/bpi2302-myakishev` от skeleton `0ab3a34`; ссылка на PR — после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: код lab2 перенесён из своей `lab2/mtusi2026lab2`. Библиотеки: FastAPI, SQLAlchemy 2.1, Alembic, psycopg 3, PyJWT, argon2-cffi (версии в `requirements.txt`); HAProxy 3.0, PostgreSQL 17. Документация: Compose services, HAProxy 3.0 Configuration Manual (`server-template`, `resolvers`, `http-check`, `retry-on`, `option redispatch`), PostgreSQL `INSERT … ON CONFLICT`. При разработке использовался ИИ-ассистент Claude: генерация кода, конфигурации; запуски и измерения выполнены на стенде автора, все числа ниже взяты из файлов в `evidence/`.
|
||||
|
||||
Все проверки выполнены 2026-10-29 на одной машине: AMD Ryzen 5 5500U (12 логических CPU), Windows 11, Docker Desktop 29.8 (WSL2, Docker доступно 12 CPU / 7.4 GiB), Compose 5.5, Python 3.14. Лимиты контейнеров — в разделе «Архитектура».
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Prerequisites: Docker Engine 29.x + Compose 5.x, Python 3.9+ для публичных тестов, свободный `127.0.0.1:8080` (и `127.0.0.1:5432` для своих pytest). На Windows для Python-скриптов задайте `PYTHONUTF8=1` (иначе `check_contract.py` читает JSON в cp1251). Команды для Windows; на Linux вместо `.cmd`/`.ps1` используйте одноимённые `.sh`.
|
||||
|
||||
```sh
|
||||
# 1. Локальные секреты и .env: случайные postgres_password и jwt_secret в secrets/ (вне Git); существующие файлы не перезаписываются
|
||||
powershell -ExecutionPolicy Bypass -File scripts\init-secrets.ps1 # Linux: sh scripts/init-secrets.sh
|
||||
|
||||
# 2. Проверка и запуск: postgres (healthy) -> migrate (один раз, exit 0) -> api x2 (healthy) -> lb (healthy)
|
||||
docker compose --env-file .env config --quiet
|
||||
docker compose --env-file .env up -d --build --wait
|
||||
|
||||
# 3. Проверки
|
||||
python scripts/check_contract.py # = make check
|
||||
python tests/smoke.py --base-url http://localhost:8080 # = make test
|
||||
python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
scripts\restore-replicas.cmd # вернуть реплику, остановленную хуком
|
||||
|
||||
# 4. Свои тесты (БД messenger_test создаётся сама в том же PostgreSQL)
|
||||
python -m venv .venv
|
||||
.venv\Scripts\python.exe -m pip install -r requirements-test.txt
|
||||
.venv\Scripts\python.exe -m pytest -q
|
||||
|
||||
# 5. Эксперименты уровня 4 (пишут evidence/*.json, без токенов)
|
||||
python scripts/experiments.py idempotency
|
||||
python scripts/experiments.py failover
|
||||
python scripts/experiments.py rolling
|
||||
```
|
||||
|
||||
Остановка без потери данных: `docker compose down` (volume `postgres_data` остаётся). Намеренное удаление учебных данных: `docker compose down -v`. Число реплик: `API_REPLICAS=3` в `.env` или `docker compose up -d --scale api=3 --wait`. Действия оператора: только шаг 1 (секреты) и, после failover-теста, `restore-replicas`. **Обновление версии без простоя** — `python scripts/experiments.py rolling` (или те же шаги вручную, см. ниже). Обычный `docker compose up -d --build` пересоздаёт все реплики разом, во время пересоздания доступных backend нет.
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
клиент ──127.0.0.1:8080──> lb (HAProxy 3.0) ──roundrobin──> api #1 ┐
|
||||
│ health: GET /health/ready └──> api #2 ┼──> postgres 17 (volume postgres_data)
|
||||
│ каждую 1 с, DNS `api` / 2 с ┘ users, chats, chat_members, messages,
|
||||
migrate: разовый контейнер, `alembic upgrade head` до старта api sessions, idempotency_keys
|
||||
```
|
||||
|
||||
| Роль | Service | Образ (версия/digest) | Health / readiness | Volume | Сеть / порт | Credentials |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| load balancer | `lb` | `haproxy:3.0.29-alpine@sha256:56b887da…` | `wget /health/live` через свой порт (healthy, если жив хоть один backend) | нет | **единственный published**: `127.0.0.1:8080` | нет |
|
||||
| api (2+) | `api` | сборка из `Dockerfile` на `python:3.14.3-slim@sha256:5e59aae3…` | образ: `/health/live` каждые 5 с; LB: `/health/ready` | нет (stateless) | только внутренняя сеть, `expose 8080` | Docker secrets `postgres_password`, `jwt_secret` |
|
||||
| миграции | `migrate` | тот же образ | exit code 0 (HTTP-healthcheck отключён) | нет | внутренняя | secret `postgres_password` |
|
||||
| база | `postgres` | `postgres:17.10-alpine@sha256:979c4379…` | `pg_isready` каждые 5 с | `postgres_data` | внутренняя + `127.0.0.1:5432` (только для своих pytest из venv) | secret `postgres_password` |
|
||||
|
||||
**Источник истины.** Всё, от чего зависит корректность, — в PostgreSQL: пользователи, чаты, сообщения, **сессии** (`sessions`: SHA-256 токена и срок; JWT дополнительно проверяется подписью общим ключом `jwt_secret`) и **ключи идемпотентности**. В процессе API состояния нет, поэтому любая реплика принимает токен, выданный другой, и балансировка идёт без привязки клиента (`balance roundrobin`, без sticky). Отдельное хранилище сессий не используется: требование «общее хранилище сессий» выполняет та же БД (`SESSION_STORE=postgres`).
|
||||
|
||||
**Что перенесено из lab2:** приложение (FastAPI + SQLAlchemy), миграция `0001`, Argon2id, JWT + серверная сессия, секреты через файлы, свои тесты (31). **Что изменено в lab3:**
|
||||
|
||||
| Изменение | Файлы |
|
||||
| --- | --- |
|
||||
| `X-Instance-Id` во всех ответах API: hostname контейнера = короткий ID, которым оперирует `docker stop` | `app/utils/middleware.py`, `app/core/config.py` |
|
||||
| Обязательный `Idempotency-Key`, таблица `idempotency_keys`, транзакционная логика | `app/api/v1/messages_rout.py`, `app/database_functions/messages_functions.py`, `app/models/idempotency_models.py`, `migrations/versions/0002_idempotency_keys.py` |
|
||||
| Двухэтапный graceful shutdown: SIGTERM → `ready`=503 → через 3 с закрытие порта | `app/main.py` (`DrainingServer`), `app/utils/lifecycle.py`, `app/api/health.py` |
|
||||
| Argon2id `parallelism=1` (было 4) под лимит 1 CPU | `app/utils/security.py` |
|
||||
| Стенд: LB + 2 реплики + разовый migrate, лимиты, без портов API | `compose.yaml`, `infra/haproxy/haproxy.cfg`, `Dockerfile` (`APP_VERSION`) |
|
||||
| Хуки и эксперименты | `scripts/stop-one.*`, `scripts/restore-replicas.*`, `scripts/restart.*`, `scripts/experiments.py` |
|
||||
| Свои тесты: +13 (идемпотентность, параллельность, `X-Instance-Id`, drain), ключ во всех POST сообщения | `student-tests/` |
|
||||
|
||||
**Идемпотентность.** Область уникальности по контракту: пользователь + метод + путь чата + ключ. Метод у маршрута один (POST), путь однозначно задан `chat_id`, поэтому первичный ключ таблицы — `(user_id, chat_id, key)`. Всё в **одной транзакции** (READ COMMITTED):
|
||||
1. Сначала проверяются сессия и членство в чате (повтор чужим пользователем → 401/404, ключ не раскрывает чужое сообщение). Неверный или отсутствующий ключ (`^[A-Za-z0-9._:-]{1,128}$`) → 400.
|
||||
2. `INSERT INTO idempotency_keys … ON CONFLICT (user_id, chat_id, key) DO UPDATE … WHERE created_at < now − 24 h RETURNING` — «заявка» на ключ. Параллельный запрос с тем же ключом через другую реплику **блокируется на уникальном индексе** до COMMIT/ROLLBACK первого, поэтому создать два сообщения невозможно.
|
||||
3. Строку не вернули → ключ уже использован: сравнивается SHA-256 канонического JSON тела (порядок ключей и пробелы не важны). Совпало → тот же 201 и то же сообщение (id, `created_at`); не совпало → 409.
|
||||
4. Иначе: INSERT сообщения → `message_id` в строку ключа → COMMIT. Побочный эффект и запись результата атомарны: при сбое (5xx, обрыв соединения, остановка реплики) откатывается всё, ключ не остаётся «занятым», клиент повторяет с тем же ключом.
|
||||
5. Ключ старше `IDEMPOTENCY_TTL_SECONDS` (24 ч) перехватывается следующим запросом (`DO UPDATE … WHERE` просрочен).
|
||||
|
||||
**Балансировщик, health-check и время обнаружения (`infra/haproxy/haproxy.cfg`).**
|
||||
- Backend-адреса: `server-template api 6 api:8080` + `resolvers docker` (127.0.0.11), кэш 2 с. Пересозданные и масштабированные контейнеры подхватываются без перезапуска LB (проверено rolling update: новые ID попали в ротацию автоматически).
|
||||
- Проверка: `GET /health/ready` каждую **1 с** (`inter`), после первой неудачи каждые 0.5 с (`fastinter`), таймаут 0.5 с, исключение после **2** неудач (`fall`), возврат после 2 успехов (`rise`). **Максимальное время обнаружения по конфигурации ≈2.5 с** (`fall×inter + timeout check`), типично ≈1.5 с. Измерено: до исправления (ниже) обрывы приходились на 1.3–1.6 с после SIGTERM, то есть на момент исключения.
|
||||
- В окне обнаружения сбойное соединение повторяется на другой реплике: `retries 3`, `option redispatch 1` (redispatch на каждой попытке), `retry-on conn-failure empty-response response-timeout 502 503 504`. HTTP-повтор POST разрешён **только** для `POST /api/v1/chats/{id}/messages` (защищён ключом); регистрация, вход, создание чата не повторяются LB (`http-request disable-l7-retry`), чтобы не создать дубль.
|
||||
- Таймауты: connect 1 с; server 10 с (> `statement_timeout` БД 5 с: API сам отвечает 503, а не получает обрыв от LB); client 15 с; http-request 10 с; keep-alive 5 с; queue 5 с.
|
||||
|
||||
**Readiness и SIGTERM.** `live` — только жив ли процесс; `ready` — доступен ли PostgreSQL (критичная зависимость синхронного пути) и не останавливается ли реплика. На SIGTERM (`docker stop`):
|
||||
1. `t=0`: `ready` → 503, сервер продолжает принимать и обслуживать запросы;
|
||||
2. `t≈1.5 с` (≤2.5 с): LB исключает реплику, новые запросы идут на другие;
|
||||
3. `t=3 с` (`drain_seconds`): uvicorn закрывает порт и дообслуживает начатые запросы (`graceful_timeout` 5 с);
|
||||
4. `stop_grace_period: 15s` — запас до SIGKILL (хук `stop-one` использует `docker stop -t 8`, чтобы уложиться в бюджет теста 10 с).
|
||||
|
||||
Согласованность: `drain 3 с > обнаружение ≤2.5 с`; `timeout server 10 с > statement_timeout 5 с`; `drain + graceful = 8 с < stop_grace_period 15 с`. Измерено: `docker stop` реплики занимает 4.3–5.0 с, контейнер завершается с кодом 0.
|
||||
|
||||
**Миграции.** Их выполняет только одноразовый сервис `migrate` (`depends_on: condition: service_completed_successfully`); реплики API миграции не запускают, гонки нет (дополнительно `pg_advisory_xact_lock` в `migrations/env.py`). Повторный `alembic upgrade head` безопасен (при rolling update выполнен ещё раз: «Will assume transactional DDL», изменений нет). Миграция `0002` только **добавляет** таблицу (expand), поэтому предыдущая версия приложения работает на новой схеме. Ограничение: обратно-несовместимое изменение схемы потребует двух релизов (expand → переключение кода → contract); на время смешанных версий старые реплики не требуют `Idempotency-Key`.
|
||||
|
||||
**Лимиты и изоляция** (`deploy.resources.limits`, см. `evidence/compose-config.txt`): api 1 CPU / 512 MiB / 200 pids на реплику; lb 0.5 CPU / 128 MiB; postgres 1 CPU / 512 MiB; migrate 0.5 CPU / 256 MiB. Пул реплики 5+5 соединений; при `max_connections=100` до ~9 реплик не исчерпывают БД. У `api` нет `ports` и `container_name` (масштабирование `--scale` проверено: rolling update поднимал до 4 реплик). Секреты только там, где нужны: `jwt_secret` — только `api`; пароль БД — `api`, `migrate`, `postgres`; у `lb` секретов нет. Образы закреплены по digest.
|
||||
|
||||
**Соответствие `.env.example`:** `APP_HOST/APP_PORT` → `APP_CONFIG__RUN__HOST/PORT`; `DATABASE_URL_FILE` → `APP_CONFIG__DB__HOST/PORT/NAME/USER` + `APP_CONFIG__DB__PASSWORD_FILE`; `SESSION_TTL_SECONDS` → `APP_CONFIG__JWT__SESSION_TTL_SECONDS`; `SESSION_STORE=postgres`; `IDEMPOTENCY_TTL_SECONDS` → `APP_CONFIG__IDEMPOTENCY__TTL_SECONDS`; `API_REPLICAS` → `compose.yaml`; `CACHE_URL_FILE` не используется (Redis нет).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
| Публичный smoke | `python scripts/check_contract.py`; `python tests/smoke.py` | оба `PASS`, exit 0 | `evidence/smoke.txt` |
|
||||
| 3: стенд в одном `compose.yaml`, ≥2 API, одна точка входа, одна БД и общее хранилище сессий | `docker compose up -d --build --wait`; `docker compose config` | 2 реплики `api`, published только `lb:8080` (и `postgres` на loopback), сессии в PostgreSQL | `evidence/compose-config.txt` |
|
||||
| 3: запросы одного пользователя обслуживают разные реплики без повторного входа | 20× `GET /users/me` с одним токеном через LB | `X-Instance-Id` 3de2fdb570eb ×10, 945116f913cc ×10; ID совпадают с `docker ps` | `evidence/instances.txt` |
|
||||
| 3: остановка любой реплики, запись ≤10 с, подтверждённые данные целы | `resilience.py --mode failover` + `stop-one.cmd`, повтор для второй реплики после `restore-replicas` | 2 прогона, остановлены **обе** реплики (`Exited (0)` в `docker ps`): `first_recovery_seconds` 5.46 и 4.85, `failed_attempts` 0 и 0, `PASS` | `evidence/failover-hook.txt` |
|
||||
| 3: повтор с тем же ключом без дубля, включая конкурентный и смену реплики | smoke (replay, 409, без ключа 400); pytest; эксперимент `idempotency` | см. строку ниже | `evidence/pytest.txt`, `evidence/idempotency-parallel.json` |
|
||||
| 3: persistence | `resilience.py --mode persistence` + `restart.cmd` | `PASS` за 41 с; новый `StartedAt` у lb, api×2, postgres; volume тот же (CreatedAt 07:16:23Z); прежний токен действителен | `evidence/persistence.txt` |
|
||||
| Свои тесты | `pytest -q` | **44 passed** (31 из lab2 + 13 новых) | `evidence/pytest.txt` |
|
||||
| 4: health/readiness, timeouts, SIGTERM согласованы; LB исключает недоступный backend; период и время обнаружения | конфигурация + эксперимент `failover` | период 1 с, обнаружение ≤2.5 с по конфигу; после возврата `docker stop` реплика обслужила **0** запросов; `ready`=503 при drain — тест `test_ready_returns_503_while_draining` | `infra/haproxy/haproxy.cfg`, `evidence/failover-sequential*.json` |
|
||||
| 4: последовательная остановка каждой реплики, RPS, ошибки, время восстановления | `python scripts/experiments.py failover` (2 серии) | 4 остановки из 4 — **0 ошибок**, 0 потерь, 0 дублей; RPS 39–47 → 23–28 на одной реплике → 31–33 после возврата; реплика возвращается в ротацию за 20–22 с | `evidence/failover-sequential.json`, `…-repeat.json` |
|
||||
| 4: общая идемпотентность параллельными запросами с одним ключом | `python scripts/experiments.py idempotency` | 30 раундов × 16 одновременных POST = 480 запросов: все 201, в каждом раунде **один** message id, 30/30 раундов обслужены обеими репликами, max 1 сообщение на ключ, другой body → 409 | `evidence/idempotency-parallel.json` |
|
||||
| 4: CPU/RAM-лимиты, нет портов API наружу, нет `container_name`, секреты по месту | `docker compose config`; `docker stats` под нагрузкой | `api: published_ports=[] container_name=None limits={cpus: 1, memory: 512M, pids: 200}`; под нагрузкой api ≤104% CPU, 81–101 MiB | `evidence/compose-config.txt`, `resource_snapshot_under_load` в `failover-sequential.json` |
|
||||
| 4: rolling update под нагрузкой; миграции однократно и совместимо | `python scripts/experiments.py rolling` (2 прогона) | замена 2 → 4 → 2 реплики на новый образ (другой image id): **0 ошибок** из 5641 и 6375 запросов, 0 потерь, 0 дублей; migrate — один `docker compose run` | `evidence/rolling-update.json` |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
Какой процесс действительно остановлен, видно в `failover-hook.txt` (`docker ps -a` до/после: у нужного контейнера `Exited (0)`) и в `failover-sequential*.json` (`stopped_replica` = ID контейнера = `X-Instance-Id`). Хук останавливает только контейнер с метками `com.docker.compose.project=mtusi-lab3` и `service=api`, ID проверяется на hex.
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
**Профиль нагрузки** (`scripts/experiments.py`): замкнутый цикл, 8 потоков; каждый поток пишет сообщение (при ошибке повторяет **тот же** ключ до 10 раз, пауза 0.1 с) и читает страницу истории; таймаут клиента 2 с, без прозрачных повторов на клиенте, кроме повтора с ключом. Прогрев 5 с, реплика остановлена 15 с, затем `restore` и ещё 8 с нагрузки. Латентность — только успешные запросы. Клиент на той же машине и ограничен своим CPU (closed loop, coordinated omission), поэтому RPS — это пропускная способность этого клиента на этом стенде, а не ёмкость системы.
|
||||
|
||||
**Гипотеза 1: остановка одной реплики не прерывает запись, подтверждённые данные не теряются.** Подтверждена.
|
||||
|
||||
| Серия / реплика | RPS до | RPS при одной реплике | p95 при одной реплике | Ошибки (причина) | Последняя ошибка после SIGTERM | Возврат в ротацию | Подтверждено / потеряно / дубли |
|
||||
| --- | --- | --- | --- | --- | --- | --- | --- |
|
||||
| до исправления, bb53391a07f1 | 48.0 | 26.7 | 573 мс | 6 (`RemoteDisconnected`) | 1.55 с | 19.5 с | 903 / 0 / 0 |
|
||||
| до исправления, e51c99b5436b | 41.6 | 26.3 | 540 мс | 0 | — | 19.8 с | 903 / 0 / 0 |
|
||||
| серия 1, bb53391a07f1 | 43.8 | 23.8 | 602 мс | **0** | — | 20.9 с | 850 / 0 / 0 |
|
||||
| серия 1, e51c99b5436b | 38.7 | 23.3 | 603 мс | **0** | — | 21.8 с | 863 / 0 / 0 |
|
||||
| серия 2, 3de2fdb570eb | 46.5 | 27.8 | 516 мс | **0** | — | 20.1 с | 911 / 0 / 0 |
|
||||
| серия 2, 945116f913cc | 41.6 | 26.3 | 579 мс | **0** | — | 21.2 с | 894 / 0 / 0 |
|
||||
|
||||
- **Подтверждённое улучшение.** В первых прогонах (с `on-marked-down shutdown-sessions` в HAProxy) остановка давала ошибки в 3 из 4 случаев: 8, 6, 6 и 0. Это были `RemoteDisconnected` через 1.3–1.6 с после SIGTERM, ровно в момент, когда LB помечал реплику down и **обрывал начатые на ней запросы**, хотя реплика в режиме drain их бы дообслужила. Без этой опции — 0 ошибок в 4 из 4 остановок (2 серии). Прогон «до» сохранён в `failover-sequential-before-fix.json` (первая серия с 8/6 ошибками была перезаписана при исправлении метрики в скрипте, её числа взяты из вывода консоли).
|
||||
- RPS при одной реплике падает примерно до 60%: оставшаяся реплика упирается в свой лимит 1 CPU (`docker stats`: 100–104%), PostgreSQL загружен на 25–57%. После возврата RPS 31–33, ниже исходного. Вероятная причина — растущая история и прогрев новой реплики; отдельно не исследовалось.
|
||||
- «Возврат в ротацию» 20–22 с — в основном `docker compose up --wait`: Compose повторно запускает `migrate` и ждёт healthcheck. Сам LB возвращает реплику через `rise 2` ≈ 2 с.
|
||||
- Публичный failover-тест: `first_recovery_seconds` 4.85–5.46 — это время работы хука (запуск `cmd`, `docker ps`, `docker stop` ≈4.4 с с drain 3 с). Записи в это время не прерывались (`failed_attempts` 0). До уменьшения drain с 5 до 3 с и `inter` с 2 до 1 с было 7.5 с из допустимых 10.
|
||||
|
||||
**Гипотеза 2: параллельные запросы с одним ключом через разные реплики создают одно сообщение.** Подтверждена: 480 запросов (30 × 16, одновременный старт по barrier), в каждом раунде запросы попали на обе реплики, все 201 с одним id, по одному сообщению на ключ; тот же ключ с другим телом → 409. На уровне приложения то же проверяют тесты `test_concurrent_requests_with_one_key_create_one_message` (16 потоков) и `test_concurrent_different_bodies_one_key_only_one_wins`.
|
||||
|
||||
**Гипотеза 3: rolling update под нагрузкой проходит без ошибок и потерь.** Подтверждена (2 прогона). Порядок: `docker compose run --rm migrate` (один раз) → сборка новой версии (`APP_VERSION=rolling-…`, новый image id) → `up --no-recreate --scale api=4` → ожидание `healthy` и появления новых ID за LB → `docker stop -t 15` + `docker rm` старых реплик по одной с паузой 3 с.
|
||||
|
||||
| Прогон | Длительность обновления | Запросов за обновление | Ошибки | RPS / p95 | Первый ответ новой / последний старой | Потеряно / дубли |
|
||||
| --- | --- | --- | --- | --- | --- | --- |
|
||||
| 1 | 133.6 с (из них ~110 с сборка образа) | 6375 | 0 | 47.7 / 427 мс | 116 с / 131 с | 0 / 0 |
|
||||
| 2 | 137.1 с | 5641 | 0 | 41.1 / 542 мс | 115.7 с / 128.5 с | 0 / 0 |
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `docker stop` одной из 2 реплик (публичный тест, обе реплики) | запись продолжается ≤10 с | PASS ×2, `failed_attempts` 0 | хук 4.9–5.5 с; потерь нет |
|
||||
| `docker stop` каждой реплики под нагрузкой (2 серии) | единичные ошибки в окне обнаружения | 0 ошибок в 4 из 4 после исправления (до: 8, 6, 6, 0) | исключение из LB ≤2.5 с; потерь и дублей нет |
|
||||
| 16 одинаковых POST с одним ключом × 30 | 1 сообщение на ключ | 1 сообщение на ключ, 0 ошибок | не применимо |
|
||||
| rolling update 2 → 4 → 2 под нагрузкой | 0 потерь | 0 ошибок, 0 потерь (2 прогона) | новая версия обслуживает с 116 с |
|
||||
| рестарт lb + api + postgres с volume | данные и токен сохранены | PASS | весь тест (bootstrap + рестарт + проверка) 41 с; потерь нет |
|
||||
| регистрация/вход при лимите 0.5 CPU и Argon2 `p=4` (найдено при проверке) | ответ <2 с (таймаут теста) | 1.0–1.7 с, один таймаут в persistence-тесте | исправлено: `p=1` и 1 CPU → 0.46–0.65 с (0.97–1.01 с единичные) |
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- **Выдерживается:** отказ (остановка) одного процесса API из двух, перезапуск стенда с volume, конкурентные повторы с одним ключом, поочерёдная замена версии. **Не выдерживается:** отказ контейнера `lb` или `postgres` и отказ всей машины — это единые точки отказа; репликации БД нет, выживание узла не заявляется.
|
||||
- Запрос, оборванный в момент отказа, имеет неопределённый результат; клиент обязан повторить его с тем же ключом. Остальные POST (регистрация, вход, создание чата) LB не повторяет; при обрыве клиент может создать второй чат (по контракту допустимо).
|
||||
- Строки `idempotency_keys` старше TTL удаляются только при повторном использовании ключа; периодической очистки нет, таблица растёт. Следующий шаг — периодический `DELETE … WHERE created_at < now() − interval '24 hours'`.
|
||||
- Ответы, созданные самим LB (503, если нет живых backend), без `X-Request-Id`/`X-Instance-Id` (контракт допускает отсутствие `X-Instance-Id`).
|
||||
- Rolling update — скрипт на одной машине, а не оркестратор с автоматическим откатом. Обычный `docker compose up --build` пересоздаёт все реплики одновременно, с кратким простоем.
|
||||
- Измерения: по одной-две серии на сценарий, клиент на той же машине; числа ошибок зависят от фазы health-check в момент остановки. Для статистических выводов нужна большая серия. Отказ процесса через `docker kill` (без drain) отдельно не измерялся: в этом случае начатые на реплике запросы обрываются, а повтор LB защищает только новые соединения.
|
||||
- CI-job со сборкой стенда и `make test` не добавлен (workflow skeleton проверяет только контракт); проверки выполнены локально, вывод в `evidence/`.
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# ASCII only: Alembic reads this file in the OS locale encoding.
|
||||
# Database URL is built in env.py from app/core/config.py.
|
||||
|
||||
[alembic]
|
||||
script_location = %(here)s/migrations
|
||||
file_template = %%(rev)s_%%(slug)s
|
||||
prepend_sys_path = .
|
||||
path_separator = os
|
||||
output_encoding = utf-8
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARNING
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARNING
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
@@ -0,0 +1,57 @@
|
||||
from fastapi import Depends, HTTPException, Request, Security
|
||||
from fastapi.security import HTTPBasic, HTTPBearer
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from core.db import get_db
|
||||
from database_functions.sessions_functions import get_active_session
|
||||
from models import AuthSession, User
|
||||
from utils.security import decode_access_token, token_hash
|
||||
|
||||
BEARER_CHALLENGE = {"WWW-Authenticate": 'Bearer realm="messenger"'}
|
||||
|
||||
|
||||
# Security-схемы нужны, чтобы в Swagger появилась кнопка Authorize.
|
||||
# Заголовок разбираем сами: стандартный HTTPBasic понимает пароль только в ASCII.
|
||||
class RawBasicHeader(HTTPBasic):
|
||||
async def __call__(self, request: Request) -> str | None:
|
||||
return request.headers.get("Authorization")
|
||||
|
||||
|
||||
class RawBearerHeader(HTTPBearer):
|
||||
async def __call__(self, request: Request) -> str | None:
|
||||
return request.headers.get("Authorization")
|
||||
|
||||
|
||||
basic_scheme = RawBasicHeader(scheme_name="basicAuth", auto_error=False)
|
||||
bearer_scheme = RawBearerHeader(scheme_name="bearerAuth", auto_error=False)
|
||||
|
||||
|
||||
async def require_empty_body(request: Request) -> None:
|
||||
# Контракт задаёт вход без тела: лишний JSON не игнорируем молча, а отклоняем.
|
||||
if await request.body():
|
||||
raise HTTPException(status_code=400, detail="Тело запроса не допускается")
|
||||
|
||||
|
||||
def unauthorized() -> HTTPException:
|
||||
return HTTPException(status_code=401, detail="Нужен действительный Bearer-токен", headers=BEARER_CHALLENGE)
|
||||
|
||||
|
||||
def get_current_session(authorization: str | None = Security(bearer_scheme),
|
||||
db: Session = Depends(get_db)) -> AuthSession:
|
||||
scheme, _, token = (authorization or "").partition(" ")
|
||||
if scheme.lower() != "bearer" or not token:
|
||||
raise unauthorized()
|
||||
|
||||
# Подпись и exp проверяются без базы, отзыв (logout) — по таблице sessions.
|
||||
payload = decode_access_token(token)
|
||||
if payload is None:
|
||||
raise unauthorized()
|
||||
|
||||
auth_session = get_active_session(db, token_hash(token))
|
||||
if auth_session is None or str(auth_session.user_id) != payload["sub"]:
|
||||
raise unauthorized()
|
||||
return auth_session
|
||||
|
||||
|
||||
def get_current_user(auth_session: AuthSession = Depends(get_current_session)) -> User:
|
||||
return auth_session.user
|
||||
@@ -0,0 +1,23 @@
|
||||
from fastapi import APIRouter, HTTPException
|
||||
|
||||
from core.db import check_database
|
||||
from schemas.health_schemas import HealthResponse
|
||||
from utils.lifecycle import is_draining
|
||||
|
||||
router = APIRouter(prefix="/health", tags=["health"])
|
||||
|
||||
|
||||
@router.get("/live", response_model=HealthResponse)
|
||||
def live_api():
|
||||
return HealthResponse(status="ok")
|
||||
|
||||
|
||||
# В отличие от live, ready зависит от PostgreSQL и от того, не останавливается ли реплика:
|
||||
# после SIGTERM ответ 503 выводит её из балансировки, пока она ещё дообслуживает запросы.
|
||||
@router.get("/ready", response_model=HealthResponse)
|
||||
def ready_api():
|
||||
if is_draining():
|
||||
raise HTTPException(status_code=503, detail="Реплика останавливается")
|
||||
if not check_database():
|
||||
raise HTTPException(status_code=503, detail="База данных недоступна")
|
||||
return HealthResponse(status="ok")
|
||||
@@ -0,0 +1,66 @@
|
||||
import base64
|
||||
import binascii
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Response, Security
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from api.dependencies import basic_scheme, get_current_session, require_empty_body
|
||||
from core.config import settings
|
||||
from core.db import get_db
|
||||
from core.logger import logger
|
||||
from database_functions.sessions_functions import create_session, delete_session
|
||||
from database_functions.users_functions import get_user_by_username
|
||||
from models import AuthSession
|
||||
from schemas.auth_schemas import SessionResponse
|
||||
from schemas.users_schemas import UserResponse
|
||||
from utils.security import create_access_token, token_hash, verify_password
|
||||
|
||||
roter = APIRouter(
|
||||
prefix=settings.api.prefix + settings.api.v1.prefix + settings.api.v1.auth,
|
||||
tags=["auth"],
|
||||
)
|
||||
|
||||
BASIC_CHALLENGE = {"WWW-Authenticate": 'Basic realm="messenger", charset="UTF-8"'}
|
||||
|
||||
|
||||
def invalid_credentials() -> HTTPException:
|
||||
# Один ответ и для неизвестного username, и для неверного пароля.
|
||||
return HTTPException(status_code=401, detail="Неверное имя пользователя или пароль", headers=BASIC_CHALLENGE)
|
||||
|
||||
|
||||
def parse_basic(authorization: str | None) -> tuple[str, str]:
|
||||
scheme, _, encoded = (authorization or "").partition(" ")
|
||||
if scheme.lower() != "basic" or not encoded:
|
||||
raise invalid_credentials()
|
||||
try:
|
||||
decoded = base64.b64decode(encoded, validate=True).decode("utf-8")
|
||||
except (binascii.Error, UnicodeDecodeError):
|
||||
raise invalid_credentials()
|
||||
# По первому двоеточию: в пароле двоеточие допустимо.
|
||||
username, separator, password = decoded.partition(":")
|
||||
if not separator:
|
||||
raise invalid_credentials()
|
||||
return username, password
|
||||
|
||||
|
||||
# Тело проверяется до пароля: зависимости маршрута выполняются раньше параметров.
|
||||
@roter.post("/sessions", status_code=201, response_model=SessionResponse, dependencies=[Depends(require_empty_body)])
|
||||
def create_session_api(authorization: str | None = Security(basic_scheme), db: Session = Depends(get_db)):
|
||||
username, password = parse_basic(authorization)
|
||||
|
||||
user = get_user_by_username(db, username)
|
||||
if not verify_password(user.password_hash if user else None, password):
|
||||
logger.info("Неудачная попытка входа")
|
||||
raise invalid_credentials()
|
||||
|
||||
token, issued_at, expires_at = create_access_token(user.id)
|
||||
create_session(db, token_hash(token), user.id, created_at=issued_at, expires_at=expires_at)
|
||||
logger.info(f"Вход выполнен: user_id={user.id}")
|
||||
return SessionResponse(token=token, expires_at=expires_at, user=UserResponse.model_validate(user))
|
||||
|
||||
|
||||
@roter.delete("/sessions/current", status_code=204)
|
||||
def delete_session_api(auth_session: AuthSession = Depends(get_current_session), db: Session = Depends(get_db)):
|
||||
delete_session(db, auth_session.token_hash)
|
||||
logger.info(f"Выход выполнен: user_id={auth_session.user_id}")
|
||||
return Response(status_code=204)
|
||||
@@ -0,0 +1,54 @@
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from api.dependencies import get_current_user
|
||||
from core.config import settings
|
||||
from core.db import get_db
|
||||
from core.logger import logger
|
||||
from database_functions.chats_functions import create_chat, get_member_chat, list_user_chats
|
||||
from models import User
|
||||
from schemas.chats_schemas import ChatPageResponse, ChatResponse, CreateChatRequest
|
||||
from utils.pagination import read_cursor, split_page
|
||||
|
||||
roter = APIRouter(
|
||||
prefix=settings.api.prefix + settings.api.v1.prefix + settings.api.v1.chats,
|
||||
tags=["chats"],
|
||||
)
|
||||
|
||||
|
||||
@roter.post("", status_code=201, response_model=ChatResponse)
|
||||
def create_chat_api(request: CreateChatRequest, current_user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_db)):
|
||||
if current_user.id not in request.member_ids:
|
||||
raise HTTPException(status_code=400, detail="Создатель должен быть участником чата")
|
||||
|
||||
chat = create_chat(db, title=request.title, member_ids=request.member_ids)
|
||||
if chat is None:
|
||||
raise HTTPException(status_code=400, detail="Неизвестный участник")
|
||||
logger.info(f"Чат создан: id={chat.id} участников={len(request.member_ids)}")
|
||||
return ChatResponse.model_validate(chat)
|
||||
|
||||
|
||||
@roter.get("", response_model=ChatPageResponse)
|
||||
def list_chats_api(
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None, min_length=1),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
scope = f"chats:{current_user.id}"
|
||||
chats = list_user_chats(db, current_user.id, limit=limit, after=read_cursor(scope, cursor))
|
||||
page, next_cursor = split_page(chats, limit, scope)
|
||||
return ChatPageResponse(items=[ChatResponse.model_validate(chat) for chat in page], next_cursor=next_cursor)
|
||||
|
||||
|
||||
@roter.get("/{chat_id}", response_model=ChatResponse)
|
||||
def get_chat_api(chat_id: uuid.UUID, current_user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_db)):
|
||||
chat = get_member_chat(db, chat_id, current_user.id)
|
||||
if chat is None:
|
||||
# Чужой чат неотличим от несуществующего.
|
||||
raise HTTPException(status_code=404, detail="Чат не найден")
|
||||
return ChatResponse.model_validate(chat)
|
||||
@@ -0,0 +1,63 @@
|
||||
import hashlib
|
||||
import json
|
||||
import uuid
|
||||
|
||||
from fastapi import APIRouter, Depends, Header, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from api.dependencies import get_current_user
|
||||
from core.config import settings
|
||||
from core.db import get_db
|
||||
from database_functions.chats_functions import is_chat_member
|
||||
from database_functions.messages_functions import IdempotencyKeyReused, create_message_once, list_messages
|
||||
from models import User
|
||||
from schemas.messages_schemas import CreateMessageRequest, MessagePageResponse, MessageResponse
|
||||
from utils.pagination import read_cursor, split_page
|
||||
|
||||
roter = APIRouter(
|
||||
prefix=settings.api.prefix + settings.api.v1.prefix + settings.api.v1.chats,
|
||||
tags=["messages"],
|
||||
)
|
||||
|
||||
|
||||
def ensure_member(db: Session, chat_id: uuid.UUID, user: User):
|
||||
if not is_chat_member(db, chat_id, user.id):
|
||||
raise HTTPException(status_code=404, detail="Чат не найден")
|
||||
|
||||
|
||||
def hash_request(request: CreateMessageRequest) -> bytes:
|
||||
# Канонический вид: порядок ключей JSON и пробелы в запросе на хеш не влияют.
|
||||
canonical = json.dumps(request.model_dump(), sort_keys=True, ensure_ascii=False, separators=(",", ":"))
|
||||
return hashlib.sha256(canonical.encode("utf-8")).digest()
|
||||
|
||||
|
||||
@roter.post("/{chat_id}/messages", status_code=201, response_model=MessageResponse)
|
||||
def create_message_api(chat_id: uuid.UUID, request: CreateMessageRequest,
|
||||
idempotency_key: str = Header(pattern=r"^[A-Za-z0-9._:-]{1,128}$"),
|
||||
current_user: User = Depends(get_current_user), db: Session = Depends(get_db)):
|
||||
# Повтор тоже проходит проверку сессии и членства: ключ не даёт доступа к чужому чату.
|
||||
ensure_member(db, chat_id, current_user)
|
||||
try:
|
||||
message = create_message_once(
|
||||
db, chat_id=chat_id, sender_id=current_user.id, text=request.text,
|
||||
key=idempotency_key, request_hash=hash_request(request),
|
||||
ttl_seconds=settings.idempotency.ttl_seconds,
|
||||
)
|
||||
except IdempotencyKeyReused:
|
||||
raise HTTPException(status_code=409, detail="Idempotency-Key уже использован с другим телом запроса")
|
||||
return MessageResponse.model_validate(message)
|
||||
|
||||
|
||||
@roter.get("/{chat_id}/messages", response_model=MessagePageResponse)
|
||||
def list_messages_api(
|
||||
chat_id: uuid.UUID,
|
||||
limit: int = Query(default=50, ge=1, le=100),
|
||||
cursor: str | None = Query(default=None, min_length=1),
|
||||
current_user: User = Depends(get_current_user),
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
ensure_member(db, chat_id, current_user)
|
||||
scope = f"chat:{chat_id}"
|
||||
messages = list_messages(db, chat_id, limit=limit, after=read_cursor(scope, cursor))
|
||||
page, next_cursor = split_page(messages, limit, scope)
|
||||
return MessagePageResponse(items=[MessageResponse.model_validate(m) for m in page], next_cursor=next_cursor)
|
||||
@@ -0,0 +1,31 @@
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from api.dependencies import get_current_user
|
||||
from core.config import settings
|
||||
from core.db import get_db
|
||||
from core.logger import logger
|
||||
from database_functions.users_functions import create_user
|
||||
from models import User
|
||||
from schemas.users_schemas import CreateUserRequest, UserResponse
|
||||
from utils.security import hash_password
|
||||
|
||||
roter = APIRouter(
|
||||
prefix=settings.api.prefix + settings.api.v1.prefix + settings.api.v1.users,
|
||||
tags=["users"],
|
||||
)
|
||||
|
||||
|
||||
@roter.post("", status_code=201, response_model=UserResponse)
|
||||
def register_user_api(request: CreateUserRequest, db: Session = Depends(get_db)):
|
||||
user = create_user(db, username=request.username, display_name=request.display_name,
|
||||
password_hash=hash_password(request.password))
|
||||
if user is None:
|
||||
raise HTTPException(status_code=409, detail="Username уже занят")
|
||||
logger.info(f"Пользователь создан: id={user.id}")
|
||||
return UserResponse.model_validate(user)
|
||||
|
||||
|
||||
@roter.get("/me", response_model=UserResponse)
|
||||
def get_me_api(current_user: User = Depends(get_current_user)):
|
||||
return UserResponse.model_validate(current_user)
|
||||
@@ -0,0 +1,102 @@
|
||||
import socket
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
# Корень проекта (в контейнере код лежит в /app, и это будет "/").
|
||||
BASE_DIR = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class RunConfig(BaseModel):
|
||||
host: str = "0.0.0.0"
|
||||
port: int = 8080
|
||||
# Docker кладёт в hostname короткий ID контейнера: по X-Instance-Id его можно остановить через `docker stop`.
|
||||
instance_id: str = socket.gethostname()
|
||||
# SIGTERM -> сразу /health/ready=503, но порт закрывается только через drain_seconds:
|
||||
# за это время балансировщик успевает вывести реплику из ротации
|
||||
# (должно быть больше худшего времени обнаружения в haproxy.cfg: fall*inter + timeout check = 2.5 с).
|
||||
drain_seconds: int = 3
|
||||
# Версия сборки (build-arg APP_VERSION): видна в логе, различает старую и новую реплику при rolling update.
|
||||
version: str = "dev"
|
||||
# Время на завершение уже принятых запросов после закрытия порта.
|
||||
graceful_timeout: int = 5
|
||||
|
||||
|
||||
class LoggingConfig(BaseModel):
|
||||
level: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO"
|
||||
format: str = "[%(asctime)s] [%(levelname)s] [%(name)s] %(message)s"
|
||||
datefmt: str = "%Y-%m-%d %H:%M:%S"
|
||||
|
||||
|
||||
class DatabaseConfig(BaseModel):
|
||||
# Значения по умолчанию — для запуска внутри Compose.
|
||||
host: str = "postgres"
|
||||
port: int = 5432
|
||||
name: str = "messenger"
|
||||
user: str = "messenger"
|
||||
password_file: Path = Path("/run/secrets/postgres_password")
|
||||
|
||||
pool_size: int = 5
|
||||
max_overflow: int = 5
|
||||
pool_timeout: int = 5
|
||||
connect_timeout: int = 3
|
||||
statement_timeout_ms: int = 5000
|
||||
echo: bool = False
|
||||
|
||||
def read_password(self) -> str:
|
||||
return read_secret(self.password_file)
|
||||
|
||||
|
||||
class JWTConfig(BaseModel):
|
||||
# Ключ должен сохраняться между перезапусками, иначе выданные токены станут недействительны.
|
||||
secret_file: Path = Path("/run/secrets/jwt_secret")
|
||||
algorithm: str = "HS256"
|
||||
session_ttl_seconds: int = 3600
|
||||
|
||||
def read_secret_key(self) -> str:
|
||||
return read_secret(self.secret_file)
|
||||
|
||||
|
||||
class IdempotencyConfig(BaseModel):
|
||||
# Контракт: повтор с тем же ключом в течение 24 часов возвращает исходный ответ.
|
||||
ttl_seconds: int = 86400
|
||||
|
||||
|
||||
def read_secret(path: Path) -> str:
|
||||
"""Прочитать файл-секрет. Относительный путь считается от корня проекта."""
|
||||
full_path = path if path.is_absolute() else BASE_DIR / path
|
||||
return full_path.read_text(encoding="utf-8").strip()
|
||||
|
||||
|
||||
class ApiV1Prefix(BaseModel):
|
||||
prefix: str = "/v1"
|
||||
users: str = "/users"
|
||||
chats: str = "/chats"
|
||||
auth: str = "/auth"
|
||||
|
||||
|
||||
class ApiPrefix(BaseModel):
|
||||
prefix: str = "/api"
|
||||
v1: ApiV1Prefix = ApiV1Prefix()
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=BASE_DIR / ".env",
|
||||
case_sensitive=False,
|
||||
env_nested_delimiter="__",
|
||||
env_prefix="APP_CONFIG__",
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
run: RunConfig = RunConfig()
|
||||
api: ApiPrefix = ApiPrefix()
|
||||
logging: LoggingConfig = LoggingConfig()
|
||||
db: DatabaseConfig = DatabaseConfig()
|
||||
jwt: JWTConfig = JWTConfig()
|
||||
idempotency: IdempotencyConfig = IdempotencyConfig()
|
||||
|
||||
|
||||
settings = Settings()
|
||||
@@ -0,0 +1,57 @@
|
||||
from collections.abc import Iterator
|
||||
|
||||
from sqlalchemy import URL, create_engine, text
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from core.config import settings
|
||||
from core.logger import logger
|
||||
|
||||
|
||||
def build_url() -> URL:
|
||||
return URL.create(
|
||||
drivername="postgresql+psycopg",
|
||||
username=settings.db.user,
|
||||
password=settings.db.read_password(),
|
||||
host=settings.db.host,
|
||||
port=settings.db.port,
|
||||
database=settings.db.name,
|
||||
)
|
||||
|
||||
|
||||
engine = create_engine(
|
||||
build_url(),
|
||||
pool_size=settings.db.pool_size,
|
||||
max_overflow=settings.db.max_overflow,
|
||||
pool_timeout=settings.db.pool_timeout,
|
||||
# Проверять соединение перед выдачей из пула (нужно после рестарта PostgreSQL).
|
||||
pool_pre_ping=True,
|
||||
connect_args={
|
||||
"connect_timeout": settings.db.connect_timeout,
|
||||
"options": f"-c statement_timeout={settings.db.statement_timeout_ms}",
|
||||
},
|
||||
echo=settings.db.echo,
|
||||
)
|
||||
|
||||
SessionLocal = sessionmaker(bind=engine, expire_on_commit=False)
|
||||
|
||||
|
||||
def get_db() -> Iterator[Session]:
|
||||
"""Одна ORM-сессия на HTTP-запрос."""
|
||||
with SessionLocal() as session:
|
||||
yield session
|
||||
|
||||
|
||||
def check_database() -> bool:
|
||||
try:
|
||||
with engine.connect() as connection:
|
||||
connection.execute(text("SELECT 1"))
|
||||
return True
|
||||
except SQLAlchemyError as exc:
|
||||
# Только тип ошибки: в тексте может оказаться адрес БД.
|
||||
logger.warning(f"PostgreSQL недоступен: {type(exc).__name__}")
|
||||
return False
|
||||
|
||||
|
||||
def log_database_target():
|
||||
logger.info(f"База данных: {engine.url.render_as_string(hide_password=True)}")
|
||||
@@ -0,0 +1,15 @@
|
||||
import logging
|
||||
|
||||
from core.config import settings
|
||||
|
||||
|
||||
def setup_logger():
|
||||
logging.basicConfig(
|
||||
level=settings.logging.level,
|
||||
format=settings.logging.format,
|
||||
datefmt=settings.logging.datefmt,
|
||||
)
|
||||
return logging.getLogger("messenger")
|
||||
|
||||
|
||||
logger = setup_logger()
|
||||
@@ -0,0 +1,50 @@
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import func, select, tuple_
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models import Chat, ChatMember, User
|
||||
from utils.pagination import Position
|
||||
|
||||
|
||||
def create_chat(db: Session, title: str, member_ids: list[uuid.UUID]) -> Chat | None:
|
||||
"""Чат и участники — одна транзакция. None — есть несуществующий участник."""
|
||||
found = db.scalar(select(func.count()).select_from(User).where(User.id.in_(member_ids)))
|
||||
if found != len(member_ids):
|
||||
return None
|
||||
|
||||
chat = Chat(title=title, members=[ChatMember(user_id=user_id) for user_id in sorted(member_ids)])
|
||||
db.add(chat)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
return None
|
||||
return chat
|
||||
|
||||
|
||||
def is_chat_member(db: Session, chat_id: uuid.UUID, user_id: uuid.UUID) -> bool:
|
||||
return db.get(ChatMember, (chat_id, user_id)) is not None
|
||||
|
||||
|
||||
def get_member_chat(db: Session, chat_id: uuid.UUID, user_id: uuid.UUID) -> Chat | None:
|
||||
query = (
|
||||
select(Chat)
|
||||
.join(ChatMember, ChatMember.chat_id == Chat.id)
|
||||
.where(Chat.id == chat_id, ChatMember.user_id == user_id)
|
||||
)
|
||||
return db.scalar(query)
|
||||
|
||||
|
||||
def list_user_chats(db: Session, user_id: uuid.UUID, limit: int, after: Position | None) -> list[Chat]:
|
||||
query = (
|
||||
select(Chat)
|
||||
.join(ChatMember, ChatMember.chat_id == Chat.id)
|
||||
.where(ChatMember.user_id == user_id)
|
||||
.order_by(Chat.created_at, Chat.id)
|
||||
.limit(limit + 1)
|
||||
)
|
||||
if after is not None:
|
||||
query = query.where(tuple_(Chat.created_at, Chat.id) > tuple_(*after))
|
||||
return list(db.scalars(query))
|
||||
@@ -0,0 +1,65 @@
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from sqlalchemy import func, select, tuple_, update
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models import IdempotencyKey, Message
|
||||
from utils.pagination import Position
|
||||
|
||||
|
||||
class IdempotencyKeyReused(Exception):
|
||||
"""Тот же Idempotency-Key уже использован с другим телом запроса."""
|
||||
|
||||
|
||||
def create_message_once(db: Session, chat_id: uuid.UUID, sender_id: uuid.UUID, text: str,
|
||||
key: str, request_hash: bytes, ttl_seconds: int) -> Message:
|
||||
"""Создать сообщение ровно один раз для (пользователь, чат, ключ), в том числе при конкурентных повторах.
|
||||
|
||||
Всё в одной транзакции: «заявка» на ключ, сообщение и ссылка ключа на сообщение фиксируются вместе.
|
||||
Если транзакция откатится (5xx, обрыв), ключ не сохранится и клиент безопасно повторит запрос.
|
||||
Параллельный повтор через другую реплику блокируется на уникальном индексе до COMMIT первой
|
||||
транзакции, затем видит её строку и возвращает то же сообщение.
|
||||
"""
|
||||
cutoff = datetime.now(timezone.utc) - timedelta(seconds=ttl_seconds)
|
||||
claim = (
|
||||
insert(IdempotencyKey)
|
||||
.values(user_id=sender_id, chat_id=chat_id, key=key, request_hash=request_hash)
|
||||
# Строка старше TTL считается свободной ключом: перехватываем её, остальные конфликты не меняем.
|
||||
.on_conflict_do_update(
|
||||
index_elements=[IdempotencyKey.user_id, IdempotencyKey.chat_id, IdempotencyKey.key],
|
||||
set_={"request_hash": request_hash, "message_id": None, "created_at": func.clock_timestamp()},
|
||||
where=IdempotencyKey.created_at < cutoff,
|
||||
)
|
||||
.returning(IdempotencyKey.key)
|
||||
)
|
||||
if db.execute(claim).first() is None:
|
||||
existing = db.scalars(select(IdempotencyKey).where(
|
||||
IdempotencyKey.user_id == sender_id, IdempotencyKey.chat_id == chat_id, IdempotencyKey.key == key,
|
||||
)).one()
|
||||
if existing.request_hash != request_hash:
|
||||
raise IdempotencyKeyReused()
|
||||
return db.get_one(Message, existing.message_id)
|
||||
|
||||
message = Message(chat_id=chat_id, sender_id=sender_id, text=text)
|
||||
db.add(message)
|
||||
db.flush()
|
||||
db.execute(update(IdempotencyKey).where(
|
||||
IdempotencyKey.user_id == sender_id, IdempotencyKey.chat_id == chat_id, IdempotencyKey.key == key,
|
||||
).values(message_id=message.id))
|
||||
db.commit()
|
||||
return message
|
||||
|
||||
|
||||
def list_messages(db: Session, chat_id: uuid.UUID, limit: int, after: Position | None) -> list[Message]:
|
||||
# Keyset-пагинация по индексу messages_chat_history_idx (chat_id, created_at, id).
|
||||
query = (
|
||||
select(Message)
|
||||
.where(Message.chat_id == chat_id)
|
||||
.order_by(Message.created_at, Message.id)
|
||||
.limit(limit + 1)
|
||||
)
|
||||
if after is not None:
|
||||
query = query.where(tuple_(Message.created_at, Message.id) > tuple_(*after))
|
||||
return list(db.scalars(query))
|
||||
@@ -0,0 +1,29 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import delete, func, select
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models import AuthSession
|
||||
|
||||
|
||||
def create_session(db: Session, token_hash: bytes, user_id: uuid.UUID,
|
||||
created_at: datetime, expires_at: datetime) -> AuthSession:
|
||||
auth_session = AuthSession(token_hash=token_hash, user_id=user_id, created_at=created_at, expires_at=expires_at)
|
||||
db.add(auth_session)
|
||||
db.commit()
|
||||
return auth_session
|
||||
|
||||
|
||||
def get_active_session(db: Session, token_hash: bytes) -> AuthSession | None:
|
||||
query = select(AuthSession).where(
|
||||
AuthSession.token_hash == token_hash,
|
||||
AuthSession.expires_at > func.now(),
|
||||
)
|
||||
return db.scalar(query)
|
||||
|
||||
|
||||
def delete_session(db: Session, token_hash: bytes) -> bool:
|
||||
result = db.execute(delete(AuthSession).where(AuthSession.token_hash == token_hash))
|
||||
db.commit()
|
||||
return result.rowcount > 0
|
||||
@@ -0,0 +1,30 @@
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from models import User
|
||||
|
||||
|
||||
def create_user(db: Session, username: str, display_name: str, password_hash: str) -> User | None:
|
||||
"""None — username уже занят."""
|
||||
user = User(username=username, display_name=display_name, password_hash=password_hash)
|
||||
db.add(user)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError as exc:
|
||||
db.rollback()
|
||||
# Уникальность проверяет сама БД, поэтому гонка регистрации невозможна.
|
||||
if exc.orig.diag.constraint_name == "users_username_key":
|
||||
return None
|
||||
raise
|
||||
return user
|
||||
|
||||
|
||||
def get_user(db: Session, user_id: uuid.UUID) -> User | None:
|
||||
return db.get(User, user_id)
|
||||
|
||||
|
||||
def get_user_by_username(db: Session, username: str) -> User | None:
|
||||
return db.scalar(select(User).where(User.username == username))
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
import signal
|
||||
import threading
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import uvicorn
|
||||
from fastapi import FastAPI
|
||||
from starlette.middleware.base import BaseHTTPMiddleware
|
||||
|
||||
from api.health import router as health_router
|
||||
from api.v1.auth_rout import roter as auth_router
|
||||
from api.v1.chats_rout import roter as chats_router
|
||||
from api.v1.messages_rout import roter as messages_router
|
||||
from api.v1.users_rout import roter as users_router
|
||||
from core.config import settings
|
||||
from core.db import engine, log_database_target
|
||||
from core.logger import logger
|
||||
from utils.errors import register_error_handlers
|
||||
from utils.lifecycle import is_draining, start_draining
|
||||
from utils.middleware import request_id_middleware
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
log_database_target()
|
||||
logger.info(f"Экземпляр API: {settings.run.instance_id}, версия {settings.run.version}")
|
||||
yield
|
||||
# Остановка (в том числе по SIGTERM): закрываем соединения пула.
|
||||
engine.dispose()
|
||||
logger.info("Соединения с базой данных закрыты")
|
||||
|
||||
|
||||
app = FastAPI(title="MTUSI Messenger lab3", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(BaseHTTPMiddleware, dispatch=request_id_middleware)
|
||||
register_error_handlers(app)
|
||||
|
||||
app.include_router(health_router)
|
||||
app.include_router(auth_router)
|
||||
app.include_router(users_router)
|
||||
app.include_router(chats_router)
|
||||
app.include_router(messages_router)
|
||||
|
||||
|
||||
class DrainingServer(uvicorn.Server):
|
||||
"""Graceful shutdown в два этапа.
|
||||
|
||||
1. SIGTERM: /health/ready начинает отвечать 503, балансировщик выводит реплику из ротации,
|
||||
а сервер продолжает принимать и обслуживать запросы.
|
||||
2. Через drain_seconds uvicorn перестаёт принимать соединения и дообслуживает начатые запросы
|
||||
(не дольше graceful_timeout).
|
||||
Повторный сигнал (или Ctrl+C) останавливает сервер сразу.
|
||||
"""
|
||||
|
||||
def handle_exit(self, sig, frame):
|
||||
if sig == signal.SIGTERM and not is_draining() and settings.run.drain_seconds > 0:
|
||||
start_draining()
|
||||
logger.info(f"SIGTERM: реплика выводится из балансировки, остановка через {settings.run.drain_seconds} c")
|
||||
threading.Timer(settings.run.drain_seconds, super().handle_exit, args=(sig, frame)).start()
|
||||
return
|
||||
super().handle_exit(sig, frame)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logger.info(f"Запуск сервера на {settings.run.host}:{settings.run.port}")
|
||||
config = uvicorn.Config(
|
||||
"main:app",
|
||||
host=settings.run.host,
|
||||
port=settings.run.port,
|
||||
timeout_graceful_shutdown=settings.run.graceful_timeout,
|
||||
access_log=False, # запросы логирует наш middleware
|
||||
)
|
||||
DrainingServer(config).run()
|
||||
@@ -0,0 +1,52 @@
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import create_engine, pool, text
|
||||
|
||||
from core.config import settings
|
||||
from core.db import build_url
|
||||
from models import Base
|
||||
|
||||
config = context.config
|
||||
|
||||
if config.config_file_name is not None:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
target_metadata = Base.metadata
|
||||
|
||||
# Ключ advisory lock: параллельные запуски миграций выполнятся по очереди.
|
||||
MIGRATION_LOCK_KEY = 2_026_002
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(
|
||||
url=build_url(),
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
engine = create_engine(
|
||||
build_url(),
|
||||
poolclass=pool.NullPool,
|
||||
connect_args={"connect_timeout": settings.db.connect_timeout},
|
||||
)
|
||||
with engine.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
compare_server_default=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
connection.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": MIGRATION_LOCK_KEY})
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
${downgrades if downgrades else "pass"}
|
||||
@@ -0,0 +1,85 @@
|
||||
"""initial schema
|
||||
|
||||
Revision ID: 0001
|
||||
Revises:
|
||||
Create Date: 2026-10-15 17:30:37.825446
|
||||
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '0001'
|
||||
down_revision: Union[str, Sequence[str], None] = None
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table('chats',
|
||||
sa.Column('id', sa.Uuid(), server_default=sa.text('gen_random_uuid()'), nullable=False),
|
||||
sa.Column('title', sa.Text(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('clock_timestamp()'), nullable=False),
|
||||
sa.CheckConstraint('char_length(title) BETWEEN 1 AND 100', name=op.f('chats_title_len')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('chats_pkey'))
|
||||
)
|
||||
op.create_table('users',
|
||||
sa.Column('id', sa.Uuid(), server_default=sa.text('gen_random_uuid()'), nullable=False),
|
||||
sa.Column('username', sa.Text(), nullable=False),
|
||||
sa.Column('display_name', sa.Text(), nullable=False),
|
||||
sa.Column('password_hash', sa.Text(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('clock_timestamp()'), nullable=False),
|
||||
sa.CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name=op.f('users_username_format')),
|
||||
sa.CheckConstraint('char_length(display_name) BETWEEN 1 AND 100', name=op.f('users_display_name_len')),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('users_pkey')),
|
||||
sa.UniqueConstraint('username', name=op.f('users_username_key'))
|
||||
)
|
||||
op.create_table('chat_members',
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('user_id', sa.Uuid(), nullable=False),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], name=op.f('chat_members_chat_id_fkey')),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('chat_members_user_id_fkey')),
|
||||
sa.PrimaryKeyConstraint('chat_id', 'user_id', name=op.f('chat_members_pkey'))
|
||||
)
|
||||
op.create_index(op.f('chat_members_user_id_chat_id_idx'), 'chat_members', ['user_id', 'chat_id'], unique=False)
|
||||
op.create_table('sessions',
|
||||
sa.Column('token_hash', sa.LargeBinary(), nullable=False),
|
||||
sa.Column('user_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('clock_timestamp()'), nullable=False),
|
||||
sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False),
|
||||
sa.CheckConstraint('expires_at > created_at', name=op.f('sessions_expiry_after_creation')),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('sessions_user_id_fkey'), ondelete='CASCADE'),
|
||||
sa.PrimaryKeyConstraint('token_hash', name=op.f('sessions_pkey'))
|
||||
)
|
||||
op.create_index(op.f('sessions_expires_at_idx'), 'sessions', ['expires_at'], unique=False)
|
||||
op.create_table('messages',
|
||||
sa.Column('id', sa.Uuid(), server_default=sa.text('gen_random_uuid()'), nullable=False),
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('sender_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('clock_timestamp()'), nullable=False),
|
||||
sa.Column('text', sa.Text(), nullable=False),
|
||||
sa.CheckConstraint('char_length(text) BETWEEN 1 AND 4000', name=op.f('messages_text_len')),
|
||||
sa.ForeignKeyConstraint(['chat_id', 'sender_id'], ['chat_members.chat_id', 'chat_members.user_id'], name='messages_sender_is_member'),
|
||||
sa.PrimaryKeyConstraint('id', name=op.f('messages_pkey'))
|
||||
)
|
||||
op.create_index('messages_chat_history_idx', 'messages', ['chat_id', 'created_at', 'id'], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index('messages_chat_history_idx', table_name='messages')
|
||||
op.drop_table('messages')
|
||||
op.drop_index(op.f('sessions_expires_at_idx'), table_name='sessions')
|
||||
op.drop_table('sessions')
|
||||
op.drop_index(op.f('chat_members_user_id_chat_id_idx'), table_name='chat_members')
|
||||
op.drop_table('chat_members')
|
||||
op.drop_table('users')
|
||||
op.drop_table('chats')
|
||||
# ### end Alembic commands ###
|
||||
@@ -0,0 +1,44 @@
|
||||
"""idempotency keys
|
||||
|
||||
Revision ID: 0002
|
||||
Revises: 0001
|
||||
Create Date: 2026-10-28 12:00:00.000000
|
||||
|
||||
Только добавляет таблицу: версия приложения до этой миграции её не использует и продолжает работать,
|
||||
поэтому миграцию можно применять перед rolling update (expand-фаза).
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision: str = '0002'
|
||||
down_revision: Union[str, Sequence[str], None] = '0001'
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
"""Upgrade schema."""
|
||||
op.create_table('idempotency_keys',
|
||||
sa.Column('user_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('chat_id', sa.Uuid(), nullable=False),
|
||||
sa.Column('key', sa.Text(), nullable=False),
|
||||
sa.Column('request_hash', sa.LargeBinary(), nullable=False),
|
||||
sa.Column('message_id', sa.Uuid(), nullable=True),
|
||||
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('clock_timestamp()'), nullable=False),
|
||||
sa.CheckConstraint("key ~ '^[A-Za-z0-9._:-]{1,128}$'", name=op.f('idempotency_keys_key_format')),
|
||||
sa.ForeignKeyConstraint(['user_id'], ['users.id'], name=op.f('idempotency_keys_user_id_fkey')),
|
||||
sa.ForeignKeyConstraint(['chat_id'], ['chats.id'], name=op.f('idempotency_keys_chat_id_fkey')),
|
||||
sa.ForeignKeyConstraint(['message_id'], ['messages.id'], name=op.f('idempotency_keys_message_id_fkey')),
|
||||
sa.PrimaryKeyConstraint('user_id', 'chat_id', 'key', name=op.f('idempotency_keys_pkey'))
|
||||
)
|
||||
op.create_index('idempotency_keys_created_at_idx', 'idempotency_keys', ['created_at'], unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
"""Downgrade schema."""
|
||||
op.drop_index('idempotency_keys_created_at_idx', table_name='idempotency_keys')
|
||||
op.drop_table('idempotency_keys')
|
||||
@@ -0,0 +1,9 @@
|
||||
# Все модели импортируются здесь, чтобы Base.metadata (и Alembic) видел все таблицы.
|
||||
from models.base import Base
|
||||
from models.chats_models import Chat, ChatMember
|
||||
from models.idempotency_models import IdempotencyKey
|
||||
from models.messages_models import Message
|
||||
from models.sessions_models import AuthSession
|
||||
from models.users_models import User
|
||||
|
||||
__all__ = ["Base", "User", "AuthSession", "Chat", "ChatMember", "Message", "IdempotencyKey"]
|
||||
@@ -0,0 +1,15 @@
|
||||
from sqlalchemy import MetaData
|
||||
from sqlalchemy.orm import DeclarativeBase
|
||||
|
||||
# Предсказуемые имена ограничений: по ним разбираем ошибки БД (users_username_key -> 409).
|
||||
NAMING_CONVENTION = {
|
||||
"pk": "%(table_name)s_pkey",
|
||||
"uq": "%(table_name)s_%(column_0_name)s_key",
|
||||
"ck": "%(table_name)s_%(constraint_name)s",
|
||||
"fk": "%(table_name)s_%(column_0_name)s_fkey",
|
||||
"ix": "%(table_name)s_%(column_0_N_name)s_idx",
|
||||
}
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
metadata = MetaData(naming_convention=NAMING_CONVENTION)
|
||||
@@ -0,0 +1,35 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from models.base import Base
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = "chats"
|
||||
__table_args__ = (
|
||||
CheckConstraint("char_length(title) BETWEEN 1 AND 100", name="title_len"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, server_default=text("gen_random_uuid()"))
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=text("clock_timestamp()"))
|
||||
|
||||
# selectin: участники всех чатов страницы грузятся одним запросом (без N+1).
|
||||
members: Mapped[list["ChatMember"]] = relationship(lazy="selectin", order_by="ChatMember.user_id")
|
||||
|
||||
@property
|
||||
def member_ids(self) -> list[uuid.UUID]:
|
||||
return [member.user_id for member in self.members]
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = "chat_members"
|
||||
__table_args__ = (
|
||||
Index(None, "user_id", "chat_id"), # для «моих чатов»
|
||||
)
|
||||
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("chats.id"), primary_key=True)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("users.id"), primary_key=True)
|
||||
@@ -0,0 +1,29 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, LargeBinary, Text, Uuid, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from models.base import Base
|
||||
|
||||
|
||||
class IdempotencyKey(Base):
|
||||
"""Результат отправки сообщения по Idempotency-Key.
|
||||
|
||||
Область уникальности по контракту: пользователь + метод + путь чата + ключ.
|
||||
Метод у маршрута один (POST), путь однозначно задаётся chat_id, поэтому ключ таблицы — (user_id, chat_id, key).
|
||||
"""
|
||||
__tablename__ = "idempotency_keys"
|
||||
__table_args__ = (
|
||||
CheckConstraint(r"key ~ '^[A-Za-z0-9._:-]{1,128}$'", name="key_format"),
|
||||
Index("idempotency_keys_created_at_idx", "created_at"),
|
||||
)
|
||||
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("users.id"), primary_key=True)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("chats.id"), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(Text, primary_key=True)
|
||||
# SHA-256 канонического тела запроса: тот же ключ с другим телом -> 409.
|
||||
request_hash: Mapped[bytes] = mapped_column(LargeBinary)
|
||||
# Заполняется в той же транзакции, что и вставка сообщения; видимой строки с NULL не бывает.
|
||||
message_id: Mapped[uuid.UUID | None] = mapped_column(Uuid, ForeignKey("messages.id"))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=text("clock_timestamp()"))
|
||||
@@ -0,0 +1,30 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKeyConstraint, Index, Text, Uuid
|
||||
# Псевдоним: столбец `text` ниже перекрыл бы функцию text().
|
||||
from sqlalchemy import text as sql_text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from models.base import Base
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "messages"
|
||||
__table_args__ = (
|
||||
CheckConstraint("char_length(text) BETWEEN 1 AND 4000", name="text_len"),
|
||||
# Отправитель обязан быть участником чата.
|
||||
ForeignKeyConstraint(
|
||||
["chat_id", "sender_id"],
|
||||
["chat_members.chat_id", "chat_members.user_id"],
|
||||
name="messages_sender_is_member",
|
||||
),
|
||||
Index("messages_chat_history_idx", "chat_id", "created_at", "id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, server_default=sql_text("gen_random_uuid()"))
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(Uuid)
|
||||
sender_id: Mapped[uuid.UUID] = mapped_column(Uuid)
|
||||
created_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), server_default=sql_text("clock_timestamp()"))
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
@@ -0,0 +1,25 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, ForeignKey, Index, LargeBinary, Uuid, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
|
||||
from models.base import Base
|
||||
from models.users_models import User
|
||||
|
||||
|
||||
class AuthSession(Base):
|
||||
"""Сессия входа (не путать с ORM-сессией SQLAlchemy)."""
|
||||
__tablename__ = "sessions"
|
||||
__table_args__ = (
|
||||
CheckConstraint("expires_at > created_at", name="expiry_after_creation"),
|
||||
Index(None, "expires_at"),
|
||||
)
|
||||
|
||||
# SHA-256 от токена, сам токен не храним.
|
||||
token_hash: Mapped[bytes] = mapped_column(LargeBinary, primary_key=True)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(Uuid, ForeignKey("users.id", ondelete="CASCADE"))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=text("clock_timestamp()"))
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
user: Mapped[User] = relationship(lazy="joined")
|
||||
@@ -0,0 +1,21 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import CheckConstraint, DateTime, Text, Uuid, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from models.base import Base
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
__table_args__ = (
|
||||
CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'", name="username_format"),
|
||||
CheckConstraint("char_length(display_name) BETWEEN 1 AND 100", name="display_name_len"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(Uuid, primary_key=True, server_default=text("gen_random_uuid()"))
|
||||
username: Mapped[str] = mapped_column(Text, unique=True)
|
||||
display_name: Mapped[str] = mapped_column(Text)
|
||||
password_hash: Mapped[str] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=text("clock_timestamp()"))
|
||||
@@ -0,0 +1,13 @@
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from schemas.users_schemas import UserResponse
|
||||
from utils.dates import UtcDatetime
|
||||
|
||||
|
||||
class SessionResponse(BaseModel):
|
||||
token: str
|
||||
token_type: Literal["Bearer"] = "Bearer"
|
||||
expires_at: UtcDatetime
|
||||
user: UserResponse
|
||||
@@ -0,0 +1,38 @@
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
|
||||
|
||||
from utils.dates import UtcDatetime
|
||||
|
||||
|
||||
class CreateChatRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
title: StrictStr = Field(min_length=1, max_length=100)
|
||||
member_ids: list[UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
@field_validator("title")
|
||||
def validate_title(cls, v):
|
||||
if not v.strip():
|
||||
raise ValueError("title must not be blank")
|
||||
return v
|
||||
|
||||
@field_validator("member_ids")
|
||||
def validate_member_ids(cls, v):
|
||||
if len(set(v)) != len(v):
|
||||
raise ValueError("member_ids must be unique")
|
||||
return v
|
||||
|
||||
|
||||
class ChatResponse(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: UUID
|
||||
title: str
|
||||
member_ids: list[UUID]
|
||||
created_at: UtcDatetime
|
||||
|
||||
|
||||
class ChatPageResponse(BaseModel):
|
||||
items: list[ChatResponse]
|
||||
next_cursor: str | None
|
||||
@@ -0,0 +1,11 @@
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class ErrorBody(BaseModel):
|
||||
code: str
|
||||
message: str
|
||||
request_id: str
|
||||
|
||||
|
||||
class ErrorResponse(BaseModel):
|
||||
error: ErrorBody
|
||||
@@ -0,0 +1,7 @@
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class HealthResponse(BaseModel):
|
||||
status: Literal["ok"]
|
||||
@@ -0,0 +1,33 @@
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
|
||||
|
||||
from utils.dates import UtcDatetime
|
||||
|
||||
|
||||
class CreateMessageRequest(BaseModel):
|
||||
# extra="forbid": клиент не может прислать sender_id.
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
text: StrictStr = Field(min_length=1, max_length=4000)
|
||||
|
||||
@field_validator("text")
|
||||
def validate_text(cls, v):
|
||||
if not v.strip():
|
||||
raise ValueError("text must not be blank")
|
||||
return v
|
||||
|
||||
|
||||
class MessageResponse(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: UUID
|
||||
chat_id: UUID
|
||||
sender_id: UUID
|
||||
text: str
|
||||
created_at: UtcDatetime
|
||||
|
||||
|
||||
class MessagePageResponse(BaseModel):
|
||||
items: list[MessageResponse]
|
||||
next_cursor: str | None
|
||||
@@ -0,0 +1,29 @@
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
|
||||
|
||||
from utils.dates import UtcDatetime
|
||||
|
||||
|
||||
class CreateUserRequest(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
username: StrictStr = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: StrictStr = Field(min_length=1, max_length=100)
|
||||
password: StrictStr = Field(min_length=12, max_length=128)
|
||||
|
||||
@field_validator("display_name")
|
||||
def validate_display_name(cls, v):
|
||||
if not v.strip():
|
||||
raise ValueError("display_name must not be blank")
|
||||
return v
|
||||
|
||||
|
||||
class UserResponse(BaseModel):
|
||||
# Собирается из ORM-объекта; password_hash сюда не попадает.
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: UtcDatetime
|
||||
@@ -0,0 +1,18 @@
|
||||
from datetime import datetime, timezone
|
||||
from typing import Annotated
|
||||
|
||||
from pydantic import PlainSerializer
|
||||
|
||||
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%fZ"
|
||||
|
||||
|
||||
def format_utc(value: datetime) -> str:
|
||||
return value.astimezone(timezone.utc).strftime(TIME_FORMAT)
|
||||
|
||||
|
||||
def parse_utc(value: str) -> datetime:
|
||||
return datetime.strptime(value, TIME_FORMAT).replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
# datetime, который в JSON превращается в строку вида 2026-09-01T12:00:00.123456Z.
|
||||
UtcDatetime = Annotated[datetime, PlainSerializer(format_utc, return_type=str)]
|
||||
@@ -0,0 +1,50 @@
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.exc import TimeoutError as PoolTimeoutError
|
||||
from starlette.exceptions import HTTPException
|
||||
|
||||
from core.logger import logger
|
||||
from schemas.error_schemas import ErrorBody, ErrorResponse
|
||||
|
||||
ERROR_CODES = {
|
||||
400: "invalid_request",
|
||||
401: "unauthorized",
|
||||
404: "not_found",
|
||||
409: "conflict",
|
||||
413: "payload_too_large",
|
||||
415: "unsupported_media_type",
|
||||
429: "rate_limited",
|
||||
500: "internal_error",
|
||||
503: "unavailable",
|
||||
}
|
||||
|
||||
|
||||
def error_response(request: Request, status_code: int, message: str,
|
||||
headers: dict[str, str] | None = None) -> JSONResponse:
|
||||
code = ERROR_CODES.get(status_code, "internal_error" if status_code >= 500 else "invalid_request")
|
||||
body = ErrorResponse(error=ErrorBody(code=code, message=message, request_id=request.state.request_id))
|
||||
return JSONResponse(status_code=status_code, content=body.model_dump(), headers=headers)
|
||||
|
||||
|
||||
def http_error_handler(request: Request, exc: HTTPException) -> JSONResponse:
|
||||
message = exc.detail if isinstance(exc.detail, str) and exc.detail else "Ошибка запроса"
|
||||
return error_response(request, exc.status_code, message, headers=exc.headers)
|
||||
|
||||
|
||||
def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
return error_response(request, 400, "Неверный запрос")
|
||||
|
||||
|
||||
# БД недоступна, statement_timeout или пул исчерпан -> 503 без подробностей.
|
||||
def database_error_handler(request: Request, exc: Exception) -> JSONResponse:
|
||||
logger.warning(f"Ошибка базы данных: {type(exc).__name__} request_id={request.state.request_id}")
|
||||
return error_response(request, 503, "База данных недоступна")
|
||||
|
||||
|
||||
def register_error_handlers(app: FastAPI):
|
||||
app.add_exception_handler(HTTPException, http_error_handler)
|
||||
app.add_exception_handler(RequestValidationError, validation_error_handler)
|
||||
app.add_exception_handler(OperationalError, database_error_handler)
|
||||
app.add_exception_handler(PoolTimeoutError, database_error_handler)
|
||||
@@ -0,0 +1,13 @@
|
||||
import threading
|
||||
|
||||
# Флаг живёт в отдельном модуле: main.py запускается как __main__ и ещё раз импортируется как "main",
|
||||
# а общее состояние должно быть одно.
|
||||
_draining = threading.Event()
|
||||
|
||||
|
||||
def start_draining() -> None:
|
||||
_draining.set()
|
||||
|
||||
|
||||
def is_draining() -> bool:
|
||||
return _draining.is_set()
|
||||
@@ -0,0 +1,30 @@
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from core.config import settings
|
||||
from core.logger import logger
|
||||
from utils.errors import error_response
|
||||
|
||||
|
||||
async def request_id_middleware(request: Request, call_next):
|
||||
request.state.request_id = str(uuid.uuid4())
|
||||
start = time.monotonic()
|
||||
|
||||
try:
|
||||
response = await call_next(request)
|
||||
except Exception:
|
||||
# Подробности пишем только в лог сервера, клиенту — общий текст.
|
||||
logger.exception(f"Непредвиденная ошибка request_id={request.state.request_id}")
|
||||
response = error_response(request, 500, "Внутренняя ошибка сервера")
|
||||
|
||||
response.headers["X-Request-Id"] = request.state.request_id
|
||||
# Какая реплика ответила (стабильно на всё время жизни процесса).
|
||||
response.headers["X-Instance-Id"] = settings.run.instance_id
|
||||
|
||||
# Тело запроса не логируем, чтобы текст сообщений не попадал в логи.
|
||||
duration_ms = (time.monotonic() - start) * 1000
|
||||
logger.info(f"{request.method} {request.url.path} {response.status_code} "
|
||||
f"{duration_ms:.1f}ms request_id={request.state.request_id} instance={settings.run.instance_id}")
|
||||
return response
|
||||
@@ -0,0 +1,35 @@
|
||||
import base64
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from utils.dates import format_utc, parse_utc
|
||||
|
||||
# Позиция в списке: (created_at, id) последнего выданного элемента.
|
||||
Position = tuple[datetime, uuid.UUID]
|
||||
|
||||
|
||||
def make_cursor(scope: str, created_at: datetime, item_id: uuid.UUID) -> str:
|
||||
data = json.dumps({"scope": scope, "created_at": format_utc(created_at), "id": str(item_id)})
|
||||
return base64.urlsafe_b64encode(data.encode()).decode()
|
||||
|
||||
|
||||
def read_cursor(scope: str, cursor: str | None) -> Position | None:
|
||||
if cursor is None:
|
||||
return None
|
||||
try:
|
||||
data = json.loads(base64.urlsafe_b64decode(cursor.encode()))
|
||||
if data["scope"] != scope:
|
||||
raise ValueError("cursor from another resource")
|
||||
return parse_utc(data["created_at"]), uuid.UUID(data["id"])
|
||||
except Exception:
|
||||
raise HTTPException(status_code=400, detail="Неверный cursor")
|
||||
|
||||
|
||||
def split_page(rows: list, limit: int, scope: str) -> tuple[list, str | None]:
|
||||
"""rows получены с LIMIT limit + 1: лишняя строка значит, что есть следующая страница."""
|
||||
page = rows[:limit]
|
||||
next_cursor = make_cursor(scope, page[-1].created_at, page[-1].id) if len(rows) > limit else None
|
||||
return page, next_cursor
|
||||
@@ -0,0 +1,65 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
import uuid
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
import jwt
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import InvalidHashError, VerificationError
|
||||
|
||||
from core.config import settings
|
||||
|
||||
# Argon2id: 64 MiB, 3 прохода (как по умолчанию), но 1 поток вместо 4: реплика ограничена 1 CPU,
|
||||
# и 4 потока только ждали бы квоту. Параметры записаны в самом хеше, старые хеши проверяются как раньше.
|
||||
password_hasher = PasswordHasher(time_cost=3, memory_cost=65536, parallelism=1)
|
||||
|
||||
# Проверяется, когда пользователя нет: время ответа такое же, как при неверном пароле.
|
||||
DUMMY_PASSWORD_HASH = password_hasher.hash(secrets.token_urlsafe(16))
|
||||
|
||||
|
||||
def hash_password(password: str) -> str:
|
||||
return password_hasher.hash(password)
|
||||
|
||||
|
||||
def verify_password(password_hash: str | None, password: str) -> bool:
|
||||
try:
|
||||
password_hasher.verify(password_hash or DUMMY_PASSWORD_HASH, password)
|
||||
return password_hash is not None
|
||||
except (VerificationError, InvalidHashError):
|
||||
return False
|
||||
|
||||
|
||||
JWT_SECRET_KEY = settings.jwt.read_secret_key()
|
||||
|
||||
|
||||
def create_access_token(user_id: uuid.UUID) -> tuple[str, datetime, datetime]:
|
||||
"""Возвращает (токен, время выдачи, время истечения)."""
|
||||
# Целые секунды, чтобы expires_at совпадал с exp в токене.
|
||||
issued_at = datetime.now(timezone.utc).replace(microsecond=0)
|
||||
expires_at = issued_at + timedelta(seconds=settings.jwt.session_ttl_seconds)
|
||||
payload = {
|
||||
"sub": str(user_id),
|
||||
"iat": issued_at,
|
||||
"exp": expires_at,
|
||||
"jti": secrets.token_urlsafe(32), # 256 бит случайности, токен всегда уникален
|
||||
}
|
||||
token = jwt.encode(payload, JWT_SECRET_KEY, algorithm=settings.jwt.algorithm)
|
||||
return token, issued_at, expires_at
|
||||
|
||||
|
||||
def decode_access_token(token: str) -> dict | None:
|
||||
"""None — подпись неверна, токен испорчен или истёк."""
|
||||
try:
|
||||
return jwt.decode(
|
||||
token,
|
||||
JWT_SECRET_KEY,
|
||||
algorithms=[settings.jwt.algorithm], # явно, чтобы не принять alg=none
|
||||
options={"require": ["sub", "exp", "iat", "jti"]},
|
||||
)
|
||||
except jwt.InvalidTokenError:
|
||||
return None
|
||||
|
||||
|
||||
def token_hash(token: str) -> bytes:
|
||||
# В БД хранится только хеш токена.
|
||||
return hashlib.sha256(token.encode()).digest()
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
# Запуск: docker compose up -d --build --wait (или scripts/init-secrets.* перед первым запуском)
|
||||
# Остановка: docker compose down — данные остаются в volume
|
||||
# Удаление данных намеренно: docker compose down -v
|
||||
#
|
||||
# Клиент -> lb (HAProxy, единственный опубликованный порт) -> api x N (без портов на хосте) -> postgres
|
||||
# Порядок старта: postgres (healthy) -> migrate (один раз, exit 0) -> api (healthy) -> lb (healthy).
|
||||
name: mtusi-lab3
|
||||
|
||||
x-app-build: &app-build
|
||||
context: .
|
||||
args:
|
||||
APP_VERSION: ${APP_VERSION:-dev}
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:17.10-alpine@sha256:979c4379dd698aba0b890599a6104e082035f98ef31d9b9291ec22f2b13059ca
|
||||
environment:
|
||||
POSTGRES_DB: messenger
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
secrets:
|
||||
- postgres_password
|
||||
volumes:
|
||||
- postgres_data:/var/lib/postgresql/data
|
||||
# Только loopback, для своих pytest-тестов из venv. API-реплики ходят в БД по внутренней сети.
|
||||
ports:
|
||||
- "127.0.0.1:${POSTGRES_HOST_PORT:-5432}:5432"
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 10
|
||||
start_period: 10s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
restart: unless-stopped
|
||||
|
||||
# Разовая задача: ОДИН контейнер применяет миграции и завершается с кодом 0.
|
||||
# Реплики API миграции не запускают, поэтому гонки при старте нет (плюс advisory lock в migrations/env.py).
|
||||
migrate:
|
||||
build: *app-build
|
||||
image: mtusi-messenger:lab3
|
||||
command: ["python", "-m", "alembic", "upgrade", "head"]
|
||||
# HEALTHCHECK образа проверяет HTTP-сервер API, а здесь сервера нет: иначе статус был бы unhealthy.
|
||||
healthcheck:
|
||||
disable: true
|
||||
environment: &app-env
|
||||
APP_CONFIG__DB__HOST: postgres
|
||||
APP_CONFIG__DB__PORT: "5432"
|
||||
APP_CONFIG__DB__NAME: messenger
|
||||
APP_CONFIG__DB__USER: messenger
|
||||
APP_CONFIG__DB__PASSWORD_FILE: /run/secrets/postgres_password
|
||||
APP_CONFIG__JWT__SECRET_FILE: /run/secrets/jwt_secret
|
||||
APP_CONFIG__JWT__SESSION_TTL_SECONDS: ${APP_CONFIG__JWT__SESSION_TTL_SECONDS:-3600}
|
||||
APP_CONFIG__LOGGING__LEVEL: ${APP_CONFIG__LOGGING__LEVEL:-INFO}
|
||||
secrets:
|
||||
- postgres_password
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 256M
|
||||
restart: "no"
|
||||
|
||||
# Масштабируется: API_REPLICAS (по умолчанию 2). Нет container_name и нет published ports.
|
||||
# Локального состояния нет: пользователи, чаты, сообщения, сессии и ключи идемпотентности лежат в PostgreSQL.
|
||||
api:
|
||||
build: *app-build
|
||||
image: mtusi-messenger:lab3
|
||||
# Без env_file: в .env адреса для запуска из venv, а не из контейнера.
|
||||
environment: *app-env
|
||||
secrets:
|
||||
- postgres_password
|
||||
- jwt_secret
|
||||
expose:
|
||||
- "8080"
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
deploy:
|
||||
replicas: ${API_REPLICAS:-2}
|
||||
resources:
|
||||
limits:
|
||||
# Argon2id (64 MiB на каждый одновременный хеш) — самое тяжёлое место: на 0.5 CPU вход занимал 1.2–1.7 с.
|
||||
cpus: "1.0"
|
||||
memory: 512M
|
||||
pids: 200
|
||||
# SIGTERM: drain_seconds (3) + graceful_timeout (5) = 8 с, запас до SIGKILL.
|
||||
stop_grace_period: 15s
|
||||
restart: unless-stopped
|
||||
|
||||
lb:
|
||||
image: haproxy:3.0.29-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6
|
||||
volumes:
|
||||
- ./infra/haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro
|
||||
ports:
|
||||
- "127.0.0.1:8080:8080"
|
||||
depends_on:
|
||||
api:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
# Через собственный порт LB: 200 только если есть хотя бы одна живая реплика.
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1:8080/health/live"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 5
|
||||
start_period: 5s
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: "0.5"
|
||||
memory: 128M
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password
|
||||
jwt_secret:
|
||||
file: ./secrets/jwt_secret
|
||||
@@ -0,0 +1,12 @@
|
||||
# 2026-10-29T10:53:54 docker compose config (resolved) + docker compose ps
|
||||
$ docker compose --env-file .env config --quiet
|
||||
exit=0
|
||||
api: image=mtusi-messenger:lab3 replicas=2 published_ports=[] container_name=None limits={'cpus': 1, 'memory': '536870912', 'pids': 200} secrets=['postgres_password', 'jwt_secret'] stop_grace_period=15s
|
||||
lb: image=haproxy:3.0.29-alpine@sha256:56b887da77428b7a6621e59e480cdbd330cc805c22d3cedb66ceea76ffdea2c6 replicas=None published_ports=['8080'] container_name=None limits={'cpus': 0.5, 'memory': '134217728'} secrets=[] stop_grace_period=None
|
||||
migrate: image=mtusi-messenger:lab3 replicas=None published_ports=[] container_name=None limits={'cpus': 0.5, 'memory': '268435456'} secrets=['postgres_password'] stop_grace_period=None
|
||||
postgres: image=postgres:17.10-alpine@sha256:979c4379dd698aba0b890599a6104e082035f98ef31d9b9291ec22f2b13059ca replicas=None published_ports=['5432'] container_name=None limits={'cpus': 1, 'memory': '536870912'} secrets=['postgres_password'] stop_grace_period=None
|
||||
$ docker compose ps
|
||||
mtusi-lab3-api-5 | Up 52 seconds (healthy) | 8080/tcp
|
||||
mtusi-lab3-api-6 | Up About a minute (healthy) | 8080/tcp
|
||||
mtusi-lab3-lb-1 | Up 13 minutes (healthy) | 127.0.0.1:8080->8080/tcp
|
||||
mtusi-lab3-postgres-1 | Up 25 minutes (healthy) | 127.0.0.1:5432->5432/tcp
|
||||
@@ -0,0 +1,33 @@
|
||||
# 2026-10-29T10:51:49 public failover test (final config), repeated until every replica was stopped once; restore between runs
|
||||
replicas: 3de2fdb570eb, 945116f913cc
|
||||
## run 1 — containers BEFORE
|
||||
945116f913cc mtusi-lab3-api-5 Up 36 seconds (healthy)
|
||||
3de2fdb570eb mtusi-lab3-api-6 Up About a minute (healthy)
|
||||
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 91, "failed_attempts": 0, "first_recovery_seconds": 5.459}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## run 1 — containers AFTER
|
||||
945116f913cc mtusi-lab3-api-5 Up 57 seconds (healthy)
|
||||
3de2fdb570eb mtusi-lab3-api-6 Exited (0) 10 seconds ago
|
||||
stopped by hook: 3de2fdb570eb
|
||||
$ scripts\restore-replicas.cmd
|
||||
exit=0
|
||||
|
||||
## run 2 — containers BEFORE
|
||||
945116f913cc mtusi-lab3-api-5 Up About a minute (healthy)
|
||||
3de2fdb570eb mtusi-lab3-api-6 Up 11 seconds (healthy)
|
||||
$ python tests/resilience.py --mode failover --action-script scripts/stop-one.cmd
|
||||
Executing explicit failover hook; hook output suppressed to avoid leaking secrets.
|
||||
{"observed_instances_before": 2, "successful_writes": 90, "failed_attempts": 0, "first_recovery_seconds": 4.847}
|
||||
PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0
|
||||
## run 2 — containers AFTER
|
||||
945116f913cc mtusi-lab3-api-5 Exited (0) 11 seconds ago
|
||||
3de2fdb570eb mtusi-lab3-api-6 Up 32 seconds (healthy)
|
||||
stopped by hook: 945116f913cc
|
||||
$ scripts\restore-replicas.cmd
|
||||
exit=0
|
||||
|
||||
replicas stopped at least once: 945116f913cc, 3de2fdb570eb of 2
|
||||
@@ -0,0 +1,700 @@
|
||||
{
|
||||
"method": "closed loop 8 workers: POST message (retry same Idempotency-Key on error) + GET page; client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next",
|
||||
"replicas": [
|
||||
"bb53391a07f1",
|
||||
"e51c99b5436b"
|
||||
],
|
||||
"warmup_seconds": 5,
|
||||
"seconds_down": 15,
|
||||
"runs": [
|
||||
{
|
||||
"stopped_replica": "bb53391a07f1",
|
||||
"baseline_before_stop": {
|
||||
"seconds": 7.62,
|
||||
"successful_requests": 366,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 48.0,
|
||||
"latency_ms": {
|
||||
"p50": 131.3,
|
||||
"p95": 390.6,
|
||||
"p99": 501.0
|
||||
}
|
||||
},
|
||||
"stop_to_restore": {
|
||||
"seconds": 19.67,
|
||||
"successful_requests": 526,
|
||||
"failed_requests": 6,
|
||||
"error_reasons": {
|
||||
"read: connection/TLS failure (RemoteDisconnected)": 1,
|
||||
"write: connection/TLS failure (RemoteDisconnected)": 5
|
||||
},
|
||||
"error_ratio": 0.0113,
|
||||
"successful_rps": 26.7,
|
||||
"latency_ms": {
|
||||
"p50": 276.3,
|
||||
"p95": 573.1,
|
||||
"p99": 693.5
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 27.91,
|
||||
"successful_requests": 913,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 32.7,
|
||||
"latency_ms": {
|
||||
"p50": 239.9,
|
||||
"p95": 516.6,
|
||||
"p99": 684.7
|
||||
}
|
||||
},
|
||||
"docker_stop_seconds": 4.67,
|
||||
"last_error_seconds_after_stop": 1.55,
|
||||
"first_success_on_other_replica_seconds": 0.0,
|
||||
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
||||
"seconds_until_replica_back_in_rotation": 19.5,
|
||||
"instances_seen": [
|
||||
"bb53391a07f1",
|
||||
"e51c99b5436b"
|
||||
],
|
||||
"resource_snapshot_under_load": [
|
||||
"mtusi-lab3-api-1: cpu 39.24%, mem 80.89MiB / 512MiB",
|
||||
"mtusi-lab3-api-2: cpu 102.77%, mem 83.61MiB / 512MiB",
|
||||
"mtusi-lab3-lb-1: cpu 9.63%, mem 14.8MiB / 128MiB",
|
||||
"mtusi-lab3-postgres-1: cpu 36.60%, mem 55.37MiB / 512MiB"
|
||||
],
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 37,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 51,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 63,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 60,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 30,
|
||||
"errors": 6
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 17,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 23,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 20,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 23,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 18,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 17,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 20,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 47,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 37,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 43,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 58,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 60,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 62,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 9,
|
||||
"errors": 0
|
||||
}
|
||||
],
|
||||
"acked_messages": 903,
|
||||
"messages_in_history": 903,
|
||||
"lost_acked_messages": 0,
|
||||
"duplicate_messages": 0
|
||||
},
|
||||
{
|
||||
"stopped_replica": "e51c99b5436b",
|
||||
"baseline_before_stop": {
|
||||
"seconds": 7.7,
|
||||
"successful_requests": 320,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 41.6,
|
||||
"latency_ms": {
|
||||
"p50": 140.9,
|
||||
"p95": 485.4,
|
||||
"p99": 605.6
|
||||
}
|
||||
},
|
||||
"stop_to_restore": {
|
||||
"seconds": 19.39,
|
||||
"successful_requests": 510,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 26.3,
|
||||
"latency_ms": {
|
||||
"p50": 289.3,
|
||||
"p95": 540.3,
|
||||
"p99": 666.9
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 27.92,
|
||||
"successful_requests": 976,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 35.0,
|
||||
"latency_ms": {
|
||||
"p50": 228.7,
|
||||
"p95": 493.7,
|
||||
"p99": 661.6
|
||||
}
|
||||
},
|
||||
"docker_stop_seconds": 4.39,
|
||||
"last_error_seconds_after_stop": 0.0,
|
||||
"first_success_on_other_replica_seconds": 0.09,
|
||||
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
||||
"seconds_until_replica_back_in_rotation": 19.75,
|
||||
"instances_seen": [
|
||||
"bb53391a07f1",
|
||||
"e51c99b5436b"
|
||||
],
|
||||
"resource_snapshot_under_load": [
|
||||
"mtusi-lab3-api-1: cpu 71.15%, mem 83.38MiB / 512MiB",
|
||||
"mtusi-lab3-api-2: cpu 35.64%, mem 81.46MiB / 512MiB",
|
||||
"mtusi-lab3-lb-1: cpu 9.00%, mem 15.11MiB / 128MiB",
|
||||
"mtusi-lab3-postgres-1: cpu 46.72%, mem 56.47MiB / 512MiB"
|
||||
],
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 18,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 43,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 60,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 59,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 47,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 23,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 13,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 18,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 17,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 14,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 58,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 51,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 58,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 60,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 1,
|
||||
"errors": 0
|
||||
}
|
||||
],
|
||||
"acked_messages": 903,
|
||||
"messages_in_history": 1806,
|
||||
"lost_acked_messages": 0,
|
||||
"duplicate_messages": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,702 @@
|
||||
{
|
||||
"method": "closed loop 8 workers: POST message (retry same Idempotency-Key on error) + GET page; client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next",
|
||||
"replicas": [
|
||||
"3de2fdb570eb",
|
||||
"945116f913cc"
|
||||
],
|
||||
"warmup_seconds": 5,
|
||||
"seconds_down": 15,
|
||||
"runs": [
|
||||
{
|
||||
"stopped_replica": "3de2fdb570eb",
|
||||
"baseline_before_stop": {
|
||||
"seconds": 7.72,
|
||||
"successful_requests": 359,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 46.5,
|
||||
"latency_ms": {
|
||||
"p50": 136.4,
|
||||
"p95": 433.5,
|
||||
"p99": 628.8
|
||||
}
|
||||
},
|
||||
"stop_to_restore": {
|
||||
"seconds": 19.38,
|
||||
"successful_requests": 538,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 27.8,
|
||||
"latency_ms": {
|
||||
"p50": 281.4,
|
||||
"p95": 515.9,
|
||||
"p99": 694.2
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 28.26,
|
||||
"successful_requests": 925,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 32.7,
|
||||
"latency_ms": {
|
||||
"p50": 244.6,
|
||||
"p95": 537.0,
|
||||
"p99": 698.6
|
||||
}
|
||||
},
|
||||
"docker_stop_seconds": 4.38,
|
||||
"last_error_seconds_after_stop": 0.0,
|
||||
"first_success_on_other_replica_seconds": 0.03,
|
||||
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
||||
"seconds_until_replica_back_in_rotation": 20.11,
|
||||
"instances_seen": [
|
||||
"3de2fdb570eb",
|
||||
"945116f913cc"
|
||||
],
|
||||
"resource_snapshot_under_load": [
|
||||
"mtusi-lab3-api-5: cpu 99.82%, mem 80.75MiB / 512MiB",
|
||||
"mtusi-lab3-api-6: cpu 46.77%, mem 81.65MiB / 512MiB",
|
||||
"mtusi-lab3-lb-1: cpu 7.09%, mem 17.03MiB / 128MiB",
|
||||
"mtusi-lab3-postgres-1: cpu 25.45%, mem 64.93MiB / 512MiB"
|
||||
],
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 58,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 60,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 59,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 60,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 41,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 56,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 23,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 17,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 17,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 18,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 17,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 17,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 14,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 19,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 50,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 36,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 59,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 56,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 20,
|
||||
"errors": 0
|
||||
}
|
||||
],
|
||||
"acked_messages": 911,
|
||||
"messages_in_history": 911,
|
||||
"lost_acked_messages": 0,
|
||||
"duplicate_messages": 0
|
||||
},
|
||||
{
|
||||
"stopped_replica": "945116f913cc",
|
||||
"baseline_before_stop": {
|
||||
"seconds": 7.66,
|
||||
"successful_requests": 319,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 41.6,
|
||||
"latency_ms": {
|
||||
"p50": 139.5,
|
||||
"p95": 516.2,
|
||||
"p99": 694.5
|
||||
}
|
||||
},
|
||||
"stop_to_restore": {
|
||||
"seconds": 19.29,
|
||||
"successful_requests": 507,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 26.3,
|
||||
"latency_ms": {
|
||||
"p50": 289.9,
|
||||
"p95": 578.7,
|
||||
"p99": 688.3
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 29.42,
|
||||
"successful_requests": 962,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 32.7,
|
||||
"latency_ms": {
|
||||
"p50": 239.1,
|
||||
"p95": 560.3,
|
||||
"p99": 699.2
|
||||
}
|
||||
},
|
||||
"docker_stop_seconds": 4.29,
|
||||
"last_error_seconds_after_stop": 0.0,
|
||||
"first_success_on_other_replica_seconds": 0.06,
|
||||
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
||||
"seconds_until_replica_back_in_rotation": 21.19,
|
||||
"instances_seen": [
|
||||
"3de2fdb570eb",
|
||||
"945116f913cc"
|
||||
],
|
||||
"resource_snapshot_under_load": [
|
||||
"mtusi-lab3-api-5: cpu 101.75%, mem 94.56MiB / 512MiB",
|
||||
"mtusi-lab3-api-6: cpu 68.81%, mem 81.32MiB / 512MiB",
|
||||
"mtusi-lab3-lb-1: cpu 15.13%, mem 17.34MiB / 128MiB",
|
||||
"mtusi-lab3-postgres-1: cpu 34.54%, mem 63.73MiB / 512MiB"
|
||||
],
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 50,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 59,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 59,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 43,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 23,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 20,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 18,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 19,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 55,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 56,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 41,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 63,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 56,
|
||||
"ok": 20,
|
||||
"errors": 0
|
||||
}
|
||||
],
|
||||
"acked_messages": 894,
|
||||
"messages_in_history": 1805,
|
||||
"lost_acked_messages": 0,
|
||||
"duplicate_messages": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,712 @@
|
||||
{
|
||||
"method": "closed loop 8 workers: POST message (retry same Idempotency-Key on error) + GET page; client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next",
|
||||
"replicas": [
|
||||
"bb53391a07f1",
|
||||
"e51c99b5436b"
|
||||
],
|
||||
"warmup_seconds": 5,
|
||||
"seconds_down": 15,
|
||||
"runs": [
|
||||
{
|
||||
"stopped_replica": "bb53391a07f1",
|
||||
"baseline_before_stop": {
|
||||
"seconds": 7.73,
|
||||
"successful_requests": 339,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 43.8,
|
||||
"latency_ms": {
|
||||
"p50": 147.3,
|
||||
"p95": 398.0,
|
||||
"p99": 597.8
|
||||
}
|
||||
},
|
||||
"stop_to_restore": {
|
||||
"seconds": 19.62,
|
||||
"successful_requests": 467,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 23.8,
|
||||
"latency_ms": {
|
||||
"p50": 314.1,
|
||||
"p95": 602.1,
|
||||
"p99": 699.8
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 29.0,
|
||||
"successful_requests": 894,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 30.8,
|
||||
"latency_ms": {
|
||||
"p50": 253.7,
|
||||
"p95": 590.4,
|
||||
"p99": 704.4
|
||||
}
|
||||
},
|
||||
"docker_stop_seconds": 4.62,
|
||||
"last_error_seconds_after_stop": 0.0,
|
||||
"first_success_on_other_replica_seconds": 0.02,
|
||||
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
||||
"seconds_until_replica_back_in_rotation": 20.86,
|
||||
"instances_seen": [
|
||||
"bb53391a07f1",
|
||||
"e51c99b5436b"
|
||||
],
|
||||
"resource_snapshot_under_load": [
|
||||
"mtusi-lab3-api-1: cpu 90.61%, mem 81.91MiB / 512MiB",
|
||||
"mtusi-lab3-api-2: cpu 103.60%, mem 82.59MiB / 512MiB",
|
||||
"mtusi-lab3-lb-1: cpu 15.84%, mem 13.01MiB / 128MiB",
|
||||
"mtusi-lab3-postgres-1: cpu 57.27%, mem 54.5MiB / 512MiB"
|
||||
],
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 34,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 45,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 45,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 50,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 23,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 18,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 18,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 13,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 18,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 24,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 19,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 51,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 56,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 37,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 56,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 56,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
}
|
||||
],
|
||||
"acked_messages": 850,
|
||||
"messages_in_history": 850,
|
||||
"lost_acked_messages": 0,
|
||||
"duplicate_messages": 0
|
||||
},
|
||||
{
|
||||
"stopped_replica": "e51c99b5436b",
|
||||
"baseline_before_stop": {
|
||||
"seconds": 7.62,
|
||||
"successful_requests": 295,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 38.7,
|
||||
"latency_ms": {
|
||||
"p50": 169.1,
|
||||
"p95": 520.0,
|
||||
"p99": 644.0
|
||||
}
|
||||
},
|
||||
"stop_to_restore": {
|
||||
"seconds": 19.96,
|
||||
"successful_requests": 465,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 23.3,
|
||||
"latency_ms": {
|
||||
"p50": 321.7,
|
||||
"p95": 602.9,
|
||||
"p99": 694.0
|
||||
}
|
||||
},
|
||||
"after_restore": {
|
||||
"seconds": 29.98,
|
||||
"successful_requests": 966,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 32.2,
|
||||
"latency_ms": {
|
||||
"p50": 237.7,
|
||||
"p95": 579.4,
|
||||
"p99": 767.1
|
||||
}
|
||||
},
|
||||
"docker_stop_seconds": 4.96,
|
||||
"last_error_seconds_after_stop": 0.0,
|
||||
"first_success_on_other_replica_seconds": 0.0,
|
||||
"requests_served_by_stopped_replica_after_stop_returned": 0,
|
||||
"seconds_until_replica_back_in_rotation": 21.78,
|
||||
"instances_seen": [
|
||||
"bb53391a07f1",
|
||||
"e51c99b5436b"
|
||||
],
|
||||
"resource_snapshot_under_load": [
|
||||
"mtusi-lab3-api-1: cpu 103.71%, mem 82.34MiB / 512MiB",
|
||||
"mtusi-lab3-api-2: cpu 89.86%, mem 90.22MiB / 512MiB",
|
||||
"mtusi-lab3-lb-1: cpu 24.15%, mem 13.37MiB / 128MiB",
|
||||
"mtusi-lab3-postgres-1: cpu 34.28%, mem 54.15MiB / 512MiB"
|
||||
],
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 19,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 52,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 24,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 19,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 17,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 24,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 13,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 13,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 24,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 20,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 15,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 22,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 24,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 13,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 13,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 16,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 24,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 39,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 51,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 50,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 34,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 47,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 55,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 56,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 57,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
}
|
||||
],
|
||||
"acked_messages": 863,
|
||||
"messages_in_history": 1713,
|
||||
"lost_acked_messages": 0,
|
||||
"duplicate_messages": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
{
|
||||
"method": "30 rounds; each round 16 simultaneous identical POSTs with one Idempotency-Key through the LB (separate connections, released by a barrier)",
|
||||
"rounds": 30,
|
||||
"parallel_requests_per_round": 16,
|
||||
"instances_seen": [
|
||||
"bb53391a07f1",
|
||||
"e51c99b5436b"
|
||||
],
|
||||
"all_rounds_single_message_id": true,
|
||||
"failed_requests": 0,
|
||||
"rounds_with_requests_on_2plus_replicas": 30,
|
||||
"messages_created_per_key_max": 1,
|
||||
"same_key_other_body_status": 409,
|
||||
"per_round": [
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
},
|
||||
{
|
||||
"responses_201": 16,
|
||||
"failures": 0,
|
||||
"distinct_message_ids": 1,
|
||||
"distinct_instances": 2
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-10-29T10:53:36
|
||||
20 x GET /api/v1/users/me with ONE bearer token (no re-login) through LB:
|
||||
X-Instance-Id 3de2fdb570eb -> 10 responses
|
||||
X-Instance-Id 945116f913cc -> 10 responses
|
||||
$ docker ps (api containers: short ID = hostname = X-Instance-Id; no published ports)
|
||||
945116f913cc mtusi-lab3-api-5 Up 36 seconds (healthy) ports=8080/tcp
|
||||
3de2fdb570eb mtusi-lab3-api-6 Up About a minute (healthy) ports=8080/tcp
|
||||
@@ -0,0 +1,19 @@
|
||||
# 2026-10-29T10:54:06 persistence: restart lb, api, postgres with volumes kept; same bearer token after restart
|
||||
## containers BEFORE
|
||||
/mtusi-lab3-api-5 running StartedAt=2026-10-29T07:53:03.171345935Z
|
||||
/mtusi-lab3-api-6 running StartedAt=2026-10-29T07:52:20.424136452Z
|
||||
/mtusi-lab3-migrate-1 exited StartedAt=2026-10-29T07:52:54.407649276Z
|
||||
/mtusi-lab3-lb-1 running StartedAt=2026-10-29T07:40:06.38472124Z
|
||||
/mtusi-lab3-postgres-1 running StartedAt=2026-10-29T07:28:35.676622955Z
|
||||
volume mtusi-lab3_postgres_data CreatedAt=2026-10-29T07:16:23Z
|
||||
$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd
|
||||
Executing explicit persistence hook; hook output suppressed to avoid leaking secrets.
|
||||
PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence.
|
||||
exit=0 (test took 41s)
|
||||
## containers AFTER (new StartedAt = really restarted, volume CreatedAt unchanged)
|
||||
/mtusi-lab3-migrate-1 exited StartedAt=2026-10-29T07:54:25.913938428Z
|
||||
/mtusi-lab3-api-5 running StartedAt=2026-10-29T07:54:35.437454812Z
|
||||
/mtusi-lab3-api-6 running StartedAt=2026-10-29T07:54:35.935961297Z
|
||||
/mtusi-lab3-lb-1 running StartedAt=2026-10-29T07:54:42.453081277Z
|
||||
/mtusi-lab3-postgres-1 running StartedAt=2026-10-29T07:54:19.666540466Z
|
||||
volume mtusi-lab3_postgres_data CreatedAt=2026-10-29T07:16:23Z
|
||||
@@ -0,0 +1,5 @@
|
||||
# 2026-10-29T10:29:42 student tests against local PostgreSQL (compose), DB messenger_test
|
||||
$ .venv\Scripts\python.exe -m pytest -q
|
||||
............................................ [100%]
|
||||
44 passed in 62.86s (0:01:02)
|
||||
exit=0
|
||||
@@ -0,0 +1,842 @@
|
||||
{
|
||||
"method": "closed loop 8 workers under rolling replacement: start N new replicas of the new build, wait until healthy and routed by LB, then stop+remove old replicas one at a time",
|
||||
"replicas_before": [
|
||||
"0a88ad74a16f",
|
||||
"d47dafe15036"
|
||||
],
|
||||
"replicas_after": [
|
||||
"3de2fdb570eb",
|
||||
"945116f913cc"
|
||||
],
|
||||
"new_replicas": [
|
||||
"3de2fdb570eb",
|
||||
"945116f913cc"
|
||||
],
|
||||
"old_image_ids": {
|
||||
"0a88ad74a16f": "sha256:5188aa6e7392cab4e9fbcf712a04a24e7f6e7f0e610676633cbaf67efc39cfc5",
|
||||
"d47dafe15036": "sha256:5188aa6e7392cab4e9fbcf712a04a24e7f6e7f0e610676633cbaf67efc39cfc5"
|
||||
},
|
||||
"new_image_ids": {
|
||||
"3de2fdb570eb": "sha256:35f4b43a6b981783ec1342221ba5f066f8e630dddc7ab2c424c8cbe208fa9921",
|
||||
"945116f913cc": "sha256:35f4b43a6b981783ec1342221ba5f066f8e630dddc7ab2c424c8cbe208fa9921"
|
||||
},
|
||||
"app_version_build_arg": "rolling-104629",
|
||||
"update_seconds": 137.13,
|
||||
"steps": [
|
||||
{
|
||||
"replaced": "0a88ad74a16f",
|
||||
"seconds_into_update": 118.09
|
||||
},
|
||||
{
|
||||
"replaced": "d47dafe15036",
|
||||
"seconds_into_update": 126.86
|
||||
}
|
||||
],
|
||||
"before_update": {
|
||||
"seconds": 5.29,
|
||||
"successful_requests": 183,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 34.6,
|
||||
"latency_ms": {
|
||||
"p50": 139.4,
|
||||
"p95": 509.9,
|
||||
"p99": 716.6
|
||||
}
|
||||
},
|
||||
"during_update": {
|
||||
"seconds": 137.13,
|
||||
"successful_requests": 5641,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 41.1,
|
||||
"latency_ms": {
|
||||
"p50": 134.9,
|
||||
"p95": 542.3,
|
||||
"p99": 696.1
|
||||
}
|
||||
},
|
||||
"after_update": {
|
||||
"seconds": 8.25,
|
||||
"successful_requests": 384,
|
||||
"failed_requests": 0,
|
||||
"error_reasons": {},
|
||||
"error_ratio": 0.0,
|
||||
"successful_rps": 46.5,
|
||||
"latency_ms": {
|
||||
"p50": 165.1,
|
||||
"p95": 409.6,
|
||||
"p99": 603.4
|
||||
}
|
||||
},
|
||||
"first_new_replica_response_seconds_into_update": 115.74,
|
||||
"last_old_replica_response_seconds_into_update": 128.45,
|
||||
"migrate_output_tail": [
|
||||
" Container mtusi-lab3-migrate-run-b1814ed4ca5d Created ",
|
||||
"INFO [alembic.runtime.migration] Context impl PostgresqlImpl.",
|
||||
"INFO [alembic.runtime.migration] Will assume transactional DDL."
|
||||
],
|
||||
"timeline_per_second": [
|
||||
{
|
||||
"t": 0,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 1,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 2,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 3,
|
||||
"ok": 45,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 4,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 5,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 6,
|
||||
"ok": 44,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 7,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 8,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 9,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 10,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 11,
|
||||
"ok": 44,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 12,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 13,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 14,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 15,
|
||||
"ok": 50,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 16,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 17,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 18,
|
||||
"ok": 43,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 19,
|
||||
"ok": 34,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 20,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 21,
|
||||
"ok": 43,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 22,
|
||||
"ok": 58,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 23,
|
||||
"ok": 45,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 24,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 25,
|
||||
"ok": 37,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 26,
|
||||
"ok": 52,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 27,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 28,
|
||||
"ok": 34,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 29,
|
||||
"ok": 55,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 30,
|
||||
"ok": 55,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 31,
|
||||
"ok": 36,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 32,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 33,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 34,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 35,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 36,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 37,
|
||||
"ok": 59,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 38,
|
||||
"ok": 42,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 39,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 40,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 41,
|
||||
"ok": 36,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 42,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 43,
|
||||
"ok": 34,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 44,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 45,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 46,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 47,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 48,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 49,
|
||||
"ok": 36,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 50,
|
||||
"ok": 27,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 51,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 52,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 53,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 54,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 55,
|
||||
"ok": 37,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 56,
|
||||
"ok": 41,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 57,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 58,
|
||||
"ok": 46,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 59,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 60,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 61,
|
||||
"ok": 34,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 62,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 63,
|
||||
"ok": 50,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 64,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 65,
|
||||
"ok": 45,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 66,
|
||||
"ok": 58,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 67,
|
||||
"ok": 55,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 68,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 69,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 70,
|
||||
"ok": 50,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 71,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 72,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 73,
|
||||
"ok": 60,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 74,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 75,
|
||||
"ok": 41,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 76,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 77,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 78,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 79,
|
||||
"ok": 26,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 80,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 81,
|
||||
"ok": 43,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 82,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 83,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 84,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 85,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 86,
|
||||
"ok": 35,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 87,
|
||||
"ok": 46,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 88,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 89,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 90,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 91,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 92,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 93,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 94,
|
||||
"ok": 41,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 95,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 96,
|
||||
"ok": 56,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 97,
|
||||
"ok": 41,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 98,
|
||||
"ok": 28,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 99,
|
||||
"ok": 42,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 100,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 101,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 102,
|
||||
"ok": 46,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 103,
|
||||
"ok": 56,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 104,
|
||||
"ok": 51,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 105,
|
||||
"ok": 25,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 106,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 107,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 108,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 109,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 110,
|
||||
"ok": 49,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 111,
|
||||
"ok": 46,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 112,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 113,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 114,
|
||||
"ok": 21,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 115,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 116,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 117,
|
||||
"ok": 43,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 118,
|
||||
"ok": 56,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 119,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 120,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 121,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 122,
|
||||
"ok": 65,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 123,
|
||||
"ok": 51,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 124,
|
||||
"ok": 66,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 125,
|
||||
"ok": 68,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 126,
|
||||
"ok": 36,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 127,
|
||||
"ok": 30,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 128,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 129,
|
||||
"ok": 34,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 130,
|
||||
"ok": 44,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 131,
|
||||
"ok": 72,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 132,
|
||||
"ok": 48,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 133,
|
||||
"ok": 38,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 134,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 135,
|
||||
"ok": 42,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 136,
|
||||
"ok": 54,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 137,
|
||||
"ok": 53,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 138,
|
||||
"ok": 47,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 139,
|
||||
"ok": 52,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 140,
|
||||
"ok": 33,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 141,
|
||||
"ok": 29,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 142,
|
||||
"ok": 31,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 143,
|
||||
"ok": 55,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 144,
|
||||
"ok": 57,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 145,
|
||||
"ok": 55,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 146,
|
||||
"ok": 52,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 147,
|
||||
"ok": 40,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 148,
|
||||
"ok": 32,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 149,
|
||||
"ok": 39,
|
||||
"errors": 0
|
||||
},
|
||||
{
|
||||
"t": 150,
|
||||
"ok": 34,
|
||||
"errors": 0
|
||||
}
|
||||
],
|
||||
"acked_messages": 3104,
|
||||
"messages_in_history": 3104,
|
||||
"lost_acked_messages": 0,
|
||||
"duplicate_messages": 0
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
# 2026-10-29T10:53:14 base commit 0ab3a34 (+ solution); stack: lb + api x2 (1 CPU/512M each) + postgres
|
||||
$ python scripts/check_contract.py
|
||||
PASS: offline skeleton structure, local refs and Python syntax; application NOT tested.
|
||||
exit=0
|
||||
$ python tests/smoke.py --base-url http://localhost:8080
|
||||
PASS: lab3 public API scenario; infrastructure and grading evidence remain separate.
|
||||
exit=0
|
||||
@@ -0,0 +1,66 @@
|
||||
# Единственная публичная точка входа. Реплики API находятся по DNS-имени сервиса `api`
|
||||
# (Docker DNS отдаёт IP всех реплик) и перечитываются каждые ~2 с, поэтому новые/пересозданные
|
||||
# контейнеры подхватываются без перезапуска LB.
|
||||
#
|
||||
# Обнаружение отказа: проверка GET /health/ready каждую 1 с (inter), 2 неудачи подряд (fall),
|
||||
# после первой неудачи период 0.5 с (fastinter) => реплика исключается за ~1.5 с, в худшем случае
|
||||
# за inter + fastinter + timeout check = 1 + 0.5 + 0.5 ≈ 2 с (оценка сверху fall*inter + timeout = 2.5 с).
|
||||
# Возврат в ротацию после 2 успешных проверок (rise), т.е. ~2 с после готовности реплики.
|
||||
# Пока реплика не исключена, соединение к ней обрывается/отказывает и запрос сразу
|
||||
# повторяется на другой реплике (retries + redispatch), поэтому клиент видит единицы ошибок.
|
||||
|
||||
global
|
||||
log stdout format raw local0
|
||||
maxconn 2000
|
||||
|
||||
defaults
|
||||
mode http
|
||||
log global
|
||||
option httplog
|
||||
option dontlognull
|
||||
timeout connect 1s
|
||||
# Больше statement_timeout БД (5 с): API успевает ответить 503 сам, а не получить обрыв от LB.
|
||||
timeout server 10s
|
||||
timeout client 15s
|
||||
timeout http-request 10s
|
||||
timeout http-keep-alive 5s
|
||||
timeout queue 5s
|
||||
timeout check 500ms
|
||||
retries 3
|
||||
# Повторять на другой реплике при каждой попытке (а не только на последней).
|
||||
option redispatch 1
|
||||
retry-on conn-failure empty-response response-timeout 502 503 504
|
||||
|
||||
resolvers docker
|
||||
nameserver dns 127.0.0.11:53
|
||||
accepted_payload_size 8192
|
||||
resolve_retries 3
|
||||
timeout resolve 1s
|
||||
timeout retry 1s
|
||||
hold valid 2s
|
||||
hold nx 2s
|
||||
hold other 2s
|
||||
hold refused 2s
|
||||
hold timeout 2s
|
||||
hold obsolete 2s
|
||||
|
||||
frontend public
|
||||
bind :8080
|
||||
default_backend api
|
||||
|
||||
backend api
|
||||
balance roundrobin
|
||||
# Никакой привязки клиента к реплике (sticky): состояние общее, сессии лежат в PostgreSQL.
|
||||
|
||||
# Повтор на уровне HTTP безопасен только для GET и для POST сообщения (защищён Idempotency-Key).
|
||||
# Остальные POST (регистрация, вход, создание чата) повторять нельзя: сервер мог уже выполнить запрос.
|
||||
acl is_message_post path_reg ^/api/v1/chats/[^/]+/messages$
|
||||
http-request disable-l7-retry if METH_POST !is_message_post
|
||||
|
||||
option httpchk
|
||||
http-check send meth GET uri /health/ready ver HTTP/1.1 hdr Host api
|
||||
http-check expect status 200
|
||||
# Без `on-marked-down shutdown-sessions`: при выводе реплики из ротации начатые запросы не обрываются,
|
||||
# реплика (drain) их дообслуживает. С этой опцией эксперимент давал 6–8 RemoteDisconnected на каждую остановку.
|
||||
default-server check inter 1s fastinter 500ms downinter 1s fall 2 rise 2 resolvers docker resolve-prefer ipv4 init-addr none
|
||||
server-template api 6 api:8080
|
||||
@@ -0,0 +1,5 @@
|
||||
[pytest]
|
||||
testpaths = student-tests
|
||||
pythonpath = app
|
||||
filterwarnings =
|
||||
ignore:Using .httpx. with .starlette.testclient. is deprecated
|
||||
@@ -0,0 +1,3 @@
|
||||
-r requirements.txt
|
||||
pytest==9.1.1
|
||||
httpx==0.28.1
|
||||
@@ -0,0 +1,26 @@
|
||||
fastapi==0.141.1
|
||||
starlette==1.7.0
|
||||
pydantic==2.13.5
|
||||
pydantic_core==2.46.5
|
||||
pydantic-settings==2.15.0
|
||||
python-dotenv==1.2.3
|
||||
uvicorn==0.54.0
|
||||
annotated-doc==0.0.5
|
||||
annotated-types==0.8.0
|
||||
anyio==4.15.1
|
||||
click==8.5.0
|
||||
h11==0.16.0
|
||||
idna==3.20
|
||||
typing-inspection==0.4.4
|
||||
typing_extensions==4.16.0
|
||||
SQLAlchemy==2.1.4
|
||||
psycopg==3.3.6
|
||||
psycopg-binary==3.3.6
|
||||
alembic==1.20.0
|
||||
Mako==1.4.3
|
||||
MarkupSafe==3.0.4
|
||||
argon2-cffi==25.1.0
|
||||
argon2-cffi-bindings==26.1.0
|
||||
cffi==2.1.1
|
||||
pycparser==3.11
|
||||
PyJWT==2.15.1
|
||||
@@ -0,0 +1,413 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Эксперименты уровня 4 lab3. Стенд (docker compose up -d --build --wait) должен быть уже запущен.
|
||||
|
||||
failover останавливает каждую API-реплику по очереди под нагрузкой, восстанавливает её и
|
||||
только потом берёт следующую; считает RPS, ошибки и время восстановления
|
||||
idempotency параллельно шлёт один Idempotency-Key через LB (разные реплики): должно быть ровно 1 сообщение
|
||||
rolling поочерёдная замена всех реплик на новую сборку под нагрузкой (migrate запускается один раз)
|
||||
|
||||
Результаты (без токенов) пишутся в evidence/*.json. Данные создаются синтетические и не удаляются.
|
||||
Использует публичный tests/client.py и docker CLI из каталога проекта.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from urllib.parse import quote
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / 'tests'))
|
||||
from client import Client, Failure, check # noqa: E402
|
||||
|
||||
PROJECT = 'mtusi-lab3'
|
||||
|
||||
|
||||
# ---------- docker ----------
|
||||
|
||||
def docker(*args, env=None, timeout=180, ok=True, with_stderr=False):
|
||||
proc = subprocess.run(['docker', *args], cwd=ROOT, capture_output=True, text=True, encoding='utf-8',
|
||||
errors='replace', timeout=timeout, env=env)
|
||||
if ok and proc.returncode != 0:
|
||||
raise Failure('docker ' + ' '.join(args[:3]) + ' failed: ' + proc.stderr.strip()[:300])
|
||||
return (proc.stdout + proc.stderr).strip() if with_stderr else proc.stdout.strip()
|
||||
|
||||
|
||||
def api_containers(all_states=False):
|
||||
"""Короткие ID контейнеров сервиса api. Совпадают с X-Instance-Id (hostname контейнера)."""
|
||||
args = ['ps', '-q', '--filter', 'label=com.docker.compose.project=' + PROJECT,
|
||||
'--filter', 'label=com.docker.compose.service=api']
|
||||
if all_states:
|
||||
args.insert(1, '-a')
|
||||
return sorted(x for x in docker(*args).split() if x)
|
||||
|
||||
|
||||
def wait_healthy(ids, timeout=90):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
states = {i: docker('inspect', '-f', '{{.State.Health.Status}}', i) for i in ids}
|
||||
if all(s == 'healthy' for s in states.values()):
|
||||
return
|
||||
check(time.monotonic() < deadline, 'containers did not become healthy: ' + str(states))
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def wait_instances(c, actor, expected, timeout=60):
|
||||
"""Ждать, пока через LB ответят все ожидаемые реплики (т.е. LB включил их в ротацию)."""
|
||||
deadline = time.monotonic() + timeout
|
||||
seen = set()
|
||||
while True:
|
||||
for _ in range(20):
|
||||
try:
|
||||
seen.add(c.call('GET', '/api/v1/users/me', actor=actor)['headers'].get('X-Instance-Id'))
|
||||
except Failure:
|
||||
pass
|
||||
if set(expected) <= seen:
|
||||
return
|
||||
check(time.monotonic() < deadline, 'LB did not route to all replicas: missing ' + str(set(expected) - seen))
|
||||
time.sleep(0.5)
|
||||
|
||||
|
||||
def resource_snapshot():
|
||||
out = docker('stats', '--no-stream', '--format', '{{.Name}}: cpu {{.CPUPerc}}, mem {{.MemUsage}}', ok=False)
|
||||
return [line for line in out.splitlines() if line]
|
||||
|
||||
|
||||
# ---------- нагрузка ----------
|
||||
|
||||
class Load:
|
||||
"""Замкнутая нагрузка: запись сообщения (с ретраями ТОГО ЖЕ ключа при ошибке) + чтение страницы истории."""
|
||||
|
||||
def __init__(self, base_url, ca_file, actor, chat, workers):
|
||||
self.base_url, self.ca_file, self.actor, self.chat, self.workers = base_url, ca_file, actor, chat, workers
|
||||
self.events, self.acked = [], [] # events: (t_start, t_end, kind, ok, instance, error_reason)
|
||||
self.stop_event = threading.Event()
|
||||
self.lock = threading.Lock()
|
||||
self.threads = []
|
||||
self.t0 = time.monotonic()
|
||||
|
||||
def now(self):
|
||||
return time.monotonic() - self.t0
|
||||
|
||||
def record(self, start, kind, ok, instance, error=None):
|
||||
# Причина без пути (в нём id чата): "expected [201], received 503" или "connection/TLS failure (...)".
|
||||
reason = str(error).split(': ', 1)[-1] if error else None
|
||||
with self.lock:
|
||||
self.events.append((start, self.now(), kind, ok, instance, reason))
|
||||
|
||||
def worker(self, wid):
|
||||
c = Client(self.base_url, self.ca_file, timeout=2)
|
||||
path = '/api/v1/chats/' + self.chat['id'] + '/messages'
|
||||
i = 0
|
||||
while not self.stop_event.is_set():
|
||||
key = uuid.uuid4().hex
|
||||
text = 'exp-%d-%d-%s' % (wid, i, key)
|
||||
i += 1
|
||||
for _ in range(10):
|
||||
start = self.now()
|
||||
try:
|
||||
r = c.message(self.actor, self.chat, text, key=key)
|
||||
self.record(start, 'write', True, r['headers'].get('X-Instance-Id'))
|
||||
with self.lock:
|
||||
self.acked.append(r['json'])
|
||||
break
|
||||
except Failure as exc:
|
||||
self.record(start, 'write', False, None, exc) # неопределённый результат: повтор с тем же ключом
|
||||
if self.stop_event.wait(0.1):
|
||||
break
|
||||
start = self.now()
|
||||
try:
|
||||
r = c.call('GET', path + '?limit=1', actor=self.actor, schema='MessagePage')
|
||||
self.record(start, 'read', True, r['headers'].get('X-Instance-Id'))
|
||||
except Failure as exc:
|
||||
self.record(start, 'read', False, None, exc)
|
||||
|
||||
def start(self):
|
||||
self.threads = [threading.Thread(target=self.worker, args=(n,), daemon=True) for n in range(self.workers)]
|
||||
for t in self.threads:
|
||||
t.start()
|
||||
|
||||
def stop(self):
|
||||
self.stop_event.set()
|
||||
for t in self.threads:
|
||||
t.join(timeout=15)
|
||||
|
||||
|
||||
def percentile(sorted_values, p):
|
||||
if not sorted_values:
|
||||
return None
|
||||
return round(1000 * sorted_values[max(0, math.ceil(p * len(sorted_values)) - 1)], 1)
|
||||
|
||||
|
||||
def window(events, lo, hi):
|
||||
"""Метрики по запросам, завершившимся в [lo, hi)."""
|
||||
sel = [e for e in events if lo <= e[1] < hi]
|
||||
ok = [e for e in sel if e[3]]
|
||||
errors = [e for e in sel if not e[3]]
|
||||
lat = sorted(e[1] - e[0] for e in ok)
|
||||
span = max(hi - lo, 1e-9)
|
||||
reasons = {}
|
||||
for e in errors:
|
||||
key = e[2] + ': ' + (e[5] or 'unknown')
|
||||
reasons[key] = reasons.get(key, 0) + 1
|
||||
return {
|
||||
'seconds': round(span, 2),
|
||||
'successful_requests': len(ok), 'failed_requests': len(errors), 'error_reasons': reasons,
|
||||
'error_ratio': round(len(errors) / len(sel), 4) if sel else None,
|
||||
'successful_rps': round(len(ok) / span, 1),
|
||||
'latency_ms': {'p50': percentile(lat, .5), 'p95': percentile(lat, .95), 'p99': percentile(lat, .99)},
|
||||
}
|
||||
|
||||
|
||||
def timeline(events, end):
|
||||
rows = []
|
||||
for s in range(int(end) + 1):
|
||||
sel = [e for e in events if s <= e[1] < s + 1]
|
||||
rows.append({'t': s, 'ok': sum(1 for e in sel if e[3]), 'errors': sum(1 for e in sel if not e[3])})
|
||||
return rows
|
||||
|
||||
|
||||
def last_error_after(events, moment):
|
||||
late = [e[1] for e in events if not e[3] and e[1] >= moment]
|
||||
return round(max(late) - moment, 2) if late else 0.0
|
||||
|
||||
|
||||
def history(c, actor, chat):
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100'
|
||||
result, seen = [], set()
|
||||
for _ in range(100000):
|
||||
page = c.call('GET', path, actor=actor, schema='MessagePage')['json']
|
||||
result.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return result
|
||||
check(cursor not in seen, 'cursor cycle')
|
||||
seen.add(cursor)
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(cursor, safe='')
|
||||
raise Failure('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def verify(c, actor, chat, acked):
|
||||
"""Подтверждённые сообщения не потеряны, а повторы с тем же ключом не создали дублей."""
|
||||
messages = history(c, actor, chat)
|
||||
by_id = {m['id']: m for m in messages}
|
||||
lost = sum(1 for a in acked if by_id.get(a['id']) != a)
|
||||
texts = [m['text'] for m in messages if m['text'].startswith('exp-')]
|
||||
duplicates = len(texts) - len(set(texts))
|
||||
return {'acked_messages': len(acked), 'messages_in_history': len(messages),
|
||||
'lost_acked_messages': lost, 'duplicate_messages': duplicates}
|
||||
|
||||
|
||||
def write_evidence(out_dir, name, data):
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
path = out_dir / name
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
print('written', path)
|
||||
|
||||
|
||||
def setup(args):
|
||||
c = Client(args.base_url, args.ca_file)
|
||||
alice, bob, eve, chat = c.bootstrap()
|
||||
return c, alice, bob, chat
|
||||
|
||||
|
||||
# ---------- эксперименты ----------
|
||||
|
||||
def run_failover(args):
|
||||
c, alice, bob, chat = setup(args)
|
||||
containers = api_containers()
|
||||
check(len(containers) >= 2, 'need at least 2 running API replicas, found %d' % len(containers))
|
||||
results = []
|
||||
for target in containers:
|
||||
# Предыдущая реплика уже восстановлена; убеждаемся, что в ротации все.
|
||||
docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300)
|
||||
wait_instances(c, alice, containers)
|
||||
load = Load(args.base_url, args.ca_file, alice, chat, args.workers)
|
||||
load.start()
|
||||
time.sleep(args.warmup)
|
||||
resources = resource_snapshot()
|
||||
t_stop = load.now()
|
||||
docker('stop', '-t', '8', target)
|
||||
stop_command_seconds = round(load.now() - t_stop, 2)
|
||||
time.sleep(args.after)
|
||||
t_restore = load.now()
|
||||
docker('compose', 'up', '-d', '--no-recreate', '--wait', 'api', 'lb', timeout=300)
|
||||
wait_instances(c, alice, containers)
|
||||
t_back = load.now()
|
||||
time.sleep(args.settle)
|
||||
load.stop()
|
||||
end = load.now()
|
||||
stats = verify(c, bob, chat, load.acked)
|
||||
after_stop = [e for e in load.events if e[1] >= t_stop and e[3]]
|
||||
first_other = min((e[1] for e in after_stop if e[4] != target and e[2] == 'write'), default=None)
|
||||
# Сколько запросов остановленная реплика обслужила после возврата `docker stop` и до восстановления: должно быть 0.
|
||||
served_by_target_after_stop = sum(1 for e in load.events
|
||||
if t_stop + stop_command_seconds <= e[0] < t_restore and e[3] and e[4] == target)
|
||||
results.append({
|
||||
'stopped_replica': target,
|
||||
'baseline_before_stop': window(load.events, 0, t_stop),
|
||||
'stop_to_restore': window(load.events, t_stop, t_restore),
|
||||
'after_restore': window(load.events, t_restore, end),
|
||||
'docker_stop_seconds': stop_command_seconds,
|
||||
'last_error_seconds_after_stop': last_error_after(load.events, t_stop),
|
||||
'first_success_on_other_replica_seconds': round(first_other - t_stop, 2) if first_other else None,
|
||||
'requests_served_by_stopped_replica_after_stop_returned': served_by_target_after_stop,
|
||||
# Включает `docker compose up --wait` (повторный migrate, start_period healthcheck) и rise 2 в LB.
|
||||
'seconds_until_replica_back_in_rotation': round(t_back - t_restore, 2),
|
||||
'instances_seen': sorted({e[4] for e in load.events if e[4]}),
|
||||
'resource_snapshot_under_load': resources,
|
||||
'timeline_per_second': timeline(load.events, end),
|
||||
**stats,
|
||||
})
|
||||
print('replica %s: errors=%d, last error %.2fs after stop, lost=%d, dup=%d' % (
|
||||
target, results[-1]['stop_to_restore']['failed_requests'],
|
||||
results[-1]['last_error_seconds_after_stop'], stats['lost_acked_messages'], stats['duplicate_messages']))
|
||||
write_evidence(Path(args.output), 'failover-sequential.json', {
|
||||
'method': 'closed loop %d workers: POST message (retry same Idempotency-Key on error) + GET page; '
|
||||
'client timeout 2s; each replica stopped with docker stop -t 8, restored, then the next' % args.workers,
|
||||
'replicas': containers, 'warmup_seconds': args.warmup, 'seconds_down': args.after, 'runs': results})
|
||||
for r in results:
|
||||
check(r['lost_acked_messages'] == 0 and r['duplicate_messages'] == 0, 'data integrity violated')
|
||||
|
||||
|
||||
def run_idempotency(args):
|
||||
c, alice, bob, chat = setup(args)
|
||||
path = '/api/v1/chats/' + chat['id'] + '/messages'
|
||||
instances, rounds = set(), []
|
||||
for _ in range(args.rounds):
|
||||
key = uuid.uuid4().hex
|
||||
text = 'idem-' + key
|
||||
barrier = threading.Barrier(args.parallel)
|
||||
|
||||
def send(_):
|
||||
local = Client(args.base_url, args.ca_file, timeout=10)
|
||||
barrier.wait()
|
||||
try:
|
||||
r = local.message(alice, chat, text, key=key)
|
||||
return r['status'], r['json']['id'], r['headers'].get('X-Instance-Id')
|
||||
except Failure as exc:
|
||||
return None, str(exc), None
|
||||
|
||||
with ThreadPoolExecutor(max_workers=args.parallel) as pool:
|
||||
answers = list(pool.map(send, range(args.parallel)))
|
||||
ok = [a for a in answers if a[0] == 201]
|
||||
instances.update(a[2] for a in ok)
|
||||
rounds.append({'responses_201': len(ok), 'failures': len(answers) - len(ok),
|
||||
'distinct_message_ids': len({a[1] for a in ok}),
|
||||
'distinct_instances': len({a[2] for a in ok})})
|
||||
messages = history(c, bob, chat)
|
||||
texts = [m['text'] for m in messages]
|
||||
created = [texts.count(m['text']) for m in messages if m['text'].startswith('idem-')]
|
||||
# Тот же ключ с другим телом -> 409.
|
||||
key = uuid.uuid4().hex
|
||||
c.message(alice, chat, 'idem-first-' + key, key=key)
|
||||
other_body = c.call('POST', path, 409, actor=alice, body={'text': 'idem-other-' + key},
|
||||
headers={'Idempotency-Key': key})
|
||||
report = {
|
||||
'method': '%d rounds; each round %d simultaneous identical POSTs with one Idempotency-Key through the LB '
|
||||
'(separate connections, released by a barrier)' % (args.rounds, args.parallel),
|
||||
'rounds': args.rounds, 'parallel_requests_per_round': args.parallel,
|
||||
'instances_seen': sorted(i for i in instances if i),
|
||||
'all_rounds_single_message_id': all(r['distinct_message_ids'] == 1 for r in rounds),
|
||||
'failed_requests': sum(r['failures'] for r in rounds),
|
||||
'rounds_with_requests_on_2plus_replicas': sum(1 for r in rounds if r['distinct_instances'] >= 2),
|
||||
'messages_created_per_key_max': max(created) if created else 0,
|
||||
'same_key_other_body_status': other_body['status'],
|
||||
'per_round': rounds,
|
||||
}
|
||||
write_evidence(Path(args.output), 'idempotency-parallel.json', report)
|
||||
check(report['all_rounds_single_message_id'], 'a key produced several message ids')
|
||||
check(report['messages_created_per_key_max'] == 1, 'a key created several messages')
|
||||
check(report['failed_requests'] == 0, 'some parallel requests failed')
|
||||
|
||||
|
||||
def run_rolling(args):
|
||||
c, alice, bob, chat = setup(args)
|
||||
old = api_containers()
|
||||
n = len(old)
|
||||
check(n >= 2, 'need at least 2 running API replicas')
|
||||
old_image = {i: docker('inspect', '-f', '{{.Image}}', i) for i in old}
|
||||
# Migrate запускается один раз, до замены; миграции только добавляют (expand), старые реплики их переживают.
|
||||
# Alembic пишет журнал в stderr.
|
||||
migrate_log = docker('compose', 'run', '--rm', 'migrate', timeout=300, with_stderr=True)
|
||||
load = Load(args.base_url, args.ca_file, alice, chat, args.workers)
|
||||
load.start()
|
||||
time.sleep(args.warmup)
|
||||
t_start = load.now()
|
||||
env = dict(os.environ, APP_VERSION='rolling-' + time.strftime('%H%M%S'))
|
||||
docker('compose', 'build', 'api', env=env, timeout=600)
|
||||
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % (2 * n), 'api',
|
||||
env=env, timeout=300)
|
||||
new = [i for i in api_containers() if i not in old]
|
||||
check(len(new) == n, 'expected %d new replicas, found %d' % (n, len(new)))
|
||||
wait_healthy(new)
|
||||
wait_instances(c, alice, new)
|
||||
steps = []
|
||||
for victim in old:
|
||||
t = load.now()
|
||||
docker('stop', '-t', '15', victim)
|
||||
docker('rm', victim)
|
||||
steps.append({'replaced': victim, 'seconds_into_update': round(t - t_start, 2)})
|
||||
time.sleep(args.step_pause)
|
||||
docker('compose', 'up', '-d', '--no-deps', '--no-recreate', '--scale', 'api=%d' % n, 'api', env=env, timeout=300)
|
||||
t_done = load.now()
|
||||
time.sleep(args.settle)
|
||||
load.stop()
|
||||
end = load.now()
|
||||
stats = verify(c, bob, chat, load.acked)
|
||||
seen_old = [e for e in load.events if e[3] and e[4] in old]
|
||||
seen_new = [e for e in load.events if e[3] and e[4] in new]
|
||||
report = {
|
||||
'method': 'closed loop %d workers under rolling replacement: start N new replicas of the new build, '
|
||||
'wait until healthy and routed by LB, then stop+remove old replicas one at a time' % args.workers,
|
||||
'replicas_before': old, 'replicas_after': api_containers(), 'new_replicas': new,
|
||||
'old_image_ids': old_image, 'new_image_ids': {i: docker('inspect', '-f', '{{.Image}}', i) for i in new},
|
||||
'app_version_build_arg': env['APP_VERSION'],
|
||||
'update_seconds': round(t_done - t_start, 2),
|
||||
'steps': steps,
|
||||
'before_update': window(load.events, 0, t_start),
|
||||
'during_update': window(load.events, t_start, t_done),
|
||||
'after_update': window(load.events, t_done, end),
|
||||
# Все моменты — секунды от начала обновления, как и steps[].seconds_into_update.
|
||||
'first_new_replica_response_seconds_into_update': round(min((e[1] for e in seen_new), default=t_start) - t_start, 2),
|
||||
'last_old_replica_response_seconds_into_update': round(max((e[1] for e in seen_old), default=t_start) - t_start, 2),
|
||||
'migrate_output_tail': migrate_log.splitlines()[-3:],
|
||||
'timeline_per_second': timeline(load.events, end),
|
||||
**stats,
|
||||
}
|
||||
write_evidence(Path(args.output), 'rolling-update.json', report)
|
||||
print('rolling: errors during update=%d, lost=%d, dup=%d' % (
|
||||
report['during_update']['failed_requests'], stats['lost_acked_messages'], stats['duplicate_messages']))
|
||||
check(stats['lost_acked_messages'] == 0 and stats['duplicate_messages'] == 0, 'data integrity violated')
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
p.add_argument('experiment', choices=['failover', 'idempotency', 'rolling'])
|
||||
p.add_argument('--base-url', default=None)
|
||||
p.add_argument('--ca-file', default=None)
|
||||
p.add_argument('--workers', type=int, default=8, help='closed-loop workers (failover, rolling)')
|
||||
p.add_argument('--warmup', type=float, default=5, help='seconds of load before the action')
|
||||
p.add_argument('--after', type=float, default=15, help='failover: seconds the replica stays down under load')
|
||||
p.add_argument('--settle', type=float, default=8, help='seconds of load after the action')
|
||||
p.add_argument('--step-pause', type=float, default=3, help='rolling: pause between replacing replicas')
|
||||
p.add_argument('--rounds', type=int, default=30, help='idempotency: number of keys')
|
||||
p.add_argument('--parallel', type=int, default=16, help='idempotency: simultaneous requests per key')
|
||||
p.add_argument('--output', default=str(ROOT / 'evidence'))
|
||||
args = p.parse_args()
|
||||
try:
|
||||
{'failover': run_failover, 'idempotency': run_idempotency, 'rolling': run_rolling}[args.experiment](args)
|
||||
except (Failure, RuntimeError, subprocess.TimeoutExpired) as exc:
|
||||
print('FAIL:', exc)
|
||||
raise SystemExit(1)
|
||||
print('PASS:', args.experiment)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,19 @@
|
||||
# Windows twin of scripts/init-secrets.sh. Existing files are kept.
|
||||
$ErrorActionPreference = 'Stop'
|
||||
Set-Location (Join-Path $PSScriptRoot '..')
|
||||
New-Item -ItemType Directory -Force secrets | Out-Null
|
||||
|
||||
function New-Secret($name, $bytes) {
|
||||
$path = Join-Path 'secrets' $name
|
||||
if ((Test-Path $path) -and (Get-Item $path).Length -gt 0) { "kept secrets/$name"; return }
|
||||
$buffer = New-Object byte[] $bytes
|
||||
[Security.Cryptography.RandomNumberGenerator]::Create().GetBytes($buffer)
|
||||
$value = [Convert]::ToBase64String($buffer) -replace '[+/=]', ''
|
||||
# ASCII, no BOM, no trailing newline.
|
||||
[IO.File]::WriteAllText((Join-Path (Get-Location) $path), $value, [Text.Encoding]::ASCII)
|
||||
"created secrets/$name"
|
||||
}
|
||||
|
||||
New-Secret 'postgres_password' 32
|
||||
New-Secret 'jwt_secret' 64
|
||||
if (-not (Test-Path .env)) { Copy-Item .env.example .env; 'created .env' }
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/bin/sh
|
||||
# Создать локальные секреты после clone. Существующие файлы не перезаписываются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p secrets
|
||||
|
||||
make_secret() {
|
||||
if [ ! -s "secrets/$1" ]; then
|
||||
head -c "$2" /dev/urandom | base64 | tr -d '\n+/=' > "secrets/$1"
|
||||
echo "created secrets/$1"
|
||||
else
|
||||
echo "kept secrets/$1"
|
||||
fi
|
||||
}
|
||||
|
||||
make_secret postgres_password 32
|
||||
make_secret jwt_secret 64
|
||||
[ -f .env ] || { cp .env.example .env; echo "created .env"; }
|
||||
@@ -0,0 +1,8 @@
|
||||
@echo off
|
||||
rem Windows twin of scripts/restart.sh: the test cannot execute .sh on Windows.
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
|
||||
docker compose stop lb api postgres || exit /b 1
|
||||
docker compose up -d --wait || exit /b 1
|
||||
docker compose ps --format "{{.Name}} {{.Status}}"
|
||||
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
# Hook для make test-persistence: перезапуск всего стенда (LB, API, PostgreSQL) с сохранением volumes.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
docker compose stop lb api postgres
|
||||
# Порядок: postgres (healthy) -> migrate (exit 0) -> api (healthy) -> lb (healthy).
|
||||
docker compose up -d --wait
|
||||
docker compose ps --format '{{.Name}} {{.Status}}'
|
||||
@@ -0,0 +1,7 @@
|
||||
@echo off
|
||||
rem Windows twin of scripts/restore-replicas.sh
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
|
||||
docker compose up -d --no-recreate --wait api lb || exit /b 1
|
||||
docker compose ps --format "{{.Name}} {{.Status}}" api lb
|
||||
@@ -0,0 +1,7 @@
|
||||
#!/bin/sh
|
||||
# Запустить остановленные реплики API (после stop-one) и дождаться готовности. Данные не затрагиваются.
|
||||
set -eu
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
docker compose up -d --no-recreate --wait api lb
|
||||
docker compose ps --format '{{.Name}} {{.Status}}' api lb
|
||||
@@ -0,0 +1,16 @@
|
||||
@echo off
|
||||
rem Windows twin of scripts/stop-one.sh: the test cannot execute .sh on Windows.
|
||||
setlocal
|
||||
cd /d "%~dp0.."
|
||||
|
||||
if "%TARGET_INSTANCE_ID%"=="" exit /b 2
|
||||
rem Только hex: значение подставляется в команду docker.
|
||||
echo %TARGET_INSTANCE_ID%|findstr /r /x "[0-9a-f][0-9a-f]*" >nul || exit /b 2
|
||||
|
||||
set "CONTAINER="
|
||||
for /f %%i in ('docker ps -q --filter "label=com.docker.compose.project=mtusi-lab3" --filter "label=com.docker.compose.service=api" --filter "id=%TARGET_INSTANCE_ID%"') do set "CONTAINER=%%i"
|
||||
if not defined CONTAINER exit /b 3
|
||||
|
||||
rem SIGTERM -> drain (3 s) -> exit; -t 8 keeps the hook inside the 10 s budget.
|
||||
docker stop -t 8 %CONTAINER% >nul || exit /b 1
|
||||
echo stopped %CONTAINER%
|
||||
@@ -0,0 +1,21 @@
|
||||
#!/bin/sh
|
||||
# Hook для make test-failover: остановить ОДНУ API-реплику и оставить её остановленной.
|
||||
# TARGET_INSTANCE_ID = X-Instance-Id реплики = hostname контейнера = первые 12 символов его ID.
|
||||
# БД, LB и volumes не трогаем. Вернуть реплику: scripts/restore-replicas.sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?test provides target API instance alias}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
""|*[!0-9a-f]*) echo "bad TARGET_INSTANCE_ID" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
# Только контейнер сервиса api нашего проекта: чужой/неизвестный ID не остановим.
|
||||
container=$(docker ps -q \
|
||||
--filter "label=com.docker.compose.project=mtusi-lab3" \
|
||||
--filter "label=com.docker.compose.service=api" \
|
||||
--filter "id=$TARGET_INSTANCE_ID")
|
||||
[ -n "$container" ] || { echo "no running api container for the instance" >&2; exit 3; }
|
||||
[ "$(printf '%s\n' "$container" | wc -l)" -eq 1 ] || { echo "ambiguous instance id" >&2; exit 4; }
|
||||
|
||||
# SIGTERM -> drain (3 с) -> завершение; -t 8: не дольше 8 с, чтобы уложиться в 10 с контракта теста.
|
||||
docker stop -t 8 "$container" >/dev/null
|
||||
echo "stopped $container"
|
||||
@@ -0,0 +1,108 @@
|
||||
import base64
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
# Отдельная тестовая база; задаётся до импорта приложения (настройки читаются при импорте).
|
||||
TEST_DB_NAME = "messenger_test"
|
||||
os.environ["APP_CONFIG__DB__NAME"] = TEST_DB_NAME
|
||||
|
||||
import pytest
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import create_engine, text
|
||||
|
||||
from core.db import build_url, engine
|
||||
from main import app
|
||||
|
||||
APP_DIR = Path(__file__).resolve().parents[1] / "app"
|
||||
PASSWORD = "Test-password-123"
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def test_database():
|
||||
# CREATE DATABASE нельзя выполнить в транзакции, поэтому AUTOCOMMIT.
|
||||
admin = create_engine(build_url().set(database="postgres"), isolation_level="AUTOCOMMIT")
|
||||
with admin.connect() as connection:
|
||||
exists = connection.scalar(text("SELECT 1 FROM pg_database WHERE datname = :name"), {"name": TEST_DB_NAME})
|
||||
if not exists:
|
||||
connection.execute(text(f'CREATE DATABASE "{TEST_DB_NAME}"'))
|
||||
admin.dispose()
|
||||
|
||||
command.upgrade(Config(str(APP_DIR / "alembic.ini")), "head")
|
||||
yield
|
||||
engine.dispose()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_database():
|
||||
with engine.begin() as connection:
|
||||
connection.execute(text("TRUNCATE users, sessions, chats, chat_members, messages, idempotency_keys"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(app)
|
||||
|
||||
|
||||
def basic(username, password):
|
||||
credential = base64.b64encode(f"{username}:{password}".encode()).decode()
|
||||
return {"Authorization": "Basic " + credential}
|
||||
|
||||
|
||||
def login(client, username, password=PASSWORD):
|
||||
response = client.post("/api/v1/auth/sessions", headers=basic(username, password))
|
||||
assert response.status_code == 201
|
||||
return response.json()
|
||||
|
||||
|
||||
def bearer(token):
|
||||
return {"Authorization": "Bearer " + token}
|
||||
|
||||
|
||||
def with_key(headers, key=None):
|
||||
"""Добавить обязательный с lab3 Idempotency-Key (по умолчанию — новый уникальный)."""
|
||||
return {**headers, "Idempotency-Key": key or uuid.uuid4().hex}
|
||||
|
||||
|
||||
def create_user(client, username):
|
||||
"""Регистрация + вход; заголовок авторизации — в поле "auth"."""
|
||||
response = client.post("/api/v1/users", json={"username": username, "display_name": "Имя " + username,
|
||||
"password": PASSWORD})
|
||||
assert response.status_code == 201
|
||||
user = response.json()
|
||||
user["token"] = login(client, username)["token"]
|
||||
user["auth"] = bearer(user["token"])
|
||||
return user
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def alice(client):
|
||||
return create_user(client, "alice")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bob(client):
|
||||
return create_user(client, "bob")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def eve(client):
|
||||
return create_user(client, "eve")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chat(client, alice, bob):
|
||||
"""Чат alice и bob; eve в него не входит."""
|
||||
response = client.post("/api/v1/chats", headers=alice["auth"],
|
||||
json={"title": "Тестовый чат", "member_ids": [alice["id"], bob["id"]]})
|
||||
assert response.status_code == 201
|
||||
return response.json()
|
||||
|
||||
|
||||
def assert_error(response, status_code, code):
|
||||
assert response.status_code == status_code
|
||||
body = response.json()
|
||||
assert body["error"]["code"] == code
|
||||
assert body["error"]["request_id"] == response.headers["X-Request-Id"]
|
||||
@@ -0,0 +1,106 @@
|
||||
import time
|
||||
|
||||
import jwt
|
||||
from sqlalchemy import text
|
||||
|
||||
from conftest import PASSWORD, assert_error, basic, bearer, login
|
||||
from core.config import settings
|
||||
from core.db import engine
|
||||
|
||||
|
||||
def assert_challenge(response, scheme):
|
||||
assert response.headers["WWW-Authenticate"].startswith(scheme + ' realm="messenger"')
|
||||
|
||||
|
||||
def test_login_returns_session(client, alice):
|
||||
session = login(client, "alice")
|
||||
assert session["token_type"] == "Bearer"
|
||||
assert len(session["token"]) >= 32
|
||||
assert session["expires_at"].endswith("Z")
|
||||
assert session["user"]["id"] == alice["id"]
|
||||
assert "password" not in session["user"] and "password_hash" not in session["user"]
|
||||
assert login(client, "alice")["token"] != session["token"]
|
||||
|
||||
|
||||
def test_jwt_payload_contains_sub_exp_jti(client, alice):
|
||||
payload = jwt.decode(alice["token"], options={"verify_signature": False})
|
||||
assert payload["sub"] == alice["id"]
|
||||
assert payload["exp"] - payload["iat"] == settings.jwt.session_ttl_seconds
|
||||
assert len(payload["jti"]) >= 43
|
||||
|
||||
|
||||
def test_wrong_password_and_unknown_user_give_same_401(client, alice):
|
||||
wrong = client.post("/api/v1/auth/sessions", headers=basic("alice", "Wrong-password-1"))
|
||||
unknown = client.post("/api/v1/auth/sessions", headers=basic("nobody", PASSWORD))
|
||||
for response in (wrong, unknown):
|
||||
assert_error(response, 401, "unauthorized")
|
||||
assert_challenge(response, "Basic")
|
||||
assert 'charset="UTF-8"' in response.headers["WWW-Authenticate"]
|
||||
assert wrong.json()["error"]["message"] == unknown.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_malformed_basic_returns_401(client):
|
||||
for header in ("", "Basic", "Basic !!!not-base64!!!", "Bearer abc", "Basic " + "YWxpY2U="): # последний — без ':'
|
||||
response = client.post("/api/v1/auth/sessions", headers={"Authorization": header})
|
||||
assert_error(response, 401, "unauthorized")
|
||||
assert_challenge(response, "Basic")
|
||||
|
||||
|
||||
def test_login_with_body_returns_400_and_creates_no_session(client, alice):
|
||||
with engine.connect() as connection:
|
||||
before = connection.scalar(text("SELECT count(*) FROM sessions"))
|
||||
for kwargs in ({"json": {"unexpected": True}}, {"json": {}}, {"content": b"x"}):
|
||||
response = client.post("/api/v1/auth/sessions", headers=basic("alice", PASSWORD), **kwargs)
|
||||
assert_error(response, 400, "invalid_request")
|
||||
with engine.connect() as connection:
|
||||
assert connection.scalar(text("SELECT count(*) FROM sessions")) == before
|
||||
# Без тела вход по-прежнему работает.
|
||||
assert login(client, "alice")["user"]["id"] == alice["id"]
|
||||
|
||||
|
||||
def test_password_with_colon_and_unicode(client):
|
||||
password = "пароль:с:двоеточиями 👋"
|
||||
client.post("/api/v1/users", json={"username": "colon", "display_name": "C", "password": password})
|
||||
assert login(client, "colon", password)["user"]["username"] == "colon"
|
||||
|
||||
|
||||
def test_protected_endpoint_requires_bearer(client, alice):
|
||||
cases = [{}, basic("alice", PASSWORD), bearer("not-a-jwt"), {"Authorization": "Bearer"}]
|
||||
for headers in cases:
|
||||
response = client.get("/api/v1/users/me", headers=headers)
|
||||
assert_error(response, 401, "unauthorized")
|
||||
assert_challenge(response, "Bearer")
|
||||
|
||||
|
||||
def test_x_user_id_does_not_override_token(client, alice, eve):
|
||||
response = client.get("/api/v1/users/me", headers={**alice["auth"], "X-User-Id": eve["id"]})
|
||||
assert response.json()["id"] == alice["id"]
|
||||
|
||||
|
||||
def test_forged_token_is_rejected(client, alice):
|
||||
payload = jwt.decode(alice["token"], options={"verify_signature": False})
|
||||
forged = jwt.encode(payload, "attacker-key-attacker-key-attacker-key!!", algorithm="HS256")
|
||||
assert_error(client.get("/api/v1/users/me", headers=bearer(forged)), 401, "unauthorized")
|
||||
unsigned = jwt.encode(payload, None, algorithm="none")
|
||||
assert_error(client.get("/api/v1/users/me", headers=bearer(unsigned)), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_logout_revokes_only_current_token(client, alice):
|
||||
second = login(client, "alice")["token"]
|
||||
response = client.delete("/api/v1/auth/sessions/current", headers=alice["auth"])
|
||||
assert response.status_code == 204
|
||||
assert response.content == b""
|
||||
assert_error(client.get("/api/v1/users/me", headers=alice["auth"]), 401, "unauthorized")
|
||||
assert_error(client.delete("/api/v1/auth/sessions/current", headers=alice["auth"]), 401, "unauthorized")
|
||||
# Другая сессия того же пользователя продолжает работать.
|
||||
assert client.get("/api/v1/users/me", headers=bearer(second)).status_code == 200
|
||||
|
||||
|
||||
def test_session_expires(client, alice, monkeypatch):
|
||||
monkeypatch.setattr(settings.jwt, "session_ttl_seconds", 1)
|
||||
token = login(client, "alice")["token"]
|
||||
assert client.get("/api/v1/users/me", headers=bearer(token)).status_code == 200
|
||||
time.sleep(2.1)
|
||||
response = client.get("/api/v1/users/me", headers=bearer(token))
|
||||
assert_error(response, 401, "unauthorized")
|
||||
assert_challenge(response, "Bearer")
|
||||
@@ -0,0 +1,54 @@
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from conftest import assert_error, with_key
|
||||
from core.db import engine
|
||||
|
||||
|
||||
def count_rows(table):
|
||||
with engine.connect() as connection:
|
||||
return connection.scalar(text(f"SELECT count(*) FROM {table}"))
|
||||
|
||||
|
||||
def test_unknown_member_returns_400_and_nothing_is_saved(client, alice):
|
||||
response = client.post("/api/v1/chats", headers=alice["auth"],
|
||||
json={"title": "С призраком", "member_ids": [alice["id"], str(uuid.uuid4())]})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
assert count_rows("chats") == 0
|
||||
assert count_rows("chat_members") == 0
|
||||
|
||||
|
||||
def test_created_chat_is_stored_with_members(client, chat, alice, bob):
|
||||
assert sorted(chat["member_ids"]) == sorted([alice["id"], bob["id"]])
|
||||
assert count_rows("chats") == 1
|
||||
assert count_rows("chat_members") == 2
|
||||
again = client.get("/api/v1/chats/" + chat["id"], headers=bob["auth"]).json()
|
||||
assert again == chat
|
||||
|
||||
|
||||
def test_foreign_chat_returns_404(client, chat, eve):
|
||||
headers = eve["auth"]
|
||||
path = "/api/v1/chats/" + chat["id"]
|
||||
|
||||
assert_error(client.get(path, headers=headers), 404, "not_found")
|
||||
assert_error(client.get(path + "/messages", headers=headers), 404, "not_found")
|
||||
assert_error(client.post(path + "/messages", headers=with_key(headers), json={"text": "чужое"}), 404, "not_found")
|
||||
assert client.get("/api/v1/chats", headers=headers).json()["items"] == []
|
||||
|
||||
|
||||
def test_member_sees_chat(client, chat, bob):
|
||||
response = client.get("/api/v1/chats", headers=bob["auth"])
|
||||
assert [c["id"] for c in response.json()["items"]] == [chat["id"]]
|
||||
|
||||
|
||||
def test_creator_must_be_member(client, alice, bob):
|
||||
response = client.post("/api/v1/chats", headers=alice["auth"],
|
||||
json={"title": "Без меня", "member_ids": [bob["id"]]})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
|
||||
|
||||
def test_duplicate_members_return_400(client, alice):
|
||||
response = client.post("/api/v1/chats", headers=alice["auth"],
|
||||
json={"title": "Дубли", "member_ids": [alice["id"], alice["id"]]})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
@@ -0,0 +1,162 @@
|
||||
import datetime as dt
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import text
|
||||
|
||||
from conftest import assert_error, with_key
|
||||
from core.config import settings
|
||||
from core.db import engine
|
||||
from main import app
|
||||
from utils import lifecycle
|
||||
|
||||
|
||||
def messages_path(chat):
|
||||
return "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
|
||||
|
||||
def count_messages():
|
||||
with engine.connect() as connection:
|
||||
return connection.scalar(text("SELECT count(*) FROM messages"))
|
||||
|
||||
|
||||
def test_missing_or_invalid_key_returns_400(client, chat, alice):
|
||||
path = messages_path(chat)
|
||||
assert_error(client.post(path, headers=alice["auth"], json={"text": "без ключа"}), 400, "invalid_request")
|
||||
# Только ASCII: HTTP-заголовок с кириллицей тестовый клиент отправить не может.
|
||||
for key in ("", "a b", "x" * 129, "bad/key", "bad,key"):
|
||||
response = client.post(path, headers={**alice["auth"], "Idempotency-Key": key}, json={"text": "плохой ключ"})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
assert count_messages() == 0
|
||||
|
||||
|
||||
def test_valid_key_alphabet_and_max_length(client, chat, alice):
|
||||
path = messages_path(chat)
|
||||
for key in ("A-z_0.9:x", "k" * 128):
|
||||
assert client.post(path, headers=with_key(alice["auth"], key), json={"text": key[:5]}).status_code == 201
|
||||
|
||||
|
||||
def test_replay_returns_same_message_without_duplicate(client, chat, alice):
|
||||
path = messages_path(chat)
|
||||
headers = with_key(alice["auth"], "replay-1")
|
||||
first = client.post(path, headers=headers, json={"text": "один раз"})
|
||||
second = client.post(path, headers=headers, json={"text": "один раз"})
|
||||
assert first.status_code == second.status_code == 201
|
||||
assert first.json() == second.json()
|
||||
assert count_messages() == 1
|
||||
|
||||
|
||||
def test_json_key_order_and_whitespace_do_not_matter(client, chat, alice):
|
||||
path = messages_path(chat)
|
||||
headers = {**with_key(alice["auth"], "order-1"), "Content-Type": "application/json"}
|
||||
first = client.post(path, headers=headers, content='{"text": "тот же текст"}')
|
||||
second = client.post(path, headers=headers, content='{ "text" :"тот же текст" }')
|
||||
assert first.status_code == second.status_code == 201
|
||||
assert first.json() == second.json()
|
||||
assert count_messages() == 1
|
||||
|
||||
|
||||
def test_same_key_with_other_body_returns_409(client, chat, alice):
|
||||
path = messages_path(chat)
|
||||
headers = with_key(alice["auth"], "conflict-1")
|
||||
assert client.post(path, headers=headers, json={"text": "первое"}).status_code == 201
|
||||
assert_error(client.post(path, headers=headers, json={"text": "другое"}), 409, "conflict")
|
||||
assert count_messages() == 1
|
||||
|
||||
|
||||
def test_key_scope_is_user_and_chat(client, chat, alice, bob):
|
||||
path = messages_path(chat)
|
||||
a = client.post(path, headers=with_key(alice["auth"], "shared"), json={"text": "alice"})
|
||||
b = client.post(path, headers=with_key(bob["auth"], "shared"), json={"text": "bob"})
|
||||
assert a.status_code == b.status_code == 201
|
||||
assert a.json()["id"] != b.json()["id"]
|
||||
|
||||
other = client.post("/api/v1/chats", headers=alice["auth"],
|
||||
json={"title": "Другой", "member_ids": [alice["id"]]}).json()
|
||||
c = client.post(messages_path(other), headers=with_key(alice["auth"], "shared"), json={"text": "alice"})
|
||||
assert c.status_code == 201
|
||||
assert c.json()["id"] != a.json()["id"]
|
||||
assert count_messages() == 3
|
||||
|
||||
|
||||
def test_replay_still_checks_session_and_membership(client, chat, alice, eve):
|
||||
path = messages_path(chat)
|
||||
assert client.post(path, headers=with_key(alice["auth"], "k1"), json={"text": "секрет"}).status_code == 201
|
||||
# Посторонний с тем же ключом не получает чужое сообщение.
|
||||
assert_error(client.post(path, headers=with_key(eve["auth"], "k1"), json={"text": "секрет"}), 404, "not_found")
|
||||
# Без токена повтор тоже отклоняется.
|
||||
assert_error(client.post(path, headers={"Idempotency-Key": "k1"}, json={"text": "секрет"}), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_failed_request_does_not_consume_key(client, chat, alice):
|
||||
path = messages_path(chat)
|
||||
headers = with_key(alice["auth"], "retry-after-error")
|
||||
assert_error(client.post(path, headers=headers, json={"text": ""}), 400, "invalid_request")
|
||||
# Тот же ключ после ошибки валидации можно использовать для корректного запроса.
|
||||
assert client.post(path, headers=headers, json={"text": "теперь верно"}).status_code == 201
|
||||
|
||||
|
||||
def test_expired_key_is_reusable(client, chat, alice):
|
||||
path = messages_path(chat)
|
||||
headers = with_key(alice["auth"], "ttl-1")
|
||||
first = client.post(path, headers=headers, json={"text": "старое"}).json()
|
||||
old = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=settings.idempotency.ttl_seconds + 60)
|
||||
with engine.begin() as connection:
|
||||
connection.execute(text("UPDATE idempotency_keys SET created_at = :old"), {"old": old})
|
||||
# Спустя TTL тот же ключ может описывать уже другое сообщение.
|
||||
second = client.post(path, headers=headers, json={"text": "новое"})
|
||||
assert second.status_code == 201
|
||||
assert second.json()["id"] != first["id"]
|
||||
assert client.post(path, headers=headers, json={"text": "новое"}).json() == second.json()
|
||||
|
||||
|
||||
def test_concurrent_requests_with_one_key_create_one_message(chat, alice):
|
||||
path = messages_path(chat)
|
||||
headers = with_key(alice["auth"], "parallel-1")
|
||||
|
||||
def send(_):
|
||||
# Отдельный клиент на поток (без `with`: lifespan закрыл бы общий пул соединений).
|
||||
return TestClient(app).post(path, headers=headers, json={"text": "гонка"})
|
||||
|
||||
with ThreadPoolExecutor(max_workers=16) as pool:
|
||||
responses = list(pool.map(send, range(16)))
|
||||
|
||||
assert {r.status_code for r in responses} == {201}
|
||||
assert len({r.json()["id"] for r in responses}) == 1
|
||||
assert count_messages() == 1
|
||||
|
||||
|
||||
def test_concurrent_different_bodies_one_key_only_one_wins(chat, alice):
|
||||
path = messages_path(chat)
|
||||
headers = with_key(alice["auth"], "parallel-2")
|
||||
|
||||
def send(i):
|
||||
return TestClient(app).post(path, headers=headers, json={"text": f"вариант {i % 2}"})
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
responses = list(pool.map(send, range(8)))
|
||||
|
||||
statuses = sorted(r.status_code for r in responses)
|
||||
assert set(statuses) <= {201, 409} and 201 in statuses
|
||||
assert count_messages() == 1
|
||||
|
||||
|
||||
def test_instance_id_header_is_present_and_stable(client, alice):
|
||||
first = client.get("/health/live")
|
||||
second = client.get("/api/v1/users/me", headers=alice["auth"])
|
||||
error = client.get("/api/v1/users/me")
|
||||
assert first.headers["X-Instance-Id"] == settings.run.instance_id
|
||||
assert second.headers["X-Instance-Id"] == error.headers["X-Instance-Id"] == first.headers["X-Instance-Id"]
|
||||
assert error.status_code == 401
|
||||
|
||||
|
||||
def test_ready_returns_503_while_draining(client):
|
||||
assert client.get("/health/ready").status_code == 200
|
||||
lifecycle._draining.set()
|
||||
try:
|
||||
ready = client.get("/health/ready")
|
||||
assert_error(ready, 503, "unavailable")
|
||||
assert client.get("/health/live").status_code == 200 # процесс жив и обслуживает запросы
|
||||
finally:
|
||||
lifecycle._draining.clear()
|
||||
assert client.get("/health/ready").status_code == 200
|
||||
@@ -0,0 +1,66 @@
|
||||
from conftest import assert_error, with_key
|
||||
|
||||
|
||||
def test_empty_message_returns_400(client, chat, alice):
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
headers = with_key(alice["auth"])
|
||||
|
||||
assert_error(client.post(path, headers=headers, json={"text": ""}), 400, "invalid_request")
|
||||
assert_error(client.post(path, headers=headers, json={"text": " \n\t"}), 400, "invalid_request")
|
||||
|
||||
|
||||
def test_sender_is_taken_from_token(client, chat, alice, bob):
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
response = client.post(path, headers=with_key(bob["auth"]), json={"text": "Привет 👋"})
|
||||
assert response.status_code == 201
|
||||
assert response.json()["sender_id"] == bob["id"]
|
||||
assert response.json()["text"] == "Привет 👋"
|
||||
|
||||
spoof = client.post(path, headers=with_key(bob["auth"]), json={"text": "x", "sender_id": alice["id"]})
|
||||
assert_error(spoof, 400, "invalid_request")
|
||||
|
||||
|
||||
def test_pagination_over_several_pages(client, chat, alice):
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
headers = alice["auth"]
|
||||
sent = [client.post(path, headers=with_key(headers), json={"text": f"сообщение {i}"}).json() for i in range(7)]
|
||||
|
||||
received = []
|
||||
pages = 0
|
||||
cursor = None
|
||||
while True:
|
||||
params = {"limit": 3}
|
||||
if cursor:
|
||||
params["cursor"] = cursor
|
||||
page = client.get(path, headers=headers, params=params).json()
|
||||
received += page["items"]
|
||||
pages += 1
|
||||
cursor = page["next_cursor"]
|
||||
if cursor is None:
|
||||
break
|
||||
|
||||
assert pages == 3 # 3 + 3 + 1
|
||||
assert received == sent # тот же порядок, ничего не потеряно
|
||||
assert len({m["id"] for m in received}) == 7 # без повторов
|
||||
|
||||
|
||||
def test_cursor_from_another_chat_returns_400(client, chat, alice):
|
||||
headers = alice["auth"]
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
for i in range(2):
|
||||
client.post(path, headers=with_key(headers), json={"text": f"m{i}"})
|
||||
cursor = client.get(path, headers=headers, params={"limit": 1}).json()["next_cursor"]
|
||||
|
||||
other = client.post("/api/v1/chats", headers=headers, json={"title": "Другой", "member_ids": [alice["id"]]}).json()
|
||||
response = client.get("/api/v1/chats/" + other["id"] + "/messages", headers=headers, params={"cursor": cursor})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
|
||||
response = client.get(path, headers=headers, params={"cursor": "мусор"})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
|
||||
|
||||
def test_invalid_limit_returns_400(client, chat, alice):
|
||||
path = "/api/v1/chats/" + chat["id"] + "/messages"
|
||||
for limit in (0, 101, "abc"):
|
||||
response = client.get(path, headers=alice["auth"], params={"limit": limit})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
@@ -0,0 +1,69 @@
|
||||
import uuid
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from conftest import PASSWORD, assert_error
|
||||
from core.db import engine
|
||||
|
||||
|
||||
def test_invalid_json_returns_400(client):
|
||||
response = client.post("/api/v1/users", content=b'{"username": "bob",',
|
||||
headers={"Content-Type": "application/json"})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
|
||||
|
||||
def test_extra_field_and_wrong_type_return_400(client):
|
||||
response = client.post("/api/v1/users", json={"username": "bob", "display_name": "Bob", "password": PASSWORD,
|
||||
"admin": True})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
response = client.post("/api/v1/users", json={"username": "bob", "display_name": 123, "password": PASSWORD})
|
||||
assert_error(response, 400, "invalid_request")
|
||||
|
||||
|
||||
def test_duplicate_username_returns_409(client, alice):
|
||||
response = client.post("/api/v1/users", json={"username": "alice", "display_name": "Другая Алиса",
|
||||
"password": PASSWORD})
|
||||
assert_error(response, 409, "conflict")
|
||||
|
||||
|
||||
def test_unicode_display_name_is_kept(client):
|
||||
response = client.post("/api/v1/users", json={"username": "uni", "display_name": "Алиса 👋",
|
||||
"password": PASSWORD})
|
||||
assert response.status_code == 201
|
||||
assert response.json()["display_name"] == "Алиса 👋"
|
||||
|
||||
|
||||
def test_password_is_required_and_length_checked(client):
|
||||
for password in (None, "short", "x" * 129):
|
||||
body = {"username": "pwd", "display_name": "P"}
|
||||
if password is not None:
|
||||
body["password"] = password
|
||||
assert_error(client.post("/api/v1/users", json=body), 400, "invalid_request")
|
||||
# 12 символов (не байт) — допустимо.
|
||||
response = client.post("/api/v1/users", json={"username": "pwd", "display_name": "P", "password": "пароль123456"})
|
||||
assert response.status_code == 201
|
||||
|
||||
|
||||
def test_password_is_hashed_and_never_returned(client, alice):
|
||||
assert "password" not in alice and "password_hash" not in alice
|
||||
with engine.connect() as connection:
|
||||
stored = connection.scalar(text("SELECT password_hash FROM users WHERE username = 'alice'"))
|
||||
assert stored.startswith("$argon2id$")
|
||||
assert PASSWORD not in stored
|
||||
|
||||
|
||||
def test_me_returns_current_user(client, alice):
|
||||
response = client.get("/api/v1/users/me", headers=alice["auth"])
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {key: alice[key] for key in ("id", "username", "display_name", "created_at")}
|
||||
|
||||
|
||||
def test_x_user_id_no_longer_authenticates(client, alice):
|
||||
assert_error(client.get("/api/v1/users/me", headers={"X-User-Id": alice["id"]}), 401, "unauthorized")
|
||||
assert_error(client.get("/api/v1/users/me", headers={"X-User-Id": str(uuid.uuid4())}), 401, "unauthorized")
|
||||
|
||||
|
||||
def test_every_response_has_request_id(client):
|
||||
assert client.get("/health/live").headers["X-Request-Id"]
|
||||
assert client.get("/health/ready").headers["X-Request-Id"]
|
||||
assert_error(client.get("/no/such/path"), 404, "not_found")
|
||||
Reference in New Issue
Block a user