diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index b0a745d6..ba8580e8 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,13 @@ ## Лабораторная 3 Автор / группа: Тимофей Александров, БВТ2301 -Целевая оценка (3/4/5): 4 -Трек 5, если заявлен: не заявлен +Целевая оценка (3/4/5): 5 +Трек 5: трек 1 «Цена масштабирования» Что реализовано и что перенесено: - Перенесено из lab2: PostgreSQL источник истины, Valkey кэш сессий, Argon2id пароли, Bearer-сессии, транзакционное создание чата, keyset-пагинация, circuit breaker на Valkey. -- Новое в lab3: 2 API-реплики (api-1, api-2) с уникальным INSTANCE_ID; Nginx load balancer с health-check и автоматическим failover; Idempotency-Key в заголовке (replay возвращает исходное сообщение); X-Instance-Id в каждом ответе; /version и /info эндпоинты; housekeeping-loop для очистки истёкших сессий и ключей; таблица idempotency_keys с миграциями 004, 005, 006. +- Новое в lab3: 2 API-реплики (api-1, api-2) с уникальным INSTANCE_ID; Nginx load balancer с health-check и автоматическим failover; Idempotency-Key в заголовке (replay возвращает исходное сообщение); X-Instance-Id в каждом ответе; /version и /info эндпоинты; housekeeping-loop для очистки истёкших сессий и ключей; таблица idempotency_keys с миграциями 004–007. +- Исправлено по ревью: scope idempotency — `(actor_id, chat_id, key)`; атомарное резервирование ключа не даёт 500 при 32 concurrent replays; Bearer-сессия проверяет PostgreSQL на каждой реплике, поэтому старый cache-hit не может воскресить отозванный токен. - Исправлены импорты config/models/api/store/sessions (убраны несуществующие `config`, добавлены `IDEMPOTENCY_KEY_PATTERN`, `__version__`). - Добавлены скрипты stop-one.sh/cmd (failover) и restart.sh/cmd (persistence). @@ -20,22 +21,24 @@ python3 scripts/validate_openapi.py python3 tests/smoke.py --base-url http://127.0.0.1:8081 python3 tests/resilience.py --mode persistence --action-script scripts/restart.sh --base-url http://127.0.0.1:8081 python3 tests/resilience.py --mode failover --action-script scripts/stop-one.sh --base-url http://127.0.0.1:8081 -cd student-tests && BASE_URL=http://127.0.0.1:8081 python3 -u -B -m unittest -v test_api test_race test_degradation +cd student-tests && BASE_URL=http://127.0.0.1:8081 python3 -u -B -m unittest -v test_api test_race test_idempotency test_degradation +cd .. && python3 scripts/research-scale.py --base-url http://127.0.0.1:8081 --duration 10 --concurrency 4 ``` Результаты проверок и ссылка на REPORT.md: - Smoke: PASS: lab3 public API scenario - check_contract: PASS: offline skeleton structure, local refs and Python syntax - validate_openapi: PASS: full OpenAPI 3.1 and JSON Schema validation -- Student tests: 26/26 OK (включая duplicate_username_race, chat_creation_rolls_back, bounded_pool) +- Student tests: 28/28 OK; idempotency 2/2 OK (32 concurrent replays); degradation 2/2 OK (durable cross-replica logout) - Failover: PASS — 79 успешных записей из 83, восстановление за 2.6 с - Persistence: PASS — данные и сессия сохранены после рестарта - Degradation: OK — cache_down_p50=47 мс, db_down_write=2.05 с, recovery=1.97 с +- Трек 5: fixed total RPS 1/2/4 = 36.9 / 49.4 / 36.2; growing resources = 42.4 / 45.2 / 57.0. Raw JSON: `evidence/scale-*.json`. - Подробности: REPORT.md Ограничения / известные ошибки: - На Windows .sh-скрипты не запускаются напрямую (WinError 193); созданы эквиваленты .cmd. -- Локальный кэш отозванных токенов не разделяется между репликами (уровень 5 не заявлен). +- PostgreSQL и Nginx остаются одиночными узлами; трек не заявляет выживание их отказа. - Нагрузочный тест закрытого цикла содержит coordinated omission. Источники и использование ИИ: README/COURSE/CONTRIBUTING этого skeleton, contracts/README.md, tests/README.md; документация PostgreSQL, Valkey, Nginx; RFC 7617; OWASP Password Storage Cheat Sheet. ИИ (Codex) использовался для изучения задания, проектирования, написания исходников и тестов, проверки и подготовки отчёта. Автор проверяет результат и должен уметь объяснить и изменить его на защите. diff --git a/Makefile b/Makefile index bbce74da..9084cec7 100644 --- a/Makefile +++ b/Makefile @@ -7,13 +7,14 @@ ACTION_SCRIPT ?= TARGET_INSTANCE_ID ?= api-1 STUDENT_DIR = student-tests -.PHONY: help check validate test load test-persistence test-failover run bootstrap up down test-student explain +.PHONY: help check validate test load test-persistence test-failover run bootstrap up down test-student test-degradation explain research-scale help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @echo "bootstrap/up/down: local Compose stand; run: API without Docker (needs DATABASE_URL/CACHE_URL)" @echo "test-student: own functional+race tests; test-persistence: restart stack; test-failover: stop one API replica" @echo "explain: seed 12k messages and capture EXPLAIN (ANALYZE, BUFFERS) into evidence/" + @echo "research-scale: grade-5 1/2/4 replica experiment; set BASE_URL, DURATION and CONCURRENCY as needed" check: $(PYTHON) scripts/check_contract.py @@ -50,7 +51,13 @@ down: docker compose --env-file .env down test-student: - cd $(STUDENT_DIR) && BASE_URL="$(BASE_URL)" $(PYTHON) -B -m unittest -v test_api test_race + cd $(STUDENT_DIR) && BASE_URL="$(BASE_URL)" $(PYTHON) -B -m unittest -v test_api test_race test_idempotency + +test-degradation: + cd $(STUDENT_DIR) && BASE_URL="$(BASE_URL)" STUDENT_DOCKER_TESTS=1 $(PYTHON) -B -m unittest -v test_degradation explain: sh scripts/explain-history.sh + +research-scale: + $(PYTHON) scripts/research-scale.py --base-url "$(BASE_URL)" diff --git a/REPORT.md b/REPORT.md index 9432c861..f2ecfcd5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,7 +1,7 @@ # Отчёт — лабораторная 3 - Автор, группа: Тимофей Александров, БВТ2301 -- Целевая оценка; выбранный трек 5, если нужен: **4** (трек 5 не заявлен) +- Целевая оценка; выбранный трек 5, если нужен: **5**, трек 1 «Цена масштабирования» - Commit/PR: подготовлен к публикации - Использованные источники, библиотеки и помощь ИИ: README/COURSE/CONTRIBUTING этого skeleton, `contracts/README.md` и `tests/README.md`; документация PostgreSQL (transactions, EXPLAIN), Valkey (persistence), Nginx (upstream, health checks), RFC 7617 (Basic), OWASP Password Storage Cheat Sheet (Argon2id). Библиотеки: FastAPI/Starlette/uvicorn, psycopg 3 + psycopg-pool, redis-py, argon2-cffi, nginx 1.27. Лабораторная 3 построена на кодовой базе лабораторной 2. @@ -32,6 +32,7 @@ TARGET_INSTANCE_ID=api-1 sh scripts/stop-one.sh # остановить одну cd student-tests && BASE_URL=http://127.0.0.1:8081 \ python3 -u -B -m unittest -v test_api test_race test_degradation sh scripts/explain-history.sh # = make explain (EXPLAIN ANALYZE на 12 000 сообщений) +python3 scripts/research-scale.py --base-url http://127.0.0.1:8081 --duration 10 --concurrency 4 ``` Безопасная остановка с сохранением данных: `docker compose --env-file .env down` (volumes `pgdata`/`valkeydata` не удаляются, сессии и сообщения переживают рестарт). Намеренное удаление учебных данных: `docker compose --env-file .env down -v`. На Windows эквивалент restart-hook для `tests/resilience.py` — `scripts/restart.cmd` (тот же сценарий, что и `scripts/restart.sh`: `down` без `-v`, затем `up -d --wait`; Python не умеет запускать `.sh` напрямую, ошибка `WinError 193`). Аналогично для failover — `scripts/stop-one.cmd`. @@ -46,11 +47,11 @@ URL вида `http://127.0.0.1:8081`, а не `localhost`: Compose публик **Балансировка и отказ API-реплики (новое в lab3).** Два API-инстанса (`api-1`, `api-2`) с разными `INSTANCE_ID`. Nginx upstream балансирует round-robin с health-check (`max_fails=3`, `fail_timeout=10s`): при остановке одной реплики трафик автоматически идёт на вторую. `proxy_next_upstream` перенаправляет запрос на следующую реплику при 502/503. Порты API-реплик не публикуются наружу — только через LB. Failover-тест: 79 успешных записей из 83 попыток за 15 с при остановленной реплике, восстановление за 2.6 с. -**Источник истины и кэш.** Сессии живут в таблице `sessions` (хранится только SHA-256 токена). Valkey хранит тот же токен под `session:` максимум `CACHE_TTL_SECONDS=300` с и одновременно не дольше остатка жизни сессии. Кэш записывается при логине и при каждом cache-miss, инвалидируется логаутом через tombstone `revoked` (а не `DEL`: устаревшая запись обязана стать недействительной). Если Valkey недоступен — каждый запрос просто идёт в PostgreSQL; кэш-ошибка никогда не означает «авторизован». Плюс 5-секундный circuit breaker. +**Источник истины и кэш.** Сессии живут в таблице `sessions` (хранится только SHA-256 токена). Valkey хранит публичный снимок пользователя под `session:` максимум `CACHE_TTL_SECONDS=300` с и одновременно не дольше остатка жизни сессии. Tombstone `revoked` остаётся полезной быстрой отрицательной записью, но PostgreSQL подтверждает действительность сессии на каждом Bearer-запросе. Поэтому cache-hit не может воскресить отзыв: при остановленном Valkey, рестарте одной/обеих API-реплик и возврате старого кэш-тома токен остаётся 401 на любой реплике. Если Valkey недоступен — запрос идёт в PostgreSQL; кэш-ошибка никогда не означает «авторизован». Плюс 5-секундный circuit breaker. **Транзакционные границы.** Создание чата (`chats` + строки `chat_members`) — одна транзакция. Регистрация и отправка сообщения — по одной транзакции. Миграции — версионированный раннер `src/migrate.py`: одна транзакция + `pg_advisory_xact_lock`, таблица `schema_migrations`, повторный запуск печатает `migrations_up_to_date` и не дублирует данные. -**Idempotency-Key (новое в lab3).** Ключ передаётся в заголовке `Idempotency-Key` (обязательный, паттерн `^[A-Za-z0-9._:-]{1,128}$`). Хранится в таблице `idempotency_keys` вместе с SHA-256 тела запроса и `message_id`. Повтор с тем же ключом и тем же телом возвращает исходное сообщение (201); другой body → 409. Миграция 006 добавляет `message_id` для возврата оригинального сообщения при replay. +**Idempotency-Key (новое в lab3).** Ключ передаётся в заголовке `Idempotency-Key` (обязательный, паттерн `^[A-Za-z0-9._:-]{1,128}$`). Scope — `(actor_id, chat_id, key)`, то есть пользователь + POST + путь чата + ключ: один ключ независим между пользователями и чатами. Хранится SHA-256 тела и `message_id`; повтор с тем же телом возвращает исходное сообщение (201), другой body в той же области → 409. Миграция 006 добавляет `message_id`, миграция 007 добавляет `actor_id` и составной primary key. Резервирование ключа использует `INSERT ... ON CONFLICT DO NOTHING`, поэтому конкурентный repeat не оставляет транзакцию в aborted state и все запросы получают один и тот же Message. **Ограничения и таймауты.** Пул `DB_POOL_MIN=1 … DB_POOL_MAX=10` c `DB_POOL_TIMEOUT_SECONDS=2`, `DB_CONNECT_TIMEOUT_SECONDS=2`, серверные `statement_timeout=5000`, `lock_timeout=2000`, `idle_in_transaction_session_timeout=10000`, `timezone=UTC`. Все запросы параметризованные (psycopg3). Кэш-операции ограничены `CACHE_TIMEOUT_SECONDS=0.5`. @@ -75,6 +76,9 @@ URL вида `http://127.0.0.1:8081`, а не `localhost`: Compose публик | Ограниченный пул и параллельные чтения | там же, `bounded_pool_serves_parallel_reads` | OK; 100 записей от 10 клиентов за 2.50 с, ровно 100 строк | `evidence/student-tests.txt` | | Индекс истории, EXPLAIN ≥ 10 000 строк, без OFFSET | `sh scripts/explain-history.sh` | 12 000 сообщений: страница глубоко в истории — Index Scan; вариант OFFSET (не используется) — Seq Scan + Sort | `evidence/explain-history.txt` | | Недоступность Redis/Valkey и БД | `STUDENT_DOCKER_TESTS=1 python3 -m unittest -v test_degradation` | OK: `cached_p50=31 мс`, `cache_down_p50=47 мс`, `cache_down_p95=110 мс`, `/health/ready` остаётся 200 при падении кэша и становится 503 при падении БД, запись отказывает за 2.05 с, восстановление за 1.97 с | `evidence/test-degradation.txt` | +| Durable logout на обеих репликах | `test_logout_remains_revoked_across_replicas_after_cache_outage` | **OK**: остановка Valkey → logout → restart api-1/api-2 → запуск Valkey с прежним томом; прежний токен получает 401 на GET/POST | `evidence/test-degradation-grade5.txt` | +| Scope и конкурентность idempotency | `python -m unittest -v test_idempotency` | **2/2 OK**: одинаковый key независим для разных user/chat; 32 одинаковых POST → 32×201 одного Message, без 500 | `evidence/idempotency-grade5.txt` | +| Трек 5: 1/2/4 API-реплики | `python scripts/research-scale.py --duration 10 --concurrency 4` | fixed total saturation: 36.9 / 49.4 / 36.2 RPS; growing resources: 42.4 / 45.2 / 57.0 RPS | `evidence/scale-summary.json`, `evidence/scale-*.json` | | Нагрузочный прогон | `python3 tests/load.py …` | результаты в evidence/load.json | `evidence/load.json` | Дополнительно: `docker compose --env-file .env ps` показывает `api-1/api-2/postgres/valkey/nginx — healthy`; балансировка round-robin между двумя репликами подтверждена заголовком `X-Instance-Id`. @@ -92,10 +96,20 @@ URL вида `http://127.0.0.1:8081`, а не `localhost`: Compose публик | Гонка 20 параллельных регистраций одного username | ровно один `INSERT`, остальные 409 | 1 пользователь в БД, ответы 201/409 | не применимо | | Регистрация чата с неизвестным участником | откат всей операции | в `chats`/`chat_members` 0 строк | не применимо | +## Трек 5 — Цена масштабирования + +**Гипотеза.** При фиксированном общем бюджете CPU/RAM после двух API-реплик дополнительное размножение процессов не ускорит PostgreSQL-bound запись и повысит tail latency; при росте суммарного бюджета 4 реплики должны дать прирост. + +**Методика.** `scripts/research-scale.py` использует один closed-loop профиль `POST messages` (`tests/load.py`, 4 клиента, 10 секунд, без retry) и сохраняет raw JSON для каждой точки. В режиме `fixed_total` общий лимит API равен 2 CPU / 1024 MiB и делится между 1/2/4 репликами. В режиме `growing_total` каждая реплика получает 1 CPU / 512 MiB. Overlay `compose.scale.yaml` добавляет api-3/api-4 только для исследования; по завершении скрипт возвращает обычный двухрепличный стенд. + +**Результат.** При фиксированном бюджете: 1 → **36.921 RPS** (p95 172 мс), 2 → **49.412 RPS** (p95 172 мс), 4 → **36.171 RPS** (p95 281 мс). Saturation наблюдается после 2 реплик: PostgreSQL и конкуренция за суммарно ограниченный CPU становятся узким местом, а дополнительные процессы увеличивают p95/p99. При росте ресурсов: 1 → **42.374 RPS**, 2 → **45.174 RPS**, 4 → **57.007 RPS**; p95 у 4 реплик 125 мс. Подтверждённое улучшение — увеличение суммарного бюджета с четырьмя репликами вместо деления фиксированного бюджета: +57.6% RPS против fixed-total 4. + +**Ограничения.** Один повтор на Docker Desktop и закрытая модель нагрузки не дают production capacity claim; закрытый цикл содержит coordinated omission. CPU/RAM измерялись как назначенные Docker limits, а не как аппаратные counters. LB и PostgreSQL остаются SPOF; работа не заявляет отказоустойчивость их узлов. + ## Ограничения и следующие шаги - **Выдерживается:** отказ одной API-реплики (LB перенаправляет на вторую), отказ Valkey (полный обход кэша), кратковременный отказ PostgreSQL (ограниченная ошибка и быстрый возврат), рестарт всего стека с сохранением данных. **Не выдерживается:** одновременный выход обоих хранилищ из-под `up --wait` (стенд перестаёт готовиться — это ожидаемо), потеря PostgreSQL-тома вместе с контейнером. -- Локальный кэш отозванных токенов и circuit breaker не разделяются между репликами. Для уровня 5 с несколькими репликами нужно вынести отзыв в общее хранилище. +- Локальный pending-тombstone остаётся только оптимизацией при outage, но не является авторизационным решением: PostgreSQL проверяет revoke на каждой реплике. Поэтому отсутствие общего local cache не может воскресить токен. - Нагрузочный тест закрытого цикла содержит coordinated omission и накладные издержки клиента; это не доказательство ёмкости. Повторов и замера CPU/RAM не делалось. - `make` в среде отсутствует, поэтому все команды выполнялись напрямую; `Makefile` сохранён для Linux/CI. - На Windows `.sh`-скрипты не запускаются напрямую (WinError 193); созданы эквиваленты `.cmd` (`restart.cmd`, `stop-one.cmd`). На Linux grading-стенде используются `.sh`. diff --git a/compose.scale.yaml b/compose.scale.yaml new file mode 100644 index 00000000..8cdc09e1 --- /dev/null +++ b/compose.scale.yaml @@ -0,0 +1,20 @@ +# Optional grade-5 research overlay. It is never needed for normal lab3 +# startup; scripts/research-scale.py adds it only for the 4-replica cases. +services: + api-3: + extends: + file: compose.yaml + service: api-1 + environment: + INSTANCE_ID: api-3 + + api-4: + extends: + file: compose.yaml + service: api-1 + environment: + INSTANCE_ID: api-4 + + nginx: + volumes: + - ./nginx/nginx-scale.conf:/etc/nginx/nginx.conf:ro diff --git a/evidence/idempotency-grade5.txt b/evidence/idempotency-grade5.txt new file mode 100644 index 00000000..1ffaa457 --- /dev/null +++ b/evidence/idempotency-grade5.txt @@ -0,0 +1,8 @@ +test_concurrent_identical_replays_return_one_message_without_500 (test_idempotency.IdempotencyScope.test_concurrent_identical_replays_return_one_message_without_500) ... ok +test_same_key_is_independent_for_different_users_and_chats (test_idempotency.IdempotencyScope.test_same_key_is_independ +ent_for_different_users_and_chats) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 2.952s + +OK diff --git a/evidence/scale-fixed_total-1.json b/evidence/scale-fixed_total-1.json new file mode 100644 index 00000000..cd7826cb --- /dev/null +++ b/evidence/scale-fixed_total-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 11.484, + "concurrency": 4, + "successful_requests": 424, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 36.921, + "successful_latency_ms": { + "p50": 78.0, + "p95": 172.0, + "p99": 235.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scale-fixed_total-2.json b/evidence/scale-fixed_total-2.json new file mode 100644 index 00000000..b6558d3e --- /dev/null +++ b/evidence/scale-fixed_total-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.281, + "concurrency": 4, + "successful_requests": 508, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 49.412, + "successful_latency_ms": { + "p50": 62.0, + "p95": 172.0, + "p99": 234.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scale-fixed_total-4.json b/evidence/scale-fixed_total-4.json new file mode 100644 index 00000000..d9bf6974 --- /dev/null +++ b/evidence/scale-fixed_total-4.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.312, + "concurrency": 4, + "successful_requests": 373, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 36.171, + "successful_latency_ms": { + "p50": 63.0, + "p95": 281.0, + "p99": 515.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scale-growing_total-1.json b/evidence/scale-growing_total-1.json new file mode 100644 index 00000000..9a88e2a8 --- /dev/null +++ b/evidence/scale-growing_total-1.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 11.375, + "concurrency": 4, + "successful_requests": 482, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 42.374, + "successful_latency_ms": { + "p50": 63.0, + "p95": 172.0, + "p99": 265.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scale-growing_total-2.json b/evidence/scale-growing_total-2.json new file mode 100644 index 00000000..0b5be3fb --- /dev/null +++ b/evidence/scale-growing_total-2.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.36, + "concurrency": 4, + "successful_requests": 468, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 45.174, + "successful_latency_ms": { + "p50": 63.0, + "p95": 187.0, + "p99": 219.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scale-growing_total-4.json b/evidence/scale-growing_total-4.json new file mode 100644 index 00000000..e789b368 --- /dev/null +++ b/evidence/scale-growing_total-4.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.297, + "concurrency": 4, + "successful_requests": 587, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 57.007, + "successful_latency_ms": { + "p50": 62.0, + "p95": 125.0, + "p99": 187.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/scale-summary.json b/evidence/scale-summary.json new file mode 100644 index 00000000..3640c9d6 --- /dev/null +++ b/evidence/scale-summary.json @@ -0,0 +1,122 @@ +{ + "method": "closed-loop POST messages; no retries; latency percentiles contain successes only", + "fixed_total_budget": "2 CPU / 1024 MiB divided equally among 1, 2, 4 replicas", + "growing_total_budget": "1 CPU / 512 MiB per replica", + "results": [ + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 11.484, + "concurrency": 4, + "successful_requests": 424, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 36.921, + "successful_latency_ms": { + "p50": 78.0, + "p95": 172.0, + "p99": 235.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "regime": "fixed_total", + "replicas": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.281, + "concurrency": 4, + "successful_requests": 508, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 49.412, + "successful_latency_ms": { + "p50": 62.0, + "p95": 172.0, + "p99": 234.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "regime": "fixed_total", + "replicas": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.312, + "concurrency": 4, + "successful_requests": 373, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 36.171, + "successful_latency_ms": { + "p50": 63.0, + "p95": 281.0, + "p99": 515.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "regime": "fixed_total", + "replicas": 4 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 11.375, + "concurrency": 4, + "successful_requests": 482, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 42.374, + "successful_latency_ms": { + "p50": 63.0, + "p95": 172.0, + "p99": 265.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "regime": "growing_total", + "replicas": 1 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.36, + "concurrency": 4, + "successful_requests": 468, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 45.174, + "successful_latency_ms": { + "p50": 63.0, + "p95": 187.0, + "p99": 219.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "regime": "growing_total", + "replicas": 2 + }, + { + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 10.0, + "elapsed_seconds": 10.297, + "concurrency": 4, + "successful_requests": 587, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 57.007, + "successful_latency_ms": { + "p50": 62.0, + "p95": 125.0, + "p99": 187.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead.", + "regime": "growing_total", + "replicas": 4 + } + ], + "limitations": "Docker Desktop host scheduling and closed-loop coordinated omission; this is not a production capacity claim." +} diff --git a/evidence/test-degradation-grade5.txt b/evidence/test-degradation-grade5.txt new file mode 100644 index 00000000..5496a38d --- /dev/null +++ b/evidence/test-degradation-grade5.txt @@ -0,0 +1,10 @@ +test_logout_remains_revoked_across_replicas_after_cache_outage (test_degradation.DependencyOutages.test_logout_remains_revoked_across_replicas_after_cache_outage) ... ok +test_valkey_loss_degrades_then_postgres_loss_is_bounded (test_degradation.DependencyOutages.test_valkey_loss_degrades_t +hen_postgres_loss_is_bounded) ... ok + +---------------------------------------------------------------------- +Ran 2 tests in 47.789s + +OK + + outage summary: {"cached_p50_ms": 16.0, "cache_down_p50_ms": 31.0, "cache_down_p95_ms": 47.0, "db_down_write_seconds": 2.02, "recovery_seconds": 1.95} diff --git a/nginx/nginx-scale.conf b/nginx/nginx-scale.conf new file mode 100644 index 00000000..84ef6904 --- /dev/null +++ b/nginx/nginx-scale.conf @@ -0,0 +1,31 @@ +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Used only by scripts/research-scale.py with compose.scale.yaml. + upstream api { + server api-1:8080 max_fails=3 fail_timeout=10s; + server api-2:8080 max_fails=3 fail_timeout=10s; + server api-3:8080 max_fails=3 fail_timeout=10s; + server api-4:8080 max_fails=3 fail_timeout=10s; + } + + server { + listen 8080; + server_name _; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 2s; + proxy_read_timeout 10s; + proxy_send_timeout 10s; + proxy_next_upstream error timeout http_502; + proxy_next_upstream_tries 4; + location / { proxy_pass http://api; } + } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 5d516c54..11a9cef2 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -28,9 +28,15 @@ http { proxy_read_timeout 10s; proxy_send_timeout 10s; - # Redirect on 502/503 to the next available replica once, then give up. - proxy_next_upstream error timeout http_502 http_503; - proxy_next_upstream_tries 1; + # A refused connection/502 means the selected replica is unavailable and + # can be retried on its peer. A structured 503 is an application + # dependency failure (for example PostgreSQL down), not an unhealthy + # replica: propagating it preserves the API error envelope instead of + # marking both live peers down and manufacturing an Nginx 502. + proxy_next_upstream error timeout http_502; + # Two attempts cover both configured replicas: a request that lands on + # the stopped instance is retried once on its healthy peer. + proxy_next_upstream_tries 2; location / { proxy_pass http://api; diff --git a/scripts/research-scale.py b/scripts/research-scale.py new file mode 100644 index 00000000..2bd8baf3 --- /dev/null +++ b/scripts/research-scale.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Grade-5 track 1: repeatable 1/2/4 API-replica scaling experiment. + +The script measures the same closed-loop write workload in two regimes: +``fixed_total`` divides a fixed 2 CPU / 1 GiB API budget among replicas; +``growing_total`` assigns every replica 1 CPU / 512 MiB. It writes raw output +from tests/load.py to evidence/scale-*.json and a sanitized summary to +evidence/scale-summary.json. No secrets or tokens are written. +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import subprocess +import time + + +ROOT = Path(__file__).resolve().parents[1] +BASE_COMPOSE = ["docker", "compose", "--env-file", ".env", "-f", "compose.yaml"] +SCALE_COMPOSE = BASE_COMPOSE + ["-f", "compose.scale.yaml"] + + +def run(command: list[str], *, timeout: int = 180) -> None: + result = subprocess.run(command, cwd=ROOT, text=True, capture_output=True, timeout=timeout) + if result.returncode: + raise SystemExit( + "command failed:\n" + + " ".join(command) + + "\n" + + (result.stdout + result.stderr)[-2000:] + ) + + +def wait_ready(base_url: str) -> None: + import urllib.request + + deadline = time.monotonic() + 120 + while time.monotonic() < deadline: + try: + with urllib.request.urlopen(base_url + "/health/ready", timeout=3) as response: + if response.status == 200: + return + except OSError: + pass + time.sleep(1) + raise SystemExit("load balancer did not become ready within 120 seconds") + + +def set_budget(replicas: int, regime: str) -> None: + if regime == "fixed_total": + cpu = 2.0 / replicas + memory = 1024 // replicas + else: + cpu = 1.0 + memory = 512 + for index in range(1, replicas + 1): + run([ + "docker", "update", "--cpus", str(cpu), "--memory", str(memory) + "m", + "--memory-swap", str(memory) + "m", + "mtusi2026lab3-api-" + str(index) + "-1", + ]) + + +def configure_replicas(replicas: int) -> None: + if replicas in {1, 2}: + # The standard two-replica topology remains intact; stop api-2 only for + # the one-replica measurement. Nginx retries the healthy peer. + run(BASE_COMPOSE + ["up", "-d", "--wait", "api-1", "api-2", "nginx"]) + if replicas == 1: + run(BASE_COMPOSE + ["stop", "api-2"]) + else: + run(SCALE_COMPOSE + ["up", "-d", "--wait", "api-1", "api-2", "api-3", "api-4", "nginx"]) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--base-url", default=os.environ.get("BASE_URL", "http://127.0.0.1:8081")) + parser.add_argument("--duration", type=float, default=15.0) + parser.add_argument("--concurrency", type=int, default=4) + args = parser.parse_args() + if not 1 <= args.duration <= 300: + raise SystemExit("duration must be 1..300") + + evidence = ROOT / "evidence" + evidence.mkdir(exist_ok=True) + results: list[dict[str, object]] = [] + try: + for regime in ("fixed_total", "growing_total"): + for replicas in (1, 2, 4): + configure_replicas(replicas) + set_budget(replicas, regime) + wait_ready(args.base_url.rstrip("/")) + output = evidence / ("scale-" + regime + "-" + str(replicas) + ".json") + run([ + "python", "tests/load.py", "--base-url", args.base_url, + "--duration", str(args.duration), "--concurrency", str(args.concurrency), + "--output", str(output), + ], timeout=int(args.duration + 120)) + sample = json.loads(output.read_text(encoding="utf-8")) + sample.update({"regime": regime, "replicas": replicas}) + results.append(sample) + finally: + # Return to the normal grade-3 topology with its ordinary nginx config. + run(SCALE_COMPOSE + ["down"], timeout=120) + run(BASE_COMPOSE + ["up", "-d", "--wait"], timeout=180) + + summary = { + "method": "closed-loop POST messages; no retries; latency percentiles contain successes only", + "fixed_total_budget": "2 CPU / 1024 MiB divided equally among 1, 2, 4 replicas", + "growing_total_budget": "1 CPU / 512 MiB per replica", + "results": results, + "limitations": "Docker Desktop host scheduling and closed-loop coordinated omission; this is not a production capacity claim.", + } + (evidence / "scale-summary.json").write_text( + json.dumps(summary, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/src/api.py b/src/api.py index 11c4da58..81bcc2bc 100644 --- a/src/api.py +++ b/src/api.py @@ -334,7 +334,9 @@ async def create_message( 400, "invalid_request", "Idempotency-Key must match " "^[A-Za-z0-9._:-]{1,128}$" ) - existing = await store.get_idempotency_key(cursor, key) + existing = await store.get_idempotency_key( + cursor, actor_id=actor.user_id, chat_id=normalized, key=key + ) if existing is not None: if existing["request_sha256"] != request_sha256: raise ApiProblem(409, "conflict", "Idempotency-Key was already used") @@ -342,18 +344,27 @@ async def create_message( original = await store.message_by_id(cursor, existing["message_id"]) if original is not None: return original - try: - await store.insert_idempotency_key( - cursor, key=key, chat_id=normalized, request_sha256=request_sha256 + inserted = await store.insert_idempotency_key( + cursor, + actor_id=actor.user_id, + key=key, + chat_id=normalized, + request_sha256=request_sha256, + ) + if not inserted: + # INSERT .. ON CONFLICT keeps this transaction usable. PostgreSQL + # waited for the winning transaction, so its message is committed + # and can be returned as the exact replay result. + existing = await store.get_idempotency_key( + cursor, actor_id=actor.user_id, chat_id=normalized, key=key ) - except ApiProblem: - # Race: another request inserted the key first. Treat as replay. - existing = await store.get_idempotency_key(cursor, key) - if existing is not None and existing["request_sha256"] == request_sha256: + if existing is not None: + if existing["request_sha256"] != request_sha256: + raise ApiProblem(409, "conflict", "Idempotency-Key was already used") original = await store.message_by_id(cursor, existing["message_id"]) if original is not None: return original - raise + raise ApiProblem(409, "conflict", "Idempotency-Key is still being stored") return await store.create_message( cursor, actor_id=actor.user_id, diff --git a/src/migrations/007_idempotency_scope.sql b/src/migrations/007_idempotency_scope.sql new file mode 100644 index 00000000..81e21f64 --- /dev/null +++ b/src/migrations/007_idempotency_scope.sql @@ -0,0 +1,25 @@ +-- An idempotency key belongs to the authenticated user and target chat, not +-- to the entire service. This is the POST /chats/{chat_id}/messages path; +-- together (actor_id, chat_id, key) form the contract's user+method+path+key +-- scope. +ALTER TABLE idempotency_keys + ADD COLUMN actor_id uuid REFERENCES users (id); + +-- All completed records created by the previous schema have message_id. +-- Derive the actor from that immutable message before making the new scope +-- mandatory. Any incomplete legacy row cannot be safely replayed, so remove +-- it instead of assigning it to an arbitrary user. +UPDATE idempotency_keys AS ik +SET actor_id = m.sender_id +FROM messages AS m +WHERE ik.message_id = m.id + AND ik.actor_id IS NULL; + +DELETE FROM idempotency_keys WHERE actor_id IS NULL; + +ALTER TABLE idempotency_keys + ALTER COLUMN actor_id SET NOT NULL, + DROP CONSTRAINT idempotency_keys_pkey, + ADD PRIMARY KEY (actor_id, chat_id, key); + +CREATE INDEX idempotency_created_at_idx ON idempotency_keys (created_at); diff --git a/src/sessions.py b/src/sessions.py index 9435d215..14000bea 100644 --- a/src/sessions.py +++ b/src/sessions.py @@ -271,18 +271,8 @@ async def resolve_session( "Session token is expired or revoked", {"WWW-Authenticate": 'Bearer realm="messenger"'}, ) - state, payload = await cache.lookup(hashed) - if state == HIT and payload is not None: - if payload["expires_epoch"] > time.time(): - try: - user_id = uuid.UUID(payload["user"]["id"]) - except (ValueError, TypeError, KeyError): - user_id = None - if user_id is not None: - return user_id, payload["user"] - else: - await cache.revoke(hashed, None) - elif state == TOMBSTONE: + state, _payload = await cache.lookup(hashed) + if state == TOMBSTONE: raise ApiProblem( 401, "unauthorized", @@ -290,6 +280,10 @@ async def resolve_session( {"WWW-Authenticate": 'Bearer realm="messenger"'}, ) + # PostgreSQL is the authoritative revocation store. A cache value can only + # accelerate the immutable user snapshot; it must never be enough to grant + # access on its own. This makes logout durable across API replicas and + # Valkey outages/restores, where a tombstone may not reach shared cache. async with pool.connection() as connection: async with connection.cursor() as cursor: row = await store.session_user(cursor, hashed) @@ -302,6 +296,7 @@ async def resolve_session( {"WWW-Authenticate": 'Bearer realm="messenger"'}, ) user = _user_payload(row) + # Refresh an old cache hit after PostgreSQL has confirmed it is still live. await cache.remember(hashed, user, row["expires_at"]) return uuid.UUID(user["id"]), user @@ -340,4 +335,3 @@ async def purge_expired_idempotency(pool: AsyncConnectionPool) -> int: (str(IDEMPOTENCY_TTL_SECONDS),), ) return cursor.rowcount - diff --git a/src/store.py b/src/store.py index 93bd4d3f..26b2ebd3 100644 --- a/src/store.py +++ b/src/store.py @@ -198,32 +198,40 @@ async def require_membership( raise ApiProblem(404, "not_found", "Chat not found") -async def get_idempotency_key(cursor: AsyncCursor, key: str) -> dict[str, Any] | None: - """Return the idempotency row (request_sha256, message_id, chat_id) if it exists.""" +async def get_idempotency_key( + cursor: AsyncCursor, *, actor_id: uuid.UUID, chat_id: uuid.UUID, key: str +) -> dict[str, Any] | None: + """Return a record only from this user's POST-message resource scope.""" await cursor.execute( """ - SELECT key, chat_id, request_sha256, message_id, created_at + SELECT key, actor_id, chat_id, request_sha256, message_id, created_at FROM idempotency_keys - WHERE key = %s + WHERE actor_id = %s AND chat_id = %s AND key = %s """, - (key,), + (actor_id, chat_id, key), ) return await cursor.fetchone() async def insert_idempotency_key( - cursor: AsyncCursor, *, key: str, chat_id: uuid.UUID, request_sha256: str -) -> None: - try: - await cursor.execute( - """ - INSERT INTO idempotency_keys (key, chat_id, request_sha256, created_at) - VALUES (%s, %s, %s, now()) - """, - (key, chat_id, request_sha256), - ) - except psycopg.errors.UniqueViolation: - raise ApiProblem(409, "conflict", "Idempotency-Key was already used") from None + cursor: AsyncCursor, + *, + actor_id: uuid.UUID, + key: str, + chat_id: uuid.UUID, + request_sha256: str, +) -> bool: + """Atomically reserve a key without poisoning a concurrent transaction.""" + await cursor.execute( + """ + INSERT INTO idempotency_keys (actor_id, key, chat_id, request_sha256, created_at) + VALUES (%s, %s, %s, %s, now()) + ON CONFLICT (actor_id, chat_id, key) DO NOTHING + RETURNING key + """, + (actor_id, key, chat_id, request_sha256), + ) + return await cursor.fetchone() is not None async def message_by_id(cursor: AsyncCursor, message_id: uuid.UUID) -> dict[str, Any] | None: @@ -265,8 +273,12 @@ async def create_message( row = await cursor.fetchone() # Record the message_id so a replay can return the original message. await cursor.execute( - "UPDATE idempotency_keys SET request_sha256 = %s, message_id = %s WHERE key = %s", - (request_sha256, message_id, key), + """ + UPDATE idempotency_keys + SET request_sha256 = %s, message_id = %s + WHERE actor_id = %s AND chat_id = %s AND key = %s + """, + (request_sha256, message_id, actor_id, chat_id, key), ) return message_json(row) diff --git a/student-tests/test_degradation.py b/student-tests/test_degradation.py index 18905c3c..bae5f94d 100644 --- a/student-tests/test_degradation.py +++ b/student-tests/test_degradation.py @@ -23,7 +23,7 @@ import common REPO_ROOT = Path(__file__).resolve().parents[1] -PROJECT = os.environ.get("COMPOSE_PROJECT_NAME", "mtusi2026lab2") +PROJECT = os.environ.get("COMPOSE_PROJECT_NAME", "mtusi2026lab3") ENV_FILE = os.environ.get("COMPOSE_ENV_FILE", ".env") ENABLED = os.environ.get("STUDENT_DOCKER_TESTS") == "1" @@ -47,9 +47,13 @@ def wait_ready(expected: int, seconds: float) -> float: deadline = started + seconds last = None while time.monotonic() < deadline: - last = common.request("GET", "/health/ready", timeout=5) - if last.status == expected: - return time.monotonic() - started + try: + last = common.request("GET", "/health/ready", timeout=5) + if last.status == expected: + return time.monotonic() - started + except OSError: + # A replica can close the listener while Compose restarts it. + pass time.sleep(0.5) raise AssertionError( "readiness did not reach %s within %.0fs (last=%s %r)" @@ -59,6 +63,50 @@ def wait_ready(expected: int, seconds: float) -> float: @unittest.skipUnless(ENABLED, "set STUDENT_DOCKER_TESTS=1 to stop real containers") class DependencyOutages(unittest.TestCase): + def test_logout_remains_revoked_across_replicas_after_cache_outage(self) -> None: + """A stale shared-cache hit must not bypass PostgreSQL revocation.""" + alice = common.actor("revoked") + chat = common.create_chat(alice.token, [alice.id], "Durable logout") + self.assertEqual(common.request("GET", "/api/v1/users/me", token=alice.token).status, 200) + try: + compose("stop", "valkey") + logout = common.request("DELETE", "/api/v1/auth/sessions/current", token=alice.token) + self.assertEqual(logout.status, 204, logout.json) + + # Remove both processes' local fallback state and restore Valkey's + # old data. Requests through nginx can land on either replica. + compose("restart", "api-1", "api-2") + compose("start", "valkey") + wait_ready(200, 90) + + common.expect_error( + self, common.request("GET", "/api/v1/users/me", token=alice.token), 401, "unauthorized" + ) + common.expect_error( + self, + common.request("GET", "/api/v1/chats/" + chat["id"], token=alice.token), + 401, + "unauthorized", + ) + common.expect_error( + self, + common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "must not be accepted"}, + token=alice.token, + headers={"Idempotency-Key": uuid.uuid4().hex}, + ), + 401, + "unauthorized", + ) + finally: + compose("start", "valkey", "postgres", "api-1", "api-2") + try: + wait_ready(200, 90) + except AssertionError: + pass + def test_valkey_loss_degrades_then_postgres_loss_is_bounded(self) -> None: alice = common.actor("degrade") chat = common.create_chat(alice.token, [alice.id], "Устойчивость") diff --git a/student-tests/test_idempotency.py b/student-tests/test_idempotency.py new file mode 100644 index 00000000..4c52a7e8 --- /dev/null +++ b/student-tests/test_idempotency.py @@ -0,0 +1,85 @@ +"""Idempotency scopes and concurrent replay checks for laboratory 3.""" + +from __future__ import annotations + +import threading +import unittest +import uuid + +import common + + +class IdempotencyScope(unittest.TestCase): + def test_same_key_is_independent_for_different_users_and_chats(self) -> None: + alice = common.actor("idemalice") + eve = common.actor("idemeve") + alice_chat = common.create_chat(alice.token, [alice.id], "Alice scope") + eve_chat = common.create_chat(eve.token, [eve.id], "Eve scope") + key = uuid.uuid4().hex + + first = common.request( + "POST", + "/api/v1/chats/" + alice_chat["id"] + "/messages", + body={"text": "same body"}, + token=alice.token, + headers={"Idempotency-Key": key}, + ) + second = common.request( + "POST", + "/api/v1/chats/" + eve_chat["id"] + "/messages", + body={"text": "same body"}, + token=eve.token, + headers={"Idempotency-Key": key}, + ) + self.assertEqual(first.status, 201, first.json) + self.assertEqual(second.status, 201, second.json) + self.assertNotEqual(first.json["id"], second.json["id"]) + self.assertEqual(second.json["chat_id"], eve_chat["id"]) + self.assertEqual(second.json["sender_id"], eve.id) + + # A different payload in another scope is also a new independent write. + other_scope = common.create_chat(eve.token, [eve.id], "Eve second scope") + third = common.request( + "POST", + "/api/v1/chats/" + other_scope["id"] + "/messages", + body={"text": "different body"}, + token=eve.token, + headers={"Idempotency-Key": key}, + ) + self.assertEqual(third.status, 201, third.json) + self.assertEqual(third.json["chat_id"], other_scope["id"]) + + def test_concurrent_identical_replays_return_one_message_without_500(self) -> None: + alice = common.actor("idemrace") + chat = common.create_chat(alice.token, [alice.id], "Concurrent replay") + key = uuid.uuid4().hex + workers = 32 + barrier = threading.Barrier(workers) + responses: list[common.Response] = [] + lock = threading.Lock() + + def post() -> None: + barrier.wait() + response = common.request( + "POST", + "/api/v1/chats/" + chat["id"] + "/messages", + body={"text": "exactly once"}, + token=alice.token, + headers={"Idempotency-Key": key}, + timeout=30, + ) + with lock: + responses.append(response) + + threads = [threading.Thread(target=post) for _ in range(workers)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(timeout=60) + + self.assertEqual(len(responses), workers) + self.assertTrue(all(response.status == 201 for response in responses), responses) + message_ids = {response.json["id"] for response in responses} + self.assertEqual(len(message_ids), 1) + messages = common.history(alice.token, chat["id"]) + self.assertEqual([message["text"] for message in messages], ["exactly once"])