diff --git a/.env.example b/.env.example index 714c8076..68a026f8 100644 --- a/.env.example +++ b/.env.example @@ -1,11 +1,12 @@ -# Copy to .env for your LOCAL synthetic test environment; never commit .env. -# These are logical interface suggestions; map actual framework names in REPORT.md. +# Copy to .env; choose a local password. Never commit .env. APP_HOST=0.0.0.0 APP_PORT=8080 -APP_ENV=lab +POSTGRES_DB=messenger +POSTGRES_USER=messenger +POSTGRES_PASSWORD=replace-with-local-password SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. -DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres -CACHE_URL_FILE=/run/secrets/cache_url +CACHE_TTL_SECONDS=30 +MAX_REQUEST_BYTES=65536 +MAX_INFLIGHT=32 +# Compose sets POSTGRES_HOST=postgres and REDIS_URL=redis://redis:6379/0. +# For a manual API launch set DATABASE_URL and REDIS_URL in the environment. diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..dfdb8b77 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.sh text eol=lf diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8ceefa10..58c81db7 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,21 @@ ## Лабораторная 2 Автор / группа: TODO -Целевая оценка (3/4/5): TODO -Трек 5, если заявлен: TODO +Целевая оценка (3/4/5): 5 +Трек 5, если заявлен: 1 — согласованность кэша -Что реализовано и что перенесено: TODO -Команды запуска из чистого clone: TODO -Результаты проверок и ссылка на REPORT.md: TODO -Ограничения / известные ошибки: TODO -Источники и использование ИИ: TODO +Что реализовано и что перенесено: FastAPI и REST API из lab1; PostgreSQL, +SQL-миграции, Argon2id, Basic/Bearer, Redis-сессии с TTL и AOF, Compose. +Трек 1: кэш истории с версией из PostgreSQL, воспроизведение stale read, +защита и измерения SQL/latency; read-after-write и восстановление после рестарта Redis. +Команды запуска из чистого clone: [REPORT.md](../REPORT.md). +Результаты проверок: [REPORT.md](../REPORT.md), [evidence](../evidence). +Ограничения: один Docker-host, без TLS и резервных копий; только локальный стенд. +Источники и использование ИИ: документация библиотек и помощь Codex, подробнее в отчёте. -- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. -- [ ] Контракт и публичные тесты не ослаблены. -- [ ] make check и make test выполнены; приложен реальный вывод. -- [ ] Сценарии отказов/безопасности своего уровня воспроизведены. -- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. -- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. +- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Эквиваленты make check и make test выполнены через Python на Windows; приложен реальный вывод. +- [x] Сценарии отказов/безопасности своего уровня воспроизведены. +- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens. +- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..1616ebd0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +FROM python:3.12.8-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + APP_HOST=0.0.0.0 \ + APP_PORT=8080 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY contracts ./contracts +COPY src ./src +COPY migrations ./migrations +COPY tests ./tests +COPY student-tests ./student-tests +COPY scripts ./scripts +COPY lab.json . + +RUN useradd --create-home --shell /usr/sbin/nologin appuser +USER appuser + +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=3s --start-period=5s --retries=3 \ + CMD python -c "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:' + os.getenv('APP_PORT', '8080') + '/health/ready', timeout=2).read()" + +STOPSIGNAL SIGTERM +CMD ["python", "-m", "src"] diff --git a/Makefile b/Makefile index 9ecb223e..b38f88ad 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence test-grade4 test-track1 test-failures explain help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -24,3 +24,15 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-grade4: + docker compose exec -T api python -m unittest discover -s student-tests -v + +test-failures: + $(PYTHON) student-tests/failures.py + +explain: + docker compose exec -T api python -m scripts.explain_history + +test-track1: + $(PYTHON) scripts/track1.py diff --git a/REPORT.md b/REPORT.md index e42f2b05..130f153d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,82 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Ромашко Владимир, БВТ2301 +- Целевая оценка: 5, трек 1 — согласованность кэша +- Commit/PR: указать после публикации +- Источники и помощь ИИ: документация PostgreSQL, Redis, SQLAlchemy; использован ChatGPT ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Требуется Docker Desktop/Engine с Compose v2 и Python 3.9+. + +```powershell +Copy-Item .env.example .env +# Задать POSTGRES_PASSWORD в .env +docker compose up -d --build --wait --wait-timeout 120 +python scripts/check_contract.py +python tests/smoke.py --base-url http://127.0.0.1:8080 +docker compose exec -T api python -m unittest discover -s student-tests -v +python tests/resilience.py --mode persistence --base-url http://127.0.0.1:8080 --action-script scripts/restart.cmd +python student-tests/failures.py +python scripts/track1.py +``` + +Для Linux перед запуском: `chmod +x scripts/restart.sh`. + +Остановка без удаления данных: `docker compose down`. +Удаление учебных volumes: `docker compose down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → FastAPI → PostgreSQL и Redis. + +PostgreSQL — источник истины для пользователей, чатов, участников и сообщений. +SQLAlchemy ORM использует пул до 10 соединений, timeout пула 2 с, +connect timeout 2 с, statement timeout 3 с. + +Redis хранит сессии с TTL и кэш страниц истории. Включены AOF, +`appendfsync always` и именованный volume. + +Пароли хешируются Argon2id. Basic используется только для создания сессии, +остальные защищённые запросы используют Bearer. `X-User-Id` игнорируется. + +Миграции находятся в `migrations/`. Создание чата и участников выполняется +в одной транзакции. При создании сообщения версия истории увеличивается +в той же транзакции. + +Ключ кэша истории содержит `chat_id`, версию истории, limit и cursor. +После записи старый ключ не используется. При ошибке кэша чтение выполняется +из PostgreSQL. ## Доказательства критериев | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Публичный smoke | `python tests/smoke.py` | PASS | `evidence/track1-regression.txt` | +| Сохранность после рестарта | `tests/resilience.py` | Сессия и сообщения сохранены | `evidence/track1-regression.txt` | +| Миграции, ORM и транзакции | `student-tests` | 13 тестов OK; проверены rollback, гонка регистрации и повторные миграции | `evidence/orm-tests.txt` | +| Redis, TTL и отказы | `student-tests/failures.py` | Redis/PostgreSQL возвращают 503 и восстанавливаются | `evidence/track1-failures.txt` | +| Индекс истории | `python -m scripts.explain_history` | Index Scan по `messages_history_idx` на 10 000 сообщений | `evidence/explain.txt` | +| Трек 1: stale read | `python scripts/track1.py` | До защиты 5/5 stale read, после защиты 0/5 | `evidence/cache-experiment.json` | +| Трек 1: производительность | `python scripts/track1.py` | SQL-команды: 600 без кэша и 300 с кэшем | `evidence/cache-experiment.json` | +| Трек 1: восстановление | `python scripts/track1.py` | Redis перезапущен, read-after-write и logout сохранены | `evidence/cache-restart.json` | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Гипотеза: версия истории в ключе Redis устранит stale read при позднем заполнении кэша. + +Методика: 5 повторов управляемой конкурентной гонки; затем 3 серии по 300 чтений, +concurrency 8, страница 100 сообщений, отдельный чат с 10 000 сообщениями. | Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Конкурентная запись и чтение | Старый ключ не должен использоваться | 0/5 stale read после защиты | Не применимо | +| Redis stop/restart | Ошибка 503, затем восстановление | 503; сессия и сообщения доступны | 6.1 с, потери нет | +| PostgreSQL stop/start | Ошибка 503, затем восстановление | 503; API не доверяет анонимному запросу | 6.2 с, потери нет | +| Redis restart в треке 1 | Кэш восстановится из PostgreSQL | Read-after-write и logout прошли | 1.2 с, потери нет | ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Стенд рассчитан на один Docker-host и один экземпляр PostgreSQL/Redis. +Cache stampede не исследовался. Нагрузочные результаты относятся к слою хранения, +а не ко всему HTTP API. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..e8952368 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,60 @@ +services: + api: + build: . + dns_opt: + - timeout:1 + - attempts:1 + ports: + - "127.0.0.1:${APP_PORT:-8080}:8080" + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + POSTGRES_HOST: postgres + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + REDIS_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-30} + MAX_REQUEST_BYTES: ${MAX_REQUEST_BYTES:-65536} + MAX_INFLIGHT: ${MAX_INFLIGHT:-32} + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + cpus: 1.0 + mem_limit: 512m + + postgres: + image: postgres:17.4-bookworm + environment: + POSTGRES_DB: ${POSTGRES_DB:-messenger} + POSTGRES_USER: ${POSTGRES_USER:-messenger} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 1.0 + mem_limit: 512m + + redis: + image: redis:7.4.2-bookworm + command: ["redis-server", "--appendonly", "yes", "--appendfsync", "always", "--maxmemory", "128mb", "--maxmemory-policy", "noeviction"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 2s + timeout: 2s + retries: 30 + cpus: 0.5 + mem_limit: 256m + +volumes: + postgres_data: + redis_data: diff --git a/evidence/cache-experiment.json b/evidence/cache-experiment.json new file mode 100644 index 00000000..850bae77 --- /dev/null +++ b/evidence/cache-experiment.json @@ -0,0 +1,207 @@ +{ + "race_repeats": 5, + "race": { + "naive": [ + { + "stale": true, + "read_after_write_ms": 3.362, + "sql": 6, + "history_selects": 1 + }, + { + "stale": true, + "read_after_write_ms": 2.705, + "sql": 6, + "history_selects": 1 + }, + { + "stale": true, + "read_after_write_ms": 2.644, + "sql": 6, + "history_selects": 1 + }, + { + "stale": true, + "read_after_write_ms": 2.601, + "sql": 6, + "history_selects": 1 + }, + { + "stale": true, + "read_after_write_ms": 2.491, + "sql": 6, + "history_selects": 1 + } + ], + "versioned": [ + { + "stale": false, + "read_after_write_ms": 6.639, + "sql": 7, + "history_selects": 2 + }, + { + "stale": false, + "read_after_write_ms": 7.862, + "sql": 7, + "history_selects": 2 + }, + { + "stale": false, + "read_after_write_ms": 8.104, + "sql": 7, + "history_selects": 2 + }, + { + "stale": false, + "read_after_write_ms": 8.303, + "sql": 7, + "history_selects": 2 + }, + { + "stale": false, + "read_after_write_ms": 6.384, + "sql": 7, + "history_selects": 2 + } + ] + }, + "benchmark": { + "uncached": [ + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 2.84, + "successful_rps": 105.64, + "client_cpu_seconds": 2.878, + "client_peak_rss_mib": 132.52, + "sql": 600, + "history_selects": 300, + "p50_ms": 78.194, + "p95_ms": 108.779, + "p99_ms": 140.556 + }, + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 3.164, + "successful_rps": 94.82, + "client_cpu_seconds": 3.148, + "client_peak_rss_mib": 132.52, + "sql": 600, + "history_selects": 300, + "p50_ms": 85.254, + "p95_ms": 126.226, + "p99_ms": 150.227 + }, + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 3.134, + "successful_rps": 95.73, + "client_cpu_seconds": 3.142, + "client_peak_rss_mib": 132.52, + "sql": 600, + "history_selects": 300, + "p50_ms": 83.202, + "p95_ms": 121.385, + "p99_ms": 148.622 + } + ], + "naive": [ + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 1.683, + "successful_rps": 178.21, + "client_cpu_seconds": 1.689, + "client_peak_rss_mib": 132.52, + "sql": 300, + "history_selects": 0, + "p50_ms": 39.913, + "p95_ms": 86.212, + "p99_ms": 101.446 + }, + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 1.361, + "successful_rps": 220.37, + "client_cpu_seconds": 1.353, + "client_peak_rss_mib": 132.52, + "sql": 300, + "history_selects": 0, + "p50_ms": 28.237, + "p95_ms": 65.387, + "p99_ms": 73.071 + }, + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 1.147, + "successful_rps": 261.59, + "client_cpu_seconds": 1.17, + "client_peak_rss_mib": 132.52, + "sql": 300, + "history_selects": 0, + "p50_ms": 22.607, + "p95_ms": 58.264, + "p99_ms": 63.041 + } + ], + "versioned": [ + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 1.238, + "successful_rps": 242.35, + "client_cpu_seconds": 1.254, + "client_peak_rss_mib": 132.52, + "sql": 300, + "history_selects": 0, + "p50_ms": 25.229, + "p95_ms": 60.546, + "p99_ms": 65.823 + }, + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 1.696, + "successful_rps": 176.85, + "client_cpu_seconds": 1.709, + "client_peak_rss_mib": 132.52, + "sql": 300, + "history_selects": 0, + "p50_ms": 41.596, + "p95_ms": 73.311, + "p99_ms": 81.611 + }, + { + "requests": 300, + "concurrency": 8, + "errors": 0, + "seconds": 1.209, + "successful_rps": 248.18, + "client_cpu_seconds": 1.22, + "client_peak_rss_mib": 132.52, + "sql": 300, + "history_selects": 0, + "p50_ms": 24.794, + "p95_ms": 58.879, + "p99_ms": 64.291 + } + ] + }, + "messages": 10000, + "page_size": 100, + "warmup_reads_per_run": 1, + "scope": "store calls; real SQL and Redis; excludes HTTP, sessions and KDF; SQL excludes BEGIN/COMMIT" +} diff --git a/evidence/cache-resources.txt b/evidence/cache-resources.txt new file mode 100644 index 00000000..583821b9 --- /dev/null +++ b/evidence/cache-resources.txt @@ -0,0 +1,4 @@ +Post-experiment snapshot (not peak): +mtusi2026lab2-api-1 CPU=0.22% RAM=58.04MiB / 512MiB +mtusi2026lab2-postgres-1 CPU=7.08% RAM=38.14MiB / 512MiB +mtusi2026lab2-redis-1 CPU=0.30% RAM=3.582MiB / 256MiB diff --git a/evidence/cache-restart.json b/evidence/cache-restart.json new file mode 100644 index 00000000..d2fb524e --- /dev/null +++ b/evidence/cache-restart.json @@ -0,0 +1,11 @@ +{ + "status": "PASS", + "same_session": true, + "confirmed_messages": 3, + "cold_cache_rebuilt": true, + "read_after_write": true, + "logout_still_401": true, + "recovery_seconds": 1.161, + "redis_start_before_utc": "2026-10-09T12:36:26.91215174Z", + "redis_start_after_utc": "2026-10-09T12:47:22.636577776Z" +} diff --git a/evidence/checks.txt b/evidence/checks.txt new file mode 100644 index 00000000..2591a59c --- /dev/null +++ b/evidence/checks.txt @@ -0,0 +1,38 @@ +Date: 2026-10-09 (Europe/Moscow) + +Command: python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +Command: docker compose exec -T api python tests/smoke.py +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. + +Command: docker compose exec -T api python -m unittest discover -s student-tests -v +test_chat_rollback ... ok +test_expiry_short_ttl ... ok +test_migrations_repeat ... ok +test_password_logout_and_identity ... ok +test_registration_race ... ok +test_unicode_password ... ok +Ran 6 tests in 8.057s +OK + +Command: python tests/resilience.py --mode persistence --action-script scripts/restart.cmd +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. +Container start times: evidence/restart.txt + +Command: python student-tests/failures.py +PASS redis unavailable: 503; seconds=[3.04, 3.041, 3.036]; anonymous=401; live=200 +PASS redis recovered in 6.12s: same session and message +PASS postgres unavailable: 503; seconds=[2.032, 4.034, 4.049]; anonymous=401; live=200 +PASS postgres recovered in 6.41s: same session and message + +Initial failure test exceeded the HTTP timeout because Docker DNS waited too long +for the stopped service. Fixed with dns_opt timeout:1, attempts:1 on the API. +The timings above are from the successful rerun after this fix. + +Final verification after fixes: +python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. +Integration suite repeated: Ran 6 tests in 8.030s; OK. +docker compose ps: api, postgres and redis are healthy. diff --git a/evidence/explain.txt b/evidence/explain.txt new file mode 100644 index 00000000..0be4e813 --- /dev/null +++ b/evidence/explain.txt @@ -0,0 +1,12 @@ +Command: docker compose exec -T api python -m scripts.explain_history +Dataset: 10000 messages; page after message 9000; ORM query; limit 51 +Limit (cost=0.29..75.12 rows=51 width=1055) (actual time=0.013..0.028 rows=51 loops=1) + Buffers: shared hit=10 + -> Index Scan using messages_history_idx on messages (cost=0.29..1535.22 rows=1046 width=1055) (actual time=0.013..0.024 rows=51 loops=1) + Index Cond: ((chat_id = 'ff693a1b-4e99-43c9-bb51-68b5acdd972b'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:09+00'::timestamp with time zone, 'e35d4a84-afe1-4aae-8f55-27be1490c84e'::uuid))) + Buffers: shared hit=10 +Planning: + Buffers: shared hit=7 +Planning Time: 0.078 ms +Execution Time: 0.056 ms +Synthetic data rolled back. diff --git a/evidence/orm-tests.txt b/evidence/orm-tests.txt new file mode 100644 index 00000000..1befae36 --- /dev/null +++ b/evidence/orm-tests.txt @@ -0,0 +1,20 @@ +SQLAlchemy ORM +Command: docker compose exec -T api python -m unittest discover -s student-tests -v +test_concurrent_writers_do_not_lose_version_increments (test_cache.CacheTests.test_concurrent_writers_do_not_lose_version_increments) ... ok +test_hit_still_checks_membership_and_cache_failure_falls_back (test_cache.CacheTests.test_hit_still_checks_membership_and_cache_failure_falls_back) ... ok +test_http_read_after_write_and_cursor_pages (test_cache.CacheTests.test_http_read_after_write_and_cursor_pages) ... ok +test_late_fill_cannot_restore_old_page (test_cache.CacheTests.test_late_fill_cannot_restore_old_page) ... ok +test_read_after_write_through_another_api_process (test_cache.CacheTests.test_read_after_write_through_another_api_process) ... ok +test_version_rolls_back_with_failed_message (test_cache.CacheTests.test_version_rolls_back_with_failed_message) ... ok +test_migrate_empty_schema_and_preserve_orm_data_on_repeat (test_database.MigrationTests.test_migrate_empty_schema_and_preserve_orm_data_on_repeat) ... ok +test_chat_rollback (test_lab2.Lab2Tests.test_chat_rollback) ... ok +test_expiry_short_ttl (test_lab2.Lab2Tests.test_expiry_short_ttl) ... ok +test_migrations_repeat (test_lab2.Lab2Tests.test_migrations_repeat) ... ok +test_password_logout_and_identity (test_lab2.Lab2Tests.test_password_logout_and_identity) ... ok +test_registration_race (test_lab2.Lab2Tests.test_registration_race) ... ok +test_unicode_password (test_lab2.Lab2Tests.test_unicode_password) ... ok + +---------------------------------------------------------------------- +Ran 13 tests in 18.089s + +OK diff --git a/evidence/restart.txt b/evidence/restart.txt new file mode 100644 index 00000000..70f791a2 --- /dev/null +++ b/evidence/restart.txt @@ -0,0 +1,9 @@ +Command: docker compose restart api postgres redis +Before (UTC): +/mtusi2026lab2-api-1 2026-10-09T12:45:16.929905363Z +/mtusi2026lab2-postgres-1 2026-10-09T12:36:54.655202315Z +/mtusi2026lab2-redis-1 2026-10-09T12:47:22.636577776Z +After (UTC): +/mtusi2026lab2-api-1 2026-10-09T12:49:18.471856288Z +/mtusi2026lab2-postgres-1 2026-10-09T12:49:17.844464627Z +/mtusi2026lab2-redis-1 2026-10-09T12:49:17.810209464Z diff --git a/evidence/track1-failures.txt b/evidence/track1-failures.txt new file mode 100644 index 00000000..219dfb5b --- /dev/null +++ b/evidence/track1-failures.txt @@ -0,0 +1,13 @@ +Command: python student-tests/failures.py + Container mtusi2026lab2-redis-1 Stopping + Container mtusi2026lab2-redis-1 Stopped +PASS redis unavailable: 503; seconds=[3.043, 3.052, 3.035]; anonymous=401; live=200 + Container mtusi2026lab2-redis-1 Starting + Container mtusi2026lab2-redis-1 Started +PASS redis recovered in 6.13s: same session and message + Container mtusi2026lab2-postgres-1 Stopping + Container mtusi2026lab2-postgres-1 Stopped +PASS postgres unavailable: 503; seconds=[2.038, 4.033, 4.027]; anonymous=401; live=200 + Container mtusi2026lab2-postgres-1 Starting + Container mtusi2026lab2-postgres-1 Started +PASS postgres recovered in 6.14s: same session and message diff --git a/evidence/track1-regression.txt b/evidence/track1-regression.txt new file mode 100644 index 00000000..60ab0b8c --- /dev/null +++ b/evidence/track1-regression.txt @@ -0,0 +1,6 @@ +Command: docker compose exec -T api python tests/smoke.py +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. + +Command: .venv/Scripts/python.exe tests/resilience.py --mode persistence --base-url http://127.0.0.1:8080 --action-script scripts/restart.cmd +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/track1-tests.txt b/evidence/track1-tests.txt new file mode 100644 index 00000000..d229a468 --- /dev/null +++ b/evidence/track1-tests.txt @@ -0,0 +1,17 @@ +Command: docker compose exec -T api python -m unittest discover -s student-tests -v +test_hit_still_checks_membership_and_cache_failure_falls_back (test_cache.CacheTests.test_hit_still_checks_membership_and_cache_failure_falls_back) ... ok +test_http_read_after_write_and_cursor_pages (test_cache.CacheTests.test_http_read_after_write_and_cursor_pages) ... ok +test_late_fill_cannot_restore_old_page (test_cache.CacheTests.test_late_fill_cannot_restore_old_page) ... ok +test_read_after_write_through_another_api_process (test_cache.CacheTests.test_read_after_write_through_another_api_process) ... ok +test_version_rolls_back_with_failed_message (test_cache.CacheTests.test_version_rolls_back_with_failed_message) ... ok +test_chat_rollback (test_lab2.Lab2Tests.test_chat_rollback) ... ok +test_expiry_short_ttl (test_lab2.Lab2Tests.test_expiry_short_ttl) ... ok +test_migrations_repeat (test_lab2.Lab2Tests.test_migrations_repeat) ... ok +test_password_logout_and_identity (test_lab2.Lab2Tests.test_password_logout_and_identity) ... ok +test_registration_race (test_lab2.Lab2Tests.test_registration_race) ... ok +test_unicode_password (test_lab2.Lab2Tests.test_unicode_password) ... ok + +---------------------------------------------------------------------- +Ran 11 tests in 13.089s + +OK diff --git a/infra/schema.md b/infra/schema.md new file mode 100644 index 00000000..cc9dfb31 --- /dev/null +++ b/infra/schema.md @@ -0,0 +1,25 @@ +# Схема и транзакции (план реализации) + +- `users`: UUID PK, уникальный username, display_name, password_hash, created_at. +- `chats`: UUID PK, title, created_at. +- `chat_members`: составной PK (chat_id, user_id), FK на chats и users. +- `messages`: UUID PK, FK chat_id и sender_id, text, created_at. +- Индекс `messages(chat_id, created_at, id)` обслуживает историю: + `WHERE chat_id = %s AND (created_at, id) > (%s, %s) + ORDER BY created_at, id LIMIT %s`. Берём limit + 1 для next_cursor. +- Индекс `chat_members(user_id, chat_id)` — список чатов пользователя. +- Создание чата и всех участников — одна транзакция; ошибка любого FK + откатывает также сам чат. Регистрация и запись сообщения — отдельные транзакции. +- UNIQUE username разрешает гонку регистрации на стороне PostgreSQL. +- Версии SQL-миграций записываются в `schema_migrations` в той же транзакции, + что и DDL; advisory lock защищает одновременный запуск миграций. +- Redis — источник истины для сессий: ключ с SHA-256 случайного токена, + значение user_id и expires_at, абсолютный TTL. Logout удаляет ключ. + AOF + appendfsync always и именованный volume сохраняют действующие сессии. + При отказе Redis защищённый запрос получает 503, обхода аутентификации нет. +- Трек 1: migration 002 добавляет `chats.history_version`. Запись сообщения + увеличивает версию в той же транзакции (сначала UPDATE, затем INSERT). + Redis хранит страницы по ключу chat_id + version + hash(limit, cursor), TTL 30 с. + Членство и версия всегда читаются из PostgreSQL. Старые ключи удаляются по TTL; + логическая инвалидация происходит атомарно при COMMIT версии и сообщения. + Опоздавшее заполнение старого ключа не влияет на новую версию. diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..aee31d8f --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); + +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX chat_members_user_idx ON chat_members (user_id, chat_id); + +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id), + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE INDEX messages_history_idx ON messages (chat_id, created_at, id); +CREATE INDEX chats_order_idx ON chats (created_at, id); diff --git a/migrations/002_history_version.sql b/migrations/002_history_version.sql new file mode 100644 index 00000000..7b7972c1 --- /dev/null +++ b/migrations/002_history_version.sql @@ -0,0 +1 @@ +ALTER TABLE chats ADD COLUMN history_version bigint NOT NULL DEFAULT 0; diff --git a/requirements-experiment.txt b/requirements-experiment.txt new file mode 100644 index 00000000..ab4bcbb5 --- /dev/null +++ b/requirements-experiment.txt @@ -0,0 +1,2 @@ +# Only needed to draw the saved experiment data, not by the API. +matplotlib==3.10.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..7702e0b3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,17 @@ +fastapi==0.115.6 +uvicorn==0.34.0 +pydantic==2.13.5 +starlette==0.41.3 +annotated-types==0.8.0 +anyio==4.15.1 +click==8.5.0 +colorama==0.4.6; sys_platform == "win32" +h11==0.16.0 +idna==3.20 +pydantic_core==2.46.5 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +psycopg[binary]==3.2.6 +SQLAlchemy==2.0.41 +redis==5.2.1 +argon2-cffi==23.1.0 diff --git a/scripts/cache_experiment.py b/scripts/cache_experiment.py new file mode 100644 index 00000000..8371d124 --- /dev/null +++ b/scripts/cache_experiment.py @@ -0,0 +1,168 @@ +"""Track 1: deterministic stale-fill race and measured reads on real PostgreSQL/Redis. + +The unsafe cache exists ONLY in this experiment; API always uses versioned keys. +""" +import json +import math +import resource +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from uuid import uuid4 + +from sqlalchemy import delete, event, select +from sqlalchemy.orm import sessionmaker + +from src.auth import sessions +from src.cache import HistoryCache +from src.database import create_db_engine +from src.models import User, Chat, ChatMember +from src.store import PostgresStore +from .fixtures import seed_messages + + +class NaiveCache(HistoryCache): + def key(self, chat_id, version, limit, after): + return super().key(chat_id, 0, limit, after) + + +class NoCache: + def get_or_load(self, chat_id, version, limit, after, load): + return load() + + +class QueryCounter: + lock = threading.Lock() + counts = {'sql': 0, 'history_selects': 0} + + @classmethod + def record(cls, conn, cursor, statement, parameters, context, executemany): + with cls.lock: + cls.counts['sql'] += 1 + normalized = ' '.join(statement.split()).lower() + if normalized.startswith('select ') and ' from messages ' in normalized: + cls.counts['history_selects'] += 1 + + @classmethod + def reset(cls): + with cls.lock: + cls.counts = {'sql': 0, 'history_selects': 0} + + +def make_store(engine, cache): + store = PostgresStore(cache) + store.sessions = sessionmaker(engine, expire_on_commit=False) + return store + + +def stale_race(reader, writer, actor_id, chat_id, naive=False): + """Pause AFTER the real history SELECT, commit a write, then finish cache SET.""" + loaded, resume = threading.Event(), threading.Event() + original_load = reader._load_messages + + def paused_load(*args): + page = original_load(*args) + loaded.set() + if not resume.wait(10): + raise AssertionError('Race writer did not finish') + return page + + reader._load_messages = paused_load + try: + with ThreadPoolExecutor(max_workers=1) as executor: + pending = executor.submit(reader.list_messages, chat_id, actor_id, 100, None) + try: + assert loaded.wait(10), 'Reader did not reach the cache-fill barrier' + message = writer.create_message(chat_id, actor_id, 'committed during cache fill') + if naive: + reader.cache.client.delete(reader.cache.key(chat_id, 0, 100, None)) + finally: + resume.set() + overlapping_page = pending.result(timeout=10) + finally: + reader._load_messages = original_load + assert message['id'] not in {item['id'] for item in overlapping_page['items']} + started = time.perf_counter() + page = reader.list_messages(chat_id, actor_id, 100, None) + return {'stale': message['id'] not in {item['id'] for item in page['items']}, + 'read_after_write_ms': round((time.perf_counter() - started) * 1000, 3)} + + +def benchmark(store, actor_id, chat_id, requests=300, concurrency=8): + if isinstance(store.cache, HistoryCache): + for key in store.cache.client.scan_iter(match=f'{store.cache.prefix}:{chat_id}:*'): + store.cache.client.delete(key) + expected = store.list_messages(chat_id, actor_id, 100, None) + QueryCounter.reset() + started = time.perf_counter() + cpu_started = time.process_time() + + def read(_): + tick = time.perf_counter() + page = store.list_messages(chat_id, actor_id, 100, None) + assert page == expected + return (time.perf_counter() - tick) * 1000 + + with ThreadPoolExecutor(max_workers=concurrency) as executor: + latency = sorted(executor.map(read, range(requests))) + elapsed = time.perf_counter() - started + result = { + 'requests': requests, 'concurrency': concurrency, 'errors': 0, + 'seconds': round(elapsed, 3), 'successful_rps': round(requests / elapsed, 2), + 'client_cpu_seconds': round(time.process_time() - cpu_started, 3), + 'client_peak_rss_mib': round(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1024, 2), + **QueryCounter.counts, + } + for percentile in (50, 95, 99): + result[f'p{percentile}_ms'] = round(latency[math.ceil(len(latency) * percentile / 100) - 1], 3) + return result + + +def main(): + prefix = 'experiment:' + uuid4().hex + engine = create_db_engine() + event.listen(engine, 'before_cursor_execute', QueryCounter.record) + writer = make_store(engine, NoCache()) + user = writer.create_user('exp_' + uuid4().hex[:12], 'Experiment', 'Synthetic-password-123') + result = {'race_repeats': 5, 'race': {}, 'benchmark': {}, 'messages': 10000, + 'page_size': 100, 'warmup_reads_per_run': 1, + 'scope': 'store calls; real SQL and Redis; excludes HTTP, sessions and KDF; SQL excludes BEGIN/COMMIT'} + try: + for name, cache_type in [('naive', NaiveCache), ('versioned', HistoryCache)]: + results = [] + for n in range(5): + chat = writer.create_chat(user['id'], 'Race', [user['id']]) + reader = make_store(engine, cache_type(sessions.client, prefix=f'{prefix}:{name}:{n}')) + QueryCounter.reset() + row = stale_race(reader, writer, user['id'], chat['id'], naive=name == 'naive') + row.update(QueryCounter.counts) + assert row['stale'] == (name == 'naive') + results.append(row) + result['race'][name] = results + + chat = writer.create_chat(user['id'], 'Benchmark', [user['id']]) + with writer.sessions.begin() as session: + seed_messages(session, chat['id'], user['id']) + modes = { + 'uncached': make_store(engine, NoCache()), + 'naive': make_store(engine, NaiveCache(sessions.client, prefix=prefix + ':bench-naive')), + 'versioned': make_store(engine, HistoryCache(sessions.client, prefix=prefix + ':bench-versioned')), + } + result['benchmark'] = {name: [] for name in modes} + names = list(modes) + for repeat in range(3): + for name in names[repeat:] + names[:repeat]: + result['benchmark'][name].append(benchmark(modes[name], user['id'], chat['id'])) + print(json.dumps(result, ensure_ascii=False, indent=2)) + finally: + for key in sessions.client.scan_iter(match=prefix + ':*'): + sessions.client.delete(key) + with writer.sessions.begin() as session: + session.execute(delete(Chat).where(Chat.id.in_( + select(ChatMember.chat_id).where(ChatMember.user_id == user['id'])))) + session.execute(delete(User).where(User.id == user['id'])) + engine.dispose() + + +if __name__ == '__main__': + main() diff --git a/scripts/explain_history.py b/scripts/explain_history.py new file mode 100644 index 00000000..cb6916f6 --- /dev/null +++ b/scripts/explain_history.py @@ -0,0 +1,40 @@ +"""EXPLAIN the actual ORM history query on 10,000 rows, then roll back fixtures.""" +from uuid import uuid4 + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from src.database import create_db_engine +from src.models import User, Chat, Message +from src.store import history_query +from .fixtures import seed_messages + + +def main(): + engine = create_db_engine() + try: + with Session(engine) as session: + user = User(username='explain_' + uuid4().hex[:12], display_name='Explain', password_hash='unused') + session.add(user) + session.flush() + chat = Chat(title='Explain') + session.add(chat) + session.flush() + seed_messages(session, chat.id, user.id) + conn = session.connection() + conn.exec_driver_sql('ANALYZE messages') + boundary = session.execute(select(Message.created_at, Message.id).where( + Message.chat_id == chat.id).order_by(Message.created_at, Message.id).offset(8999).limit(1)).one() + after = (boundary.created_at.isoformat(), boundary.id) + compiled = history_query(chat.id, 50, after).compile(dialect=engine.dialect) + print('Dataset: 10000 messages; page after message 9000; ORM query; limit 51') + for row in conn.exec_driver_sql('EXPLAIN (ANALYZE, BUFFERS) ' + str(compiled), compiled.params): + print(row[0]) + session.rollback() + print('Synthetic data rolled back.') + finally: + engine.dispose() + + +if __name__ == '__main__': + main() diff --git a/scripts/fixtures.py b/scripts/fixtures.py new file mode 100644 index 00000000..6fcf6eaf --- /dev/null +++ b/scripts/fixtures.py @@ -0,0 +1,17 @@ +"""Synthetic message history shared by EXPLAIN and cache measurements.""" +from datetime import datetime, timedelta, timezone + +from sqlalchemy import insert, update + +from src.models import Chat, Message + + +def seed_messages(session, chat_id, user_id, count=10000): + start = datetime(2026, 1, 1, tzinfo=timezone.utc) + session.execute(insert(Message), [ + {'chat_id': chat_id, 'sender_id': user_id, 'text': 'synthetic ' * 100, + 'created_at': start + timedelta(milliseconds=n)} + for n in range(1, count + 1) + ]) + session.execute(update(Chat).where(Chat.id == chat_id).values( + history_version=Chat.history_version + 1)) diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..95d967db --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts\restart.py diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..c797090a --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,28 @@ +"""Restart the whole lab stack, retaining volumes and recording actual start times.""" +import subprocess +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def snapshot(): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], cwd=ROOT, text=True).split() + return subprocess.check_output( + ['docker', 'inspect', '--format', '{{.Name}} {{.State.StartedAt}}', *ids], + cwd=ROOT, text=True, + ) + + +def main(): + before = snapshot() + subprocess.run(['docker', 'compose', 'restart', 'api', 'postgres', 'redis'], + cwd=ROOT, check=True, timeout=60) + after = snapshot() + (ROOT / 'evidence').mkdir(exist_ok=True) + (ROOT / 'evidence' / 'restart.txt').write_text( + 'Command: docker compose restart api postgres redis\nBefore (UTC):\n' + before + + 'After (UTC):\n' + after, encoding='utf-8') + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..d504e1e1 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +python3 scripts/restart.py diff --git a/scripts/track1.py b/scripts/track1.py new file mode 100644 index 00000000..0f9cf1a0 --- /dev/null +++ b/scripts/track1.py @@ -0,0 +1,91 @@ +"""Run on the host: research measurements and a real Redis restart through HTTP.""" +import json +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + + +def command(*args): + return subprocess.check_output(list(args), cwd=ROOT, text=True, encoding='utf-8', timeout=180) + + +def cache_state(chat_id, clear=False): + code = """import json, sys +from uuid import UUID +from src.auth import sessions +keys = list(sessions.client.scan_iter(match='history:' + str(UUID(sys.argv[1])) + ':*')) +print(json.dumps({'keys': len(keys)})) +if sys.argv[2] == 'clear' and keys: + sessions.client.delete(*keys) +""" + return json.loads(command('docker', 'compose', 'exec', '-T', 'api', 'python', '-c', code, + chat_id, 'clear' if clear else 'inspect')) + + +def restart_check(): + client = Client('http://127.0.0.1:8080') + actor = client.user('restartcache') + chat = client.call('POST', '/api/v1/chats', 201, actor=actor, + body={'title': 'Cache restart', 'member_ids': [actor['id']]}, schema='Chat')['json'] + path = f"/api/v1/chats/{chat['id']}/messages" + def read(): + return client.call('GET', path, actor=actor, schema='MessagePage')['json'] + first = client.message(actor, chat, 'before Redis restart')['json'] + assert read()['items'] == [first] + cached_before = cache_state(chat['id'])['keys'] + assert cached_before > 0 + second = client.message(actor, chat, 'committed after cache fill')['json'] + redis_id = command('docker', 'compose', 'ps', '-q', 'redis').strip() + before_start = command('docker', 'inspect', '--format', '{{.State.StartedAt}}', redis_id).strip() + started = time.monotonic() + command('docker', 'compose', 'restart', 'redis') + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + try: + client.call('GET', '/health/ready') + break + except AssertionError: + time.sleep(0.5) + else: + raise AssertionError('Redis recovery timed out') + after_start = command('docker', 'inspect', '--format', '{{.State.StartedAt}}', redis_id).strip() + assert before_start != after_start + assert read()['items'] == [first, second] + recovery = round(time.monotonic() - started, 3) + cache_state(chat['id'], clear=True) + assert cache_state(chat['id'])['keys'] == 0 + assert read()['items'] == [first, second] + assert cache_state(chat['id'])['keys'] > 0 + third = client.message(actor, chat, 'after Redis recovery')['json'] + assert read()['items'] == [first, second, third] + client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + client.call('GET', path, 401, actor=actor) + return {'status': 'PASS', 'same_session': True, 'confirmed_messages': 3, + 'cold_cache_rebuilt': True, 'read_after_write': True, 'logout_still_401': True, + 'recovery_seconds': recovery, 'redis_start_before_utc': before_start, + 'redis_start_after_utc': after_start} + + +def main(): + evidence = ROOT / 'evidence' + evidence.mkdir(exist_ok=True) + result = json.loads(command('docker', 'compose', 'exec', '-T', 'api', + 'python', '-m', 'scripts.cache_experiment')) + (evidence / 'cache-experiment.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') + result = restart_check() + (evidence / 'cache-restart.json').write_text(json.dumps(result, indent=2) + '\n', encoding='utf-8') + ids = command('docker', 'compose', 'ps', '-q').split() + snapshot = command('docker', 'stats', '--no-stream', '--format', '{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}', *ids) + (evidence / 'cache-resources.txt').write_text('Post-experiment snapshot (not peak):\n' + snapshot, encoding='utf-8') + print('PASS: stale race reproduced before protection and prevented after; SQL/latency measured.') + print('PASS: Redis restart, same session, read-after-write, cold cache rebuild, logout.') + print('Evidence: cache-experiment.json, cache-restart.json, cache-resources.txt') + + +if __name__ == '__main__': + main() diff --git a/scripts/validate_openapi.py b/scripts/validate_openapi.py index 1dfe4a98..ab5f72cf 100644 --- a/scripts/validate_openapi.py +++ b/scripts/validate_openapi.py @@ -9,7 +9,6 @@ root = Path(__file__).resolve().parents[1] spec = json.loads((root/'contracts/openapi.json').read_text()) validate(spec) from referencing import Registry, Resource -# The base URI resolves local #/components references inside examples. resource = dict(spec, **{'$schema': 'https://json-schema.org/draft/2020-12/schema'}) registry = Registry().with_resource('urn:mtusi:openapi', Resource.from_contents(resource)) for name, example in json.loads((root/'contracts/examples.json').read_text()).items(): diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/__main__.py b/src/__main__.py new file mode 100644 index 00000000..d5a34072 --- /dev/null +++ b/src/__main__.py @@ -0,0 +1,13 @@ +import os + +import uvicorn + + +if __name__ == '__main__': + uvicorn.run( + 'src.main:app', + host=os.getenv('APP_HOST', '0.0.0.0'), + port=int(os.getenv('APP_PORT', '8080')), + workers=1, + timeout_graceful_shutdown=5, + ) diff --git a/src/auth.py b/src/auth.py new file mode 100644 index 00000000..ab93a9fd --- /dev/null +++ b/src/auth.py @@ -0,0 +1,68 @@ +import hashlib +import json +import os +import re +import secrets +from datetime import datetime, timedelta, timezone + +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from redis import Redis + +from .errors import APIError + +password_hasher = PasswordHasher() +dummy_hash = password_hasher.hash(secrets.token_urlsafe(32)) + + +def verify_password(encoded, password): + try: + return password_hasher.verify(encoded or dummy_hash, password) and encoded is not None + except VerificationError: + return False + + +def unauthorized(): + return APIError(401, 'unauthorized', 'Нет действительной идентификации') + + +class Sessions: + def __init__(self): + self.client = Redis.from_url( + os.getenv('REDIS_URL', 'redis://redis:6379/0'), decode_responses=True, + socket_connect_timeout=1, socket_timeout=1, max_connections=20, + retry_on_timeout=False, + ) + self.ttl = int(os.getenv('SESSION_TTL_SECONDS', '3600')) + if self.ttl < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') + + @staticmethod + def key(token): + if not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise unauthorized() + return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest() + + def create(self, user): + token = secrets.token_urlsafe(32) + expires = datetime.now(timezone.utc) + timedelta(seconds=self.ttl) + expires_at = expires.isoformat(timespec='microseconds').replace('+00:00', 'Z') + self.client.set(self.key(token), json.dumps({'user_id': user['id'], 'expires_at': expires_at}), + pxat=int(expires.timestamp() * 1000)) + return {'token': token, 'token_type': 'Bearer', 'expires_at': expires_at, 'user': user} + + def user_id(self, token): + raw = self.client.get(self.key(token)) + if raw is None: + raise unauthorized() + session = json.loads(raw) + if datetime.fromisoformat(session['expires_at'].replace('Z', '+00:00')) <= datetime.now(timezone.utc): + raise unauthorized() + return session['user_id'] + + def delete(self, token): + if not self.client.delete(self.key(token)): + raise unauthorized() + + +sessions = Sessions() diff --git a/src/cache.py b/src/cache.py new file mode 100644 index 00000000..af851c20 --- /dev/null +++ b/src/cache.py @@ -0,0 +1,40 @@ +"""History pages are disposable; PostgreSQL owns both data and their version.""" +import hashlib +import json +import os + +from redis.exceptions import RedisError + +from .auth import sessions + + +class HistoryCache: + def __init__(self, client, ttl=30, prefix='history'): + self.client = client + self.ttl = ttl + self.prefix = prefix + if ttl < 1: + raise ValueError('CACHE_TTL_SECONDS must be positive') + + def key(self, chat_id, version, limit, after): + page = json.dumps([limit, after], separators=(',', ':')) + digest = hashlib.sha256(page.encode()).hexdigest() + return f'{self.prefix}:{chat_id}:{version}:{digest}' + + def get_or_load(self, chat_id, version, limit, after, load): + key = self.key(chat_id, version, limit, after) + try: + cached = self.client.get(key) + except RedisError: + return load() + if cached is not None: + return json.loads(cached) + page = load() + try: + self.client.set(key, json.dumps(page, ensure_ascii=False), ex=self.ttl) + except RedisError: + pass + return page + + +history_cache = HistoryCache(sessions.client, int(os.getenv('CACHE_TTL_SECONDS', '30'))) diff --git a/src/database.py b/src/database.py new file mode 100644 index 00000000..e98b4d4c --- /dev/null +++ b/src/database.py @@ -0,0 +1,51 @@ +import os +from pathlib import Path + +from sqlalchemy import create_engine, func, insert, select +from sqlalchemy.engine import URL, make_url + +from .models import SchemaMigration + + +def database_url(): + if os.getenv('DATABASE_URL'): + url = make_url(os.environ['DATABASE_URL']) + if url.drivername in ('postgres', 'postgresql'): + url = url.set(drivername='postgresql+psycopg') + if url.drivername != 'postgresql+psycopg': + raise ValueError('DATABASE_URL must use PostgreSQL with psycopg') + return url + return URL.create( + 'postgresql+psycopg', host=os.getenv('POSTGRES_HOST', 'postgres'), + database=os.getenv('POSTGRES_DB', 'messenger'), + username=os.getenv('POSTGRES_USER', 'messenger'), + password=os.environ['POSTGRES_PASSWORD'], + ) + + +def create_db_engine(): + return create_engine( + database_url(), pool_size=10, max_overflow=0, pool_timeout=2, pool_pre_ping=True, + connect_args={'connect_timeout': 2, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}, + ) + + +def migrate(engine): + with engine.begin() as conn: + conn.execute(select(func.pg_advisory_xact_lock(202602))) + SchemaMigration.__table__.create(conn, checkfirst=True) + applied = set(conn.scalars(select(SchemaMigration.version))) + for path in sorted((Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')): + if path.name not in applied: + conn.exec_driver_sql(path.read_text(encoding='utf-8')) + conn.execute(insert(SchemaMigration).values(version=path.name)) + + +if __name__ == '__main__': + engine = create_db_engine() + try: + migrate(engine) + print('Migrations applied') + finally: + engine.dispose() diff --git a/src/dependencies.py b/src/dependencies.py new file mode 100644 index 00000000..03aac92c --- /dev/null +++ b/src/dependencies.py @@ -0,0 +1,31 @@ +from typing import Optional + +from fastapi import Depends, Header, Query + +from .auth import sessions, unauthorized +from .errors import APIError +from .store import store + + +def page_limit(limit: str = Query(default="50")) -> int: + if not limit.isascii() or not limit.isdecimal() or len(limit) > 3: + raise APIError(400, "invalid_request", "Неверный limit") + number = int(limit) + if not 1 <= number <= 100: + raise APIError(400, "invalid_request", "Неверный limit") + return number + + + +def bearer_token(authorization: Optional[str] = Header(default=None)) -> str: + parts = (authorization or '').split() + if len(parts) != 2 or parts[0].lower() != 'bearer': + raise unauthorized() + return parts[1] + + +def current_user(token: str = Depends(bearer_token)) -> dict: + user = store.get_user(sessions.user_id(token)) + if user is None: + raise unauthorized() + return user diff --git a/src/errors.py b/src/errors.py new file mode 100644 index 00000000..de05211c --- /dev/null +++ b/src/errors.py @@ -0,0 +1,59 @@ +from fastapi import Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + + +class APIError(Exception): + def __init__(self, status_code: int, code: str, message: str): + self.status_code = status_code + self.code = code + self.message = message + + +def error_response(request: Request, status_code: int, code: str, message: str) -> JSONResponse: + request_id = getattr(request.state, "request_id", "") + return JSONResponse( + status_code=status_code, + headers={"X-Request-Id": request_id}, + content={ + "error": { + "code": code, + "message": message, + "request_id": request_id, + } + }, + ) + + +def api_error_handler(request: Request, exc: APIError) -> JSONResponse: + response = error_response(request, exc.status_code, exc.code, exc.message) + if exc.status_code == 429: + response.headers['Retry-After'] = '1' + if exc.status_code == 401: + response.headers['WWW-Authenticate'] = ( + 'Basic realm="messenger", charset="UTF-8"' + if request.url.path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"' + ) + return response + + +def dependency_error_handler(request: Request, exc: Exception) -> JSONResponse: + return error_response(request, 503, 'unavailable', 'Хранилище временно недоступно') + + +def validation_error_handler(request: Request, exc: RequestValidationError) -> JSONResponse: + return error_response(request, 400, "invalid_request", "Неверный запрос") + + +def http_error_handler(request: Request, exc: StarletteHTTPException) -> JSONResponse: + if exc.status_code == 404: + return error_response(request, 404, "not_found", "Объект не найден") + if exc.status_code == 405: + return error_response(request, 400, "invalid_request", "Метод не поддерживается") + else: + return error_response(request, exc.status_code, "invalid_request", "Неверный запрос") + + +def internal_error_handler(request: Request, exc: Exception) -> JSONResponse: + return error_response(request, 500, "internal_error", "Внутренняя ошибка") diff --git a/src/limits.py b/src/limits.py new file mode 100644 index 00000000..e4f3b8a2 --- /dev/null +++ b/src/limits.py @@ -0,0 +1,95 @@ +import asyncio +import os +from dataclasses import dataclass + +from starlette.requests import Request + +from .errors import error_response + + +@dataclass(frozen=True) +class Limits: + max_users: int = 1000 + max_chats: int = 1000 + max_messages: int = 10000 + max_text_bytes: int = 16 * 1024 * 1024 + max_request_bytes: int = 64 * 1024 + max_inflight: int = 32 + message_rate: int = 1000 + message_burst: int = 1000 + + @classmethod + def from_env(cls): + values = {} + for name, default in cls().__dict__.items(): + value = int(os.getenv(name.upper(), str(default))) + if value < 1: + raise ValueError(f'{name.upper()} must be positive') + values[name] = value + return cls(**values) + + +limits = Limits.from_env() + + +class ResourceLimitMiddleware: + def __init__(self, app, settings=limits): + self.app = app + self.settings = settings + self.active = 0 + + async def __call__(self, scope, receive, send): + if scope['type'] != 'http' or scope['path'].startswith('/health/'): + return await self.app(scope, receive, send) + + request = Request(scope) + + async def reject(status, code, message): + response = error_response(request, status, code, message) + if status == 503: + response.headers['Retry-After'] = '1' + await response(scope, receive, send) + + if self.active >= self.settings.max_inflight: + return await reject(503, 'unavailable', 'Слишком много одновременных запросов') + self.active += 1 + try: + length = request.headers.get('content-length') + if length is not None: + try: + size = int(length) + if size < 0: + raise ValueError() + except ValueError: + return await reject(400, 'invalid_request', 'Неверный Content-Length') + if size > self.settings.max_request_bytes: + return await reject(413, 'payload_too_large', 'Тело запроса слишком большое') + + body = bytearray() + try: + async with asyncio.timeout(5): + while True: + message = await receive() + if message['type'] == 'http.disconnect': + return + chunk = message.get('body', b'') + if len(body) + len(chunk) > self.settings.max_request_bytes: + return await reject(413, 'payload_too_large', 'Тело запроса слишком большое') + body.extend(chunk) + if not message.get('more_body', False): + break + except TimeoutError: + return await reject(503, 'unavailable', 'Истекло время чтения запроса') + + delivered = False + + async def buffered_receive(): + nonlocal delivered + if not delivered: + delivered = True + return {'type': 'http.request', 'body': bytes(body), 'more_body': False} + return await receive() + + await self.app(scope, buffered_receive, send) + finally: + self.active -= 1 diff --git a/src/main.py b/src/main.py new file mode 100644 index 00000000..a0a7d184 --- /dev/null +++ b/src/main.py @@ -0,0 +1,47 @@ +from fastapi import FastAPI +from fastapi.exceptions import RequestValidationError +from starlette.exceptions import HTTPException +from contextlib import asynccontextmanager +from sqlalchemy.exc import OperationalError, InterfaceError, TimeoutError as PoolTimeout +from sqlalchemy.orm import sessionmaker +from redis.exceptions import RedisError +from starlette.concurrency import run_in_threadpool + +from .errors import APIError, api_error_handler, validation_error_handler, http_error_handler, internal_error_handler, dependency_error_handler +from .database import create_db_engine, migrate +from .store import store +from .auth import sessions +from .middleware import RequestIdMiddleware +from .limits import ResourceLimitMiddleware +from .routers.health import router as health_router +from .routers.users import router as users_router +from .routers.chats import router as chats_router +from .routers.auth import router as auth_router + + +@asynccontextmanager +async def lifespan(app): + engine = create_db_engine() + try: + await run_in_threadpool(migrate, engine) + store.sessions = sessionmaker(engine, expire_on_commit=False) + yield + finally: + engine.dispose() + sessions.client.close() + + +app = FastAPI(title="MTUSI Messenger", lifespan=lifespan) +app.add_middleware(ResourceLimitMiddleware) +app.add_middleware(RequestIdMiddleware) +app.add_exception_handler(APIError, api_error_handler) +app.add_exception_handler(RequestValidationError, validation_error_handler) +app.add_exception_handler(HTTPException, http_error_handler) +app.add_exception_handler(Exception, internal_error_handler) +for error in (OperationalError, InterfaceError, PoolTimeout, RedisError): + app.add_exception_handler(error, dependency_error_handler) + +app.include_router(health_router) +app.include_router(users_router) +app.include_router(chats_router) +app.include_router(auth_router) diff --git a/src/middleware.py b/src/middleware.py new file mode 100644 index 00000000..f19a3df4 --- /dev/null +++ b/src/middleware.py @@ -0,0 +1,32 @@ +from uuid import uuid4 +from starlette.responses import JSONResponse +from starlette.middleware.base import BaseHTTPMiddleware +from starlette.requests import Request + + +class RequestIdMiddleware(BaseHTTPMiddleware): + async def dispatch(self, request: Request, call_next): + request_id = request.headers.get("X-Request-Id", "").strip() or str(uuid4()) + request.state.request_id = request_id + + if (request.method in {"POST", "PUT", "PATCH"} + and request.url.path.startswith("/api/") + and request.url.path != '/api/v1/auth/sessions'): + content_type = request.headers.get("content-type", "").split(";", 1)[0].strip().lower() + if content_type != "application/json": + response = JSONResponse( + status_code=415, + content={ + "error": { + "code": "unsupported_media_type", + "message": "Ожидается Content-Type application/json", + "request_id": request_id, + } + }, + ) + response.headers["X-Request-Id"] = request_id + return response + + response = await call_next(request) + response.headers["X-Request-Id"] = request_id + return response diff --git a/src/models.py b/src/models.py new file mode 100644 index 00000000..418dc306 --- /dev/null +++ b/src/models.py @@ -0,0 +1,60 @@ +from datetime import datetime +from uuid import uuid4 + +from sqlalchemy import BigInteger, CheckConstraint, DateTime, ForeignKey, Index, Text, Uuid, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column + + +class Base(DeclarativeBase): + pass + + +class User(Base): + __tablename__ = 'users' + __table_args__ = ( + CheckConstraint("username ~ '^[a-z0-9_]{3,32}$'"), + CheckConstraint('char_length(display_name) BETWEEN 1 AND 100'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + username: Mapped[str] = mapped_column(Text, unique=True) + display_name: Mapped[str] = mapped_column(Text) + password_hash: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + + +class Chat(Base): + __tablename__ = 'chats' + __table_args__ = ( + CheckConstraint('char_length(title) BETWEEN 1 AND 100'), + Index('chats_order_idx', 'created_at', 'id'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + title: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + history_version: Mapped[int] = mapped_column(BigInteger, server_default='0') + + +class ChatMember(Base): + __tablename__ = 'chat_members' + __table_args__ = (Index('chat_members_user_idx', 'user_id', 'chat_id'),) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE'), primary_key=True) + user_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id'), primary_key=True) + + +class Message(Base): + __tablename__ = 'messages' + __table_args__ = ( + CheckConstraint('char_length(text) BETWEEN 1 AND 4000'), + Index('messages_history_idx', 'chat_id', 'created_at', 'id'), + ) + id: Mapped[str] = mapped_column(Uuid(as_uuid=False), primary_key=True, default=lambda: str(uuid4())) + chat_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('chats.id', ondelete='CASCADE')) + sender_id: Mapped[str] = mapped_column(Uuid(as_uuid=False), ForeignKey('users.id')) + text: Mapped[str] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.clock_timestamp()) + + +class SchemaMigration(Base): + __tablename__ = 'schema_migrations' + version: Mapped[str] = mapped_column(Text, primary_key=True) + applied_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), nullable=True) diff --git a/src/routers/__init__.py b/src/routers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/routers/auth.py b/src/routers/auth.py new file mode 100644 index 00000000..bb4beaa4 --- /dev/null +++ b/src/routers/auth.py @@ -0,0 +1,34 @@ +import base64 + +from fastapi import APIRouter, Depends, Header, Request, Response +from starlette.concurrency import run_in_threadpool + +from ..auth import sessions, unauthorized +from ..dependencies import bearer_token, current_user +from ..errors import APIError +from ..store import store + +router = APIRouter(prefix='/api/v1/auth/sessions', tags=['auth']) + + +@router.post('', status_code=201) +async def create_session(request: Request, authorization: str = Header(default='')): + if await request.body(): + raise APIError(400, 'invalid_request', 'Вход не принимает тело запроса') + try: + scheme, encoded = authorization.split() + if scheme.lower() != 'basic': + raise ValueError() + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not username.isascii() or not 12 <= len(password) <= 128 or '\x00' in username: + raise ValueError() + except (ValueError, UnicodeError): + raise unauthorized() from None + user = await run_in_threadpool(store.login, username, password) + return await run_in_threadpool(sessions.create, user) + + +@router.delete('/current', status_code=204) +def logout(token: str = Depends(bearer_token), user: dict = Depends(current_user)): + sessions.delete(token) + return Response(status_code=204) diff --git a/src/routers/chats.py b/src/routers/chats.py new file mode 100644 index 00000000..87085067 --- /dev/null +++ b/src/routers/chats.py @@ -0,0 +1,44 @@ +from typing import Optional + +from fastapi import APIRouter, Depends, Query, status + +from ..dependencies import current_user, page_limit +from ..schemas.chat import Chat, ChatPage, CreateChat +from ..schemas.message import CreateMessage, Message, MessagePage +from ..store import store + +router = APIRouter(prefix="/api/v1/chats", tags=["chats"]) + + +@router.post("", response_model=Chat, status_code=status.HTTP_201_CREATED) +def create_chat(payload: CreateChat, user: dict = Depends(current_user)): + return store.create_chat(user["id"], payload.title, payload.member_ids) + + +@router.get("", response_model=ChatPage) +def list_chats( + limit: int = Depends(page_limit), + cursor: Optional[str] = Query(default=None, min_length=1), + user: dict = Depends(current_user), +): + return store.list_chats(user["id"], limit, cursor) + + +@router.get("/{chat_id}", response_model=Chat) +def get_chat(chat_id: str, user: dict = Depends(current_user)): + return store.get_chat_for_user(chat_id, user["id"]) + + +@router.post("/{chat_id}/messages", response_model=Message, status_code=status.HTTP_201_CREATED) +def create_message(chat_id: str, payload: CreateMessage, user: dict = Depends(current_user)): + return store.create_message(chat_id, user["id"], payload.text) + + +@router.get("/{chat_id}/messages", response_model=MessagePage) +def list_messages( + chat_id: str, + limit: int = Depends(page_limit), + cursor: Optional[str] = Query(default=None, min_length=1), + user: dict = Depends(current_user), +): + return store.list_messages(chat_id, user["id"], limit, cursor) diff --git a/src/routers/health.py b/src/routers/health.py new file mode 100644 index 00000000..b0871bda --- /dev/null +++ b/src/routers/health.py @@ -0,0 +1,21 @@ +from fastapi import APIRouter +from sqlalchemy import select + +from ..schemas.common import Health +from ..auth import sessions +from ..store import store + +router = APIRouter(prefix='/health') + + +@router.get("/live", response_model=Health) +def health_live(): + return {"status": "ok"} + + +@router.get("/ready", response_model=Health) +def health_ready(): + with store.sessions() as session: + session.execute(select(1)) + sessions.client.ping() + return {"status": "ok"} diff --git a/src/routers/users.py b/src/routers/users.py new file mode 100644 index 00000000..2c8b69c9 --- /dev/null +++ b/src/routers/users.py @@ -0,0 +1,19 @@ +from fastapi import APIRouter, Depends, status + +from ..dependencies import current_user +from ..schemas.user import CreateUser, User +from ..store import store + + + +router = APIRouter(prefix="/api/v1/users", tags=["users"]) + + +@router.post("", response_model=User, status_code=status.HTTP_201_CREATED) +def create_user(payload: CreateUser): + return store.create_user(payload.username, payload.display_name, payload.password) + + +@router.get("/me", response_model=User) +def get_current_user(user: dict = Depends(current_user)): + return user diff --git a/src/schemas/__init__.py b/src/schemas/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/schemas/chat.py b/src/schemas/chat.py new file mode 100644 index 00000000..b1a3629d --- /dev/null +++ b/src/schemas/chat.py @@ -0,0 +1,35 @@ +from typing import List, Optional + +from pydantic import Field, StrictStr, field_validator + +from .common import ContractModel + + + +class CreateChat(ContractModel): + title: StrictStr = Field(min_length=1, max_length=100) + member_ids: List[StrictStr] = Field(min_length=1, max_length=100) + + @field_validator("title") + def validate_title(cls, value): + if not value.strip(): + raise ValueError("blank title") + return value + + @field_validator("member_ids") + def validate_member_ids(cls, value): + if len(value) != len(set(value)): + raise ValueError("duplicate member_ids") + return value + + +class Chat(ContractModel): + id: str + title: str + member_ids: List[str] + created_at: str + + +class ChatPage(ContractModel): + items: List[Chat] + next_cursor: Optional[str] diff --git a/src/schemas/common.py b/src/schemas/common.py new file mode 100644 index 00000000..06a7bc20 --- /dev/null +++ b/src/schemas/common.py @@ -0,0 +1,17 @@ +from pydantic import BaseModel, ConfigDict, field_validator + + +class ContractModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + @field_validator('*') + def validate_text_encoding(cls, value, info): + if isinstance(value, str): + value.encode('utf-8') + if info.field_name != 'password' and '\x00' in value: + raise ValueError('NUL is not supported in PostgreSQL text') + return value + + +class Health(ContractModel): + status: str diff --git a/src/schemas/message.py b/src/schemas/message.py new file mode 100644 index 00000000..3c755028 --- /dev/null +++ b/src/schemas/message.py @@ -0,0 +1,29 @@ +from typing import List, Optional + +from pydantic import Field, StrictStr, field_validator + +from .common import ContractModel + + + +class CreateMessage(ContractModel): + text: StrictStr = Field(min_length=1, max_length=4000) + + @field_validator("text") + def validate_text(cls, value): + if not value.strip(): + raise ValueError("blank text") + return value + + +class Message(ContractModel): + id: str + chat_id: str + sender_id: str + text: str + created_at: str + + +class MessagePage(ContractModel): + items: List[Message] + next_cursor: Optional[str] diff --git a/src/schemas/user.py b/src/schemas/user.py new file mode 100644 index 00000000..bd18d5bc --- /dev/null +++ b/src/schemas/user.py @@ -0,0 +1,34 @@ +import re + +from pydantic import Field, StrictStr, field_validator + +from .common import ContractModel + + + +USERNAME_RE = re.compile(r"^[a-z0-9_]{3,32}$") + + +class CreateUser(ContractModel): + username: StrictStr + display_name: StrictStr = Field(min_length=1, max_length=100) + password: StrictStr = Field(min_length=12, max_length=128) + + @field_validator("username") + def validate_username(cls, value): + if not USERNAME_RE.fullmatch(value): + raise ValueError("invalid username") + return value + + @field_validator("display_name") + def validate_display_name(cls, value): + if not value.strip(): + raise ValueError("blank display_name") + return value + + +class User(ContractModel): + id: str + username: str + display_name: str = Field(min_length=1, max_length=100) + created_at: str diff --git a/src/store.py b/src/store.py new file mode 100644 index 00000000..3f82ac2c --- /dev/null +++ b/src/store.py @@ -0,0 +1,207 @@ +import base64 +import json +from datetime import datetime, timezone +from typing import Tuple +from uuid import UUID + +from sqlalchemy import select, tuple_, update +from sqlalchemy.exc import IntegrityError + +from .auth import password_hasher, verify_password, unauthorized +from .errors import APIError +from .cache import history_cache +from .models import User, Chat, ChatMember, Message + + +def ensure_uuid(value): + try: + if not isinstance(value, str): + raise ValueError() + return str(UUID(value)) + except (TypeError, ValueError) as exc: + raise APIError(400, 'invalid_request', 'Неверный UUID') from exc + + +def serialize(model, *fields): + if model is None: + return None + result = {} + for field in fields: + value = getattr(model, field) + if isinstance(value, datetime): + value = value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + result[field] = value + return result + + +USER_FIELDS = ('id', 'username', 'display_name', 'created_at') +CHAT_FIELDS = ('id', 'title', 'created_at') +MESSAGE_FIELDS = ('id', 'chat_id', 'sender_id', 'text', 'created_at') + + +def after_condition(model, after): + return tuple_(model.created_at, model.id) > ( + datetime.fromisoformat(after[0].replace('Z', '+00:00')), after[1]) + + +def history_query(chat_id, limit, after): + query = select(Message).where(Message.chat_id == chat_id) + if after: + query = query.where(after_condition(Message, after)) + return query.order_by(Message.created_at, Message.id).limit(limit + 1) + + +class PostgresStore: + def __init__(self, cache=history_cache): + self.sessions = None + self.cache = cache + + def create_user(self, username, display_name, password): + user = User(username=username, display_name=display_name, password_hash=password_hasher.hash(password)) + try: + with self.sessions.begin() as session: + session.add(user) + session.flush() + result = serialize(user, *USER_FIELDS) + return result + except IntegrityError as exc: + if getattr(exc.orig, 'sqlstate', None) == '23505': + raise APIError(409, 'conflict', 'Имя пользователя уже занято') from exc + raise + + def get_user(self, user_id): + with self.sessions() as session: + return serialize(session.get(User, ensure_uuid(user_id)), *USER_FIELDS) + + def login(self, username, password): + with self.sessions() as session: + user = session.scalar(select(User).where(User.username == username)) + encoded = user.password_hash if user else None + result = serialize(user, *USER_FIELDS) + if not verify_password(encoded, password): + raise unauthorized() + return result + + def create_chat(self, actor_id, title, member_ids): + members = [ensure_uuid(member) for member in member_ids] + if len(members) != len(set(members)) or actor_id not in members: + raise APIError(400, 'invalid_request', 'Неверный список участников') + try: + with self.sessions.begin() as session: + chat = Chat(title=title) + session.add(chat) + session.flush() + session.add_all([ChatMember(chat_id=chat.id, user_id=member) for member in members]) + result = serialize(chat, *CHAT_FIELDS) + result['member_ids'] = sorted(members) + return result + except IntegrityError as exc: + if getattr(exc.orig, 'sqlstate', None) == '23503': + raise APIError(400, 'invalid_request', 'Неизвестный участник') from exc + raise + + @staticmethod + def _chat(session, chat_id, actor_id): + chat = session.scalar(select(Chat).join(ChatMember).where( + Chat.id == chat_id, ChatMember.user_id == actor_id)) + if chat is None: + raise APIError(404, 'not_found', 'Объект не найден') + return chat + + @staticmethod + def _members(session, chat): + result = serialize(chat, *CHAT_FIELDS) + result['member_ids'] = list(session.scalars(select(ChatMember.user_id).where( + ChatMember.chat_id == chat.id).order_by(ChatMember.user_id))) + return result + + def get_chat_for_user(self, chat_id, actor_id): + with self.sessions() as session: + return self._members(session, self._chat(session, ensure_uuid(chat_id), actor_id)) + + def list_chats(self, actor_id, limit, cursor): + after = self._decode_cursor(cursor, 'chats', actor_id) if cursor else None + query = select(Chat).join(ChatMember).where(ChatMember.user_id == actor_id) + if after: + query = query.where(after_condition(Chat, after)) + query = query.order_by(Chat.created_at, Chat.id).limit(limit + 1) + with self.sessions() as session: + chats = list(session.scalars(query)) + items = [self._members(session, chat) for chat in chats[:limit]] + return self._page(items, len(chats) > limit, 'chats', actor_id) + + def create_message(self, chat_id, actor_id, text): + chat_id = ensure_uuid(chat_id) + with self.sessions.begin() as session: + self._chat(session, chat_id, actor_id) + session.execute(update(Chat).where(Chat.id == chat_id).values( + history_version=Chat.history_version + 1)) + message = Message(chat_id=chat_id, sender_id=actor_id, text=text) + session.add(message) + session.flush() + result = serialize(message, *MESSAGE_FIELDS) + return result + + def list_messages(self, chat_id, actor_id, limit, cursor): + chat_id = ensure_uuid(chat_id) + after = self._decode_cursor(cursor, 'messages', chat_id) if cursor else None + with self.sessions() as session: + version = self._chat(session, chat_id, actor_id).history_version + return self.cache.get_or_load( + chat_id, version, limit, after, + lambda: self._load_messages(chat_id, limit, after), + ) + + def _load_messages(self, chat_id, limit, after): + with self.sessions() as session: + messages = list(session.scalars(history_query(chat_id, limit, after))) + items = [serialize(message, *MESSAGE_FIELDS) for message in messages[:limit]] + return self._page(items, len(messages) > limit, 'messages', chat_id) + + def _page(self, items, has_more, resource, owner_id): + cursor = None + if has_more: + last = items[-1] + cursor = self._encode_cursor(resource, owner_id, last['created_at'], last['id']) + return {'items': items, 'next_cursor': cursor} + + @staticmethod + def _encode_cursor(resource: str, owner_id: str, created_at: str, item_id: str) -> str: + payload = { + "resource": resource, + "owner_id": owner_id, + "created_at": created_at, + "id": item_id, + } + raw = json.dumps(payload, separators=(",", ":"), ensure_ascii=False).encode("utf-8") + return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=") + + @staticmethod + def _decode_cursor(cursor: str, resource: str, owner_id: str) -> Tuple[str, str]: + try: + padded = cursor + "=" * (-len(cursor) % 4) + payload = json.loads(base64.b64decode(padded.encode("ascii"), altchars=b"-_", validate=True).decode("utf-8")) + except (ValueError, UnicodeError, json.JSONDecodeError) as exc: + raise APIError(400, "invalid_request", "Неверный cursor") from exc + + if not isinstance(payload, dict): + raise APIError(400, "invalid_request", "Неверный cursor") + + if payload.get("resource") != resource or payload.get("owner_id") != owner_id: + raise APIError(400, "invalid_request", "Неверный cursor") + + item_id = ensure_uuid(payload.get("id")) + created_at = payload.get("created_at") + if not isinstance(created_at, str) or not created_at: + raise APIError(400, "invalid_request", "Неверный cursor") + try: + parsed = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + if not created_at.endswith("Z") or parsed.tzinfo != timezone.utc: + raise ValueError("Expected UTC timestamp") + created_at = parsed.isoformat(timespec="microseconds").replace("+00:00", "Z") + except ValueError as exc: + raise APIError(400, "invalid_request", "Неверный cursor") from exc + return created_at, item_id + + +store = PostgresStore() diff --git a/student-tests/failures.py b/student-tests/failures.py new file mode 100644 index 00000000..9ebd2253 --- /dev/null +++ b/student-tests/failures.py @@ -0,0 +1,53 @@ +"""Explicitly stop each storage service on the local lab stack, then restore it.""" +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client + + +def compose(*args): + subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, timeout=60) + + +def main(): + client = Client(timeout=6) + actor = client.user('failure') + chat = client.call('POST', '/api/v1/chats', 201, actor=actor, + body={'title': 'Failure test', 'member_ids': [actor['id']]}, schema='Chat')['json'] + message = client.message(actor, chat, 'Must survive storage failure')['json'] + for service in ('redis', 'postgres'): + try: + compose('stop', service) + elapsed = [] + for _ in range(3): + result = client.call('GET', '/api/v1/users/me', 503, actor=actor) + assert result['elapsed'] < 5, 'Dependency error took too long' + elapsed.append(round(result['elapsed'], 3)) + client.call('GET', '/health/ready', 503) + client.call('GET', '/health/live', 200, schema='Health') + client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']}) + client.call('GET', '/api/v1/users/me', 401, actor={'token': 'unknown'}) + print(f'PASS {service} unavailable: 503; seconds={elapsed}; anonymous=401; live=200', flush=True) + finally: + compose('start', service) + start = time.monotonic() + while time.monotonic() - start < 60: + try: + client.call('GET', '/health/ready') + break + except AssertionError: + time.sleep(0.5) + else: + raise AssertionError('Storage recovery timed out') + client.call('GET', '/api/v1/users/me', actor=actor, schema='User') + page = client.call('GET', f"/api/v1/chats/{chat['id']}/messages", actor=actor, schema='MessagePage')['json'] + assert page['items'] == [message] + print(f'PASS {service} recovered in {time.monotonic() - start:.2f}s: same session and message', flush=True) + + +if __name__ == '__main__': + main() diff --git a/student-tests/test_cache.py b/student-tests/test_cache.py new file mode 100644 index 00000000..3c21be2f --- /dev/null +++ b/student-tests/test_cache.py @@ -0,0 +1,135 @@ +import os +import subprocess +import sys +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from unittest.mock import patch +from urllib.parse import quote +from uuid import uuid4 + +from redis.exceptions import ConnectionError +from sqlalchemy import select +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Session + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client +from scripts.cache_experiment import make_store, stale_race +from src.auth import sessions +from src.cache import HistoryCache, history_cache +from src.database import create_db_engine +from src.errors import APIError +from src.models import Chat + + +class CacheTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.engine = create_db_engine() + + @classmethod + def tearDownClass(cls): + cls.engine.dispose() + + def setUp(self): + self.client = Client('http://127.0.0.1:8080') + self.actor = self.client.user('cache') + self.chat = self.client.call('POST', '/api/v1/chats', 201, actor=self.actor, + body={'title': 'Cache', 'member_ids': [self.actor['id']]}, schema='Chat')['json'] + self.prefix = 'test-history:' + uuid4().hex + self.cache = HistoryCache(sessions.client, prefix=self.prefix) + self.reader = make_store(self.engine, self.cache) + self.writer = make_store(self.engine, history_cache) + + def tearDown(self): + for key in sessions.client.scan_iter(match=self.prefix + ':*'): + sessions.client.delete(key) + + def test_late_fill_cannot_restore_old_page(self): + result = stale_race(self.reader, self.writer, self.actor['id'], self.chat['id']) + self.assertFalse(result['stale']) + + def test_hit_still_checks_membership_and_cache_failure_falls_back(self): + args = (self.chat['id'], self.actor['id'], 100, None) + self.writer.create_message(self.chat['id'], self.actor['id'], 'hello') + page = self.reader.list_messages(*args) + with patch.object(self.reader, '_load_messages', side_effect=AssertionError('Cache miss')): + self.assertEqual(self.reader.list_messages(*args), page) + with self.assertRaises(APIError) as error: + self.reader.list_messages(self.chat['id'], str(uuid4()), 100, None) + self.assertEqual(error.exception.status_code, 404) + with patch.object(self.cache.client, 'get', side_effect=ConnectionError('unavailable')): + self.assertEqual(self.reader.list_messages(*args), page) + self.cache.client.delete(self.cache.key(self.chat['id'], 1, 100, None)) + with patch.object(self.cache.client, 'set', side_effect=ConnectionError('unavailable')): + self.assertEqual(self.reader.list_messages(*args), page) + + def test_version_rolls_back_with_failed_message(self): + with self.assertRaises(IntegrityError) as error: + self.writer.create_message(self.chat['id'], self.actor['id'], '') + self.assertEqual(error.exception.orig.sqlstate, '23514') + with Session(self.engine) as session: + self.assertEqual(session.get(Chat, self.chat['id']).history_version, 0) + + def test_concurrent_writers_do_not_lose_version_increments(self): + self.assertEqual(self.reader.list_messages(self.chat['id'], self.actor['id'], 100, None)['items'], []) + def write(n): + return self.writer.create_message(self.chat['id'], self.actor['id'], f'message {n}') + with ThreadPoolExecutor(max_workers=4) as executor: + messages = list(executor.map(write, range(8))) + with Session(self.engine) as session: + self.assertEqual(session.get(Chat, self.chat['id']).history_version, 8) + page = self.reader.list_messages(self.chat['id'], self.actor['id'], 100, None) + self.assertEqual({item['id'] for item in page['items']}, {item['id'] for item in messages}) + + def test_http_read_after_write_and_cursor_pages(self): + path = f"/api/v1/chats/{self.chat['id']}/messages" + def read(query=''): + return self.client.call('GET', path + query, actor=self.actor, schema='MessagePage')['json'] + self.assertEqual(read()['items'], []) + first = self.client.message(self.actor, self.chat, 'one')['json'] + self.assertEqual(read()['items'], [first]) + self.client.message(self.actor, self.chat, 'two') + page = read('?limit=1') + cursor = quote(page['next_cursor'], safe='') + tail = read('?limit=1&cursor=' + cursor) + self.assertIsNone(tail['next_cursor']) + third = self.client.message(self.actor, self.chat, 'three')['json'] + tail = read('?limit=1&cursor=' + cursor) + self.assertIsNotNone(tail['next_cursor']) + self.assertEqual(read('?limit=1&cursor=' + quote(tail['next_cursor'], safe=''))['items'], [third]) + self.assertEqual(len(read()['items']), 3) + with Session(self.engine) as session: + version = session.scalar(select(Chat.history_version).where(Chat.id == self.chat['id'])) + key = history_cache.key(self.chat['id'], version, 50, None) + self.assertGreater(sessions.client.ttl(key), 0) + self.assertLessEqual(sessions.client.ttl(key), history_cache.ttl) + + def test_read_after_write_through_another_api_process(self): + process = subprocess.Popen([sys.executable, '-m', 'src'], + env={**os.environ, 'APP_HOST': '127.0.0.1', 'APP_PORT': '8082'}, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + try: + replica = Client('http://127.0.0.1:8082') + for _ in range(100): + try: + replica.call('GET', '/health/ready') + break + except AssertionError: + self.assertIsNone(process.poll()) + time.sleep(0.1) + else: + self.fail('Second API did not become ready') + path = f"/api/v1/chats/{self.chat['id']}/messages" + self.assertEqual(replica.call('GET', path, actor=self.actor, schema='MessagePage')['json']['items'], []) + message = self.client.message(self.actor, self.chat, 'written through first API')['json'] + self.assertEqual(replica.call('GET', path, actor=self.actor, schema='MessagePage')['json']['items'], [message]) + finally: + process.terminate() + process.wait(timeout=10) + + +if __name__ == '__main__': + unittest.main() diff --git a/student-tests/test_database.py b/student-tests/test_database.py new file mode 100644 index 00000000..13595357 --- /dev/null +++ b/student-tests/test_database.py @@ -0,0 +1,38 @@ +import unittest +from uuid import uuid4 + +from sqlalchemy import create_engine, inspect, select +from sqlalchemy.schema import CreateSchema, DropSchema + +from scripts.cache_experiment import NoCache, make_store +from src.database import create_db_engine, migrate +from src.models import SchemaMigration + + +class MigrationTests(unittest.TestCase): + def test_migrate_empty_schema_and_preserve_orm_data_on_repeat(self): + engine = create_db_engine() + schema = 'orm_test_' + uuid4().hex + isolated = create_engine(engine.url, connect_args={ + 'connect_timeout': 2, 'options': f'-c search_path={schema} -c statement_timeout=3000'}) + try: + with engine.begin() as conn: + conn.execute(CreateSchema(schema)) + migrate(isolated) + store = make_store(isolated, NoCache()) + user = store.create_user('migration_user', 'Migration', 'Synthetic-password-123') + chat = store.create_chat(user['id'], 'Migration', [user['id']]) + message = store.create_message(chat['id'], user['id'], 'must survive migration repeat') + migrate(isolated) + self.assertEqual(store.list_messages(chat['id'], user['id'], 100, None)['items'], [message]) + with isolated.connect() as conn: + self.assertEqual(set(conn.scalars(select(SchemaMigration.version))), + {'001_initial.sql', '002_history_version.sql'}) + indexes = {index['name'] for index in inspect(isolated).get_indexes('messages')} + self.assertIn('messages_history_idx', indexes) + self.assertEqual(len(inspect(isolated).get_foreign_keys('messages')), 2) + finally: + isolated.dispose() + with engine.begin() as conn: + conn.execute(DropSchema(schema, cascade=True, if_exists=True)) + engine.dispose() diff --git a/student-tests/test_lab2.py b/student-tests/test_lab2.py new file mode 100644 index 00000000..bda481f8 --- /dev/null +++ b/student-tests/test_lab2.py @@ -0,0 +1,127 @@ +"""Run inside the API container: python -m unittest discover -s student-tests -v.""" +import base64 +import os +import subprocess +import sys +import time +import unittest +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from uuid import uuid4 +from sqlalchemy import func, select +from sqlalchemy.orm import sessionmaker + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client +from src.auth import sessions +from src.database import create_db_engine, migrate +from src.models import User, Chat, ChatMember, SchemaMigration + + +class Lab2Tests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.engine = create_db_engine() + cls.sessions = sessionmaker(cls.engine) + + @classmethod + def tearDownClass(cls): + cls.engine.dispose() + + def setUp(self): + self.client = Client('http://127.0.0.1:8080') + + def test_registration_race(self): + body = {'username': 'race_' + uuid4().hex[:12], 'display_name': 'Race', + 'password': 'Synthetic-password-123'} + def register(_): + return Client().call('POST', '/api/v1/users', [201, 409], body=body)['status'] + with ThreadPoolExecutor(max_workers=4) as executor: + codes = list(executor.map(register, range(4))) + self.assertEqual(sorted(codes), [201, 409, 409, 409]) + with self.sessions() as session: + count = session.scalar(select(func.count()).select_from(User).where(User.username == body['username'])) + self.assertEqual(count, 1) + + def test_chat_rollback(self): + actor = self.client.user('rollback') + title = 'rollback_' + uuid4().hex + self.client.call('POST', '/api/v1/chats', 400, actor=actor, + body={'title': title, 'member_ids': [actor['id'], str(uuid4())]}) + with self.sessions() as session: + self.assertEqual(session.scalar(select(func.count()).select_from(Chat).where(Chat.title == title)), 0) + self.assertEqual(session.scalar(select(func.count()).select_from(ChatMember).where( + ChatMember.user_id == actor['id'])), 0) + + def test_password_logout_and_identity(self): + actor = self.client.user('auth') + credential = base64.b64encode((actor['user']['username'] + ':Wrong-password-123').encode()).decode() + self.client.call('POST', '/api/v1/auth/sessions', 401, + headers={'Authorization': 'Basic ' + credential}) + self.client.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Basic ' + credential}) + self.client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']}) + me = self.client.call('GET', '/api/v1/users/me', actor=actor, + headers={'X-User-Id': str(uuid4())}, schema='User')['json'] + self.assertEqual(me['id'], actor['id']) + ttl = sessions.client.ttl(sessions.key(actor['token'])) + self.assertGreater(ttl, 0) + self.assertLessEqual(ttl, 3600) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + self.client.call('GET', '/api/v1/users/me', 401, actor=actor) + self.client.call('DELETE', '/api/v1/auth/sessions/current', 401, actor=actor) + self.assertIsNone(sessions.client.get(sessions.key(actor['token']))) + with self.sessions() as session: + encoded = session.get(User, actor['id']).password_hash + self.assertTrue(encoded.startswith('$argon2id$')) + self.assertNotIn(actor['registration']['password'], encoded) + + def test_unicode_password(self): + username = 'unicode_' + uuid4().hex[:12] + password = '🙂:я' * 40 + self.client.call('POST', '/api/v1/users', 201, + body={'username': username, 'display_name': 'Unicode', 'password': password}) + credential = base64.b64encode((username + ':' + password).encode()).decode() + result = self.client.call('POST', '/api/v1/auth/sessions', 201, + headers={'Authorization': 'Basic ' + credential}, schema='Session')['json'] + self.assertEqual(result['user']['username'], username) + + def test_migrations_repeat(self): + with self.sessions() as session: + before = session.scalar(select(func.count()).select_from(User)) + migrate(self.engine) + migrate(self.engine) + with self.sessions() as session: + self.assertEqual(session.scalar(select(func.count()).select_from(User)), before) + versions = set(session.scalars(select(SchemaMigration.version))) + expected = {path.name for path in (Path(__file__).resolve().parents[1] / 'migrations').glob('*.sql')} + self.assertEqual(versions, expected) + + def test_expiry_short_ttl(self): + process = subprocess.Popen( + [sys.executable, '-m', 'src'], + env={**os.environ, 'SESSION_TTL_SECONDS': '2', 'APP_PORT': '8081', 'APP_HOST': '127.0.0.1'}, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + try: + client = Client('http://127.0.0.1:8081') + for _ in range(100): + try: + client.call('GET', '/health/ready') + break + except AssertionError: + self.assertIsNone(process.poll(), 'Short-TTL API exited') + time.sleep(0.1) + else: + self.fail('Short-TTL API did not become ready') + actor = client.user('expiry') + client.call('GET', '/api/v1/users/me', actor=actor, schema='User') + time.sleep(2.2) + client.call('GET', '/api/v1/users/me', 401, actor=actor) + self.assertIsNone(sessions.client.get(sessions.key(actor['token']))) + finally: + process.terminate() + process.wait(timeout=10) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_api.py b/tests/test_api.py new file mode 100644 index 00000000..9f3e9d9f --- /dev/null +++ b/tests/test_api.py @@ -0,0 +1,201 @@ +import json +import os +import socket +import subprocess +import sys +import time +import unittest +import base64 +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import patch +from pathlib import Path +from urllib.error import HTTPError +from urllib.request import Request, urlopen +from uuid import uuid4 +from urllib.parse import urlencode + +ROOT = Path(__file__).resolve().parents[1] + + +class APITests(unittest.TestCase): + @classmethod + def setUpClass(cls): + with socket.socket() as sock: + sock.bind(('127.0.0.1', 0)) + port = sock.getsockname()[1] + cls.base_url = f'http://127.0.0.1:{port}' + cls.server = subprocess.Popen( + [sys.executable, '-m', 'src'], cwd=ROOT, + env={**os.environ, 'APP_HOST': '127.0.0.1', 'APP_PORT': str(port)}, + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + cls.addClassCleanup(cls.stop_server) + for _ in range(100): + try: + with urlopen(cls.base_url + '/health/ready', timeout=1): + return + except OSError: + if cls.server.poll() is not None: + raise RuntimeError('Server stopped before startup') + time.sleep(0.1) + raise RuntimeError('Server startup timed out') + + @classmethod + def stop_server(cls): + cls.server.terminate() + try: + cls.server.wait(timeout=10) + except subprocess.TimeoutExpired: + cls.server.kill() + cls.server.wait() + + def call(self, method, path, status=200, body=None, actor=None, raw=None, content_type='application/json'): + headers = {'Content-Type': content_type} + if actor: + headers['X-User-Id'] = actor + data = json.dumps(body, ensure_ascii=False).encode() if body is not None else raw + request = Request(self.base_url + path, data=data, headers=headers, method=method) + try: + response = urlopen(request, timeout=10) + except HTTPError as error: + response = error + with response: + result = json.load(response) + self.assertEqual(response.status, status, result) + self.assertTrue(response.headers.get('X-Request-Id')) + if status >= 400: + self.assertEqual(result['error']['request_id'], response.headers['X-Request-Id']) + return result + + def user(self): + return self.call('POST', '/api/v1/users', 201, + {'username': 'u_' + uuid4().hex[:20], 'display_name': 'Алиса 👋'})['id'] + + def test_users(self): + body = {'username': 'u_' + uuid4().hex[:20], 'display_name': 'Боб 👋'} + user = self.call('POST', '/api/v1/users', 201, body) + self.assertEqual(user['display_name'], body['display_name']) + self.assertTrue(user['created_at'].endswith('Z')) + self.assertEqual(self.call('GET', '/api/v1/users/me', actor=user['id']), user) + self.call('POST', '/api/v1/users', 409, body) + for actor in [None, 'bad-uuid', str(uuid4())]: + self.call('GET', '/api/v1/users/me', 401, actor=actor) + + def test_validation_and_health(self): + for path in ['/health/live', '/health/ready']: + self.assertEqual(self.call('GET', path), {'status': 'ok'}) + self.call('POST', '/api/v1/users', 400, raw=b'{broken') + self.call('POST', '/api/v1/users', 415, raw=b'{}', content_type='text/plain') + for body in [None, {}, {'username': 123, 'display_name': 'Name'}, + {'username': 'valid', 'display_name': ' '}, + {'username': 'UPPER', 'display_name': 'Name'}, + {'username': 'valid', 'display_name': 'Name', 'extra': True}]: + self.call('POST', '/api/v1/users', 400, body) + self.call('GET', '/missing', 404) + + def chat(self, actor, members=None): + return self.call('POST', '/api/v1/chats', 201, + {'title': 'Учебный чат', 'member_ids': members or [actor]}, actor) + + def test_chats(self): + alice, bob, eve = self.user(), self.user(), self.user() + chat = self.chat(alice, [alice, bob]) + path = '/api/v1/chats/' + chat['id'] + self.assertEqual(self.call('GET', path, actor=bob), chat) + self.call('GET', path, 404, actor=eve) + self.call('GET', path, 401) + self.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=alice) + self.call('GET', '/api/v1/chats/not-a-uuid', 401) + self.assertEqual(self.call('GET', '/api/v1/chats', actor=eve)['items'], []) + for members in [[alice, alice], [bob], [alice, str(uuid4())], [], [alice, 42]]: + self.call('POST', '/api/v1/chats', 400, + {'title': 'Чат', 'member_ids': members}, alice) + self.assertEqual(len(self.call('GET', '/api/v1/chats', actor=alice)['items']), 1) + + def test_chat_pages(self): + actor = self.user() + expected = [self.chat(actor) for _ in range(4)] + expected.sort(key=lambda item: (item['created_at'], item['id'])) + page = self.call('GET', '/api/v1/chats?limit=1', actor=actor) + self.assertEqual(page['items'], expected[:1]) + query = urlencode({'limit': 100, 'cursor': page['next_cursor']}) + last = self.call('GET', '/api/v1/chats?' + query, actor=actor) + self.assertEqual(last['items'], expected[1:]) + self.assertIsNone(last['next_cursor']) + self.call('GET', '/api/v1/chats?' + query, 400, actor=self.user()) + for query in ['limit=0', 'limit=101', 'limit=1.0', 'limit=true', 'cursor=', 'cursor=bad']: + self.call('GET', '/api/v1/chats?' + query, 400, actor=actor) + + def test_messages_and_cursors(self): + alice, bob, eve = self.user(), self.user(), self.user() + chat = self.chat(alice, [alice, bob]) + path = '/api/v1/chats/' + chat['id'] + '/messages' + self.assertEqual(self.call('GET', path, actor=alice), {'items': [], 'next_cursor': None}) + messages = [self.call('POST', path, 201, {'text': text}, alice) + for text in ['Привет 👋', 'Second', '🙂' * 4000]] + self.assertEqual(messages[0]['sender_id'], alice) + self.assertEqual(messages[-1]['text'], '🙂' * 4000) + messages.sort(key=lambda item: (item['created_at'], item['id'])) + first = self.call('GET', path + '?limit=1', actor=bob) + query = urlencode({'limit': 2, 'cursor': first['next_cursor']}) + last = self.call('GET', path + '?' + query, actor=bob) + self.assertEqual(first['items'] + last['items'], messages) + self.assertIsNone(last['next_cursor']) + other = '/api/v1/chats/' + self.chat(alice)['id'] + '/messages' + self.call('GET', other + '?' + query, 400, actor=alice) + for body in [{'text': ''}, {'text': ' \n\t'}, {'text': 'a' * 4001}, + {'text': 123}, {'text': 'Hi', 'sender_id': bob}]: + self.call('POST', path, 400, body, alice) + for method, body in [('GET', None), ('POST', {'text': 'No access'})]: + self.call(method, path, 404, body, eve) + self.call(method, path, 401, body) + for payload in [{}, {'resource': 'messages', 'owner_id': chat['id'], + 'id': None, 'created_at': 'bad'}, + {'resource': 'messages', 'owner_id': chat['id'], + 'id': messages[0]['id'], 'created_at': 'bad'}]: + cursor = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode() + self.call('GET', path + '?' + urlencode({'cursor': cursor}), 400, actor=alice) + + def test_concurrent_messages(self): + actor = self.user() + chat = self.chat(actor) + path = '/api/v1/chats/' + chat['id'] + '/messages' + def send(number): + return self.call('POST', path, 201, {'text': str(number)}, actor)['id'] + with ThreadPoolExecutor(max_workers=10) as pool: + sent = list(pool.map(send, range(100))) + page = self.call('GET', path + '?limit=100', actor=actor) + self.assertEqual(len(page['items']), 100) + self.assertEqual(len(set(sent)), 100) + self.assertEqual({item['id'] for item in page['items']}, set(sent)) + self.assertIsNone(page['next_cursor']) + + def test_same_timestamp_order(self): + from src.store import InMemoryStore + store = InMemoryStore() + actor = store.create_user('alice', 'Alice')['id'] + chat = store.create_chat(actor, 'Chat', [actor])['id'] + with patch('src.store.utc_now', return_value='2026-09-01T12:00:00.000000Z'): + sent = [store.create_message(chat, actor, str(i)) for i in range(5)] + received, cursor = [], None + while True: + page = store.list_messages(chat, actor, 2, cursor) + received.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + break + self.assertEqual(received, sorted(sent, key=lambda item: item['id'])) + + def test_restart_clears_data(self): + actor = self.user() + chat = self.chat(actor) + path = '/api/v1/chats/' + chat['id'] + '/messages' + message = self.call('POST', path, 201, {'text': 'Before restart'}, actor) + self.assertEqual(self.call('GET', path, actor=actor)['items'], [message]) + self.stop_server() + self.setUpClass() + self.call('GET', '/api/v1/users/me', 401, actor=actor) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 00000000..44c9d54a --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,45 @@ +import asyncio +import json +import unittest +from unittest.mock import patch + +from src.main import app + + +class ErrorTests(unittest.TestCase): + def test_unexpected_error_has_request_id(self): + async def request(): + sent = [] + received = False + + async def receive(): + nonlocal received + if not received: + received = True + return {'type': 'http.request', 'body': b'', 'more_body': False} + await asyncio.Event().wait() + + async def send(message): + sent.append(message) + + scope = { + 'type': 'http', 'asgi': {'version': '3.0'}, 'http_version': '1.1', + 'method': 'GET', 'scheme': 'http', 'path': '/api/v1/users/me', + 'raw_path': b'/api/v1/users/me', 'query_string': b'', + 'headers': [(b'x-user-id', b'test'), (b'x-request-id', b'failure-test')], + 'server': ('localhost', 8080), 'client': ('localhost', 1234), + } + with patch('src.dependencies.store.require_user', side_effect=RuntimeError('private detail')): + try: + await app(scope, receive, send) + except RuntimeError: + pass + return sent + + sent = asyncio.run(request()) + start = next(item for item in sent if item['type'] == 'http.response.start') + body = b''.join(item.get('body', b'') for item in sent if item['type'] == 'http.response.body') + self.assertEqual(start['status'], 500) + self.assertEqual(dict(start['headers'])[b'x-request-id'], b'failure-test') + self.assertEqual(json.loads(body)['error']['request_id'], 'failure-test') + self.assertNotIn(b'private detail', body) diff --git a/tests/test_limits.py b/tests/test_limits.py new file mode 100644 index 00000000..bc7f715c --- /dev/null +++ b/tests/test_limits.py @@ -0,0 +1,106 @@ +import asyncio +import json +import unittest +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace +from unittest.mock import patch + +from src.errors import APIError +from src.limits import Limits, ResourceLimitMiddleware +from src.store import InMemoryStore + + +class StoreLimitTests(unittest.TestCase): + def make_store(self, **values): + store = InMemoryStore(replace(Limits(), **values)) + actor = store.create_user('alice', 'Alice')['id'] + chat = store.create_chat(actor, 'Chat', [actor])['id'] + return store, actor, chat + + def test_capacity_is_atomic_and_does_not_evict(self): + store, actor, chat = self.make_store(max_messages=20) + def send(i): + try: + return store.create_message(chat, actor, str(i)) + except APIError as exc: + self.assertEqual(exc.status_code, 503) + return None + with ThreadPoolExecutor(max_workers=10) as pool: + accepted = [item for item in pool.map(send, range(100)) if item] + self.assertEqual(len(accepted), 20) + page = store.list_messages(chat, actor, 100, None) + self.assertEqual({item['id'] for item in page['items']}, {item['id'] for item in accepted}) + + def test_byte_limit_counts_utf8_and_other_objects_are_bounded(self): + store, actor, chat = self.make_store(max_text_bytes=4, max_users=1, max_chats=1) + saved = store.create_message(chat, actor, '🙂') + for operation in [lambda: store.create_message(chat, actor, 'a'), + lambda: store.create_user('bob', 'Bob'), + lambda: store.create_chat(actor, 'Other', [actor])]: + with self.assertRaises(APIError) as error: + operation() + self.assertEqual(error.exception.status_code, 503) + self.assertEqual(store.list_messages(chat, actor, 100, None)['items'], [saved]) + + def test_rate_recovers_without_losing_messages(self): + with patch('src.store.time.monotonic', return_value=0) as clock: + store, actor, chat = self.make_store(message_rate=2, message_burst=1) + first = store.create_message(chat, actor, 'one') + with self.assertRaises(APIError) as error: + store.create_message(chat, actor, 'rejected') + self.assertEqual(error.exception.status_code, 429) + clock.return_value = 0.5 + second = store.create_message(chat, actor, 'two') + self.assertEqual(store.list_messages(chat, actor, 100, None)['items'], + sorted([first, second], key=lambda item: (item['created_at'], item['id']))) + + +class MiddlewareLimitTests(unittest.IsolatedAsyncioTestCase): + async def call(self, app, chunks, headers=()): + messages = iter(chunks) + output = [] + async def receive(): + return next(messages) + async def send(message): + output.append(message) + scope = {'type': 'http', 'path': '/api/v1/users', 'method': 'POST', + 'headers': headers, 'state': {'request_id': 'limit-test'}} + await app(scope, receive, send) + return output + + async def test_body_limit_with_and_without_content_length(self): + async def should_not_run(scope, receive, send): + self.fail('Oversized body reached the API') + app = ResourceLimitMiddleware(should_not_run, replace(Limits(), max_request_bytes=4)) + for headers, chunks in [([(b'content-length', b'5')], []), ([], [ + {'type': 'http.request', 'body': b'123', 'more_body': True}, + {'type': 'http.request', 'body': b'45', 'more_body': False}])]: + result = await self.call(app, chunks, headers) + self.assertEqual(result[0]['status'], 413) + self.assertEqual(json.loads(result[1]['body'])['error']['request_id'], 'limit-test') + self.assertEqual(app.active, 0) + + async def test_admission_recovers_after_inflight_request_finishes(self): + entered, release = asyncio.Event(), asyncio.Event() + async def slow_app(scope, receive, send): + entered.set() + await release.wait() + await send({'type': 'http.response.start', 'status': 200, 'headers': []}) + app = ResourceLimitMiddleware(slow_app, replace(Limits(), max_inflight=1)) + body = [{'type': 'http.request', 'body': b'', 'more_body': False}] + first = asyncio.create_task(self.call(app, body)) + await entered.wait() + overloaded = await self.call(app, body) + self.assertEqual(overloaded[0]['status'], 503) + release.set() + await first + self.assertEqual((await self.call(app, body))[0]['status'], 200) + self.assertEqual(app.active, 0) + + async def test_exception_releases_slot(self): + async def broken_app(scope, receive, send): + raise RuntimeError('test') + app = ResourceLimitMiddleware(broken_app) + with self.assertRaises(RuntimeError): + await self.call(app, [{'type': 'http.request', 'body': b''}]) + self.assertEqual(app.active, 0)