From ef96551b7d351a360887154b7db312919a0f9f83 Mon Sep 17 00:00:00 2001 From: dokidura Date: Mon, 26 Oct 2026 23:25:37 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9B=D0=A0=202:=20PostgreSQL,=20Redis-=D1=81?= =?UTF-8?q?=D0=B5=D1=81=D1=81=D0=B8=D0=B8=20=D0=B8=20=D0=BF=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B5=D1=80=D0=BA=D0=B8=20=D0=BD=D0=B0=20=D0=BE=D1=86?= =?UTF-8?q?=D0=B5=D0=BD=D0=BA=D1=83=204?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .dockerignore | 1 + .env.example | 3 +- .gitattributes | 5 + .github/workflows/application.yml | 28 +++ .gitignore | 5 + Dockerfile | 16 ++ Makefile | 9 +- REPORT.md | 149 ++++++++++++-- compose.yaml | 67 ++++++ evidence/check.txt | 2 + evidence/clean-clone.txt | 100 +++++++++ evidence/compose.txt | 1 + evidence/database.txt | 13 ++ evidence/demo.txt | 8 + evidence/durability-postgres.txt | 15 ++ evidence/durability-redis.txt | 5 + evidence/environment.json | 22 ++ evidence/expiry.txt | 4 + evidence/explain.json | 81 ++++++++ evidence/outages.txt | 39 ++++ evidence/own-api.txt | 10 + evidence/persistence.txt | 3 + evidence/resources.txt | 4 + evidence/restart.json | 56 +++++ evidence/smoke.txt | 2 + evidence/validate.txt | 2 + infra/SCHEMA.md | 17 ++ infra/redis.conf | 10 + migrations/001_initial.sql | 30 +++ requirements.txt | 5 + scripts/bootstrap.py | 24 +++ scripts/check_expiry.py | 18 ++ scripts/clean_check.py | 49 +++++ scripts/demo.py | 24 +++ scripts/explain_history.py | 36 ++++ scripts/restart.cmd | 4 + scripts/restart.py | 38 ++++ scripts/restart.sh | 4 + scripts/verify.py | 74 +++++++ src/messenger/__init__.py | 1 + src/messenger/core.py | 301 +++++++++++++++++++++++++++ src/messenger/migrate.py | 30 +++ src/messenger/server.py | 331 ++++++++++++++++++++++++++++++ student-tests/test_api.py | 89 ++++++++ student-tests/test_database.py | 59 ++++++ student-tests/test_expiry.py | 14 ++ student-tests/test_outages.py | 50 +++++ 47 files changed, 1839 insertions(+), 19 deletions(-) create mode 100644 .gitattributes create mode 100644 .github/workflows/application.yml create mode 100644 Dockerfile create mode 100644 compose.yaml create mode 100644 evidence/check.txt create mode 100644 evidence/clean-clone.txt create mode 100644 evidence/compose.txt create mode 100644 evidence/database.txt create mode 100644 evidence/demo.txt create mode 100644 evidence/durability-postgres.txt create mode 100644 evidence/durability-redis.txt create mode 100644 evidence/environment.json create mode 100644 evidence/expiry.txt create mode 100644 evidence/explain.json create mode 100644 evidence/outages.txt create mode 100644 evidence/own-api.txt create mode 100644 evidence/persistence.txt create mode 100644 evidence/resources.txt create mode 100644 evidence/restart.json create mode 100644 evidence/smoke.txt create mode 100644 evidence/validate.txt create mode 100644 infra/SCHEMA.md create mode 100644 infra/redis.conf create mode 100644 migrations/001_initial.sql create mode 100644 requirements.txt create mode 100644 scripts/bootstrap.py create mode 100644 scripts/check_expiry.py create mode 100644 scripts/clean_check.py create mode 100644 scripts/demo.py create mode 100644 scripts/explain_history.py create mode 100644 scripts/restart.cmd create mode 100644 scripts/restart.py create mode 100755 scripts/restart.sh create mode 100644 scripts/verify.py create mode 100644 src/messenger/__init__.py create mode 100644 src/messenger/core.py create mode 100644 src/messenger/migrate.py create mode 100644 src/messenger/server.py create mode 100644 student-tests/test_api.py create mode 100644 student-tests/test_database.py create mode 100644 student-tests/test_expiry.py create mode 100644 student-tests/test_outages.py diff --git a/.dockerignore b/.dockerignore index bc16e556..d58aec81 100644 --- a/.dockerignore +++ b/.dockerignore @@ -17,4 +17,5 @@ backups __pycache__ node_modules evidence +output .DS_Store diff --git a/.env.example b/.env.example index 714c8076..c2ee6f1d 100644 --- a/.env.example +++ b/.env.example @@ -7,5 +7,6 @@ SESSION_TTL_SECONDS=3600 # Choose credentials locally. File paths contain no secret values. DATABASE_URL_FILE=/run/secrets/database_url # Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +SESSION_STORE=redis CACHE_URL_FILE=/run/secrets/cache_url +CURSOR_KEY_FILE=/run/secrets/cursor_key diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..2657d46a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +* text=auto +*.sh text eol=lf +*.py text eol=lf +*.sql text eol=lf +*.conf text eol=lf diff --git a/.github/workflows/application.yml b/.github/workflows/application.yml new file mode 100644 index 00000000..c8b538a5 --- /dev/null +++ b/.github/workflows/application.yml @@ -0,0 +1,28 @@ +name: Lab 2 application +on: [push, pull_request] +permissions: + contents: read +jobs: + application: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Local synthetic secrets and stack + run: | + python scripts/bootstrap.py + docker compose --env-file .env up -d --build --wait + - name: Grade 4 checks + run: make verify PYTHON=python + - name: Evidence without secrets + if: always() + uses: actions/upload-artifact@v4 + with: + name: lab2-evidence + path: evidence/ + - name: Stop stack, preserve volumes + if: always() + run: docker compose down diff --git a/.gitignore b/.gitignore index b5a71896..2df13762 100644 --- a/.gitignore +++ b/.gitignore @@ -28,5 +28,10 @@ coverage/ .DS_Store .idea/ .vscode/ +output/ # Keep small, sanitized evidence summaries in evidence/ under version control. # Inspect git diff --cached: ignore rules are not a secret scanner. +# Personal study notes stay local +DEFENSE.md +DEFENSE_STEPS.md +MR_DRAFT.md diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..bedb441d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.13.7-slim-bookworm +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app/src +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app +COPY src ./src +COPY migrations ./migrations +COPY contracts ./contracts +COPY tests ./tests +COPY student-tests ./student-tests +COPY scripts ./scripts +COPY lab.json requirements-dev.txt ./ +USER app +EXPOSE 8080 +HEALTHCHECK --interval=5s --timeout=4s --start-period=10s --retries=5 CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health/ready', timeout=3)"] +CMD ["python", "-m", "messenger.server"] diff --git a/Makefile b/Makefile index 9ecb223e..88eb82ec 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence test-own verify help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" @@ -24,3 +24,10 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +test-own: + $(PYTHON) student-tests/test_api.py + docker compose exec -T api python student-tests/test_database.py + +verify: + $(PYTHON) scripts/verify.py diff --git a/REPORT.md b/REPORT.md index e42f2b05..2962d3b5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,152 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор: **Осокин Глеб**, группа **БВТ2302**, Gitea **glebossokin**. +- Целевая оценка: **4**. +- Основа: skeleton `65003b6`; ветка `solution/BVT2302-Osokin`. +- HTTP-сервер и валидация адаптированы из собственной lab1; помощь ИИ использовалась для разбора требований, проверки результатов и подготовки отчёта. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Нужны Git, работающий Docker с Linux containers, Compose с поддержкой `--wait`, Python 3.9+ на клиентской машине. На Windows используется PowerShell, GNU Make необязателен. Runtime: Python **3.13.7** (`python:3.13.7-slim-bookworm`), PostgreSQL **17.6** (`postgres:17.6-bookworm`), Redis **7.4.5** (`redis:7.4.5-bookworm`). В [requirements.txt](requirements.txt) закреплены psycopg/psycopg-binary 3.2.10, psycopg-pool 3.2.6, redis-py 6.4.0, typing_extensions 4.15.0. Публичные проверки используют стандартную библиотеку Python. + +Из корня `lab2`: + +```powershell +python scripts/bootstrap.py +docker compose --env-file .env config --quiet +docker compose --env-file .env up -d --build --wait +Invoke-RestMethod http://127.0.0.1:8080/health/ready +$env:BASE_URL = 'http://127.0.0.1:8080' +python scripts/check_contract.py +python tests/smoke.py --base-url http://127.0.0.1:8080 +python scripts/verify.py +``` + +`bootstrap.py` генерирует локальный случайный пароль PostgreSQL, URL подключения и постоянный ключ cursor в `secrets/`, создаёт `.env`. Повторный запуск сохраняет существующие значения. Эти файлы исключены из Git и Docker build context. Оператор должен сохранять `secrets/` между запусками с существующими volumes: новый пароль в файле не меняет пароль уже инициализированной PostgreSQL. + +Compose дожидается PostgreSQL, запускает одноразовый `migrate`, после успешного завершения — API. Создаётся `schema_migrations` и атомарно применяется `migrations/001_initial.sql`. Повторный запуск сверяет SHA-256 применённых миграций и ничего не дублирует: + +```powershell +docker compose run --rm --no-deps migrate +docker compose run --rm --no-deps migrate +``` + +В Linux/macOS после запуска стенда доступны Make-эквиваленты: + +```sh +make check +make test BASE_URL=http://127.0.0.1:8080 +make test-own +chmod +x scripts/restart.sh +make test-persistence ACTION_SCRIPT=scripts/restart.sh BASE_URL=http://127.0.0.1:8080 +make verify +``` + +В этой Windows-среде `check`/`test` выполнялись соответствующими Python-командами; persistence использовал `restart.cmd`, вызывающий тот же `scripts/restart.py`, что и `restart.sh`. Публичные проверки не изменены. Полная валидация OpenAPI отдельно: `make validate` после установки `requirements-dev.txt`; это проверка контракта, а не самого API. + +Безопасная остановка: `docker compose down`. Следующий `docker compose up -d --wait` использует прежние volumes. **Намеренное удаление всех учебных данных:** `docker compose down --volumes` — уничтожает пользователей, чаты, сообщения и сессии этого проекта. Для persistence так останавливать нельзя. Скрипты проверки не удаляют основные volumes. + +Демонстрационный сценарий API: `python scripts/demo.py`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +Клиент → HTTP API → PostgreSQL и Redis. Проект таблиц, ограничений и транзакционных границ перед реализацией SQL: [infra/SCHEMA.md](infra/SCHEMA.md). + +**PostgreSQL** — источник истины пользователей, password KDF, чатов, членства и сообщений. Есть UUID PK, UNIQUE username, FK, CHECK длины/формата. Составной FK сообщения на членство запрещает отправителя вне чата. Создание чата и всех участников — одна транзакция: FK violation отменяет весь результат. HTTP 201 регистрации/чата/сообщения выдаётся только после успешного COMMIT. Пользовательские значения передаются SQL-параметрами `%s`, не включаются в текст SQL. + +**Redis** — единственное хранилище сессий, не кэш истории. Ключ `session:`, значение UUID пользователя. Токен имеет 256 бит случайности (`secrets.token_urlsafe(32)`), в исходном виде на сервере не сохраняется. `SET ... PXAT` назначает абсолютный expiry; чтение не продлевает TTL. Logout — DEL; expiry удаляет ключ автоматически. AOF включён с `appendfsync always`, `/data` находится на именованном volume `redisdata`. `maxmemory=64mb`, `noeviction`: при нехватке памяти вход завершится 503 вместо вытеснения действующих сессий. Redis не публикует порт на host; отсутствие Redis-аутентификации — учебное упрощение изолированной сети. + +Пароль обрабатывается scrypt: N=16384, r=8, p=1, соль 16 bytes, dklen 32 bytes. Поддерживается 12–128 Unicode code points, включая двоеточие, без усечения по 72 bytes. Одинаковые пароли имеют разные salted hashes. Basic только для создания сессии, бизнес-endpoint требуют Bearer. `X-User-Id` игнорируется. Чужой и отсутствующий чат дают одинаковый 404 после аутентификации. + +История не кэшируется: следующая операция чтения обращается к primary PostgreSQL и видит завершённый COMMIT. Индекс `messages_history(chat_id, created_at, id)` обслуживает keyset-пагинацию. Cursor содержит последнюю пару `(created_at,id)`, связан с чатом/пользователем и подписан HMAC постоянным ключом. Запрашивается `limit+1` строк; `next_cursor=null`, если продолжения нет. При конкурентных транзакциях поздний COMMIT с timestamp до уже пройденной позиции может быть пропущен обходом. Snapshot между HTTP-запросами и глобальный монотонный порядок COMMIT не обещаются; неизменная история обходится без потерь/повторов. + +| Ресурс | Ограничение | +| --- | --- | +| HTTP workers / очередь | 16 / 32; переполнение → 503 | +| HTTP socket / JSON body | 2 s / 65 536 bytes | +| PostgreSQL pool | min 1, max 8, максимум 16 ожидающих, ожидание 1 s | +| PostgreSQL connect / statement / lock | 2 s / 2 s / 1 s; idle transaction 5 s | +| Redis pool | max 8, connect/read по 0.5 s, без автоматических retry | +| DNS внутри API | timeout 1 s, attempts 1; учитывает остановленный Docker service | + +`/health/live` проверяет процесс. `/health/ready` проверяет PostgreSQL и Redis: обе зависимости критичны для полного синхронного пути. При отказе возвращается JSON 503 с request_id/Retry-After, без SQL, credentials и stack trace. Анонимный пользователь не становится доверенным. Регистрация технически может работать без Redis, но полный API в этом состоянии неготов. + +API работает от UID 10001. Логи содержат method/status/request_id/latency без текста сообщений и заголовков авторизации. SIGTERM прекращает приём, завершает ограниченные workers и закрывает пулы. PostgreSQL/Redis используют именованные volumes; API локального состояния не имеет. Из lab1 перенесён обязательный REST API, заменены in-memory структуры и X-User-Id; исследования уровня 5 не переносились. + +### Соответствие .env.example + +| Переменная | Реальное использование | +| --- | --- | +| APP_HOST | Bind внутри API, по умолчанию 0.0.0.0 | +| APP_PORT | Публикуемый host port; внутри контейнера всегда 8080 | +| APP_ENV | Маркер учебного окружения, не меняет хранение | +| SESSION_TTL_SECONDS | TTL новых сессий, default 3600; для smoke/persistence ≥300 | +| SESSION_STORE | В этом решении redis, Compose фиксирует выбор | +| DATABASE_URL_FILE | `/run/secrets/database_url` из локального secrets/database_url | +| CACHE_URL_FILE | `/run/secrets/cache_url` из secrets/cache_url | +| CURSOR_KEY_FILE | `/run/secrets/cursor_key` из secrets/cursor_key | + +Пути secrets в Compose фиксированы и совпадают с примером; значения URL не публикуются в tracked `.env.example`. ## Доказательства критериев -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +Полный прогон: `python scripts/verify.py`. Все команды из корня `lab2`, при работающем стенде. -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +| Критерий README | Команда/сценарий | Наблюдение | Доказательство | +| --- | --- | --- | --- | +| Структура контракта | `python scripts/check_contract.py` | PASS; публичные tests/contracts сохранены | [check.txt](evidence/check.txt) | +| Полная валидация OpenAPI | `python scripts/validate_openapi.py` с requirements-dev | PASS: OpenAPI 3.1 и JSON Schema | [validate.txt](evidence/validate.txt) | +| Запуск из чистой копии с нуля | `python scripts/clean_check.py` | Локальный git clone без .env/secrets, no-cache build, новые volumes, smoke и 3 проверки БД OK | [clean-clone.txt](evidence/clean-clone.txt) | +| Smoke и минимум lab1 с новым auth | `python tests/smoke.py --base-url http://127.0.0.1:8080` | PASS: lab2 public API scenario | [smoke.txt](evidence/smoke.txt) | +| PostgreSQL, FK, UNIQUE, атомарность чата | `docker compose exec -T api python student-tests/test_database.py` | Ошибка второго участника: числа chats/chat_members не изменились | [database.txt](evidence/database.txt), [SQL](migrations/001_initial.sql) | +| Миграции с нуля / повторно | `docker compose up -d --build --wait`; test_repeat_migrations | Первичная миграция; два повтора Already applied, количества сохранены | [database.txt](evidence/database.txt) | +| Сохранность данных и сессий | Публичный resilience с restart.cmd / restart.sh | PASS, прежний токен читает прежнее сообщение; все 3 процесса перезапущены | [persistence.txt](evidence/persistence.txt), [restart.json](evidence/restart.json) | +| Basic/Bearer/logout/X-User-Id | Smoke, собственные API-тесты | Неверный пароль/Basic на business/X-User-Id → 401; logout 204, затем 401 | [smoke.txt](evidence/smoke.txt), [own-api.txt](evidence/own-api.txt) | +| Password KDF | test_password_salted_kdf; test_unicode_password_and_logout | Разные scrypt hashes одинаковых паролей; пароль 128 Unicode code points работает | [database.txt](evidence/database.txt), [own-api.txt](evidence/own-api.txt) | +| Единый запуск / volumes / secrets вне Git | `docker compose --env-file .env config --quiet` | Конфигурация успешна; 3 сервиса и одноразовая миграция | [compose.txt](evidence/compose.txt), [compose.yaml](compose.yaml) | +| Redis с TTL и persistence | `python scripts/check_expiry.py`; persistence | TTL=2: сначала 200, через 2.2 s → 401; обычная сессия пережила restart | [expiry.txt](evidence/expiry.txt), [redis.conf](infra/redis.conf) | +| Настройки durability | SHOW fsync/synchronous_commit/full_page_writes; Redis CONFIG GET | PostgreSQL: on/on/on; Redis: appendonly=yes, appendfsync=always | [PostgreSQL](evidence/durability-postgres.txt), [Redis](evidence/durability-redis.txt) | +| SQL parameters / bounded pools / timeouts | Реальный HTTP-путь, отказ зависимостей | 503 в пределах бюджета вместо зависания | [outages.txt](evidence/outages.txt), [core.py](src/messenger/core.py) | +| Гонка username | test_registration_race, 10 клиентов | Ровно 1×201 и 9×409 | [own-api.txt](evidence/own-api.txt) | +| EXPLAIN ANALYZE BUFFERS на ≥10 000 | `docker compose exec -T api python scripts/explain_history.py` | Index Scan messages_history, 51 строка, 4 shared hit blocks, без Seq Scan | [explain.json](evidence/explain.json) | +| Автоматические отказы Redis/БД | `python student-tests/test_outages.py` | По 3×503; live 200, ready 503; старые сессия/сообщение сохранены после восстановления | [outages.txt](evidence/outages.txt) | +| Собственные негативные/конкурентные тесты | `python student-tests/test_api.py` | 4 теста OK; 100 уникальных сообщений от 10 клиентов и обход страниц | [own-api.txt](evidence/own-api.txt) | ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. +Проверялась корректность уровня 4: durable stores сохраняют подтверждённое сообщение/сессию при перезапуске, а потеря зависимости ограниченно блокирует защищённые операции. Это не исследовательский трек 5 и не измерение максимальной нагрузки. -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +Среда: Windows 10 build 19045, Docker Desktop Engine 29.8.2, Linux/WSL2 x86_64, 8 логических CPU и около 7.71 GiB RAM для Docker. Лимиты: API 2 CPU/512 MiB, PostgreSQL 2 CPU/512 MiB, Redis 1 CPU/128 MiB. Клиент Python 3.14.3, runtime 3.13.7. Digest хранилищ и сведения о машине: [environment.json](evidence/environment.json). + +Профиль: последовательный smoke; гонка регистрации 10 клиентов; 100 POST сообщений от 10 клиентов; expiry; по три последовательных чтения при отказе зависимости; один план на новом чате с 10 000 коротких сообщений, cursor после позиции 9000, limit=50. Прогрев — создание fixtures и ANALYZE; план измерен на уже прогретых страницах. Повторяемость обеспечена скриптами. Статистические серии throughput не проводились, поэтому RPS/p50/p95/p99 не заявляются. [resources.txt](evidence/resources.txt) — единичный снимок CPU/RAM после проверок, не среднее/пик под нагрузкой. + +| Воздействие | Ожидаем | Наблюдали | Восстановление / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| Restart api/db/redis | Старый токен и сообщение действуют после ready | Persistence PASS; StartedAt всех 3 изменились, volumes прежние | В пределах лимитов публичного сценария; потерь нет | +| TTL=2 в отдельном API | После expiry → 401 | До expiry 200, через 2.2 s — 401 | Основной TTL=3600 не менялся | +| Logout | Токен недействителен | 204, затем 401 | Новый вход создаёт новую сессию | +| Redis stop | 503; анонимный вход запрещён | 3 ошибки за 1.005–1.027 s; X-User-Id без сессии → 401 | 0.604 s от начала start до ready/me; сессия/сообщение сохранены | +| PostgreSQL stop | 503; процесс жив | 3 ошибки за 1.005–1.008 s; live 200, ready 503 | 4.599 s от начала start до ready/me; сессия/сообщение сохранены | +| Неизвестный участник после INSERT chats | Полный rollback | Числа chats/chat_members до/после равны | Частичного чата нет | + +SQL Execution Time **0.049 ms**, Index Scan отдал 51 строку, Shared Hit Blocks=4, Shared Read Blocks=0. Это один локальный прогретый SQL-запрос без HTTP/аутентификации: подтверждение использования индекса, не SLA. Не сравнивались альтернативные индексы/100 000 сообщений/backup — это задания оценки 5. + +Первый сценарий отказа выявил DNS-ожидание остановленного контейнера дольше socket timeout. В итоговый Compose добавлены `dns_opt: timeout:1, attempts:1`; результаты выше получены после исправления. ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +- Один host, по одному PostgreSQL/Redis: проверяемый restart выдерживается, уничтожение диска и HA не покрыты. При отказе хранилища бизнес-путь временно недоступен. +- FLUSHDB/удаление Redis volume/повреждение AOF теряют сессии; нужно войти заново. PostgreSQL volume содержит бизнес-данные. Потеря cursor_key аннулирует прежние cursors. +- После COMMIT может оборваться транспорт: клиент не получит 201, хотя сообщение записано. Идемпотентность появляется в lab3 и здесь не реализована. +- Уже аутентифицированный конкурентный запрос может завершиться после logout; последующие проверки токена получают 401. +- HTTP опубликован только на 127.0.0.1, данные синтетические. Для удалённого использования нужны TLS и усиление конфигурации. В учебной БД используется роль, созданная официальным образом; production role management не реализован. +- Snapshot между страницами и упорядочивание конкурентных COMMIT не обещаются. +- Не реализованы треки 5: исследование согласованности кэша/stampede с измерениями, сравнение индексов на ≥100 000 и backup/restore. + +## Источники + +- PostgreSQL: [транзакции](https://www.postgresql.org/docs/17/tutorial-transactions.html), [WAL durability](https://www.postgresql.org/docs/17/wal-reliability.html), [EXPLAIN](https://www.postgresql.org/docs/17/using-explain.html). +- Redis: [persistence и AOF](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [PEXPIREAT](https://redis.io/docs/latest/commands/pexpireat/). +- Psycopg: [пулы](https://www.psycopg.org/psycopg3/docs/advanced/pool.html), [SQL parameters](https://www.psycopg.org/psycopg3/docs/basic/params.html). +- Python: [hashlib.scrypt](https://docs.python.org/3.13/library/hashlib.html#hashlib.scrypt), [secrets](https://docs.python.org/3.13/library/secrets.html). +- [RFC 7617 — Basic](https://www.rfc-editor.org/rfc/rfc7617.html), контракт и публичные проверки этого репозитория, собственная lab1, помощь OpenAI Codex, раскрытая в начале отчёта. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..1c0d70cd --- /dev/null +++ b/compose.yaml @@ -0,0 +1,67 @@ +name: mtusi-lab2-osokin +x-app: &app + image: mtusi-messenger:lab2 + build: . + environment: &app-env + APP_HOST: ${APP_HOST:-0.0.0.0} + APP_PORT: 8080 + APP_ENV: ${APP_ENV:-lab} + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + SESSION_STORE: redis + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL_FILE: /run/secrets/cache_url + CURSOR_KEY_FILE: /run/secrets/cursor_key + secrets: [database_url, cache_url, cursor_key] + mem_limit: 512m + cpus: 2 + dns_opt: ['timeout:1', 'attempts:1'] +services: + db: + image: postgres:17.6-bookworm + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, 'pg_isready -U messenger -d messenger'] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 512m + cpus: 2 + redis: + image: redis:7.4.5-bookworm + command: [redis-server, /usr/local/etc/redis/redis.conf] + volumes: + - redisdata:/data + - ./infra/redis.conf:/usr/local/etc/redis/redis.conf:ro + healthcheck: + test: [CMD, redis-cli, ping] + interval: 2s + timeout: 2s + retries: 30 + mem_limit: 128m + cpus: 1 + migrate: + <<: *app + command: [python, -m, messenger.migrate] + healthcheck: {disable: true} + depends_on: + db: {condition: service_healthy} + api: + <<: *app + ports: ['127.0.0.1:${APP_PORT:-8080}:8080'] + depends_on: + migrate: {condition: service_completed_successfully} + redis: {condition: service_healthy} + stop_grace_period: 8s +volumes: + pgdata: + redisdata: +secrets: + db_password: {file: ./secrets/db_password} + database_url: {file: ./secrets/database_url} + cache_url: {file: ./secrets/cache_url} + cursor_key: {file: ./secrets/cursor_key} diff --git a/evidence/check.txt b/evidence/check.txt new file mode 100644 index 00000000..9180d646 --- /dev/null +++ b/evidence/check.txt @@ -0,0 +1,2 @@ +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/clean-clone.txt b/evidence/clean-clone.txt new file mode 100644 index 00000000..a50097ee --- /dev/null +++ b/evidence/clean-clone.txt @@ -0,0 +1,100 @@ +$ git clone --no-hardlinks \output\clean-clone +Cloning into 'C:\Users\blobb\OneDrive\Документи\ChatGPT\высоконагружноут\lab2\output\clean-clone'... +done. + +$ python scripts/bootstrap.py +Local .env and secrets ready (existing values preserved). + +$ docker compose -p mtusi-lab2-osokin-clean --env-file .env build --no-cache --quiet + Image mtusi-messenger:lab2 Building + Image mtusi-messenger:lab2 Building + Image mtusi-messenger:lab2 Built + Image mtusi-messenger:lab2 Built + +$ docker compose -p mtusi-lab2-osokin-clean --env-file .env up -d --wait + Volume mtusi-lab2-osokin-clean_pgdata Creating + Network mtusi-lab2-osokin-clean_default Creating + Volume mtusi-lab2-osokin-clean_redisdata Creating + Volume mtusi-lab2-osokin-clean_pgdata Creating + Volume mtusi-lab2-osokin-clean_redisdata Creating + Network mtusi-lab2-osokin-clean_default Creating + Volume mtusi-lab2-osokin-clean_pgdata Created + Volume mtusi-lab2-osokin-clean_pgdata Created + Volume mtusi-lab2-osokin-clean_redisdata Created + Volume mtusi-lab2-osokin-clean_redisdata Created + Network mtusi-lab2-osokin-clean_default Created + Network mtusi-lab2-osokin-clean_default Created + Container mtusi-lab2-osokin-clean-redis-1 Creating + Container mtusi-lab2-osokin-clean-db-1 Creating + Container mtusi-lab2-osokin-clean-redis-1 Created + Container mtusi-lab2-osokin-clean-db-1 Created + Container mtusi-lab2-osokin-clean-migrate-1 Creating + Container mtusi-lab2-osokin-clean-migrate-1 Created + Container mtusi-lab2-osokin-clean-api-1 Creating + Container mtusi-lab2-osokin-clean-api-1 Created + Container mtusi-lab2-osokin-clean-redis-1 Starting + Container mtusi-lab2-osokin-clean-db-1 Starting + Container mtusi-lab2-osokin-clean-redis-1 Started + Container mtusi-lab2-osokin-clean-db-1 Started + Container mtusi-lab2-osokin-clean-db-1 Waiting + Container mtusi-lab2-osokin-clean-db-1 Healthy + Container mtusi-lab2-osokin-clean-migrate-1 Starting + Container mtusi-lab2-osokin-clean-migrate-1 Started + Container mtusi-lab2-osokin-clean-redis-1 Waiting + Container mtusi-lab2-osokin-clean-migrate-1 Waiting + Container mtusi-lab2-osokin-clean-redis-1 Healthy + Container mtusi-lab2-osokin-clean-migrate-1 Exited + Container mtusi-lab2-osokin-clean-api-1 Starting + Container mtusi-lab2-osokin-clean-api-1 Started + Container mtusi-lab2-osokin-clean-redis-1 Waiting + Container mtusi-lab2-osokin-clean-migrate-1 Waiting + Container mtusi-lab2-osokin-clean-api-1 Waiting + Container mtusi-lab2-osokin-clean-db-1 Waiting + Container mtusi-lab2-osokin-clean-redis-1 Healthy + Container mtusi-lab2-osokin-clean-db-1 Healthy + Container mtusi-lab2-osokin-clean-migrate-1 Exited + Container mtusi-lab2-osokin-clean-api-1 Healthy + +$ python scripts/check_contract.py +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. + +$ python tests/smoke.py --base-url http://127.0.0.1:18082 +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. + +$ docker compose -p mtusi-lab2-osokin-clean --env-file .env exec -T api python student-tests/test_database.py +test_password_salted_kdf (__main__.Database.test_password_salted_kdf) ... ok +test_repeat_migrations (__main__.Database.test_repeat_migrations) ... Already applied: 001_initial.sql +Migrations committed. +Already applied: 001_initial.sql +Migrations committed. +ok +test_rollback_after_chat_insert (__main__.Database.test_rollback_after_chat_insert) ... ok + +---------------------------------------------------------------------- +Ran 3 tests in 2.993s + +OK + +$ docker compose -p mtusi-lab2-osokin-clean --env-file .env logs --no-log-prefix migrate +Applied: 001_initial.sql +Migrations committed. + +$ docker compose -p mtusi-lab2-osokin-clean --env-file .env down + Container mtusi-lab2-osokin-clean-api-1 Stopping + Container mtusi-lab2-osokin-clean-api-1 Stopped + Container mtusi-lab2-osokin-clean-api-1 Removing + Container mtusi-lab2-osokin-clean-api-1 Removed + Container mtusi-lab2-osokin-clean-migrate-1 Stopping + Container mtusi-lab2-osokin-clean-redis-1 Stopping + Container mtusi-lab2-osokin-clean-migrate-1 Stopped + Container mtusi-lab2-osokin-clean-migrate-1 Removing + Container mtusi-lab2-osokin-clean-migrate-1 Removed + Container mtusi-lab2-osokin-clean-db-1 Stopping + Container mtusi-lab2-osokin-clean-redis-1 Stopped + Container mtusi-lab2-osokin-clean-redis-1 Removing + Container mtusi-lab2-osokin-clean-redis-1 Removed + Container mtusi-lab2-osokin-clean-db-1 Stopped + Container mtusi-lab2-osokin-clean-db-1 Removing + Container mtusi-lab2-osokin-clean-db-1 Removed + Network mtusi-lab2-osokin-clean_default Removing + Network mtusi-lab2-osokin-clean_default Removed diff --git a/evidence/compose.txt b/evidence/compose.txt new file mode 100644 index 00000000..8b8dc967 --- /dev/null +++ b/evidence/compose.txt @@ -0,0 +1 @@ +$ docker compose --env-file .env config --quiet diff --git a/evidence/database.txt b/evidence/database.txt new file mode 100644 index 00000000..763c3d09 --- /dev/null +++ b/evidence/database.txt @@ -0,0 +1,13 @@ +$ docker compose exec -T api python student-tests/test_database.py +test_password_salted_kdf (__main__.Database.test_password_salted_kdf) ... ok +test_repeat_migrations (__main__.Database.test_repeat_migrations) ... Already applied: 001_initial.sql +Migrations committed. +Already applied: 001_initial.sql +Migrations committed. +ok +test_rollback_after_chat_insert (__main__.Database.test_rollback_after_chat_insert) ... ok + +---------------------------------------------------------------------- +Ran 3 tests in 2.310s + +OK diff --git a/evidence/demo.txt b/evidence/demo.txt new file mode 100644 index 00000000..bc42aaad --- /dev/null +++ b/evidence/demo.txt @@ -0,0 +1,8 @@ +$ python scripts/demo.py +1. Readiness 200; registered three users; Basic login returned Bearer sessions. +2. Created chat: c2817969-606b-4551-bff4-c98ba084c1b4 ; members: Alice and Bob. +3. POST message -> 201; message id: ac5d70bf-4bd4-4da8-a17b-e8505600bc15 +4. Bob reads Alice's committed message -> 200: Привет, Боб! Это лабораторная 2. +5. Eve is authenticated but not a chat member -> 404. +6. X-User-Id without Bearer -> 401. +7. Logout -> 204; previous token -> 401. Demo PASS. diff --git a/evidence/durability-postgres.txt b/evidence/durability-postgres.txt new file mode 100644 index 00000000..c8680398 --- /dev/null +++ b/evidence/durability-postgres.txt @@ -0,0 +1,15 @@ +$ docker compose exec -T db psql -U messenger -d messenger -c SHOW fsync; -c SHOW synchronous_commit; -c SHOW full_page_writes; + fsync +------- + on +(1 row) + + synchronous_commit +-------------------- + on +(1 row) + + full_page_writes +------------------ + on +(1 row) diff --git a/evidence/durability-redis.txt b/evidence/durability-redis.txt new file mode 100644 index 00000000..b60752ac --- /dev/null +++ b/evidence/durability-redis.txt @@ -0,0 +1,5 @@ +$ docker compose exec -T redis redis-cli CONFIG GET appendonly appendfsync +appendfsync +always +appendonly +yes diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..8268f241 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,22 @@ +{ + "verified_at_utc": "2026-10-26T12:25:05.273386+00:00", + "os": "Windows-10-10.0.19045-SP0", + "client_python": "3.14.3", + "docker": { + "ServerVersion": "29.8.2", + "OperatingSystem": "Docker Desktop", + "KernelVersion": "6.18.40.1-microsoft-standard-WSL2", + "NCPU": 8, + "MemTotal": 8275755008, + "Architecture": "x86_64" + }, + "images": { + "postgres:17.6-bookworm": [ + "postgres@sha256:f3bd19c606e442c3d7bdfa8002e03fe260a1023351e0ea4598032022b68dd6e3" + ], + "redis:7.4.5-bookworm": [ + "redis@sha256:90e7a336d044f1abc9e9dbc05d65566850896d11453bbd1dd0fb7e5059f0e8fb" + ] + }, + "runtime_python": "Python 3.13.7" +} diff --git a/evidence/expiry.txt b/evidence/expiry.txt new file mode 100644 index 00000000..e8dda6ab --- /dev/null +++ b/evidence/expiry.txt @@ -0,0 +1,4 @@ +$ python scripts/check_expiry.py + Container mtusi-lab2-osokin-api-run-caf7bb185347 Creating + Container mtusi-lab2-osokin-api-run-caf7bb185347 Created +PASS: session accepted before expiry; rejected after 2.2s with SESSION_TTL_SECONDS=2. diff --git a/evidence/explain.json b/evidence/explain.json new file mode 100644 index 00000000..53b5f62d --- /dev/null +++ b/evidence/explain.json @@ -0,0 +1,81 @@ +{ + "messages": 10000, + "cursor_position": 9000, + "page_limit": 50, + "fetched_with_lookahead": 51, + "sql": "SELECT id, chat_id, sender_id, text, created_at FROM messages WHERE chat_id = %s AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s", + "explain": [ + { + "Plan": { + "Node Type": "Limit", + "Parallel Aware": false, + "Async Capable": false, + "Startup Cost": 0.29, + "Total Cost": 17.77, + "Plan Rows": 51, + "Plan Width": 68, + "Actual Startup Time": 0.008, + "Actual Total Time": 0.025, + "Actual Rows": 51, + "Actual Loops": 1, + "Shared Hit Blocks": 4, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0, + "Plans": [ + { + "Node Type": "Index Scan", + "Parent Relationship": "Outer", + "Parallel Aware": false, + "Async Capable": false, + "Scan Direction": "Forward", + "Index Name": "messages_history", + "Relation Name": "messages", + "Alias": "messages", + "Startup Cost": 0.29, + "Total Cost": 473.36, + "Plan Rows": 1380, + "Plan Width": 68, + "Actual Startup Time": 0.007, + "Actual Total Time": 0.021, + "Actual Rows": 51, + "Actual Loops": 1, + "Index Cond": "((chat_id = '137c9dcf-9f70-465c-862c-595c9a50d5c3'::uuid) AND (ROW(created_at, id) > ROW('2026-01-01 00:00:00.009+00'::timestamp with time zone, '3873da86-1a3e-2129-c84e-1daf5c084a0d'::uuid)))", + "Rows Removed by Index Recheck": 0, + "Shared Hit Blocks": 4, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + } + ] + }, + "Planning": { + "Shared Hit Blocks": 6, + "Shared Read Blocks": 0, + "Shared Dirtied Blocks": 0, + "Shared Written Blocks": 0, + "Local Hit Blocks": 0, + "Local Read Blocks": 0, + "Local Dirtied Blocks": 0, + "Local Written Blocks": 0, + "Temp Read Blocks": 0, + "Temp Written Blocks": 0 + }, + "Planning Time": 0.079, + "Triggers": [], + "Execution Time": 0.049 + } + ] +} diff --git a/evidence/outages.txt b/evidence/outages.txt new file mode 100644 index 00000000..8355f8c2 --- /dev/null +++ b/evidence/outages.txt @@ -0,0 +1,39 @@ +$ python student-tests/test_outages.py + Container mtusi-lab2-osokin-redis-1 Stopping + Container mtusi-lab2-osokin-redis-1 Stopped + Container mtusi-lab2-osokin-redis-1 Starting + Container mtusi-lab2-osokin-redis-1 Started + Container mtusi-lab2-osokin-db-1 Stopping + Container mtusi-lab2-osokin-db-1 Stopped + Container mtusi-lab2-osokin-db-1 Starting + Container mtusi-lab2-osokin-db-1 Started +[ + { + "service": "redis", + "errors": 3, + "status": 503, + "seconds": [ + 1.0267414001282305, + 1.0045876000076532, + 1.007952299900353 + ], + "recovery_seconds": 0.604, + "original_session_preserved": true, + "original_message_preserved": true, + "anonymous_status": 401 + }, + { + "service": "db", + "errors": 3, + "status": 503, + "seconds": [ + 1.0076619999017566, + 1.0045177999418229, + 1.0052553000859916 + ], + "recovery_seconds": 4.599, + "original_session_preserved": true, + "original_message_preserved": true, + "anonymous_status": 401 + } +] diff --git a/evidence/own-api.txt b/evidence/own-api.txt new file mode 100644 index 00000000..f8443be6 --- /dev/null +++ b/evidence/own-api.txt @@ -0,0 +1,10 @@ +$ python student-tests/test_api.py +test_concurrent_messages (__main__.API.test_concurrent_messages) ... ok +test_registration_race (__main__.API.test_registration_race) ... ok +test_unicode_password_and_logout (__main__.API.test_unicode_password_and_logout) ... ok +test_validation_access_and_pagination (__main__.API.test_validation_access_and_pagination) ... ok + +---------------------------------------------------------------------- +Ran 4 tests in 4.016s + +OK diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..40f4c66c --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,3 @@ +$ python tests/resilience.py --mode persistence --action-script scripts/restart.cmd --base-url http://127.0.0.1:8080 +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/resources.txt b/evidence/resources.txt new file mode 100644 index 00000000..6d40b324 --- /dev/null +++ b/evidence/resources.txt @@ -0,0 +1,4 @@ +$ docker stats --no-stream --format {{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}} mtusi-lab2-osokin-api-1 mtusi-lab2-osokin-db-1 mtusi-lab2-osokin-redis-1 +mtusi-lab2-osokin-api-1 CPU=51.26% RAM=158.7MiB / 512MiB +mtusi-lab2-osokin-db-1 CPU=8.94% RAM=27.76MiB / 512MiB +mtusi-lab2-osokin-redis-1 CPU=0.44% RAM=4.301MiB / 128MiB diff --git a/evidence/restart.json b/evidence/restart.json new file mode 100644 index 00000000..d51e1ecd --- /dev/null +++ b/evidence/restart.json @@ -0,0 +1,56 @@ +{ + "before": { + "api": { + "id": "b6b4e6f8c09ee890165395f2af5e40945dd74ce270a9c35c09b946db8ed160ef", + "started_at": "2026-10-26T20:13:56.437921569Z", + "mounts": [] + }, + "db": { + "id": "9a25c8ce8dd3a0a64d4def2f27765f792fb00739774c0e0926fe0cb1017c27fd", + "started_at": "2026-10-26T20:13:49.316227457Z", + "mounts": [ + { + "name": "lab2-defense-0eaa9a8d_pgdata", + "destination": "/var/lib/postgresql/data" + } + ] + }, + "redis": { + "id": "8009d787a3e25c88d024c121ff5dd00a613e77b785a30fbe4b08b1dcb57a021e", + "started_at": "2026-10-26T20:13:49.903677339Z", + "mounts": [ + { + "name": "lab2-defense-0eaa9a8d_redisdata", + "destination": "/data" + } + ] + } + }, + "after": { + "api": { + "id": "b6b4e6f8c09ee890165395f2af5e40945dd74ce270a9c35c09b946db8ed160ef", + "started_at": "2026-10-26T20:15:24.833818504Z", + "mounts": [] + }, + "db": { + "id": "9a25c8ce8dd3a0a64d4def2f27765f792fb00739774c0e0926fe0cb1017c27fd", + "started_at": "2026-10-26T20:15:24.683887137Z", + "mounts": [ + { + "name": "lab2-defense-0eaa9a8d_pgdata", + "destination": "/var/lib/postgresql/data" + } + ] + }, + "redis": { + "id": "8009d787a3e25c88d024c121ff5dd00a613e77b785a30fbe4b08b1dcb57a021e", + "started_at": "2026-10-26T20:15:24.575559122Z", + "mounts": [ + { + "name": "lab2-defense-0eaa9a8d_redisdata", + "destination": "/data" + } + ] + } + } +} diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..9b3a43b2 --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1,2 @@ +$ python tests/smoke.py --base-url http://127.0.0.1:8080 +PASS: lab2 public API scenario; infrastructure and grading evidence remain separate. diff --git a/evidence/validate.txt b/evidence/validate.txt new file mode 100644 index 00000000..2fecdac1 --- /dev/null +++ b/evidence/validate.txt @@ -0,0 +1,2 @@ +$ ../lab1/.venv/Scripts/python.exe scripts/validate_openapi.py +PASS: full OpenAPI 3.1 and JSON Schema validation; application NOT tested. diff --git a/infra/SCHEMA.md b/infra/SCHEMA.md new file mode 100644 index 00000000..4ccdfa06 --- /dev/null +++ b/infra/SCHEMA.md @@ -0,0 +1,17 @@ +# Схема и транзакции (проект перед реализацией SQL) + +Источник истины бизнес-данных — PostgreSQL; единственное хранилище сессий — Redis. + +* users: UUID PK, username UNIQUE с CHECK формата, display_name, password_hash (scrypt), created_at timestamptz. +* chats: UUID PK, title, created_at timestamptz. +* chat_members: (chat_id, user_id) PK, FK к chats и users; position для сохранения порядка участников. +* messages: UUID PK, FK chat_id → chats; составной FK (chat_id, sender_id) → chat_members. +* schema_migrations: имя миграции PK, SHA-256 содержимого, applied_at. DDL и запись версии в одной транзакции под advisory lock. + +Индексы: UNIQUE username защищает конкурентную регистрацию. messages_history(chat_id, created_at, id) обслуживает `WHERE chat_id = $1 AND (created_at,id) > ($2,$3) ORDER BY created_at,id LIMIT $4`, без OFFSET. chat_members_user(user_id,chat_id) — список доступных чатов. chats_order(created_at,id) — порядок чатов. + +Создание чата: INSERT chats + INSERT всех участников в одной транзакции; неизвестный участник вызывает FK violation и rollback всего чата. Регистрация и отправка сообщения — отдельные транзакции; HTTP 201 только после успешного COMMIT. Чтения — READ COMMITTED с primary PostgreSQL; история не кэшируется. + +Redis: `session:` → UUID пользователя, абсолютный expiry (PXAT); TTL не обновляется при чтении. Logout — DEL. AOF + appendfsync always, именованный volume. Потеря Redis: 503, никакого доверия X-User-Id; restart сохраняет действующие сессии, FLUSHDB отзывает все. Истечение/DEL не требует инвалидации других кэшей. + +Cursor подписан HMAC-SHA256 постоянным локальным ключом, связан с chat_id либо user_id; хранит (created_at,id). Это позиция, не snapshot: конкурентная поздно закоммиченная запись со старым timestamp может оказаться за уже пройденным cursor. Неизменная история обходится без повторов; UUID разрешает равные timestamps. Часы назначает PostgreSQL, монотонность между транзакциями не обещается. diff --git a/infra/redis.conf b/infra/redis.conf new file mode 100644 index 00000000..b3842842 --- /dev/null +++ b/infra/redis.conf @@ -0,0 +1,10 @@ +bind 0.0.0.0 +protected-mode no +port 6379 +dir /data +appendonly yes +appendfsync always +save "" +maxmemory 64mb +maxmemory-policy noeviction +# Only the private Compose network can reach Redis; no host port is published. diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..edc61ab6 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,30 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name text NOT NULL CHECK (char_length(display_name) BETWEEN 1 AND 100), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title text NOT NULL CHECK (char_length(title) BETWEEN 1 AND 100), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + position smallint NOT NULL CHECK (position BETWEEN 0 AND 99), + PRIMARY KEY (chat_id, user_id), + UNIQUE (chat_id, position) +); +CREATE INDEX chat_members_user ON chat_members(user_id, chat_id); +CREATE INDEX chats_order ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL, + text text NOT NULL CHECK (char_length(text) BETWEEN 1 AND 4000), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES chat_members(chat_id, user_id) +); +CREATE INDEX messages_history ON messages(chat_id, created_at, id); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..eab1c214 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +psycopg==3.2.10 +psycopg-binary==3.2.10 +psycopg-pool==3.2.6 +redis==6.4.0 +typing_extensions==4.15.0 diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..ac4ad91c --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Create local synthetic credentials once; never print them or replace existing files.""" +from pathlib import Path +import secrets + +root = Path(__file__).resolve().parents[1] +folder = root / 'secrets' +folder.mkdir(exist_ok=True) +password_file = folder / 'db_password' +if not password_file.exists(): + password_file.write_text(secrets.token_hex(24), encoding='utf-8') +password = password_file.read_text(encoding='utf-8').strip() +values = { + 'database_url': 'postgresql://messenger:' + password + '@db:5432/messenger', + 'cache_url': 'redis://redis:6379/0', + 'cursor_key': secrets.token_hex(32), +} +for name, value in values.items(): + target = folder / name + if not target.exists(): + target.write_text(value, encoding='utf-8') +if not (root / '.env').exists(): + (root / '.env').write_text('APP_HOST=0.0.0.0\nAPP_PORT=8080\nAPP_ENV=lab\nSESSION_TTL_SECONDS=3600\nSESSION_STORE=redis\n', encoding='utf-8') +print('Local .env and secrets ready (existing values preserved).') diff --git a/scripts/check_expiry.py b/scripts/check_expiry.py new file mode 100644 index 00000000..2cb07dd9 --- /dev/null +++ b/scripts/check_expiry.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +"""Separate API container with 2 second TTL; main API configuration is unchanged.""" +from pathlib import Path +import subprocess +import sys + +root = Path(__file__).resolve().parents[1] +command = ['docker', 'compose', 'run', '-d', '--no-deps', '-e', 'SESSION_TTL_SECONDS=2', 'api'] +identifier = subprocess.check_output(command, cwd=root, text=True).strip() +try: + # Start the client in the temporary container after its HTTP server is ready. + subprocess.run(['docker', 'exec', identifier, 'python', '-c', + "import time,urllib.request; " + "\nfor _ in range(50):\n try:\n urllib.request.urlopen('http://localhost:8080/health/ready',timeout=3); break" + "\n except OSError: time.sleep(.2)\nelse: raise RuntimeError('API not ready')"], check=True) + subprocess.run(['docker', 'exec', identifier, 'python', 'student-tests/test_expiry.py'], check=True) +finally: + subprocess.run(['docker', 'rm', '-f', identifier], check=True, stdout=subprocess.DEVNULL) diff --git a/scripts/clean_check.py b/scripts/clean_check.py new file mode 100644 index 00000000..7ebf4417 --- /dev/null +++ b/scripts/clean_check.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Check committed sources in a new local clone with fresh secrets and new volumes.""" +import os +from pathlib import Path +import subprocess +import sys + +ROOT = Path(__file__).resolve().parents[1] +clone = ROOT / 'output' / 'clean-clone' +project = 'mtusi-lab2-osokin-clean' +env = dict(os.environ, PYTHONIOENCODING='utf-8', BASE_URL='http://127.0.0.1:18082') + + +def main(): + if clone.exists(): + raise SystemExit('output/clean-clone already exists; preserve or move it before repeating this check.') + clone.parent.mkdir(exist_ok=True) + log = [] + def run(command, cwd): + log.append('$ ' + ' '.join(command).replace(str(ROOT), '').replace(sys.executable, 'python')) + process = subprocess.run(command, cwd=cwd, env=env, text=True, encoding='utf-8', errors='replace', + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + log.append(process.stdout) + cleaned = '\n'.join(line.rstrip() for line in '\n'.join(log).splitlines()) + '\n' + (ROOT / 'evidence' / 'clean-clone.txt').write_text(cleaned, encoding='utf-8') + if process.returncode: + raise SystemExit('Clean clone check failed; see evidence/clean-clone.txt') + run(['git', 'clone', '--no-hardlinks', str(ROOT), str(clone)], ROOT) + assert not (clone / '.env').exists() + assert not (clone / 'secrets').exists() + run([sys.executable, 'scripts/bootstrap.py'], clone) + local_env = clone / '.env' + local_env.write_text(local_env.read_text(encoding='utf-8').replace('APP_PORT=8080', 'APP_PORT=18082'), encoding='utf-8') + compose = ['docker', 'compose', '-p', project, '--env-file', '.env'] + try: + run(compose + ['build', '--no-cache', '--quiet'], clone) + run(compose + ['up', '-d', '--wait'], clone) + run([sys.executable, 'scripts/check_contract.py'], clone) + run([sys.executable, 'tests/smoke.py', '--base-url', env['BASE_URL']], clone) + run(compose + ['exec', '-T', 'api', 'python', 'student-tests/test_database.py'], clone) + run(compose + ['logs', '--no-log-prefix', 'migrate'], clone) + finally: + # Keep even the disposable check's volumes; only stop its isolated containers. + run(compose + ['down'], clone) + print('PASS: clean local clone; no copied secrets, fresh volumes, no-cache build, smoke/database checks.') + + +if __name__ == '__main__': + main() diff --git a/scripts/demo.py b/scripts/demo.py new file mode 100644 index 00000000..d7eb5af6 --- /dev/null +++ b/scripts/demo.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Human-readable demonstration. Passwords and session tokens stay in memory.""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + +c = Client() +alice, bob, eve, chat = c.bootstrap() +print('1. Readiness 200; registered three users; Basic login returned Bearer sessions.') +print('2. Created chat:', chat['id'], '; members: Alice and Bob.') +message = c.message(alice, chat, 'Привет, Боб! Это лабораторная 2.')['json'] +print('3. POST message -> 201; message id:', message['id']) +page = c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages', actor=bob, schema='MessagePage')['json'] +assert page['items'] == [message] +print('4. Bob reads Alice\'s committed message -> 200:', message['text']) +c.call('GET', '/api/v1/chats/' + chat['id'], 404, actor=eve) +print('5. Eve is authenticated but not a chat member -> 404.') +c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': alice['id']}) +print('6. X-User-Id without Bearer -> 401.') +c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=alice) +c.call('GET', '/api/v1/users/me', 401, actor=alice) +print('7. Logout -> 204; previous token -> 401. Demo PASS.') diff --git a/scripts/explain_history.py b/scripts/explain_history.py new file mode 100644 index 00000000..9aa9d7b9 --- /dev/null +++ b/scripts/explain_history.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""One index, 10,000 messages; grade 4 evidence, not a grade 5 comparison.""" +import json +import uuid +from messenger.core import Config, HISTORY_SQL, Store + +store = Store(Config.from_env()) +try: + actor = store.create_user(dict(username='plan_' + uuid.uuid4().hex[:20], + display_name='Plan fixture', password='synthetic-plan-password'))['id'] + chat = store.create_chat(actor, dict(title='EXPLAIN 10000', member_ids=[actor])) + with store.pool.connection() as conn: + conn.execute("INSERT INTO messages(id,chat_id,sender_id,text,created_at) " + "SELECT md5(%s || i::text)::uuid, %s, %s, 'fixture ' || i, " + "timestamptz '2026-01-01 UTC' + i * interval '1 microsecond' " + "FROM generate_series(1,10000) i", (chat['id'], chat['id'], actor)) + conn.execute('ANALYZE messages') + key = conn.execute('SELECT created_at,id FROM messages WHERE chat_id=%s ' + 'ORDER BY created_at,id LIMIT 1 OFFSET 8999', (chat['id'],)).fetchone() + plan = conn.execute('EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) ' + HISTORY_SQL, + (chat['id'], key['created_at'], key['id'], 51)).fetchone()['QUERY PLAN'] + count = conn.execute('SELECT count(*) AS n FROM messages WHERE chat_id=%s', (chat['id'],)).fetchone()['n'] + # OFFSET is used once to choose a fixture position, never in the API query. + assert count == 10000 + nodes = [] + def walk(node): + nodes.append(node) + for child in node.get('Plans', []): + walk(child) + walk(plan[0]['Plan']) + assert any(n.get('Index Name') == 'messages_history' for n in nodes) + assert all(n['Node Type'] != 'Seq Scan' for n in nodes) + print(json.dumps(dict(messages=count, cursor_position=9000, page_limit=50, fetched_with_lookahead=51, + sql=HISTORY_SQL, explain=plan), ensure_ascii=False, indent=2)) +finally: + store.close() diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..a46f4dcd --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,4 @@ +@echo off +cd /d "%~dp0.." +python scripts\restart.py +exit /b %errorlevel% diff --git a/scripts/restart.py b/scripts/restart.py new file mode 100644 index 00000000..fdc52446 --- /dev/null +++ b/scripts/restart.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Restart precisely this lab's API, PostgreSQL and Redis, keeping both volumes.""" +import json +from pathlib import Path +import subprocess + +ROOT = Path(__file__).resolve().parents[1] + + +def compose(*args): + return subprocess.check_output(['docker', 'compose', '--env-file', '.env', *args], cwd=ROOT, text=True) + + +def states(): + result = {} + for service in ('api', 'db', 'redis'): + identifier = compose('ps', '-q', service).strip() + data = json.loads(subprocess.check_output(['docker', 'inspect', identifier], text=True))[0] + result[service] = dict(id=identifier, started_at=data['State']['StartedAt'], + mounts=[dict(name=m.get('Name'), destination=m['Destination']) + for m in data['Mounts'] if m['Type'] == 'volume']) + return result + + +def main(): + before = states() + subprocess.run(['docker', 'compose', '--env-file', '.env', 'restart', 'api', 'db', 'redis'], + cwd=ROOT, check=True) + after = states() + for service in before: + assert before[service]['started_at'] != after[service]['started_at'], service + ' did not restart' + assert before[service]['mounts'] == after[service]['mounts'], service + ' changed volumes' + (ROOT / 'evidence' / 'restart.json').write_text(json.dumps(dict(before=before, after=after), indent=2) + '\n') + print('Restarted api/db/redis; volume names unchanged. Evidence: evidence/restart.json') + + +if __name__ == '__main__': + main() diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..edd8888c --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/restart.py diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 00000000..9dd6bc1a --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Run the grade 4 checks and preserve small sanitized evidence files.""" +import json +import os +from pathlib import Path +import platform +import subprocess +import sys +from datetime import datetime, timezone + +ROOT = Path(__file__).resolve().parents[1] +EVIDENCE = ROOT / 'evidence' +env = dict(os.environ, BASE_URL=os.environ.get('BASE_URL', 'http://127.0.0.1:8080'), PYTHONIOENCODING='utf-8') +python = sys.executable +if hasattr(sys.stdout, 'reconfigure'): + sys.stdout.reconfigure(encoding='utf-8') + + +def run(name, command): + print('Running', name, flush=True) + process = subprocess.run(command, cwd=ROOT, env=env, text=True, encoding='utf-8', errors='replace', + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + log = '$ ' + ' '.join(command).replace(python, 'python') + '\n' + process.stdout + log = '\n'.join(line.rstrip() for line in log.splitlines()).rstrip() + '\n' + (EVIDENCE / (name + '.txt')).write_text(log, encoding='utf-8') + print(process.stdout, end='', flush=True) + if process.returncode: + raise SystemExit('Check failed: ' + name) + + +def main(): + EVIDENCE.mkdir(exist_ok=True) + run('check', [python, 'scripts/check_contract.py']) + run('compose', ['docker', 'compose', '--env-file', '.env', 'config', '--quiet']) + run('smoke', [python, 'tests/smoke.py', '--base-url', env['BASE_URL']]) + run('own-api', [python, 'student-tests/test_api.py']) + run('database', ['docker', 'compose', 'exec', '-T', 'api', 'python', 'student-tests/test_database.py']) + run('persistence', [python, 'tests/resilience.py', '--mode', 'persistence', '--action-script', + 'scripts/restart.cmd' if os.name == 'nt' else 'scripts/restart.sh', '--base-url', env['BASE_URL']]) + run('expiry', [python, 'scripts/check_expiry.py']) + run('outages', [python, 'student-tests/test_outages.py']) + run('demo', [python, 'scripts/demo.py']) + run('durability-postgres', ['docker', 'compose', 'exec', '-T', 'db', 'psql', '-U', 'messenger', + '-d', 'messenger', '-c', 'SHOW fsync;', '-c', 'SHOW synchronous_commit;', + '-c', 'SHOW full_page_writes;']) + run('durability-redis', ['docker', 'compose', 'exec', '-T', 'redis', 'redis-cli', + 'CONFIG', 'GET', 'appendonly', 'appendfsync']) + output = subprocess.check_output(['docker', 'compose', 'exec', '-T', 'api', 'python', + 'scripts/explain_history.py'], cwd=ROOT, text=True) + plan = json.loads(output) + (EVIDENCE / 'explain.json').write_text(json.dumps(plan, ensure_ascii=False, indent=2) + '\n', encoding='utf-8') + print('PASS: EXPLAIN on', plan['messages'], 'messages uses messages_history.', flush=True) + run('resources', ['docker', 'stats', '--no-stream', '--format', + '{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}', + 'mtusi-lab2-osokin-api-1', 'mtusi-lab2-osokin-db-1', 'mtusi-lab2-osokin-redis-1']) + environment = dict(verified_at_utc=datetime.now(timezone.utc).isoformat(), os=platform.platform(), + client_python=platform.python_version()) + environment['docker'] = json.loads(subprocess.check_output( + ['docker', 'info', '--format', '{{json .}}'], text=True)) + # Whitelist machine properties; never persist Docker client config/proxy details. + environment['docker'] = {k: environment['docker'][k] for k in + ('ServerVersion', 'OperatingSystem', 'KernelVersion', 'NCPU', 'MemTotal', 'Architecture')} + environment['images'] = {} + environment['runtime_python'] = subprocess.check_output( + ['docker', 'compose', 'exec', '-T', 'api', 'python', '--version'], cwd=ROOT, text=True).strip() + for image in ('postgres:17.6-bookworm', 'redis:7.4.5-bookworm'): + environment['images'][image] = json.loads(subprocess.check_output( + ['docker', 'image', 'inspect', image, '--format', '{{json .RepoDigests}}'], text=True)) + (EVIDENCE / 'environment.json').write_text(json.dumps(environment, indent=2) + '\n', encoding='utf-8') + print('PASS: all grade 4 scenarios completed; no grade 5 track was run.') + + +if __name__ == '__main__': + main() diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..2f7a605a --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1 @@ +"""Lab 1: bounded, single-process in-memory messenger.""" diff --git a/src/messenger/core.py b/src/messenger/core.py new file mode 100644 index 00000000..ac86462d --- /dev/null +++ b/src/messenger/core.py @@ -0,0 +1,301 @@ +"""Validation and persistent state. HTTP 201 is emitted after transaction exit/COMMIT.""" +import base64 +from dataclasses import dataclass, fields +from datetime import datetime, timezone +import hashlib +import hmac +import json +import os +from pathlib import Path +import re +import secrets +import uuid + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool +import redis +from redis.backoff import NoBackoff +from redis.retry import Retry + + +class APIError(Exception): + def __init__(self, status, message): + self.status, self.message = status, message + + +@dataclass +class Config: + app_host: str = '0.0.0.0' + app_port: int = 8080 + workers: int = 16 + request_queue: int = 32 + socket_timeout: int = 2 + max_body_bytes: int = 65536 + session_ttl_seconds: int = 3600 + + @classmethod + def from_env(cls): + values = {} + for field in fields(cls): + raw = os.environ.get(field.name.upper(), field.default) + values[field.name] = str(raw) if field.type is str else int(raw) + if field.type is int and values[field.name] < 1: + raise ValueError(field.name + ' must be positive') + result = cls(**values) + if result.app_port > 65535 or result.workers > 32 or result.request_queue > 128: + raise ValueError('port/workers/queue outside bounds') + if result.max_body_bytes > 1048576 or result.socket_timeout > 5: + raise ValueError('body <= 1 MiB and socket timeout <= 5 required') + if os.getenv('SESSION_STORE', 'redis') != 'redis': + raise ValueError('This solution uses Redis sessions') + return result + + +def secret_file(name): + return Path(os.environ[name]).read_text(encoding='utf-8').strip() + + +def db_connect_kwargs(): + return dict(connect_timeout=2, options='-c statement_timeout=2000 -c lock_timeout=1000 ' + '-c idle_in_transaction_session_timeout=5000', row_factory=dict_row) + + +def uuid_value(value): + if not isinstance(value, str) or not re.fullmatch( + r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value): + raise APIError(400, 'Invalid UUID') + return str(uuid.UUID(value)) + + +def string(value, maximum): + if not isinstance(value, str) or not value.strip() or len(value) > maximum or '\x00' in value: + raise APIError(400, 'Invalid string') + try: + value.encode('utf-8') + except UnicodeError: + raise APIError(400, 'Invalid Unicode') from None + return value + + +def object_fields(body, expected): + if not isinstance(body, dict) or set(body) != set(expected): + raise APIError(400, 'Invalid object fields') + + +def password_value(value): + if not isinstance(value, str) or not value.strip() or not 12 <= len(value) <= 128: + raise APIError(400, 'Invalid password length') + try: + value.encode('utf-8') + except UnicodeError: + raise APIError(400, 'Invalid Unicode') from None + return value + + +def password_hash(value, salt=None): + salt = salt or secrets.token_bytes(16) + digest = hashlib.scrypt(value.encode('utf-8'), salt=salt, n=16384, r=8, p=1, dklen=32) + return 'scrypt$16384$8$1$' + salt.hex() + '$' + digest.hex() + + +def password_matches(value, encoded): + # Only this versioned format is generated by registration. + salt = bytes.fromhex(encoded.split('$')[4]) + return hmac.compare_digest(password_hash(value, salt), encoded) + + +def stamp(value): + return value.astimezone(timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +def public_row(row): + return {key: (str(value) if isinstance(value, uuid.UUID) else stamp(value) + if isinstance(value, datetime) else value) + for key, value in row.items() if key != 'password_hash'} + + +USER_COLUMNS = 'id, username, display_name, created_at' +MESSAGE_COLUMNS = 'id, chat_id, sender_id, text, created_at' +# Shared by the real history endpoint and EXPLAIN evidence; all values are parameters. +HISTORY_SQL = ('SELECT ' + MESSAGE_COLUMNS + ' FROM messages WHERE chat_id = %s ' + 'AND (created_at, id) > (%s, %s) ORDER BY created_at, id LIMIT %s') + + +class Store: + def __init__(self, config): + self.config = config + self.secret = bytes.fromhex(secret_file('CURSOR_KEY_FILE')) + if len(self.secret) < 32: + raise ValueError('Cursor key must be at least 256 bits') + self.pool = ConnectionPool(secret_file('DATABASE_URL_FILE'), min_size=1, max_size=8, + timeout=1, max_waiting=16, kwargs=db_connect_kwargs(), + check=ConnectionPool.check_connection, open=True) + self.cache_pool = redis.ConnectionPool.from_url( + secret_file('CACHE_URL_FILE'), max_connections=8, socket_connect_timeout=.5, + socket_timeout=.5, decode_responses=True, retry=Retry(NoBackoff(), 0)) + self.cache = redis.Redis(connection_pool=self.cache_pool) + self.dummy_hash = password_hash('dummy-password-for-timing') + + def close(self): + self.pool.close(timeout=2) + self.cache_pool.disconnect() + + def ready(self): + with self.pool.connection() as conn: + conn.execute('SELECT 1') + self.cache.ping() + + def create_user(self, body): + object_fields(body, ('username', 'display_name', 'password')) + username = string(body['username'], 32) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username): + raise APIError(400, 'Invalid username') + display = string(body['display_name'], 100) + hashed = password_hash(password_value(body['password'])) + try: + with self.pool.connection() as conn: + row = conn.execute('INSERT INTO users(id, username, display_name, password_hash) ' + 'VALUES (%s,%s,%s,%s) RETURNING ' + USER_COLUMNS, + (uuid.uuid4(), username, display, hashed)).fetchone() + except psycopg.errors.UniqueViolation: + raise APIError(409, 'Username already exists') from None + return public_row(row) + + def login(self, authorization): + try: + scheme, encoded = authorization.split(' ', 1) + if scheme.lower() != 'basic' or len(encoded) > 1024: + raise ValueError() + username, password = base64.b64decode(encoded, validate=True).decode('utf-8').split(':', 1) + if not re.fullmatch(r'[a-z0-9_]{3,32}', username): + raise ValueError() + password_value(password) + except (AttributeError, ValueError, UnicodeError, APIError): + raise APIError(401, 'Invalid credentials') from None + with self.pool.connection() as conn: + row = conn.execute('SELECT * FROM users WHERE username = %s', (username,)).fetchone() + matched = password_matches(password, row['password_hash'] if row else self.dummy_hash) + if not matched or row is None: + raise APIError(401, 'Invalid credentials') + token = secrets.token_urlsafe(32) + expires_ms = int(datetime.now(timezone.utc).timestamp() * 1000) + self.config.session_ttl_seconds * 1000 + self.cache.set(self.session_key(token), str(row['id']), pxat=expires_ms) + return dict(token=token, token_type='Bearer', + expires_at=stamp(datetime.fromtimestamp(expires_ms / 1000, timezone.utc)), + user=public_row(row)) + + @staticmethod + def session_key(token): + return 'session:' + hashlib.sha256(token.encode('ascii')).hexdigest() + + def auth(self, header): + try: + scheme, token = header.split(' ', 1) + if scheme.lower() != 'bearer' or not re.fullmatch(r'[A-Za-z0-9_-]{43}', token): + raise ValueError() + except (ValueError, AttributeError): + raise APIError(401, 'Invalid session') from None + key = self.session_key(token) + actor = self.cache.get(key) + if actor is None: + raise APIError(401, 'Invalid session') + return actor, key + + def me(self, actor): + with self.pool.connection() as conn: + row = conn.execute('SELECT ' + USER_COLUMNS + ' FROM users WHERE id = %s', (actor,)).fetchone() + if row is None: + raise APIError(401, 'Invalid session') + return public_row(row) + + def chat_access(self, actor, identifier): + identifier = uuid_value(identifier) + with self.pool.connection() as conn: + row = conn.execute('SELECT 1 FROM chat_members WHERE chat_id=%s AND user_id=%s', + (identifier, actor)).fetchone() + if row is None: + raise APIError(404, 'Object not found') + return identifier + + @staticmethod + def chat_row(conn, row): + members = conn.execute('SELECT user_id FROM chat_members WHERE chat_id=%s ORDER BY position', + (row['id'],)).fetchall() + return dict(public_row(row), member_ids=[str(x['user_id']) for x in members]) + + def chat(self, chat_id): + with self.pool.connection() as conn: + row = conn.execute('SELECT * FROM chats WHERE id=%s', (chat_id,)).fetchone() + return self.chat_row(conn, row) + + def create_chat(self, actor, body): + object_fields(body, ('title', 'member_ids')) + title = string(body['title'], 100) + members = body['member_ids'] + if not isinstance(members, list) or not 1 <= len(members) <= 100: + raise APIError(400, 'Invalid members') + members = [uuid_value(x) for x in members] + if len(members) != len(set(members)) or actor not in members: + raise APIError(400, 'Invalid members') + try: + with self.pool.connection() as conn: + row = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *', + (uuid.uuid4(), title)).fetchone() + with conn.cursor() as cur: + cur.executemany('INSERT INTO chat_members(chat_id,user_id,position) VALUES (%s,%s,%s)', + [(row['id'], member, pos) for pos, member in enumerate(members)]) + except psycopg.errors.ForeignKeyViolation: + raise APIError(400, 'Invalid members') from None + return dict(public_row(row), member_ids=members) + + def create_message(self, actor, chat_id, body): + object_fields(body, ('text',)) + text = string(body['text'], 4000) + with self.pool.connection() as conn: + row = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) VALUES (%s,%s,%s,%s) ' + 'RETURNING ' + MESSAGE_COLUMNS, (uuid.uuid4(), chat_id, actor, text)).fetchone() + return public_row(row) + + def encode_cursor(self, scope, key): + payload = json.dumps([1, scope, *key], separators=(',', ':')).encode() + return base64.urlsafe_b64encode(payload + hmac.digest(self.secret, payload, 'sha256')).rstrip(b'=').decode() + + def decode_cursor(self, cursor, scope): + try: + if not re.fullmatch(r'[A-Za-z0-9_-]{1,1024}', cursor): + raise ValueError() + raw = base64.b64decode(cursor + '=' * (-len(cursor) % 4), altchars=b'-_', validate=True) + if base64.urlsafe_b64encode(raw).rstrip(b'=').decode() != cursor: + raise ValueError() + payload, signature = raw[:-32], raw[-32:] + if not hmac.compare_digest(signature, hmac.digest(self.secret, payload, 'sha256')): + raise ValueError() + version, resource, timestamp, identifier = json.loads(payload) + if version != 1 or resource != scope or not isinstance(timestamp, str) or not timestamp.endswith('Z'): + raise ValueError() + return datetime.fromisoformat(timestamp.replace('Z', '+00:00')), uuid_value(identifier) + except (ValueError, TypeError, UnicodeError, APIError): + raise APIError(400, 'Invalid cursor') from None + + def page(self, actor, chat_id, query): + if set(query) - {'limit', 'cursor'} or any(len(v) != 1 for v in query.values()): + raise APIError(400, 'Invalid query') + limit = query.get('limit', ['50'])[0] + if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100: + raise APIError(400, 'Invalid limit') + limit = int(limit) + scope = 'messages:' + chat_id if chat_id else 'chats:' + actor + key = self.decode_cursor(query['cursor'][0], scope) if 'cursor' in query else ( + datetime.min.replace(tzinfo=timezone.utc), '00000000-0000-0000-0000-000000000000') + with self.pool.connection() as conn: + if chat_id: + rows = conn.execute(HISTORY_SQL, (chat_id, *key, limit + 1)).fetchall() + items = [public_row(row) for row in rows[:limit]] + else: + rows = conn.execute('SELECT c.* FROM chats c JOIN chat_members m ON m.chat_id=c.id ' + 'WHERE m.user_id=%s AND (c.created_at,c.id) > (%s,%s) ' + 'ORDER BY c.created_at,c.id LIMIT %s', (actor, *key, limit + 1)).fetchall() + items = [self.chat_row(conn, row) for row in rows[:limit]] + cursor = self.encode_cursor(scope, (items[-1]['created_at'], items[-1]['id'])) if len(rows) > limit else None + return dict(items=items, next_cursor=cursor) diff --git a/src/messenger/migrate.py b/src/messenger/migrate.py new file mode 100644 index 00000000..e05e2556 --- /dev/null +++ b/src/messenger/migrate.py @@ -0,0 +1,30 @@ +"""Transactional, versioned migrations; repeat application is a no-op.""" +import hashlib +from pathlib import Path +import psycopg +from .core import secret_file, db_connect_kwargs + + +def main(): + with psycopg.connect(secret_file('DATABASE_URL_FILE'), **db_connect_kwargs()) as conn: + conn.execute('SELECT pg_advisory_xact_lock(2026, 2)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations ' + '(name text PRIMARY KEY, sha256 text NOT NULL, ' + 'applied_at timestamptz NOT NULL DEFAULT clock_timestamp())') + for path in sorted((Path(__file__).resolve().parents[2] / 'migrations').glob('*.sql')): + content = path.read_bytes() + digest = hashlib.sha256(content).hexdigest() + row = conn.execute('SELECT sha256 FROM schema_migrations WHERE name=%s', (path.name,)).fetchone() + if row: + if row['sha256'] != digest: + raise RuntimeError('Applied migration checksum mismatch') + print('Already applied:', path.name) + continue + conn.execute(content.decode('utf-8'), prepare=False) + conn.execute('INSERT INTO schema_migrations(name,sha256) VALUES (%s,%s)', (path.name, digest)) + print('Applied:', path.name) + print('Migrations committed.') + + +if __name__ == '__main__': + main() diff --git a/src/messenger/server.py b/src/messenger/server.py new file mode 100644 index 00000000..1a06da67 --- /dev/null +++ b/src/messenger/server.py @@ -0,0 +1,331 @@ +"""Bounded HTTP worker pool; one request per connection; JSON logs omit bodies.""" +from collections import Counter +from http.server import BaseHTTPRequestHandler, HTTPServer +import http.client +import json +import logging +import queue +import signal +import socket +import threading +import time +from urllib.parse import parse_qs, urlsplit +import uuid + +import psycopg +from psycopg_pool import PoolTimeout, TooManyRequests +import redis + +from .core import APIError, Config, Store + + +CODES = {400: "invalid_request", 401: "unauthorized", 404: "not_found", 409: "conflict", + 413: "payload_too_large", 415: "unsupported_media_type", 429: "rate_limited", + 503: "unavailable", 500: "internal_error"} + + +def error_body(status, message, request_id): + return {"error": {"code": CODES.get(status, "invalid_request"), + "message": message, "request_id": request_id}} + + +def log_event(**data): + logging.info(json.dumps(data, ensure_ascii=True)) + + +class Server(HTTPServer): + allow_reuse_address = True + + def __init__(self, config): + self.config, self.store = config, Store(config) + self.jobs = queue.Queue(maxsize=config.request_queue) + self.draining = threading.Event() + self.stats_lock = threading.Lock() + self.statuses = Counter() + self.active = 0 + self.clients = set() + self.started = time.monotonic() + self.request_queue_size = config.request_queue + super().__init__((config.app_host, config.app_port), Handler) + self.threads = [threading.Thread(target=self.worker, name="http-" + str(n), daemon=True) + for n in range(config.workers)] + for thread in self.threads: + thread.start() + + def get_request(self): + client, address = super().get_request() + client.settimeout(self.config.socket_timeout) + return client, address + + def record(self, status): + with self.stats_lock: + self.statuses[str(status)] += 1 + + def process_request(self, request, address): + try: + if self.draining.is_set(): + raise queue.Full() + self.jobs.put_nowait((request, address)) + except queue.Full: + # Transport admission before HTTP parsing: bounded time, no extra thread. + identifier = str(uuid.uuid4()) + body = json.dumps(error_body(503, "Request queue full or shutting down", identifier)).encode() + headers = ("HTTP/1.1 503 Service Unavailable\r\nContent-Type: application/json\r\n" + "Connection: close\r\nRetry-After: 1\r\nX-Request-Id: " + identifier + + "\r\nContent-Length: " + str(len(body)) + "\r\n\r\n").encode() + try: + request.settimeout(.05) + # Wait for the first request bytes: closing before a local client + # finishes sendall can otherwise discard the 503 with a TCP reset. + try: + request.recv(16384) + except OSError: + pass + request.sendall(headers + body) + request.shutdown(socket.SHUT_WR) + request.settimeout(.01) + for _ in range(5): + if not request.recv(16384): + break + except OSError: + pass + self.shutdown_request(request) + self.record(503) + log_event(event="queue_rejected", status=503, request_id=identifier) + + def worker(self): + while not self.draining.is_set(): + try: + request, address = self.jobs.get(timeout=.1) + except queue.Empty: + continue + with self.stats_lock: + self.active += 1 + self.clients.add(request) + try: + self.finish_request(request, address) + except OSError: + pass + except Exception: + log_event(event="worker_error") + finally: + self.shutdown_request(request) + with self.stats_lock: + self.active -= 1 + self.clients.discard(request) + self.jobs.task_done() + + def server_close(self): + self.draining.set() + with self.stats_lock: + clients = list(self.clients) + for client in clients: + try: + client.shutdown(socket.SHUT_RDWR) + except OSError: + pass + while True: + try: + request, _ = self.jobs.get_nowait() + except queue.Empty: + break + self.shutdown_request(request) + self.jobs.task_done() + deadline = time.monotonic() + 5 + for thread in self.threads: + thread.join(timeout=max(0, deadline - time.monotonic())) + self.store.close() + super().server_close() + + +class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "Messenger" + sys_version = "" + + def setup(self): + super().setup() + self.request_id = str(uuid.uuid4()) + self.received = time.monotonic() + + def log_message(self, *_): + pass + + def parse_request(self): + original = self.rfile + + class HeaderBudget: + remaining = 16384 + + def readline(self, limit=-1): + line = original.readline(min(limit, self.remaining + 1) if limit >= 0 else self.remaining + 1) + self.remaining -= len(line) + if self.remaining < 0: + raise http.client.LineTooLong("header section") + return line + + self.rfile = HeaderBudget() + try: + return super().parse_request() + finally: + self.rfile = original + + def send_error(self, code, message=None, explain=None): + self.reply(code if code in CODES else 400, error_body( + code if code in CODES else 400, "Invalid HTTP request", self.request_id)) + + def reply(self, status, body): + raw = b'' if status == 204 else json.dumps(body, ensure_ascii=False, allow_nan=False).encode('utf-8') + content_type = 'application/json' + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(raw))) + self.send_header("X-Request-Id", self.request_id) + self.send_header("X-Content-Type-Options", "nosniff") + self.send_header("Connection", "close") + if status == 401: + challenge = ('Basic realm="messenger", charset="UTF-8"' + if urlsplit(self.path).path == '/api/v1/auth/sessions' else 'Bearer realm="messenger"') + self.send_header('WWW-Authenticate', challenge) + if status in (429, 503): + self.send_header("Retry-After", "1") + self.end_headers() + self.close_connection = True + self.wfile.write(raw) + self.server.record(status) + # No URL query, header, username, text, or exception detail in logs. + log_event(event="request", method=self.command, status=status, + request_id=self.request_id, elapsed_ms=round(1000 * (time.monotonic() - self.received), 3)) + + def handle_expect_100(self): + # Do not acknowledge a large unauthorized request before checking identity. + return True + + def body(self): + if self.headers.get_all("Transfer-Encoding"): + raise APIError(400, "Transfer-Encoding is unsupported") + sizes = self.headers.get_all("Content-Length", []) + if len(sizes) != 1 or not sizes[0].isascii() or not sizes[0].isdigit() or len(sizes[0]) > 10: + raise APIError(400, "One valid Content-Length is required") + size = int(sizes[0]) + if size > self.server.config.max_body_bytes: + raise APIError(413, "Request body exceeds byte limit") + if self.headers.get_content_type() != "application/json": + raise APIError(415, "Expected application/json") + if self.headers.get("Expect", "").lower() == "100-continue": + self.send_response_only(100) + self.end_headers() + raw = self.rfile.read(size) + if len(raw) != size: + raise APIError(400, "Incomplete request body") + + def pairs(values): + result = {} + for key, value in values: + if key in result: + raise ValueError("duplicate key") + result[key] = value + return result + + def constant(_): + raise ValueError("non JSON constant") + + try: + return json.loads(raw.decode("utf-8"), object_pairs_hook=pairs, parse_constant=constant) + except (ValueError, UnicodeError, RecursionError): + raise APIError(400, "Invalid JSON") from None + + def dispatch(self): + try: + url = urlsplit(self.path) + except ValueError: + raise APIError(400, "Invalid request target") from None + path, method = url.path, self.command + store = self.server.store + if self.server.draining.is_set(): + raise APIError(503, "Shutting down") + if method == 'GET' and path == '/health/live': + return 200, {'status': 'ok'} + if method == 'GET' and path == '/health/ready': + store.ready() + return 200, {'status': 'ok'} + if path == '/api/v1/users' and method == 'POST': + return 201, store.create_user(self.body()) + headers = self.headers.get_all('Authorization', []) + authorization = headers[0] if len(headers) == 1 else None + if path == '/api/v1/auth/sessions' and method == 'POST': + if self.headers.get('Transfer-Encoding') or self.headers.get('Content-Length', '0') != '0': + raise APIError(400, 'Login takes no request body') + return 201, store.login(authorization) + actor, session_key = store.auth(authorization) + if method == 'DELETE' and path == '/api/v1/auth/sessions/current': + store.cache.delete(session_key) + return 204, None + if method == 'GET' and path == '/api/v1/users/me': + return 200, store.me(actor) + parts = path.strip("/").split("/") + chat_id = None + if len(parts) >= 4 and parts[:3] == ["api", "v1", "chats"]: + chat_id = store.chat_access(actor, parts[3]) + if method == "POST" and path == "/api/v1/chats": + return 201, store.create_chat(actor, self.body()) + if method == "POST" and chat_id and len(parts) == 5 and parts[4] == "messages": + return 201, store.create_message(actor, chat_id, self.body()) + if method == "GET": + if chat_id and len(parts) == 4: + return 200, store.chat(chat_id) + if path == "/api/v1/chats" or (chat_id and len(parts) == 5 and parts[4] == "messages"): + try: + query = parse_qs(url.query, keep_blank_values=True, strict_parsing=True, max_num_fields=8) + except ValueError: + raise APIError(400, "Invalid query") from None + return 200, store.page(actor, chat_id, query) + raise APIError(404, "Object not found") + + def handle_api(self): + try: + status, result = self.dispatch() + except APIError as error: + status = error.status + result = error_body(status, error.message, self.request_id) + except (psycopg.OperationalError, psycopg.InterfaceError, psycopg.errors.QueryCanceled, + psycopg.errors.LockNotAvailable, PoolTimeout, TooManyRequests, redis.RedisError): + status, result = 503, error_body(503, 'Dependency temporarily unavailable', self.request_id) + except (TimeoutError, socket.timeout): + status, result = 400, error_body(400, "Request read timeout", self.request_id) + except Exception: + status, result = 500, error_body(500, "Internal error", self.request_id) + log_event(event="internal_error", request_id=self.request_id) + try: + self.reply(status, result) + except OSError: + pass + + do_GET = do_POST = do_PUT = do_DELETE = do_PATCH = do_HEAD = do_OPTIONS = handle_api + + +def main(): + logging.basicConfig(level=logging.INFO, format="%(message)s") + # Pool diagnostics may include connection details; application emits sanitized errors. + logging.getLogger('psycopg.pool').disabled = True + config = Config.from_env() + server = Server(config) + + def stop(signum, _): + if not server.draining.is_set(): + server.draining.set() + log_event(event="shutdown_started", signal=signum) + threading.Thread(target=server.shutdown, daemon=True).start() + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + log_event(event="started", port=config.app_port, workers=config.workers) + try: + server.serve_forever(poll_interval=.1) + finally: + server.server_close() + log_event(event="shutdown_complete") + + +if __name__ == "__main__": + main() diff --git a/student-tests/test_api.py b/student-tests/test_api.py new file mode 100644 index 00000000..2de2c8b2 --- /dev/null +++ b/student-tests/test_api.py @@ -0,0 +1,89 @@ +"""Own black-box tests; synthetic data, standard library, no tokens in output.""" +import base64 +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import sys +import unittest +from urllib.parse import quote +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client + + +class API(unittest.TestCase): + def setUp(self): + self.c = Client() + + def test_registration_race(self): + body = dict(username='race_' + uuid.uuid4().hex[:20], display_name='Гонка', password='Unicode:пароль🔑123') + def register(_): + return self.c.call('POST', '/api/v1/users', [201, 409], body=body)['status'] + with ThreadPoolExecutor(max_workers=10) as pool: + statuses = list(pool.map(register, range(10))) + self.assertEqual(statuses.count(201), 1) + self.assertEqual(statuses.count(409), 9) + + def test_validation_access_and_pagination(self): + self.c.call('POST', '/api/v1/users', 400, + body=dict(username='spaces_' + uuid.uuid4().hex[:20], display_name='Invalid', password=' ' * 12)) + alice, bob, eve, chat = self.c.bootstrap() + path = '/api/v1/chats/' + chat['id'] + for raw in (b'{', b'{"text":NaN}', b'{"text":"a","text":"b"}', b'[]'): + self.c.call('POST', path + '/messages', 400, actor=alice, raw=raw, + headers={'Content-Type': 'application/json'}) + for body in ({'text': ' '}, {'text': 'hi', 'sender_id': eve['id']}, {'text': 2}, {'text': '\x00'}): + self.c.call('POST', path + '/messages', 400, actor=alice, body=body) + self.c.call('POST', path + '/messages', 404, actor=eve, body={'text': 'нет доступа'}) + self.c.call('POST', '/api/v1/users', 409, body=alice['registration']) + for members in ([alice['id'], str(uuid.uuid4())], [bob['id']], [alice['id'], alice['id']]): + self.c.call('POST', '/api/v1/chats', 400, actor=alice, + body={'title': 'Invalid membership', 'member_ids': members}) + sent = [self.c.message(alice, chat, 'Страница ' + str(i))['json'] for i in range(7)] + read, cursor = [], None + while True: + query = '?limit=2' + ('&cursor=' + quote(cursor, safe='') if cursor else '') + page = self.c.call('GET', path + '/messages' + query, actor=bob, schema='MessagePage')['json'] + read.extend(page['items']) + cursor = page['next_cursor'] + if cursor is None: + break + self.assertEqual(read, sorted(sent, key=lambda x: (x['created_at'], x['id']))) + first = self.c.call('GET', path + '/messages?limit=1', actor=alice, schema='MessagePage')['json'] + other = self.c.call('POST', '/api/v1/chats', 201, actor=alice, + body={'title': 'Другой чат', 'member_ids': [alice['id']]}, schema='Chat')['json'] + self.c.call('GET', '/api/v1/chats/' + other['id'] + '/messages?cursor=' + first['next_cursor'], 400, actor=alice) + self.c.call('GET', path + '/messages?cursor=broken', 400, actor=alice) + + def test_concurrent_messages(self): + alice, _, _, chat = self.c.bootstrap() + def send(i): + return self.c.message(alice, chat, 'Concurrent ' + str(i))['json']['id'] + with ThreadPoolExecutor(max_workers=10) as pool: + ids = list(pool.map(send, range(100))) + page = self.c.call('GET', '/api/v1/chats/' + chat['id'] + '/messages?limit=100', + actor=alice, schema='MessagePage')['json'] + self.assertEqual(len(set(ids)), 100) + self.assertEqual({x['id'] for x in page['items']}, set(ids)) + self.assertIsNone(page['next_cursor']) + + def test_unicode_password_and_logout(self): + username = 'utf_' + uuid.uuid4().hex[:20] + password = '🔑:пароль' * 16 # 128 code points, far more than bcrypt's 72 bytes + self.assertEqual(len(password), 128) + self.c.call('POST', '/api/v1/users', 201, + body=dict(username=username, display_name='Unicode', password=password), schema='User') + def login(value, expected): + auth = base64.b64encode((username + ':' + value).encode()).decode() + return self.c.call('POST', '/api/v1/auth/sessions', expected, + headers={'Authorization': 'Basic ' + auth}, schema='Session' if expected == 201 else None) + actor = {'token': login(password, 201)['json']['token']} + login('incorrect-password', 401) + self.c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + self.c.call('DELETE', '/api/v1/auth/sessions/current', 204, actor=actor) + self.c.call('GET', '/api/v1/users/me', 401, actor=actor) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/student-tests/test_database.py b/student-tests/test_database.py new file mode 100644 index 00000000..b691059e --- /dev/null +++ b/student-tests/test_database.py @@ -0,0 +1,59 @@ +"""Run inside API container: directly observe rollback and stored password KDF.""" +from pathlib import Path +import sys +import subprocess +import unittest +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +from messenger.core import APIError, Config, Store, password_matches + + +class Database(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.store = Store(Config.from_env()) + + @classmethod + def tearDownClass(cls): + cls.store.close() + + def test_rollback_after_chat_insert(self): + actor = self.store.create_user(dict(username='rollback_' + uuid.uuid4().hex[:16], + display_name='Rollback', password='rollback-password'))['id'] + with self.store.pool.connection() as conn: + before = conn.execute('SELECT count(*) AS n FROM chats').fetchone()['n'] + members_before = conn.execute('SELECT count(*) AS n FROM chat_members').fetchone()['n'] + # First membership succeeds, second violates FK after the chat was inserted. + with self.assertRaises(APIError) as raised: + self.store.create_chat(actor, dict(title='Must rollback', member_ids=[actor, str(uuid.uuid4())])) + self.assertEqual(raised.exception.status, 400) + with self.store.pool.connection() as conn: + self.assertEqual(conn.execute('SELECT count(*) AS n FROM chats').fetchone()['n'], before) + self.assertEqual(conn.execute('SELECT count(*) AS n FROM chat_members').fetchone()['n'], members_before) + + def test_password_salted_kdf(self): + password = 'identical-password' + ids = [self.store.create_user(dict(username='salt_' + uuid.uuid4().hex[:20], + display_name='Salt', password=password))['id'] for _ in range(2)] + with self.store.pool.connection() as conn: + hashes = [conn.execute('SELECT password_hash FROM users WHERE id=%s', (i,)).fetchone()['password_hash'] for i in ids] + self.assertNotEqual(*hashes) + for value in hashes: + self.assertTrue(value.startswith('scrypt$16384$8$1$')) + self.assertTrue(password_matches(password, value)) + self.assertNotEqual(value, password) + + def test_repeat_migrations(self): + def counts(): + with self.store.pool.connection() as conn: + return [conn.execute('SELECT count(*) AS n FROM ' + table).fetchone()['n'] + for table in ('users', 'chats', 'chat_members', 'messages', 'schema_migrations')] + before = counts() + for _ in range(2): + subprocess.run([sys.executable, '-m', 'messenger.migrate'], check=True) + self.assertEqual(counts(), before) + + +if __name__ == '__main__': + unittest.main(verbosity=2) diff --git a/student-tests/test_expiry.py b/student-tests/test_expiry.py new file mode 100644 index 00000000..c9795470 --- /dev/null +++ b/student-tests/test_expiry.py @@ -0,0 +1,14 @@ +"""Use a separate API process with SESSION_TTL_SECONDS=2, sharing real stores.""" +from pathlib import Path +import sys +import time + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'tests')) +from client import Client, check, timestamp + +c = Client() +actor = c.user('expiry') +response = c.call('GET', '/api/v1/users/me', actor=actor, schema='User') +time.sleep(2.2) +c.call('GET', '/api/v1/users/me', 401, actor=actor) +print('PASS: session accepted before expiry; rejected after 2.2s with SESSION_TTL_SECONDS=2.') diff --git a/student-tests/test_outages.py b/student-tests/test_outages.py new file mode 100644 index 00000000..6c5d203f --- /dev/null +++ b/student-tests/test_outages.py @@ -0,0 +1,50 @@ +"""Host-side operator test: stop one lab dependency, assert bounded failures, restore.""" +import json +from pathlib import Path +import subprocess +import sys +import time + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, Failure, check + +c = Client(timeout=4) +actor, _, _, chat = c.bootstrap() +original = c.message(actor, chat, 'Before dependency outage')['json'] +path = '/api/v1/chats/' + chat['id'] + '/messages' +summary = [] +for service in ('redis', 'db'): + subprocess.run(['docker', 'compose', 'stop', service], cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + try: + timings = [] + for _ in range(3): + response = c.call('GET', '/api/v1/users/me', 503, actor=actor) + check(response['json']['error']['code'] == 'unavailable', 'wrong dependency error') + check(response['elapsed'] < 3.5, 'failure exceeded response budget') + timings.append(response['elapsed']) + c.call('POST', path, 503, actor=actor, body={'text': 'Must not commit while unavailable'}) + c.call('GET', '/health/ready', 503) + c.call('GET', '/health/live', schema='Health') + c.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': actor['id']}) + c.call('GET', '/api/v1/users/me', 401, headers={'Authorization': 'Bearer ' + 'A' * 43}) if service == 'db' else c.call( + 'GET', '/api/v1/users/me', 503, headers={'Authorization': 'Bearer ' + 'A' * 43}) + finally: + started = time.monotonic() + subprocess.run(['docker', 'compose', 'start', service], cwd=ROOT, check=True, stdout=subprocess.DEVNULL) + deadline = time.monotonic() + 30 + while True: + try: + c.call('GET', '/health/ready', schema='Health') + c.call('GET', '/api/v1/users/me', actor=actor, schema='User') + break + except Failure: + check(time.monotonic() < deadline, 'dependency did not recover in 30 seconds') + time.sleep(.25) + recovery = time.monotonic() - started + page = c.call('GET', path, actor=actor, schema='MessagePage')['json'] + check(page['items'] == [original], 'lost confirmation or wrote during dependency outage') + summary.append(dict(service=service, errors=3, status=503, seconds=timings, + recovery_seconds=round(recovery, 3), original_session_preserved=True, + original_message_preserved=True, anonymous_status=401)) +print(json.dumps(summary, indent=2)) -- 2.54.0