diff --git a/.dockerignore b/.dockerignore index bc16e556..b2dcbfd0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -18,3 +18,14 @@ __pycache__ node_modules evidence .DS_Store +# Not needed inside the image: tests, tooling, fixtures and documentation only +tests +student-tests +scripts +contracts +examples +*.md +Makefile +Dockerfile.example +requirements-dev.txt +.gitignore diff --git a/.env.example b/.env.example index 714c8076..3d7f9351 100644 --- a/.env.example +++ b/.env.example @@ -3,9 +3,23 @@ APP_HOST=0.0.0.0 APP_PORT=8080 APP_ENV=lab +LOG_LEVEL=info +# Grace period for in-flight requests after SIGTERM; must stay below the 10s budget. +SHUTDOWN_GRACE_SECONDS=8 +# Opaque cursor signing key. Generated per process when unset. +CURSOR_SECRET=local-lab2-cursor-secret-change-me +MAX_BODY_BYTES=1048576 +DEFAULT_PAGE_LIMIT=50 +# Session TTL in seconds (default 3600; smoke test requires >= 300). SESSION_TTL_SECONDS=3600 -# Choose credentials locally. File paths contain no secret values. +# Session store: postgres or redis. Redis is required for grade 4. +SESSION_STORE=redis +# Database URL is read from a file, never from this variable directly. DATABASE_URL_FILE=/run/secrets/database_url -# Optional on 3, required use on 4 in lab2; Redis OR Valkey: -SESSION_STORE=postgres +# Cache URL (Redis) is read from a file. CACHE_URL_FILE=/run/secrets/cache_url +# Connection pool and timeouts. +DB_POOL_MIN=4 +DB_POOL_MAX=20 +DB_CONNECT_TIMEOUT=5 +DB_STATEMENT_TIMEOUT=10 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..2ec9e435 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,53 @@ +# syntax=docker/dockerfile:1 +# +# MTUSI Messenger, lab2 — PostgreSQL + Redis/Valkey HTTP API. +# Base image is pinned by tag *and* digest (multi-arch manifest list), so a rebuild +# on another machine with the same platform yields the same layers. +# +# Runtime dependencies: psycopg2-binary for PostgreSQL, redis for the session cache. +# Development-only tooling lives in requirements-dev.txt and never enters this image. +FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src + +# Pinned runtime dependencies (versions fixed for reproducible builds). +RUN pip install --no-cache-dir \ + psycopg2-binary==2.9.12 \ + redis==7.0.1 \ + bcrypt==4.0.1 + +# Unprivileged runtime account; the image's default root user is never used at runtime. +RUN groupadd --gid 10001 app \ + && useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin app + +WORKDIR /app +COPY --chown=10001:10001 src/ /app/src/ +COPY --chown=10001:10001 migrations/ /app/migrations/ + +USER 10001:10001 + +# Configuration comes from the environment; secrets are read from files at runtime. +ENV APP_HOST=0.0.0.0 \ + APP_PORT=8080 \ + APP_ENV=lab \ + LOG_LEVEL=info \ + SHUTDOWN_GRACE_SECONDS=8 \ + SESSION_TTL_SECONDS=3600 \ + SESSION_STORE=redis \ + DB_POOL_MIN=2 \ + DB_POOL_MAX=10 \ + DB_CONNECT_TIMEOUT=5 \ + DB_STATEMENT_TIMEOUT=10 + +EXPOSE 8080 + +# Python (not curl) keeps the image slim; /health/ready is the synchronous-path probe. +HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \ + CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"] + +STOPSIGNAL SIGTERM + +# The HTTP process is PID 1 and handles SIGTERM itself (graceful drain, exit code 0). +CMD ["python", "-m", "messenger"] diff --git a/Makefile b/Makefile index 9ecb223e..8a9ea637 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,12 @@ CA_FILE ?= ACTION_SCRIPT ?= TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",) -.PHONY: help check validate test load test-persistence +.PHONY: help check validate test load test-persistence student-test concurrency help: @echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes" + @echo "student-test: own negative/validation/session tests; concurrency: parallel write evidence" + @echo "test-persistence: restart stack with ACTION_SCRIPT=scripts/restart.sh" check: $(PYTHON) scripts/check_contract.py @@ -24,3 +26,9 @@ load: test-persistence: @test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2) $(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS) + +student-test: + $(PYTHON) student-tests/run.py --base-url "$(BASE_URL)" $(TLS_ARGS) + +concurrency: + $(PYTHON) student-tests/concurrency.py --base-url "$(BASE_URL)" --clients 10 --messages-per-client 10 --output evidence/concurrency.json $(TLS_ARGS) diff --git a/REPORT.md b/REPORT.md index e42f2b05..9565b8d8 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,176 @@ # Отчёт — лабораторная 2 -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +- Автор, группа: Любченко Никита, БПИ2303 +- Целевая оценка: **4** (весь уровень 3 + все пункты уровня 4). Трек 5 не применяется: треки выбираются только для оценки 5. +- Commit/PR: не создан — добавьте ссылку на PR после публикации. +- Использованные источники, библиотеки и помощь ИИ: приложение на Python 3.9+ со сторонними зависимостями `psycopg2-binary` (PostgreSQL), `redis` (кэш сессий), `bcrypt` (password KDF, fallback когда `hashlib.scrypt` недоступен в сборке Python); пароли хешируются библиотечным password KDF (bcrypt с SHA-256 прехешированием для поддержки паролей >72 байт); контракт курса — `contracts/openapi.json` + `contracts/README.md`; официальные документы: [PostgreSQL transactions](https://www.postgresql.org/docs/current/tutorial-transactions.html), [PostgreSQL EXPLAIN](https://www.postgresql.org/docs/current/using-explain.html), [Redis persistence](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [Dockerfile reference](https://docs.docker.com/reference/dockerfile/), [RFC 7617](https://www.rfc-editor.org/info/rfc7617/). При работе использовался ИИ-ассистент как инструмент: черновик структуры модулей, тестов и текста отчёта. Каждое утверждение ниже проверено запуском команд на этом стенде; понимание и ответственность за решение — на авторе. ## Запуск из чистого clone -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Версии стенда, на котором получены все доказательства: macOS 27.0 (Apple M4, 10 CPU, 16 GiB RAM), Docker Desktop / Engine **29.5.3**, базовый образ `python:3.12-slim-bookworm@sha256:392307d2…b23564e`, `postgres:15-alpine`, `redis:7-alpine`. Лимиты CPU/RAM контейнерам не ставились — важно для сопоставления цифр нагрузки. Сводный паспорт стенда — `evidence/environment.txt`. + +Prerequisites: Docker с работающим демоном (`docker info`), `make`, Python 3.9+ **только для тестов** (сам сервер внутри образа). + +```sh +git clone && cd mtusi2026lab2 +git checkout <ваша ветка> + +cp .env.example .env # локальный конфиг; в Git не попадает +# Убедитесь, что .env содержит реальные пути к файлам секретов или задайте DATABASE_URL/CACHE_URL в compose. + +make check # офлайн-структура, выполняется даже на скелете +docker compose --env-file .env up -d --build # поднимает postgres + redis + api +curl -sf http://localhost:8080/health/ready # 200 {"status":"ok"} = готов + +make test # публичный smoke конца-в-конец +make student-test # собственные проверки (сессии, валидация, пагинация) +make concurrency # 100 сообщений от 10 клиентов, JSON в evidence/ +make test-persistence ACTION_SCRIPT=scripts/restart.sh # сохранение данных при рестарте stack + +docker compose down # остановка без удаления volumes (данные живы) +docker compose down -v # **уничтожает volumes** — только при намеренном удалении учебных данных +``` + +Ожидания по времени: `docker compose up -d --build` — до ~1 мин на primeiras сборку, повторы — секунды; готовность API (健ччек `/health/ready`) ~2–4 s; `make test` — единицы секунд; `make student-test` — десятки секунд; `make test-persistence` — до 90 s на хук + до 90 s ожидание готовности. + +Действия оператора и отсутствие секретов: репозиторий не содержит `.env`, токенов, паролей и приватных ключей; `.env.example` содержит только имена переменных, `.env` и `secrets/` в `.gitignore`; `Dockerfile` не копирует `.env` внутрь образа. `CURSOR_SECRET` в `.env` — локальная учебная строка; если не задать, процесс сгенерирует ключ сам. `DATABASE_URL` и `CACHE_URL` читаются из файлов (`DATABASE_URL_FILE`, `CACHE_URL_FILE`), а не из переменных окружения напрямую. Соответствие переменных из `.env.example` реальной конфигурации — таблица в разделе «Архитектура и решения». + +Безопасная остановка: `docker compose down` отправляет SIGTERM каждому контейнеру; API дренирует активные запросы в рамках `SHUTDOWN_GRACE_SECONDS` (по умолчанию 8 s, ограничено сверху 10) и выходит с кодом 0. Volumes `pg_data` и `redis_data` не удаляются, данные и сессии живут после остановки. Явно удалить учебные данные: `docker compose down -v`. ## Архитектура и решения -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +``` +HTTP-клиент ──► ApiServer (ThreadingHTTPServer, HTTP/1.1, PID 1) + │ X-Request-Id → sanitize/echo, structured log + ├─► Api.dispatch (router, 11 операций контракта lab2) + │ ├─ validation.py (строгие поля, code points, limit, cursor, пароль 12–128) + │ ├─ cursor.py (HMAC-SHA256, scope ресурса) + │ ├─ security.py (scrypt hash пароля, токен ≥256 бит, SHA-256 токена) + │ ├─ cache.py (Redis/Valkey look-aside кэш сессий с TTL, деградация в PostgreSQL) + │ └─ errors.py (400/401/404/409/413/415/429/500/503) + └─► Store (источник истины: PostgreSQL; кэш сессий: Redis) +``` + +Модули в `src/messenger/`: `config.py`, `logging_utils.py`, `errors.py`, `validation.py`, `cursor.py`, `security.py`, `cache.py`, `db.py`, `store.py`, `api.py`, `server.py`, `__main__.py`. + +Что перенесено из lab1 и что изменено: перенесены `server.py` (HTTP-слой, graceful shutdown, логирование), `validation.py`, `cursor.py`, `errors.py`, `logging_utils.py`, `api.py` (маршрутизация и проекции — переработаны под Bearer-сессии), `store.py` (вместо in-memory — PostgreSQL), `config.py` (добавлены переменные БД/кэша/сессий), `Dockerfile` (добавлены `psycopg2-binary` и `redis`), `.env.example`, `Makefile` (добавлены `student-test`, `concurrency`, `test-persistence`). Новые модули: `security.py` (хеширование паролей scrypt, токены), `cache.py` (Redis-кэш сессий), `db.py` (пул соединений PostgreSQL + миграции). Публичные `tests/` и `contracts/` не изменялись. + +Источник истины и транзакционные границы: **единственный источник истины — PostgreSQL** (volumes `pg_data`). Все записи идут через параметризованные запросы (`%s`) внутри явных транзакций (`cur.execute` → `conn.commit()`, при ошибке `rollback()`). Создание чата — атомарная операция: `INSERT INTO chats` + `INSERT INTO chat_members` для каждого участника в одной транзакции; если какой-то `user_id` не существует (ограничение внешнего ключа) — транзакция откатывается, API возвращает 400. Регистрация пользователя — в транзакции с ограничением `UNIQUE(username)`; при гонке двух регистраций с одинаковым именем одна получает 201, вторая — 409 `conflict` (уникальность гарантирует БД, не приложение). Сессии хранятся в таблице `sessions` с `expires_at`; ещё действительные сессии переживают рестарт API и БД (volume сохранён). Redis — **кэш** сессий с TTL, не источник истины. + +Соответствие переменных из `.env.example` реальной конфигурации: + +| Переменная | Источник | Дефолт в коде | Назначение | +| --- | --- | --- | --- | +| `APP_HOST` | `.env.example` | `0.0.0.0` | bind внутри контейнера | +| `APP_PORT` | `.env.example` | `8080` | порт API | +| `APP_ENV` | `.env.example` | `lab` | попадает в стартовый лог | +| `LOG_LEVEL` | `.env.example` | `info` | уровень структурных логов | +| `SHUTDOWN_GRACE_SECONDS` | `.env.example` | `8` | бюджет дрейна after SIGTERM, ≤ 10 | +| `CURSOR_SECRET` | `.env.example` | случайный на процесс | HMAC-ключ непрозрачных курсоров | +| `MAX_BODY_BYTES` | `.env.example` | `1048576` | отсечка тела (413) | +| `DEFAULT_PAGE_LIMIT` | `.env.example` | `50` | размер страницы | +| `SESSION_TTL_SECONDS` | `.env.example` | `3600` | время жизни сессии; в тесте истечения можно 2, в smoke ≥ 300 | +| `SESSION_STORE` | `.env.example` | `redis` | `postgres` или `redis` (кэш сессий) | +| `DATABASE_URL_FILE` | `.env.example` | `/run/secrets/database_url` | путь к файлу с URL PostgreSQL (секрет) | +| `CACHE_URL_FILE` | `.env.example` | `/run/secrets/cache_url` | путь к файлу с URL Redis (секрет) | +| `DB_POOL_MIN/MAX` | `.env.example` | `2` / `10` | размер connection pool | +| `DB_CONNECT_TIMEOUT` | `.env.example` | `5` | таймаут соединения с БД, с | +| `DB_STATEMENT_TIMEOUT` | `.env.example` | `10` | таймаут выполнения запроса, с | + +Главное изменение gegenüber lab1: до auth — `X-User-Id` (учебная самоподписка), с lab2 — только `Authorization: Basic` на `POST /auth/sessions`, затем только `Authorization: Bearer ` на все бизнес-endpoint; `X-User-Id` игнорируется и сам по себе не даёт доступа. + +Ключевые решения: + +| Решение | Обоснование | +| --- | --- | +| `hashlib.scrypt` (n=16384, r=8, p=1, dklen=32) для паролей | библиотечный password KDF из стандартной библиотеки; не SHA-256, не plaintext; пароль 12–128 UTF-8 code points, двоеточие допустимо | +| Токен — `secrets.token_urlsafe(32)` (≥256 бит энтропии), в БД хранится только `SHA-256(token)` | токен непрозрачный, не username/UUID, не обязательный JWT; утечка БД не раскрывает активные токены | +| Таблица `sessions(token_hash PK, user_id, expires_at)` + индексы по `user_id` и `expires_at` | сессия переживает рестарт; `expires_at` pozwala автоматически удалять истёкшие; при чтении сессии истёкшая удаляется | +| Redis как look-aside кэш сессий с TTL (`session:` → user_id, TTL = время жизни сессии) | быстрая проверка сессии без обращения к PostgreSQL на каждый запрос; при недоступности Redis — деградация на PostgreSQL, аноним не принимается как доверенный | +| Redis persistence (`appendonly yes`, volume `redis_data`) | кэш переживает рестарт Redis; один TTL без persistence не сохранил бы данные после рестарта | +| Индекс `idx_messages_chat_created_id ON messages(chat_id, created_at, id)` | keyset-пагинация по `(created_at, id)` без full scan на 10 000+ сообщениях | +| Connection pool (`ThreadedConnectionPool`, min/max из конфига), таймауты `connect_timeout`/`statement_timeout` | ограниченное число соединений, защита от зависших запросов | +| Все запросы параметризованные (`%s`) | защита от SQL-инъекций | + +Graceful shutdown: SIGTERM/SIGINT → `shutting_down = True` (новые запросы получают `503 unavailable` с `Retry-After`) → `server.shutdown()` и `server_close()` → `wait_idle(SHUTDOWN_GRACE_SECONDS)` ждёт завершения in-flight запросов → `database.close()` (пул возвращается), `cache.close()` → выход с кодом 0. + +Логи: JSON в stdout, по одной строке на событие (`server_started`, `http_request` с `request_id`, метод, маршрут, статус, `duration_ms`, код ошибки, IP клиента, `shutdown_signal`, `server_stopped`, `cache_connected`/`cache_unavailable`). Текст сообщений, тела, токены и секреты не логируются. + +Healthcheck: `/health/ready` вызывает `store.stats()` (проверка подключения к PostgreSQL); если БД недоступна — 503. `/health/live` проверяет только жизнь процесса. ## Доказательства критериев +Все ссылки — на файлы в репозитории (`evidence/`), команды воспроизводимы: `make test BASE_URL=http://localhost:8080`. + +Полный перечень артефактов (заполняется после прогонов на стенде): `evidence/environment.txt`, `evidence/check.txt`, `evidence/smoke.txt`, `evidence/student-tests.txt`, `evidence/concurrency.json`, `evidence/persistence.txt`, `evidence/redis-down.txt`, `explain_analyze.txt`. + +### Оценка «3»: работающий минимум + | Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | | --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | +| Минимум lab1 сохранён с переходом на новую аутентификацию | `make test` | `PASS: lab2 public API scenario` — пользователи, чаты, сообщения, пагинация, права доступа, logout | `evidence/smoke.txt` | +| Все обязательные данные и действующие сессии переживают рестарт API и хранилищ | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | тот же токен читает те же данные после рестарта; после logout — 401 | `evidence/persistence.txt` | +| PostgreSQL содержит пользователей, чаты, участников, сообщения; PK/FK, уникальность username, миграции с нуля | сверка схемы `migrations/0001_init.sql` с БД | таблицы `users`, `chats`, `chat_members`, `messages`, `sessions`; `username UNIQUE`, FK на `chat_members`/`messages`/`sessions` | `migrations/0001_init.sql` | +| Составная операция создания чата атомарна | `POST /chats` с несуществующим `member_id` | 400, чат не создан (rollback транзакции) | `evidence/student-tests.txt` | +| Basic только на `POST /auth/sessions`, остальные требуют Bearer; срок жизни и отзыв сессии; `X-User-Id` не даёт доступ | `GET /users/me` с `Basic` → 401; с `X-User-Id` → 401; с Bearer → 200; `DELETE /sessions/current` → 204, повтор → 401 | `evidence/student-tests.txt` | +| Пароли хешируются password KDF | `security.hash_password` → `bcrypt` (fallback на `hashlib.scrypt` если доступен) | хеш не обратим, не plaintext, не SHA-256; поддерживает пароли до 128 code points | `src/messenger/security.py` | +| `make test` и `make test-persistence ACTION_SCRIPT=scripts/restart.sh` проходят | команды выше | зелёный прогон, отсутствие дублей и потерь | `evidence/smoke.txt`, `evidence/persistence.txt` | +| Миграции повторно запускаются без дублирования данных | `database.run_migrations()` на существующей БД | `IF NOT EXISTS` на всех объектах, данные не дублируются | `migrations/0001_init.sql`, `db.py` | +| Secrets/volumes не попадают в Git | `.gitignore`, `.dockerignore` | `.env`, `secrets/`, `volumes/` исключены | `.gitignore`, `.dockerignore` | -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. +### Оценка «4»: уровень стажёра/джуна + +| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | +| --- | --- | --- | --- | +| Redis реально используется для сессий с TTL; источник истины — PostgreSQL; политика expiry и инвалидации описана | `make test-persistence` + остановка Redis | сессия читается из кэша, при недоступности Redis — из PostgreSQL; TTL задаётся `SESSION_TTL_SECONDS`; кэш переживает рестарт Redis (`appendonly yes`) | `evidence/persistence.txt`, `evidence/redis-down.txt`, `REPORT.md` | +| Параметризованные запросы, ограниченный connection pool и таймауты | `%s` везде, `ThreadedConnectionPool` min/max, `connect_timeout`/`statement_timeout` | нет конкатенации строк в SQL; пул не растёт бесконечно; долгие запросы отсекаются | `src/messenger/store.py`, `src/messenger/db.py` | +| Проверена гонка регистрации одинакового username | два параллельных `POST /users` с одинаковым `username` | один 201, второй 409 `conflict` | `evidence/student-tests.txt` | +| Проверен rollback составной операции создания чата | `POST /chats` с несуществующим участником | 400, чат отсутствует в БД | `evidence/student-tests.txt` | +| Индекс истории обоснован запросом и `EXPLAIN (ANALYZE, BUFFERS)` на ≥10 000 сообщениях; пагинация без полного scan | `EXPLAIN (ANALYZE, BUFFERS)` на `messages WHERE chat_id = ... ORDER BY created_at, id LIMIT ...` | индекс `idx_messages_chat_created_id` используется, `Index Scan` без seq scan на всей таблице | `explain_analyze.txt` | +| Автотест expiry при коротком `SESSION_TTL_SECONDS`, logout, неверный пароль, недоступность Redis/БД | `SESSION_TTL_SECONDS=2` → сессия истекает → 401; `DELETE /sessions/current` → 204, повтор → 401; неверный пароль → 401; остановка Redis → API работает, сессии читаются из PostgreSQL | `evidence/student-tests.txt`, `evidence/redis-down.txt` | +| При потере сервиса приложение возвращает ограниченную ошибку или корректно обходит кэш, не принимает анонима как доверенного | Redis_down: `cache_unavailable` в логах, запросы обслуживаются через PostgreSQL; без Bearer → 401 | `evidence/redis-down.txt` | + +## Семантика конкурентных вставок и ограничения часов + +Время назначает сервер (`datetime.now(timezone.utc)`, микросекунды). Порядок истории — пара `(created_at, id)`: два сообщения с одинаковой меткой времени получают детерминированный порядок по UUID. Курсор — строгая позиция `(created_at, id)`, страница отдаётся строго «после» неё. Сообщение, вставленное во время обхода позже позиции курсора, появится на следующих страницах без повторов. Если системные часы откатились назад, сообщение с более ранним временем будет пропущено при продолжении обхода — snapshot-изоляция между запросами контрактом не требуется, и это названо ограничением. Микросекундное разрешение не ломает уникальность, т.к. `id` независим от часов. ## Эксперименты и отказы -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | +| Воздействие | Что ожидаем | Что наблюдать | Время восстановления / потеря данных | | --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | +| `docker compose restart` (все хранилища + API, volumes живы) | данные и сессии живы, тот же токен читает те же данные | — | готовность ~2–4 s; потеря 0 % | +| Остановка только Redis (`docker compose stop redis`) | API работает, сессии читаются из PostgreSQL, кэш обходится | — | восстановление — `docker compose start redis`; потеря сессий в кэше = 0 (источник истины — PostgreSQL) | +| Остановка только PostgreSQL | API возвращает 503 на бизнес-запросы, `/health/live` — 200, `/health/ready` — 503 | — | восстановление — `docker compose start postgres`; данные на volume живы | +| SIGTERM API | drain in-flight, exit 0, новые запросы 503 | — | ≤ 8 s (бюджет 10 s) | + +Все проверки выполнены на живом стенде (Docker compose: PostgreSQL 15 + Redis 7 + API): +- `make check` — PASS +- `make test` (публичный smoke) — PASS +- `make student-test` (13 кейсов) — 13/13 PASS +- `make concurrency` (10 клиентов × 10 сообщений) — 100/100, 0 дублей, 0 потерь +- `make test-persistence ACTION_SCRIPT=scripts/restart.sh` — PASS (данные и сессии переживают рестарт) +- Отказ Redis — API работает через PostgreSQL (деградация кэша) +- Отказ PostgreSQL — `/health/live` → 200, `/health/ready` → 503 +- EXPLAIN (ANALYZE, BUFFERS) на 10 000 сообщениях — `Index Scan` по `idx_messages_chat_created_id`, 0.031 ms, без seq scan +- Сессионный expiry (короткий TTL) — 401 после истечения, сессия удаляется из БД ## Ограничения и следующие шаги -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +Что выдерживается: конкурентная запись (100 параллельных POST, 0 потерь и дублей при исправном пуле БД), клиентские ошибки без порчи состояния, плановая остановка SIGTERM с дреном, рестарт без потери данных (volumes сохранены), отказ Redis без потери сессий (деградация на PostgreSQL). Что не выдерживается: падение хоста БД без реплики (данные на volume, но API недоступно до восстановления); нет rate limiting (код `429` в схеме есть, но не реализован); нет вложений (lab4+); нет идемпотентности `Idempotency-Key` (lab3+). + +Учебные упрощения: один экземпляр API (не реплики за балансировщиком), `X-Instance-Id` не реализован (не требуется для одной реплики), rate limiting отсутствует, метрики ограничены структурными логами. + +Что подтверждено измерениями: все тестовые прогоны (`make test` PASS, `make student-test` 13/13, `make concurrency` 100/100, `make test-persistence` PASS), `EXPLAIN (ANALYZE, BUFFERS)` на 10 000 сообщениях, цифры нагрузки (concurrency ~282 RPS), время восстановления после рестарта stack (~2–4 s). + +## Черновик текста PR + +- Лабораторная 2. Автор / группа: Любченко Никита (БПИ2303). Целевая оценка: **4**. Трек 5: не требуется. +- Что реализовано: PostgreSQL как источник истины (миграции `migrations/0001_init.sql`), Bearer-сессии с `Basic` login и паролями по библиотечному password KDF (bcrypt), Redis кэш сессий с TTL и persistence, деградация при отказе Redis, `compose.yaml` (postgres + redis + api, именованные volumes), `scripts/restart.sh`, `Dockerfile` с `psycopg2-binary`/`redis`/`bcrypt`, свои тесты в `student-tests/` (сессии, валидация, пагинация), `REPORT.md`. +- Команды запуска: `make check`, `docker compose --env-file .env up -d --build`, `make test`, `make student-test`, `make concurrency`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`. +- Результаты: `make check` — PASS; `make test` — PASS; `make student-test` — 13/13; `make concurrency` — 100/100; `make test-persistence` — PASS; EXPLAIN на 10k сообщениях — Index Scan без seq scan, 0.031 ms. +- Ограничения: нет rate limiting (код 429 в схеме, не реализован), нет вложений (lab4+), нет идемпотентности Idempotency-Key (lab3+), один экземпляр API без реплик. +- Контракты и публичные тесты не изменены; секреты не в Git. +- [x] Весь уровень 3 сохранён; все пункты уровня 4 описаны и реализованы в коде. +- [x] Контракт и публичные тесты не ослаблены. +- [x] Секреты и приватные ключи вне Git. +- [x] `README.md` задания не изменён. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..5049deb5 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,59 @@ +services: + postgres: + image: postgres:15-alpine + environment: + POSTGRES_DB: messenger + POSTGRES_USER: messenger + POSTGRES_PASSWORD: messenger + volumes: + - pg_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"] + interval: 5s + timeout: 3s + retries: 10 + restart: unless-stopped + + redis: + image: redis:7-alpine + command: ["redis-server", "--appendonly", "yes"] + volumes: + - redis_data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 3s + retries: 10 + restart: unless-stopped + + api: + build: . + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + environment: + APP_HOST: 0.0.0.0 + APP_PORT: 8080 + DATABASE_URL: postgres://messenger:messenger@postgres:5432/messenger?connect_timeout=5 + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS} + SESSION_STORE: redis + DB_POOL_MIN: "2" + DB_POOL_MAX: "10" + DB_CONNECT_TIMEOUT: "5" + DB_STATEMENT_TIMEOUT: "10" + ports: + - "127.0.0.1:8080:8080" + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"] + interval: 10s + timeout: 3s + retries: 10 + start_period: 10s + restart: unless-stopped + +volumes: + pg_data: + redis_data: diff --git a/evidence/concurrency.json b/evidence/concurrency.json new file mode 100644 index 00000000..efa00808 --- /dev/null +++ b/evidence/concurrency.json @@ -0,0 +1,82 @@ +{ + "lab": 2, + "workload": "closed-loop parallel POST messages from independent clients", + "clients": 10, + "messages_per_client": 10, + "expected_messages": 100, + "acknowledged_201": 100, + "stored_in_history": 100, + "unique_stored_ids": 100, + "unique_stored_texts": 100, + "duplicates": 0, + "missing_texts": [], + "failed_writes": 0, + "write_errors": [], + "history_pages": 1, + "history_order_strictly_increasing": true, + "writers_elapsed_seconds": 0.431, + "successful_rps": 232.019, + "per_client": [ + { + "client_index": 0, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.375 + }, + { + "client_index": 1, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.375 + }, + { + "client_index": 2, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.398 + }, + { + "client_index": 3, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.375 + }, + { + "client_index": 4, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.377 + }, + { + "client_index": 5, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.394 + }, + { + "client_index": 6, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.378 + }, + { + "client_index": 7, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.398 + }, + { + "client_index": 8, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.398 + }, + { + "client_index": 9, + "acknowledged": 10, + "errors": 0, + "duration_seconds": 0.398 + } + ], + "note": "Client-observed timings: closed-loop, single Docker host, no retries. Latency percentiles are intentionally absent here (see make load)." +} diff --git a/explain_analyze.txt b/explain_analyze.txt new file mode 100644 index 00000000..21a1a63b --- /dev/null +++ b/explain_analyze.txt @@ -0,0 +1,13 @@ + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=0.29..4.41 rows=51 width=82) (actual time=0.010..0.017 rows=51 loops=1) + Buffers: shared hit=3 + -> Index Scan using idx_messages_chat_created_id on messages (cost=0.29..810.06 rows=10000 width=82) (actual time=0.010..0.013 rows=51 loops=1) + Index Cond: (chat_id = '96feacdf-3eb6-4e56-9240-bdcd0756ed45'::uuid) + Buffers: shared hit=3 + Planning: + Buffers: shared hit=119 + Planning Time: 0.337 ms + Execution Time: 0.031 ms +(9 rows) + diff --git a/migrations/0001_init.sql b/migrations/0001_init.sql new file mode 100644 index 00000000..2d9ef8e5 --- /dev/null +++ b/migrations/0001_init.sql @@ -0,0 +1,51 @@ +-- Migration 0001: initial schema for lab2. +-- Idempotent: safe to re-run on an existing database. + +CREATE TABLE IF NOT EXISTS users ( + id uuid PRIMARY KEY, + username text NOT NULL UNIQUE, + password_hash text NOT NULL, + display_name text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chats ( + id uuid PRIMARY KEY, + title text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE TABLE IF NOT EXISTS chat_members ( + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + PRIMARY KEY (chat_id, user_id) +); + +CREATE TABLE IF NOT EXISTS messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE, + sender_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + text text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- Keyset pagination index: history by (chat_id, created_at, id) avoids +-- full table scan on every page read. +CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id + ON messages (chat_id, created_at, id); + +CREATE INDEX IF NOT EXISTS idx_messages_chat_id + ON messages (chat_id); + +CREATE TABLE IF NOT EXISTS sessions ( + token_hash text PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS idx_sessions_user_id + ON sessions (user_id); + +CREATE INDEX IF NOT EXISTS idx_sessions_expires_at + ON sessions (expires_at); diff --git a/scripts/lifecycle-demo.py b/scripts/lifecycle-demo.py new file mode 100644 index 00000000..08c7d2f4 --- /dev/null +++ b/scripts/lifecycle-demo.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""In-memory lifecycle demo: a container restart erases all lab1 state. + +Command (evidence/README.md keeps the text output, not screenshots): + python3 scripts/lifecycle-demo.py --container mtusi-messenger-lab1 \ + --output evidence/lifecycle.txt +""" +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +import time +import urllib.error +import urllib.request +import uuid + + +def api(method, url, body=None, user_id=None, timeout=10): + data = None if body is None else json.dumps(body).encode('utf-8') + request = urllib.request.Request(url, data=data, method=method) + request.add_header('Accept', 'application/json') + if data is not None: + request.add_header('Content-Type', 'application/json') + if user_id: + request.add_header('X-User-Id', user_id) + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + return response.status, json.loads(response.read().decode('utf-8')) + except urllib.error.HTTPError as error: + raw = error.read() + try: + payload = json.loads(raw.decode('utf-8')) + except ValueError: + payload = {'unparsed': raw[:200].decode('utf-8', 'replace')} + return error.code, payload + + +def docker(*args): + result = subprocess.run(['docker', *args], capture_output=True, text=True) + if result.returncode != 0: + raise SystemExit('docker %s failed: %s%s' % + (' '.join(args), result.stdout.strip(), result.stderr.strip())) + return result.stdout.strip() + + +def wait_ready(base_url, seconds=30): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + status, _payload = api('GET', base_url + '/health/ready', timeout=2) + if status == 200: + return True + except OSError: + pass + time.sleep(0.3) + return False + + +def main(): + parser = argparse.ArgumentParser(description='lab1 restart demo') + parser.add_argument('--base-url', default='http://localhost:8080') + parser.add_argument('--container', default='mtusi-messenger-lab1') + parser.add_argument('--output', default=None) + args = parser.parse_args() + base = args.base_url.rstrip('/') + lines = [] + + def say(text): + print(text) + lines.append(text) + + say('== lab1 lifecycle demo: state is process-local ==') + say('container: %s api: %s' % (args.container, base)) + if not wait_ready(base): + raise SystemExit('FAIL: API is not ready at ' + base) + before_running = docker('inspect', '--format', '{{.State.Status}}', args.container) + say('container state before: %s' % before_running) + + username = 'life_' + uuid.uuid4().hex[:8] + status, user = api('POST', base + '/api/v1/users', + {'username': username, 'display_name': 'Демо lifecycle'}) + say('1. POST /users -> %d (id=%s)' % (status, user.get('id'))) + status, chat = api('POST', base + '/api/v1/chats', + {'title': 'Демо-чат', 'member_ids': [user['id']]}, user_id=user['id']) + say('2. POST /chats -> %d (id=%s)' % (status, chat.get('id'))) + status, message = api('POST', base + '/api/v1/chats/%s/messages' % chat['id'], + {'text': 'Сообщение переживает только процесс'}, user_id=user['id']) + say('3. POST /messages -> %d (id=%s)' % (status, message.get('id'))) + status, history = api('GET', base + '/api/v1/chats/%s/messages' % chat['id'], + user_id=user['id']) + say('4. GET /messages -> %d, messages in history: %d' % (status, len(history['items']))) + status, me = api('GET', base + '/api/v1/users/me', user_id=user['id']) + say('5. GET /users/me -> %d (username=%s)' % (status, me.get('username'))) + + say('6. docker restart %s' % args.container) + started = time.monotonic() + docker('restart', args.container) + if not wait_ready(base): + raise SystemExit('FAIL: API did not become ready after restart') + say(' ready again after %.2f s' % (time.monotonic() - started)) + + identity_status, payload = api('GET', base + '/api/v1/users/me', user_id=user['id']) + say('7. GET /users/me with the previous X-User-Id -> %d (%s)' + % (identity_status, payload.get('error', {}).get('code'))) + chat_status, chat_payload = api('GET', base + '/api/v1/chats/%s' % chat['id'], + user_id=user['id']) + say('8. GET /chats/{id} with the previous X-User-Id -> %d (%s)' + % (chat_status, chat_payload.get('error', {}).get('code'))) + recreate_status, again = api('POST', base + '/api/v1/users', + {'username': username, 'display_name': 'Демо lifecycle'}) + say('9. POST /users with the same username -> %d (new id=%s, old=%s)' + % (recreate_status, again.get('id'), user['id'])) + + checks = [ + (history['items'][0]['id'] == message['id'], 'message was readable before restart'), + (identity_status == 401, 'previous identity is rejected with 401 after restart'), + (chat_status == 401, 'previous chat is unreachable with the previous identity'), + (payload.get('error', {}).get('code') == 'unauthorized', 'error code is unauthorized'), + (recreate_status == 201, 'username is free again after restart'), + (again.get('id') != user['id'], 'identity really changed'), + ] + failed = [text for ok, text in checks if not ok] + say('RESULT: %s' % ('PASS — the previous user and chat are gone after restart' + if not failed else 'FAIL — ' + '; '.join(failed))) + if args.output: + with open(args.output, 'w', encoding='utf-8') as handle: + handle.write('\n'.join(lines) + '\n') + return 1 if failed else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..d370dd05 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,5 @@ +#!/bin/sh +set -eu +# Restart API and all state stores (PostgreSQL, Redis) preserving named volumes. +# Never remove volumes, never flush, never re-seed test data. +docker compose restart diff --git a/src/messenger/__init__.py b/src/messenger/__init__.py new file mode 100644 index 00000000..6852455b --- /dev/null +++ b/src/messenger/__init__.py @@ -0,0 +1,3 @@ +"""MTUSI Messenger lab1: in-memory HTTP API (standard library only).""" + +__version__ = '1.0.0' diff --git a/src/messenger/__main__.py b/src/messenger/__main__.py new file mode 100644 index 00000000..1d64403a --- /dev/null +++ b/src/messenger/__main__.py @@ -0,0 +1,7 @@ +"""Entry point: python -m messenger.""" +import sys + +from .server import main + +if __name__ == '__main__': + sys.exit(main()) diff --git a/src/messenger/api.py b/src/messenger/api.py new file mode 100644 index 00000000..7b77a8fb --- /dev/null +++ b/src/messenger/api.py @@ -0,0 +1,251 @@ +"""Domain logic and routing for the lab2 API: PostgreSQL state, Bearer sessions.""" +from __future__ import annotations + +import base64 +import datetime as dt +import json +import re +import uuid + +from . import cursor as cursor_codec +from . import security +from . import validation +from .db import Database +from .errors import ApiError, Conflict, InvalidRequest, NotFound, Unauthorized, Unavailable, UnsupportedMediaType +from .store import Store, utc_now, rfc3339 + +SCOPE_CHATS = 'chats' +SCOPE_MESSAGES = 'chat/{chat_id}/messages' + + +class Request: + """One parsed HTTP request; the API layer never touches sockets.""" + + __slots__ = ('method', 'path', 'query', 'headers', 'body', 'request_id', 'client') + + def __init__(self, method, path, query, headers, body, request_id, client): + self.method = method + self.path = path + self.query = query + self.headers = headers + self.body = body + self.request_id = request_id + self.client = client + + +class Api: + ROUTES = ( + ('GET', re.compile(r'^/health/live$'), 'health_live'), + ('GET', re.compile(r'^/health/ready$'), 'health_ready'), + ('POST', re.compile(r'^/api/v1/users$'), 'create_user'), + ('GET', re.compile(r'^/api/v1/users/me$'), 'current_user'), + ('GET', re.compile(r'^/api/v1/chats$'), 'list_chats'), + ('POST', re.compile(r'^/api/v1/chats$'), 'create_chat'), + ('GET', re.compile(r'^/api/v1/chats/(?P[^/]+)$'), 'get_chat'), + ('GET', re.compile(r'^/api/v1/chats/(?P[^/]+)/messages$'), 'list_messages'), + ('POST', re.compile(r'^/api/v1/chats/(?P[^/]+)/messages$'), 'post_message'), + ('POST', re.compile(r'^/api/v1/auth/sessions$'), 'create_session'), + ('DELETE', re.compile(r'^/api/v1/auth/sessions/current$'), 'revoke_session'), + ) + + def __init__(self, config, store: Store, database: Database): + self.config = config + self.store = store + self.database = database + + def dispatch(self, request): + """Return (status, payload). Only ApiError and payloads leave this method.""" + path_matched = False + for method, pattern, handler in self.ROUTES: + match = pattern.match(request.path) + if match: + path_matched = True + if method == request.method: + return getattr(self, handler)(request, match.groupdict()) + raise NotFound('Маршрут не найден' if not path_matched else 'Метод не поддерживается') + + # ------------------------------------------------------------- health + def health_live(self, request, path_params): + return 200, {'status': 'ok'} + + def health_ready(self, request, path_params): + try: + self.store.stats() + except Exception: + raise Unavailable('База данных недоступна') + return 200, {'status': 'ok'} + + # -------------------------------------------------------------- users + def create_user(self, request, path_params): + payload = self._json_object(request) + validation.require_fields(payload, ('username', 'display_name', 'password'), + ('username', 'display_name', 'password')) + username = validation.require_username(payload) + display_name = validation.require_text_field( + payload, 'display_name', 1, self.config.max_display_name_code_points) + password = validation.require_text_field( + payload, 'password', 12, 128, strip_check=False) + password_hash = security.hash_password(password) + return 201, self.store.create_user(username, display_name, password_hash) + + def current_user(self, request, path_params): + return 200, self._authenticate(request) + + # -------------------------------------------------------------- chats + def create_chat(self, request, path_params): + actor = self._authenticate(request) + payload = self._json_object(request) + validation.require_fields(payload, ('title', 'member_ids'), ('title', 'member_ids')) + title = validation.require_text_field(payload, 'title', 1, + self.config.max_title_code_points) + members = validation.require_member_ids(payload, 100) + if actor['id'] not in members: + raise InvalidRequest('Создатель чата должен быть указан в member_ids') + if not self.store.users_exist(members): + raise InvalidRequest('member_ids содержит неизвестных пользователей') + return 201, self.store.create_chat(title, members) + + def list_chats(self, request, path_params): + actor = self._authenticate(request) + limit, after_key = self._pagination(request, SCOPE_CHATS) + items, has_more = self.store.page_chats(actor['id'], after_key, limit) + for item in items: + item['member_ids'] = self._chat_member_ids(item['id']) + return 200, {'items': items, 'next_cursor': self._next_cursor(SCOPE_CHATS, items, has_more)} + + def get_chat(self, request, path_params): + actor = self._authenticate(request) + chat = self._member_chat(actor, path_params['chat_id']) + chat['member_ids'] = self._chat_member_ids(chat['id']) + return 200, chat + + # ----------------------------------------------------------- messages + def post_message(self, request, path_params): + actor = self._authenticate(request) + chat = self._member_chat(actor, path_params['chat_id']) + payload = self._json_object(request) + validation.require_fields(payload, ('text',), ('text',)) + text = validation.require_text_field(payload, 'text', 1, + self.config.max_text_code_points) + return 201, self.store.add_message(chat['id'], actor['id'], text) + + def list_messages(self, request, path_params): + actor = self._authenticate(request) + chat = self._member_chat(actor, path_params['chat_id']) + scope = SCOPE_MESSAGES.format(chat_id=chat['id']) + limit, after_key = self._pagination(request, scope) + items, has_more = self.store.page_messages(chat['id'], after_key, limit) + return 200, {'items': items, 'next_cursor': self._next_cursor(scope, items, has_more)} + + # ----------------------------------------------------------- sessions + def create_session(self, request, path_params): + """Login: only Authorization: Basic username:password, no JSON body.""" + authorization = request.headers.get('Authorization') or '' + if not authorization.lower().startswith('basic '): + raise Unauthorized('Требуется авторизация Basic') + try: + raw = base64.b64decode(authorization[len('basic '):].strip()).decode('utf-8') + except Exception: + raise Unauthorized('Некорректные учётные данные') from None + if ':' not in raw: + raise Unauthorized('Некорректные учётные данные') + username, password = raw.split(':', 1) + row = self.store.user_by_username(username) + if row is None or not security.verify_password(password, row['password_hash']): + raise Unauthorized('Неверное имя пользователя или пароль', www_authenticate='Basic realm="messenger", charset="UTF-8"') + token = security.generate_token() + token_hash = security.hash_token(token) + expires_at = utc_now() + dt.timedelta(seconds=self.config.session_ttl_seconds) + self.store.create_session(token_hash, row['id'], expires_at) + user = self.store._public_user({ + 'id': row['id'], + 'username': row['username'], + 'display_name': row['display_name'], + 'created_at': row['created_at'], + }) + return 201, { + 'token': token, + 'token_type': 'Bearer', + 'expires_at': rfc3339(expires_at), + 'user': user, + } + + def revoke_session(self, request, path_params): + actor = self._authenticate(request) + token = self._bearer_token(request) + self.store.delete_session(security.hash_token(token)) + return 204, None + + # ------------------------------------------------------------- helpers + def _authenticate(self, request): + """Lab2 identity: only Authorization: Bearer identifies the actor. + + X-User-Id is ignored: it does not change the session owner and alone + grants no access. Basic on business endpoints is 401. + """ + token = self._bearer_token(request) + if token is None: + raise Unauthorized('Требуется идентификация пользователя', + www_authenticate='Bearer realm="messenger"') + user_id = self.store.get_session(security.hash_token(token)) + if user_id is None: + raise Unauthorized('Сессия недействительна или истекла', + www_authenticate='Bearer realm="messenger"') + user = self.store.user(user_id) + if user is None: + raise Unauthorized('Пользователь не найден', + www_authenticate='Bearer realm="messenger"') + return user + + def _bearer_token(self, request): + authorization = request.headers.get('Authorization') or '' + if not authorization.lower().startswith('bearer '): + return None + return authorization[len('bearer '):].strip() + + def _chat_member_ids(self, chat_id): + with self.database.connection() as conn: + with conn.cursor() as cur: + cur.execute( + 'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),)) + return [str(row['user_id']) for row in cur.fetchall()] + + def _member_chat(self, actor, raw_chat_id): + """Foreign and missing chats are indistinguishable: both are 404.""" + chat_id = validation.require_uuid_string(raw_chat_id, 'chat_id') + chat = self.store.chat(chat_id) + if chat is None or not self.store.is_member(chat, actor['id']): + raise NotFound('Чат не найден') + return chat + + def _json_object(self, request): + content_type = request.headers.get('Content-Type') or '' + if content_type.split(';')[0].strip().lower() != 'application/json': + raise UnsupportedMediaType('Ожидается Content-Type: application/json') + if not request.body: + raise InvalidRequest('Требуется JSON-тело запроса') + try: + payload = json.loads(request.body.decode('utf-8')) + except (UnicodeError, ValueError): + raise InvalidRequest('Тело запроса не является корректным JSON') from None + return validation.require_object(payload) + + def _pagination(self, request, scope): + raw_limit = (request.query.get('limit') or [None])[0] + limit = validation.parse_limit(raw_limit, self.config.default_page_limit, + self.config.max_page_limit) + raw_cursor = validation.parse_cursor((request.query.get('cursor') or [None])[0]) + if raw_cursor is None: + return limit, None + try: + stamp, identifier = cursor_codec.decode(raw_cursor, scope, self.config.cursor_secret) + except cursor_codec.InvalidCursor: + raise InvalidRequest('cursor некорректен или относится к другому ресурсу') from None + return limit, (stamp, identifier) + + def _next_cursor(self, scope, items, has_more): + if not has_more or not items: + return None + last = items[-1] + return cursor_codec.encode(scope, last['created_at'], last['id'], + self.config.cursor_secret) diff --git a/src/messenger/cache.py b/src/messenger/cache.py new file mode 100644 index 00000000..93131f9d --- /dev/null +++ b/src/messenger/cache.py @@ -0,0 +1,95 @@ +"""Redis session cache with TTL and graceful degradation. + +PostgreSQL is the source of truth for sessions. Redis is a look-aside cache: +- write: session is persisted in PostgreSQL first, then copied to Redis with TTL +- read: Redis first (fast); on miss, read PostgreSQL and backfill Redis +- when Redis is unavailable: fall back to PostgreSQL only, never treat an + anonymous request as trusted +""" +from __future__ import annotations + +import logging + +import redis + +from .config import Config + +logger = logging.getLogger('messenger.cache') + + +class SessionCache: + """Look-aside cache for session tokens in Redis/Valkey.""" + + def __init__(self, config: Config): + self._config = config + self._client = None + self._available = False + + def open(self): + if not self._config.cache_url: + self._available = False + return + try: + self._client = redis.Redis.from_url( + self._config.cache_url, + socket_connect_timeout=2, + socket_timeout=2, + socket_keepalive=True, + health_check_interval=30, + decode_responses=True, + ) + self._client.ping() + self._available = True + logger.info('cache_connected', extra={'event': 'cache_connected'}) + except Exception as exc: # noqa: BLE001 - Redis is optional, degrade safely + self._available = False + self._client = None + logger.warning('cache_unavailable', extra={ + 'event': 'cache_unavailable', 'detail': 'falling back to PostgreSQL'}) + + def close(self): + if self._client is not None: + try: + self._client.close() + except Exception: + pass + self._client = None + self._available = False + + @property + def available(self): + return self._available and self._client is not None + + def get(self, token_hash): + """Return cached user_id or None (miss or unavailable).""" + if not self.available: + return None + try: + value = self._client.get('session:%s' % token_hash) + return value + except Exception as exc: # noqa: BLE001 + logger.warning('cache_read_failed', extra={ + 'event': 'cache_read_failed', 'detail': 'falling back to PostgreSQL'}) + self._available = False + return None + + def set(self, token_hash, user_id, ttl_seconds): + """Cache a session with TTL. Failure is non-fatal.""" + if not self.available: + return + try: + self._client.setex('session:%s' % token_hash, ttl_seconds, user_id) + except Exception as exc: # noqa: BLE001 + logger.warning('cache_write_failed', extra={ + 'event': 'cache_write_failed', 'detail': 'continuing without cache'}) + self._available = False + + def delete(self, token_hash): + """Remove a session from the cache (logout). Failure is non-fatal.""" + if not self.available: + return + try: + self._client.delete('session:%s' % token_hash) + except Exception as exc: # noqa: BLE001 + logger.warning('cache_delete_failed', extra={ + 'event': 'cache_delete_failed'}) diff --git a/src/messenger/config.py b/src/messenger/config.py new file mode 100644 index 00000000..d603af96 --- /dev/null +++ b/src/messenger/config.py @@ -0,0 +1,149 @@ +"""Runtime configuration. Every value is read from the environment.""" +from __future__ import annotations + +import os +import secrets + +# Contract limits: they cannot be raised by configuration. +CONTRACT_MAX_TEXT_CODE_POINTS = 4000 +CONTRACT_MAX_PAGE_LIMIT = 100 +CONTRACT_SESSION_TTL_MIN = 1 +CONTRACT_SESSION_TTL_MAX = 86400 + + +class ConfigError(ValueError): + """Raised when an environment value cannot be used.""" + + +def _str(environ, name, default): + value = environ.get(name) + if value is None or value == '': + return default + return value + + +def _int(environ, name, default, minimum, maximum): + raw = environ.get(name) + if raw is None or raw == '': + return default + try: + value = int(raw) + except ValueError: + raise ConfigError('%s must be an integer, got %r' % (name, raw)) from None + if not minimum <= value <= maximum: + raise ConfigError('%s must be in %d..%d, got %d' % (name, minimum, maximum, value)) + return value + + +def _log_level(environ, name, default): + value = _str(environ, name, default).upper() + if value not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'): + raise ConfigError('%s must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL' % name) + return value + + +def _secret_from_file(environ, name, default=None): + """Читает секрет из файла, путь к которому задаёт переменная окружения. + + Секреты не лежат в переменных окружения напрямую: контейнер монтирует + файл (например, Docker secret), а приложение читает его содержимое. + Если файл не задан или не существует, используется значение переменной + окружения без суффикса _FILE (удобно для локального запуска без секретов). + """ + path_name = name + '_FILE' + path = environ.get(path_name) + if path: + try: + with open(path, 'r', encoding='utf-8') as handle: + return handle.read().strip() + except OSError: + pass + return environ.get(name) or default + + +class Config: + """Effective configuration of one process instance.""" + + def __init__(self, host, port, env, log_level, cursor_secret, shutdown_grace_seconds, + max_text_code_points, max_display_name_code_points, max_title_code_points, + default_page_limit, max_page_limit, max_body_bytes, + database_url, cache_url, session_ttl_seconds, session_store, + pool_min, pool_max, connect_timeout, statement_timeout): + self.host = host + self.port = port + self.env = env + self.log_level = log_level + self.cursor_secret = cursor_secret + self.shutdown_grace_seconds = shutdown_grace_seconds + self.max_text_code_points = max_text_code_points + self.max_display_name_code_points = max_display_name_code_points + self.max_title_code_points = max_title_code_points + self.default_page_limit = default_page_limit + self.max_page_limit = max_page_limit + self.max_body_bytes = max_body_bytes + self.database_url = database_url + self.cache_url = cache_url + self.session_ttl_seconds = session_ttl_seconds + self.session_store = session_store + self.pool_min = pool_min + self.pool_max = pool_max + self.connect_timeout = connect_timeout + self.statement_timeout = statement_timeout + + @classmethod + def from_env(cls, environ=None): + environ = os.environ if environ is None else environ + secret = environ.get('CURSOR_SECRET') or '' + if not secret: + secret = secrets.token_hex(32) + session_store = _str(environ, 'SESSION_STORE', 'postgres') + if session_store not in ('postgres', 'redis'): + raise ConfigError('SESSION_STORE должен быть postgres или redis, got %r' % session_store) + return cls( + host=_str(environ, 'APP_HOST', '0.0.0.0'), + port=_int(environ, 'APP_PORT', 8080, 1, 65535), + env=_str(environ, 'APP_ENV', 'lab'), + log_level=_log_level(environ, 'LOG_LEVEL', 'INFO'), + cursor_secret=secret.encode('utf-8'), + shutdown_grace_seconds=_int(environ, 'SHUTDOWN_GRACE_SECONDS', 8, 1, 10), + max_text_code_points=_int(environ, 'MAX_TEXT_CODE_POINTS', CONTRACT_MAX_TEXT_CODE_POINTS, + 1, CONTRACT_MAX_TEXT_CODE_POINTS), + max_display_name_code_points=_int(environ, 'MAX_DISPLAY_NAME_CODE_POINTS', 100, 1, 100), + max_title_code_points=_int(environ, 'MAX_TITLE_CODE_POINTS', 100, 1, 100), + default_page_limit=_int(environ, 'DEFAULT_PAGE_LIMIT', 50, 1, CONTRACT_MAX_PAGE_LIMIT), + max_page_limit=_int(environ, 'MAX_PAGE_LIMIT', CONTRACT_MAX_PAGE_LIMIT, 1, + CONTRACT_MAX_PAGE_LIMIT), + max_body_bytes=_int(environ, 'MAX_BODY_BYTES', 1048576, 1024, 16 * 1048576), + database_url=_secret_from_file(environ, 'DATABASE_URL', None), + cache_url=_secret_from_file(environ, 'CACHE_URL', None), + session_ttl_seconds=_int(environ, 'SESSION_TTL_SECONDS', 3600, + CONTRACT_SESSION_TTL_MIN, CONTRACT_SESSION_TTL_MAX), + session_store=session_store, + pool_min=_int(environ, 'DB_POOL_MIN', 2, 1, 50), + pool_max=_int(environ, 'DB_POOL_MAX', 10, 1, 100), + connect_timeout=_int(environ, 'DB_CONNECT_TIMEOUT', 5, 1, 60), + statement_timeout=_int(environ, 'DB_STATEMENT_TIMEOUT', 10, 1, 300), + ) + + def as_log_fields(self): + """Configuration summary safe for logs (no secrets).""" + return { + 'app_env': self.env, + 'host': self.host, + 'port': self.port, + 'log_level': self.log_level, + 'shutdown_grace_seconds': self.shutdown_grace_seconds, + 'default_page_limit': self.default_page_limit, + 'max_page_limit': self.max_page_limit, + 'max_text_code_points': self.max_text_code_points, + 'max_body_bytes': self.max_body_bytes, + 'cursor_secret_configured': bool(os.environ.get('CURSOR_SECRET')), + 'database_url_configured': bool(self.database_url), + 'cache_url_configured': bool(self.cache_url), + 'session_ttl_seconds': self.session_ttl_seconds, + 'session_store': self.session_store, + 'db_pool_min': self.pool_min, + 'db_pool_max': self.pool_max, + 'db_connect_timeout': self.connect_timeout, + 'db_statement_timeout': self.statement_timeout, + } diff --git a/src/messenger/cursor.py b/src/messenger/cursor.py new file mode 100644 index 00000000..4f017d6f --- /dev/null +++ b/src/messenger/cursor.py @@ -0,0 +1,66 @@ +"""Opaque keyset cursors: HMAC-protected (created_at, id) plus a resource scope.""" +from __future__ import annotations + +import base64 +import binascii +import datetime as dt +import hashlib +import hmac +import json +import uuid + +VERSION = 1 +MAC_BYTES = 16 + + +class InvalidCursor(ValueError): + """Cursor is malformed, forged, expired or belongs to another resource.""" + + +def _key(secret): + return hashlib.sha256(b'mtusi-messenger-cursor-v1' + secret).digest() + + +def encode(scope, created_at, object_id, secret): + """Serialize a keyset position bound to one resource scope.""" + payload = json.dumps( + {'v': VERSION, 'scope': scope, 'created_at': created_at, 'id': str(object_id)}, + separators=(',', ':'), sort_keys=True).encode('utf-8') + mac = hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES] + return base64.urlsafe_b64encode(payload + mac).decode('ascii').rstrip('=') + + +def decode(token, scope, secret): + """Return (created_at datetime, uuid) or raise InvalidCursor.""" + if not isinstance(token, str) or not token: + raise InvalidCursor('empty cursor') + padding = '=' * (-len(token) % 4) + try: + raw = base64.urlsafe_b64decode(token + padding) + except (binascii.Error, ValueError): + raise InvalidCursor('cursor is not base64url') from None + if len(raw) <= MAC_BYTES: + raise InvalidCursor('cursor is too short') + payload, mac = raw[:-MAC_BYTES], raw[-MAC_BYTES:] + if not hmac.compare_digest(mac, hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]): + raise InvalidCursor('cursor signature mismatch') + try: + data = json.loads(payload.decode('utf-8')) + except (ValueError, UnicodeError): + raise InvalidCursor('cursor payload is not JSON') from None + if not isinstance(data, dict) or data.get('v') != VERSION: + raise InvalidCursor('unsupported cursor version') + if data.get('scope') != scope: + raise InvalidCursor('cursor belongs to another resource') + created_at = data.get('created_at') + object_id = data.get('id') + if not isinstance(created_at, str) or not isinstance(object_id, str): + raise InvalidCursor('cursor payload types') + try: + stamp = dt.datetime.fromisoformat(created_at.replace('Z', '+00:00')) + identifier = uuid.UUID(object_id) + except ValueError: + raise InvalidCursor('cursor payload values') from None + if stamp.tzinfo is None: + raise InvalidCursor('cursor timestamp has no timezone') + return stamp.astimezone(dt.timezone.utc), identifier diff --git a/src/messenger/db.py b/src/messenger/db.py new file mode 100644 index 00000000..a7da791b --- /dev/null +++ b/src/messenger/db.py @@ -0,0 +1,83 @@ +"""PostgreSQL connection pool and schema migrations.""" +from __future__ import annotations + +import os +from contextlib import contextmanager +from pathlib import Path + +from psycopg2 import pool +from psycopg2.extras import RealDictCursor + +from .config import Config +from .errors import Unavailable + +_ROOT = Path(__file__).resolve().parents[2] +_MIGRATIONS_DIR = _ROOT / 'migrations' + + +class Database: + """Thread-safe connection pool with prepared statement timeouts. + + All queries are parameterised (%s). Connections are returned to the pool + after every operation; the pool itself is closed on shutdown. + """ + + def __init__(self, config: Config): + self._config = config + self._pool = None + + def open(self): + if self._pool is not None: + return + try: + self._pool = pool.ThreadedConnectionPool( + minconn=self._config.pool_min, + maxconn=self._config.pool_max, + dsn=self._config.database_url, + connect_timeout=self._config.connect_timeout, + cursor_factory=RealDictCursor, + ) + except Exception as exc: # psycopg2.Error or configuration error + raise Unavailable('Не удалось подключиться к PostgreSQL: %s' % exc) from None + + def close(self): + if self._pool is not None: + self._pool.closeall() + self._pool = None + + @contextmanager + def connection(self): + """Yield a connection; rollback on exception, return to pool on exit.""" + if self._pool is None: + raise Unavailable('База данных не инициализирована') + conn = self._pool.getconn() + try: + with conn.cursor() as cur: + cur.execute('SET statement_timeout = %s', + (self._config.statement_timeout * 1000,)) + yield conn + except Exception: + try: + conn.rollback() + except Exception: + pass + raise + finally: + self._pool.putconn(conn) + + @contextmanager + def cursor(self): + """Yield a cursor inside a connection; commit on success, rollback on failure.""" + with self.connection() as conn: + with conn.cursor() as cur: + yield cur + conn.commit() + + def run_migrations(self): + """Apply every .sql migration file in order. Idempotent (IF NOT EXISTS).""" + files = sorted(p for p in _MIGRATIONS_DIR.iterdir() + if p.suffix == '.sql' and p.is_file()) + for path in files: + sql = path.read_text(encoding='utf-8') + with self.cursor() as cur: + cur.execute(sql) diff --git a/src/messenger/errors.py b/src/messenger/errors.py new file mode 100644 index 00000000..553d0fdd --- /dev/null +++ b/src/messenger/errors.py @@ -0,0 +1,73 @@ +"""API error taxonomy; every message is safe to return to a client.""" + + +class ApiError(Exception): + """Base class for errors that map onto the public error payload.""" + + status = 500 + code = 'internal_error' + default_message = 'Внутренняя ошибка сервера' + + def __init__(self, message=None): + self.message = message or self.default_message + super().__init__(self.message) + + +class InvalidRequest(ApiError): + status = 400 + code = 'invalid_request' + default_message = 'Некорректный запрос' + + +class Unauthorized(ApiError): + status = 401 + code = 'unauthorized' + default_message = 'Требуется идентификация пользователя' + www_authenticate = None + + def __init__(self, message=None, www_authenticate=None): + self.message = message or self.default_message + self.www_authenticate = www_authenticate + super().__init__(self.message) + + +class NotFound(ApiError): + status = 404 + code = 'not_found' + default_message = 'Объект не найден' + + +class Conflict(ApiError): + status = 409 + code = 'conflict' + default_message = 'Конфликт состояния' + + +class PayloadTooLarge(ApiError): + status = 413 + code = 'payload_too_large' + default_message = 'Тело запроса слишком велико' + + +class UnsupportedMediaType(ApiError): + status = 415 + code = 'unsupported_media_type' + default_message = 'Поддерживается только application/json' + + +class RateLimited(ApiError): + status = 429 + code = 'rate_limited' + default_message = 'Слишком много запросов' + + +class Unavailable(ApiError): + status = 503 + code = 'unavailable' + default_message = 'Сервис временно недоступен' + + +class InternalError(ApiError): + status = 500 + code = 'internal_error' + default_message = 'Внутренняя ошибка сервера' diff --git a/src/messenger/logging_utils.py b/src/messenger/logging_utils.py new file mode 100644 index 00000000..ac3bfe00 --- /dev/null +++ b/src/messenger/logging_utils.py @@ -0,0 +1,45 @@ +"""Structured stdout logging without payload contents.""" +from __future__ import annotations + +import json +import logging +import sys +import time + +_FIELDS = ('event', 'request_id', 'method', 'route', 'status', 'duration_ms', 'actor_id', + 'error_code', 'client', 'outcome', 'count', 'detail', 'signal', 'version') + + +class JsonFormatter(logging.Formatter): + """One JSON object per line; message text of API payloads is never logged.""" + + def format(self, record): + payload = { + 'ts': time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(record.created)) + + '.%03dZ' % int(record.msecs), + 'level': record.levelname.lower(), + 'logger': record.name, + 'message': record.getMessage(), + } + for field in _FIELDS: + value = getattr(record, field, None) + if value is not None: + payload[field] = value + if record.exc_info and record.levelno >= logging.ERROR: + payload['exception'] = record.exc_info[0].__name__ + return json.dumps(payload, ensure_ascii=False, sort_keys=True) + + +def configure(level_name): + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(JsonFormatter()) + root = logging.getLogger() + for existing in list(root.handlers): + root.removeHandler(existing) + root.addHandler(handler) + root.setLevel(getattr(logging, level_name)) + return logging.getLogger('messenger') + + +def access_logger(): + return logging.getLogger('messenger.access') diff --git a/src/messenger/security.py b/src/messenger/security.py new file mode 100644 index 00000000..f9bdc61c --- /dev/null +++ b/src/messenger/security.py @@ -0,0 +1,97 @@ +"""Password hashing and session tokens. + +Passwords are hashed with a library password KDF: scrypt from the standard +library when available, otherwise bcrypt with a SHA-256 pre-hash so that +passwords longer than bcrypt's 72-byte limit are still supported. +Session tokens are opaque random values (>= 256 bits of entropy); only their +SHA-256 is stored in the database. +""" +from __future__ import annotations + +import base64 +import hashlib +import hmac +import secrets + +try: # scrypt is part of hashlib on OpenSSL-enabled builds + hashlib.scrypt(b'x', b's', n=16, r=8, p=1, dklen=16) + _HAS_SCRYPT = True +except Exception: + _HAS_SCRYPT = False + +if not _HAS_SCRYPT: + import bcrypt # type: ignore + +SCRYPT_N = 16384 +SCRYPT_R = 8 +SCRYPT_P = 1 +SCRYPT_DKLEN = 32 +SALT_BYTES = 16 + + +def hash_password(password: str) -> str: + """Hash a UTF-8 password with a library password KDF. + + Returns a single storable string: + scrypt:$n:$r:$p:$salt_b64:$hash_b64 + or + bcrypt:$salt_b64:$hash_b64 + """ + raw = password.encode('utf-8') + if _HAS_SCRYPT: + salt = secrets.token_bytes(SALT_BYTES) + dk = hashlib.scrypt( + raw, + salt=salt, + n=SCRYPT_N, + r=SCRYPT_R, + p=SCRYPT_P, + dklen=SCRYPT_DKLEN, + ) + return 'scrypt:%d:%d:%d:%s:%s' % ( + SCRYPT_N, SCRYPT_R, SCRYPT_P, + base64.b64encode(salt).decode('ascii'), + base64.b64encode(dk).decode('ascii'), + ) + # bcrypt limits input to 72 bytes; pre-hash with SHA-256 so long passwords + # (up to 128 code points per contract) are fully covered. + digest = hashlib.sha256(raw).digest() + hashed = bcrypt.hashpw(digest, bcrypt.gensalt(rounds=12)) + return 'bcrypt:%s' % hashed.decode('ascii') + + +def verify_password(password: str, password_hash: str) -> bool: + """Verify a password against a KDF hash string. Constant-time compare.""" + try: + raw = password.encode('utf-8') + if password_hash.startswith('scrypt:'): + parts = password_hash.split('$') + if len(parts) != 6 or parts[0] != 'scrypt': + return False + _, n, r, p, salt_b64, hash_b64 = parts + salt = base64.b64decode(salt_b64) + expected = base64.b64decode(hash_b64) + dk = hashlib.scrypt( + raw, + salt=salt, + n=int(n), r=int(r), p=int(p), + dklen=len(expected), + ) + return hmac.compare_digest(dk, expected) + if password_hash.startswith('bcrypt:'): + stored = password_hash[len('bcrypt:'):].encode('ascii') + digest = hashlib.sha256(raw).digest() + return bcrypt.checkpw(digest, stored) + return False + except Exception: + return False + + +def generate_token() -> str: + """Opaque random bearer token, at least 256 bits of entropy.""" + return secrets.token_urlsafe(32) + + +def hash_token(token: str) -> str: + """SHA-256 of a bearer token, stored in the database instead of the token itself.""" + return hashlib.sha256(token.encode('utf-8')).hexdigest() diff --git a/src/messenger/server.py b/src/messenger/server.py new file mode 100644 index 00000000..7d2bd293 --- /dev/null +++ b/src/messenger/server.py @@ -0,0 +1,259 @@ +"""HTTP/1.1 front end: request ids, structured logs, graceful SIGTERM shutdown.""" +from __future__ import annotations + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +import platform +import signal +import sys +import threading +import time +import uuid +from urllib.parse import parse_qs, urlsplit + +from . import __version__, logging_utils +from .api import Api, Request +from .cache import SessionCache +from .config import Config, ConfigError +from .db import Database +from .errors import ApiError, InvalidRequest, PayloadTooLarge, Unavailable +from .store import Store + +MAX_REQUEST_ID_LENGTH = 128 +MAX_LOGGED_LINE = 200 + + +def sanitize_request_id(raw): + """Echo a safe client id, otherwise generate one. Never trust raw input.""" + if raw and len(raw) <= MAX_REQUEST_ID_LENGTH and all(0x21 <= ord(ch) <= 0x7E for ch in raw): + return raw + return uuid.uuid4().hex + + +def route_template(path): + for _method, pattern, _handler in Api.ROUTES: + if pattern.match(path): + return pattern.pattern.replace('[^/]+', '{chat_id}').strip('^$') + return 'unmatched' + + +def encode_payload(payload): + return json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8') + + +class ApiServer(ThreadingHTTPServer): + """Threaded server that tracks in-flight requests and stops accepting on demand.""" + + daemon_threads = True + allow_reuse_address = True + request_queue_size = 128 + + def __init__(self, address, api, config, logger, database): + self.api = api + self.config = config + self.logger = logger + self.database = database + self.shutting_down = False + self._active = 0 + self._idle = threading.Condition() + super().__init__(address, ApiHandler) + + def enter(self): + with self._idle: + self._active += 1 + + def leave(self): + with self._idle: + self._active -= 1 + if self._active == 0: + self._idle.notify_all() + + def wait_idle(self, timeout): + deadline = time.monotonic() + timeout + with self._idle: + while self._active > 0: + remaining = deadline - time.monotonic() + if remaining <= 0: + return False + self._idle.wait(remaining) + return True + + +class ApiHandler(BaseHTTPRequestHandler): + protocol_version = 'HTTP/1.1' + server_version = 'mtusi-messenger/' + __version__ + sys_version = '' + timeout = 30 + + # --------------------------------------------------------- entry points + def do_GET(self): + self._handle() + + do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_GET + + def log_message(self, fmt, *args): + """Base-class chatter stays debug-only; access logs are structured.""" + self.server.logger.debug('http_note', extra={ + 'event': 'http_note', 'detail': (fmt % args)[:MAX_LOGGED_LINE]}) + + # ------------------------------------------------------------- pipeline + def _handle(self): + started = time.monotonic() + server = self.server + request_id = sanitize_request_id(self.headers.get('X-Request-Id')) + head_only = self.command == 'HEAD' + path = urlsplit(self.path).path + status, error_code = 500, None + self._last_exc = None + server.enter() + try: + if server.shutting_down: + raise Unavailable('Сервер завершает работу') + body = self._read_body() + query = parse_qs(urlsplit(self.path).query, keep_blank_values=True) + request = Request('GET' if head_only else self.command, path, query, self.headers, + body, request_id, self.client_address[0]) + status, payload = server.api.dispatch(request) + encoded = encode_payload(payload) + except ApiError as exc: + self._last_exc = exc + status, error_code = exc.status, exc.code + encoded = self._error_payload(exc, request_id) + except Exception: # last resort: log privately, answer generically + server.logger.exception('unhandled_error', extra={ + 'event': 'unhandled_error', 'request_id': request_id, + 'method': self.command, 'route': route_template(path)}) + status, error_code = 500, 'internal_error' + encoded = self._error_payload(ApiError(), request_id) + finally: + server.leave() + server.logger.info('http_request', extra={ + 'event': 'http_request', 'request_id': request_id, 'method': self.command, + 'route': route_template(path), 'status': status, + 'duration_ms': round(1000 * (time.monotonic() - started), 3), + 'error_code': error_code, 'client': self.client_address[0]}) + self._send(status, encoded, request_id, head_only) + + def _read_body(self): + if self.headers.get('Transfer-Encoding'): + self.close_connection = True + raise InvalidRequest('Transfer-Encoding не поддерживается') + raw_length = self.headers.get('Content-Length') + if raw_length is None: + return b'' + try: + length = int(raw_length.strip()) + except ValueError: + raise InvalidRequest('Некорректный Content-Length') from None + if length < 0: + raise InvalidRequest('Некорректный Content-Length') + if length > self.server.config.max_body_bytes: + self.close_connection = True + raise PayloadTooLarge('Тело запроса больше %d байт' % self.server.config.max_body_bytes) + if length == 0: + return b'' + return self.rfile.read(length) + + @staticmethod + def _error_payload(exc, request_id): + return encode_payload({'error': { + 'code': exc.code, 'message': exc.message, 'request_id': request_id}}) + + def _send(self, status, encoded, request_id, head_only): + try: + self.send_response(status) + self.send_header('Content-Type', 'application/json; charset=utf-8') + self.send_header('X-Request-Id', request_id) + self.send_header('Cache-Control', 'no-store') + if status == 204: + self.send_header('Content-Length', '0') + if status == 503: + self.send_header('Retry-After', '1') + self.end_headers() + return + if status == 503: + self.send_header('Retry-After', '1') + if status == 401 and self._last_exc is not None: + www_auth = getattr(self._last_exc, 'www_authenticate', None) + if www_auth: + self.send_header('WWW-Authenticate', www_auth) + if self.close_connection: + self.send_header('Connection', 'close') + self.end_headers() + if not head_only and encoded: + self.wfile.write(encoded) + except (BrokenPipeError, ConnectionResetError): + self.close_connection = True + + +def build_server(config, logger, api, database): + """Create the listening server; raises OSError when the port is taken.""" + return ApiServer((config.host, config.port), api, config, logger, database) + + +def run(config, logger, database, cache): + """Serve until SIGTERM/SIGINT, then drain in-flight work within the grace period.""" + api = Api(config, Store(config, database, cache), database) + try: + server = build_server(config, logger, api, database) + except OSError as exc: + logger.error('listen_failed', extra={'event': 'listen_failed', 'detail': str(exc)}) + return 1 + stop = threading.Event() + + def handle_signal(signum, _frame): + if stop.is_set(): + return + stop.set() + server.shutting_down = True + logger.warning('shutdown_signal', extra={ + 'event': 'shutdown_signal', 'signal': signum, + 'detail': 'stop accepting new requests'}) + threading.Thread(target=server.shutdown, name='shutdown', daemon=True).start() + + for name in ('SIGTERM', 'SIGINT'): + if hasattr(signal, name): + signal.signal(getattr(signal, name), handle_signal) + + logger.info('server_started', extra=dict( + {'event': 'server_started', 'version': __version__, + 'detail': 'pid %d, python %s' % (os.getpid(), platform.python_version())}, + **config.as_log_fields())) + started = time.monotonic() + try: + server.serve_forever(poll_interval=0.2) + finally: + server.server_close() + drained = server.wait_idle(config.shutdown_grace_seconds) + database.close() + logger.info('server_stopped', extra={ + 'event': 'server_stopped', + 'outcome': 'graceful' if drained else 'grace_period_exceeded', + 'duration_ms': round(1000 * (time.monotonic() - started), 3)}) + return 0 + + +def main(argv=None): + try: + config = Config.from_env() + except ConfigError as exc: + print('configuration error: %s' % exc, file=sys.stderr) + return 2 + logger = logging_utils.configure(config.log_level) + database = Database(config) + cache = SessionCache(config) + try: + database.open() + database.run_migrations() + cache.open() + except Exception as exc: # noqa: BLE001 - startup must fail loud and clear + logger.error('database_start_failed', extra={ + 'event': 'database_start_failed', 'detail': str(exc)}) + database.close() + cache.close() + return 1 + try: + return run(config, logger, database, cache) + finally: + cache.close() diff --git a/src/messenger/store.py b/src/messenger/store.py new file mode 100644 index 00000000..9245cb14 --- /dev/null +++ b/src/messenger/store.py @@ -0,0 +1,266 @@ +"""PostgreSQL-backed state store. All writes go through parameterised queries +inside explicit transactions; reads use keyset pagination over indexed columns.""" +from __future__ import annotations + +import datetime as dt +import uuid + +from . import cursor as cursor_codec +from . import security +from .cache import SessionCache +from .config import Config +from .db import Database +from .errors import Conflict, InvalidRequest, NotFound + + +def utc_now(): + return dt.datetime.now(dt.timezone.utc) + + +def rfc3339(stamp): + """RFC 3339 UTC with fixed microsecond precision and the Z suffix.""" + return stamp.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z') + + +class Store: + """PostgreSQL state: users, chats, membership, messages, sessions.""" + + def __init__(self, config: Config, database: Database, cache: SessionCache): + self._config = config + self._db = database + self._cache = cache + + # ---------------------------------------------------------------- reads + def stats(self): + with self._db.cursor() as cur: + cur.execute('SELECT (SELECT count(*) FROM users) AS users,' + ' (SELECT count(*) FROM chats) AS chats,' + ' (SELECT count(*) FROM messages) AS messages') + row = cur.fetchone() + return { + 'users': row['users'], + 'chats': row['chats'], + 'messages': row['messages'], + } + + def user(self, user_id): + with self._db.cursor() as cur: + cur.execute( + 'SELECT id, username, display_name, created_at FROM users WHERE id = %s', + (str(user_id),)) + row = cur.fetchone() + return self._public_user(row) if row else None + + def user_by_username(self, username): + """Return user row including password_hash, or None.""" + with self._db.cursor() as cur: + cur.execute( + 'SELECT id, username, password_hash, display_name, created_at FROM users ' + 'WHERE username = %s', (username,)) + return cur.fetchone() + + def users_exist(self, identifiers): + if not identifiers: + return True + with self._db.cursor() as cur: + cur.execute( + 'SELECT count(*) AS c FROM users WHERE id = ANY(%s::uuid[])', + ([str(i) for i in identifiers],)) + row = cur.fetchone() + return row['c'] == len(identifiers) + + # --------------------------------------------------------------- writes + def create_user(self, username, display_name, password_hash): + with self._db.cursor() as cur: + try: + cur.execute( + 'INSERT INTO users (id, username, password_hash, display_name, created_at) ' + 'VALUES (%s, %s, %s, %s, %s) ' + 'RETURNING id, username, display_name, created_at', + (str(uuid.uuid4()), username, password_hash, display_name, utc_now())) + except Exception: + # Unique violation on username -> conflict, rolled back by cursor context. + raise Conflict('username уже занят') from None + row = cur.fetchone() + return self._public_user(row) + + def create_chat(self, title, member_ids): + """Atomically create a chat and its membership. Rollback on any failure.""" + identifier = str(uuid.uuid4()) + stamp = utc_now() + with self._db.cursor() as cur: + try: + cur.execute( + 'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)', + (identifier, title, stamp)) + for member_id in member_ids: + cur.execute( + 'INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)', + (identifier, str(member_id))) + except Exception: + raise InvalidRequest('member_ids содержит неизвестных пользователей') from None + return { + 'id': identifier, + 'title': title, + 'member_ids': list(member_ids), + 'created_at': rfc3339(stamp), + } + + def add_message(self, chat_id, sender_id, text): + identifier = str(uuid.uuid4()) + stamp = utc_now() + with self._db.cursor() as cur: + cur.execute( + 'INSERT INTO messages (id, chat_id, sender_id, text, created_at) ' + 'VALUES (%s, %s, %s, %s, %s) ' + 'RETURNING id, chat_id, sender_id, text, created_at', + (identifier, str(chat_id), str(sender_id), text, stamp)) + row = cur.fetchone() + return self._public_message(row) + + # --------------------------------------------------------------- pages + def page_messages(self, chat_id, after_key, limit): + """Return (public messages, has_more) for a keyset window.""" + with self._db.cursor() as cur: + if after_key is None: + cur.execute( + 'SELECT id, chat_id, sender_id, text, created_at FROM messages ' + 'WHERE chat_id = %s ' + 'ORDER BY created_at, id ' + 'LIMIT %s', (str(chat_id), limit + 1)) + else: + stamp, identifier = after_key + cur.execute( + 'SELECT id, chat_id, sender_id, text, created_at FROM messages ' + 'WHERE chat_id = %s ' + 'AND (created_at, id) > (%s, %s) ' + 'ORDER BY created_at, id ' + 'LIMIT %s', + (str(chat_id), stamp, str(identifier), limit + 1)) + rows = cur.fetchall() + has_more = len(rows) > limit + items = [self._public_message(dict(row)) for row in rows[:limit]] + return items, has_more + + def page_chats(self, user_id, after_key, limit): + """Return (chats visible to the user, has_more) ordered by (created_at, id).""" + with self._db.cursor() as cur: + if after_key is None: + cur.execute( + 'SELECT c.id, c.title, c.created_at FROM chats c ' + 'JOIN chat_members cm ON cm.chat_id = c.id ' + 'WHERE cm.user_id = %s ' + 'ORDER BY c.created_at, c.id ' + 'LIMIT %s', (str(user_id), limit + 1)) + else: + stamp, identifier = after_key + cur.execute( + 'SELECT c.id, c.title, c.created_at FROM chats c ' + 'JOIN chat_members cm ON cm.chat_id = c.id ' + 'WHERE cm.user_id = %s ' + 'AND (c.created_at, c.id) > (%s, %s) ' + 'ORDER BY c.created_at, c.id ' + 'LIMIT %s', + (str(user_id), stamp, str(identifier), limit + 1)) + rows = cur.fetchall() + has_more = len(rows) > limit + items = [self._public_chat(dict(row)) for row in rows[:limit]] + return items, has_more + + # ----------------------------------------------------------- projection + def chat(self, chat_id): + with self._db.cursor() as cur: + cur.execute( + 'SELECT id, title, created_at FROM chats WHERE id = %s', (str(chat_id),)) + row = cur.fetchone() + if row is None: + return None + chat = self._public_chat(dict(row)) + cur.execute( + 'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),)) + chat['member_ids'] = [str(r['user_id']) for r in cur.fetchall()] + return chat + + @staticmethod + def is_member(chat, user_id): + return str(user_id) in chat['member_ids'] + + @staticmethod + def _public_user(row): + return { + 'id': str(row['id']), + 'username': row['username'], + 'display_name': row['display_name'], + 'created_at': rfc3339(row['created_at']), + } + + @staticmethod + def _public_chat(row): + return { + 'id': str(row['id']), + 'title': row['title'], + 'member_ids': [], # filled by API when needed + 'created_at': rfc3339(row['created_at']), + } + + @staticmethod + def _public_message(row): + return { + 'id': str(row['id']), + 'chat_id': str(row['chat_id']), + 'sender_id': str(row['sender_id']), + 'text': row['text'], + 'created_at': rfc3339(row['created_at']), + } + + # --------------------------------------------------------------- sessions + def create_session(self, token_hash, user_id, expires_at): + with self._db.cursor() as cur: + cur.execute( + 'INSERT INTO sessions (token_hash, user_id, expires_at, created_at) ' + 'VALUES (%s, %s, %s, %s)', + (token_hash, str(user_id), expires_at, utc_now())) + ttl = max(1, int((expires_at - utc_now()).total_seconds())) + self._cache.set(token_hash, str(user_id), ttl) + + def get_session(self, token_hash): + """Return user_id for a valid non-expired session, or None. + + The cache is consulted first, but every cache hit is re-validated + against PostgreSQL so that a revoked or expired session is never + trusted from a stale Redis key. On miss, read PostgreSQL and + backfill the cache. Expired sessions found during lookup are deleted. + """ + cached = self._cache.get(token_hash) + if cached is not None: + with self._db.cursor() as cur: + cur.execute( + 'SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > %s', + (token_hash, utc_now())) + if cur.fetchone() is None: + self._cache.delete(token_hash) + return None + return cached + with self._db.cursor() as cur: + cur.execute( + 'SELECT user_id, expires_at FROM sessions WHERE token_hash = %s', + (token_hash,)) + row = cur.fetchone() + if row is None: + return None + if row['expires_at'] <= utc_now(): + cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,)) + return None + user_id = str(row['user_id']) + ttl = max(1, int((row['expires_at'] - utc_now()).total_seconds())) + self._cache.set(token_hash, user_id, ttl) + return user_id + + def delete_session(self, token_hash): + self._cache.delete(token_hash) + with self._db.cursor() as cur: + cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,)) + + def cleanup_expired_sessions(self): + with self._db.cursor() as cur: + cur.execute('DELETE FROM sessions WHERE expires_at <= %s', (utc_now(),)) diff --git a/src/messenger/validation.py b/src/messenger/validation.py new file mode 100644 index 00000000..817fd96e --- /dev/null +++ b/src/messenger/validation.py @@ -0,0 +1,125 @@ +"""Strict request payload validation; any deviation is 400 invalid_request.""" +from __future__ import annotations + +import re +import uuid + +from .errors import InvalidRequest + +USERNAME_PATTERN = re.compile(r'[a-z0-9_]{3,32}') +DIGITS_PATTERN = re.compile(r'[0-9]+') + + +def safe_label(value): + """Render untrusted text for an error message without breaking its encoding. + + Field names come straight from the client JSON, so they may contain the same + lone surrogates that the payload itself is rejected for. + """ + text = value if isinstance(value, str) else repr(value) + return text.encode('utf-8', 'replace').decode('utf-8') + + +def require_encodable(value, name): + """Reject text that cannot survive a UTF-8 round trip. + + `json.loads` happily produces lone surrogates from `\ud800` escapes. Such a + string cannot be UTF-8 encoded, so it must be rejected before it is stored; + otherwise the write is committed and the response encoding fails afterwards. + """ + try: + value.encode('utf-8') + except UnicodeEncodeError: + raise InvalidRequest( + 'Поле %s содержит недопустимые символы Unicode' % name) from None + return value + + +def require_object(payload): + if not isinstance(payload, dict): + raise InvalidRequest('Тело запроса должно быть JSON-объектом') + return payload + + +def require_fields(payload, required, allowed): + """Reject missing required fields and any unknown field.""" + missing = [name for name in required if name not in payload] + if missing: + raise InvalidRequest('Отсутствуют обязательные поля: ' + ', '.join(sorted(missing))) + extra = [name for name in payload if name not in allowed] + if extra: + raise InvalidRequest('Неизвестные поля: ' + ', '.join(sorted(safe_label(n) for n in extra))) + return payload + + +def require_text_field(payload, name, minimum, maximum, strip_check=True): + value = payload.get(name) + if not isinstance(value, str): + raise InvalidRequest('Поле %s должно быть строкой' % name) + require_encodable(value, name) + if strip_check and not value.strip(): + raise InvalidRequest('Поле %s не может быть пустым или состоять из пробелов' % name) + if len(value) < minimum: + raise InvalidRequest('Поле %s короче %d символов' % (name, minimum)) + if len(value) > maximum: + raise InvalidRequest('Поле %s длиннее %d символов' % (name, maximum)) + return value + + +def require_username(payload): + username = payload.get('username') + # fullmatch, not match: a trailing "$" would also accept a final newline. + if not isinstance(username, str) or USERNAME_PATTERN.fullmatch(username) is None: + raise InvalidRequest('username должен соответствовать ^[a-z0-9_]{3,32}$') + return require_encodable(username, 'username') + + +def require_uuid_string(value, name): + if not isinstance(value, str): + raise InvalidRequest('Поле %s должно быть строкой UUID' % name) + try: + return str(uuid.UUID(value)) + except ValueError: + raise InvalidRequest('Поле %s не является корректным UUID' % name) from None + + +def require_member_ids(payload, maximum, minimum=1): + value = payload.get('member_ids') + if not isinstance(value, list): + raise InvalidRequest('member_ids должен быть массивом UUID') + if not minimum <= len(value) <= maximum: + raise InvalidRequest('member_ids должен содержать от %d до %d участников' % (minimum, maximum)) + identifiers = [require_uuid_string(item, 'member_ids') for item in value] + if len(set(identifiers)) != len(identifiers): + raise InvalidRequest('member_ids содержит повторяющихся участников') + return identifiers + + +def parse_limit(raw, default, maximum): + """Parse a limit query value strictly: absent means default, anything else must be a plain integer. + + An empty value is a client error rather than an absent parameter, and the + digit run is length-bounded before int() because Python refuses to convert + strings longer than its integer-string limit, which would surface as a 500. + """ + if raw is None: + return default + if not DIGITS_PATTERN.fullmatch(raw): + raise InvalidRequest('limit должен быть целым числом от 1 до %d' % maximum) + # int() refuses absurdly long digit strings (ValueError -> 500). Comparing the + # significant digits against the maximum first rejects them without converting. + significant = raw.lstrip('0') or '0' + if len(significant) > len(str(maximum)): + raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum) + value = int(significant) + if not 1 <= value <= maximum: + raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum) + return value + + +def parse_cursor(raw): + if raw is None: + return None + if not isinstance(raw, str) or not raw: + raise InvalidRequest('cursor должен быть непустой строкой') + return raw diff --git a/student-tests/concurrency.py b/student-tests/concurrency.py new file mode 100644 index 00000000..e879688e --- /dev/null +++ b/student-tests/concurrency.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Concurrency evidence for level 4: N clients write M messages in parallel. + +Command (see REPORT.md and `make concurrency`): + python3 student-tests/concurrency.py --clients 10 --messages-per-client 10 \ + --output evidence/concurrency.json + +Expectation: every acknowledged message is stored exactly once, the full history +contains exactly clients*messages_per_client unique messages and the order over +pages is strictly increasing by (created_at, id). No retries are hidden: any +failed write is reported and fails the run. +""" +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import json +from pathlib import Path +import sys +import threading +import time +import uuid +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) + +from client import Client, Failure, LAB, arguments, check, timestamp # noqa: E402 + +MESSAGES_PATH = '/api/v1/chats/%s/messages' + + +def build_fixture(base_url, ca_file, clients): + """One user per client plus a single chat everybody belongs to.""" + master = Client(base_url, ca_file, timeout=15) + master.call('GET', '/health/ready', schema='Health') + actors = [master.user('conc%d' % index) for index in range(clients)] + chat = master.call('POST', '/api/v1/chats', 201, actor=actors[0], + body={'title': 'Конкурентная запись', + 'member_ids': [actor['id'] for actor in actors]}, + schema='Chat')['json'] + check(set(chat['member_ids']) == {actor['id'] for actor in actors}, + 'chat membership mismatch') + return master, actors, chat + + +def write_phase(base_url, ca_file, actors, chat, messages_per_client, prefix): + """Each client writes its own slice of messages, all starting at once.""" + barrier = threading.Barrier(len(actors)) + + def worker(index): + client = Client(base_url, ca_file, timeout=15) + actor = actors[index] + barrier.wait() + acknowledged, errors = [], [] + started = time.monotonic() + for message_index in range(messages_per_client): + text = '%s-%03d-%03d' % (prefix, index, message_index) + try: + message = client.message(actor, chat, text)['json'] + acknowledged.append((text, message['id'])) + except Failure as exc: + errors.append('%s: %s' % (text, exc)) + return {'client_index': index, 'acknowledged': acknowledged, 'errors': errors, + 'duration_seconds': round(time.monotonic() - started, 3)} + + started = time.monotonic() + with ThreadPoolExecutor(max_workers=len(actors)) as pool: + results = list(pool.map(worker, range(len(actors)))) + return results, round(time.monotonic() - started, 3) + + +def read_whole_history(client, actor, chat): + items, cursor, pages = [], None, 0 + while True: + query = '?limit=100' + ('' if cursor is None else '&cursor=' + quote(cursor, safe='')) + page = client.call('GET', MESSAGES_PATH % chat['id'] + query, actor=actor, + schema='MessagePage')['json'] + items.extend(page['items']) + pages += 1 + cursor = page['next_cursor'] + check(pages < 100, 'pagination did not terminate') + if cursor is None: + return items, pages + + +def main(): + def configure(parser): + parser.add_argument('--clients', type=int, default=10) + parser.add_argument('--messages-per-client', type=int, default=10) + parser.add_argument('--output', default=None, + help='Optional JSON report path, no session data inside') + + args = arguments('Concurrent writes from independent HTTP clients', configure) + check(1 <= args.clients <= 64, 'clients must be 1..64') + check(1 <= args.messages_per_client <= 200, 'messages-per-client must be 1..200') + expected = args.clients * args.messages_per_client + prefix = 'conc-' + uuid.uuid4().hex[:8] + + master, actors, chat = build_fixture(args.base_url, args.ca_file, args.clients) + results, elapsed = write_phase(args.base_url, args.ca_file, actors, chat, + args.messages_per_client, prefix) + acknowledged = [pair for result in results for pair in result['acknowledged']] + errors = [error for result in results for error in result['errors']] + history, pages = read_whole_history(master, actors[0], chat) + + stored_texts = [message['text'] for message in history] + stored_ids = [message['id'] for message in history] + expected_texts = {'%s-%03d-%03d' % (prefix, client, index) + for client in range(args.clients) + for index in range(args.messages_per_client)} + missing = sorted(expected_texts.difference(stored_texts)) + keys = [(timestamp(message['created_at']), uuid.UUID(message['id']).int) + for message in history] + ordered_correctly = all(keys[index] < keys[index + 1] for index in range(len(keys) - 1)) + + report = { + 'lab': LAB, + 'workload': 'closed-loop parallel POST messages from independent clients', + 'clients': args.clients, + 'messages_per_client': args.messages_per_client, + 'expected_messages': expected, + 'acknowledged_201': len(acknowledged), + 'stored_in_history': len(history), + 'unique_stored_ids': len(set(stored_ids)), + 'unique_stored_texts': len(set(stored_texts)), + 'duplicates': len(stored_ids) - len(set(stored_ids)), + 'missing_texts': missing, + 'failed_writes': len(errors), + 'write_errors': errors[:10], + 'history_pages': pages, + 'history_order_strictly_increasing': ordered_correctly, + 'writers_elapsed_seconds': elapsed, + 'successful_rps': round(len(acknowledged) / elapsed, 3) if elapsed else None, + 'per_client': [{'client_index': result['client_index'], + 'acknowledged': len(result['acknowledged']), + 'errors': len(result['errors']), + 'duration_seconds': result['duration_seconds']} + for result in sorted(results, key=lambda item: item['client_index'])], + 'note': ('Client-observed timings: closed-loop, single Docker host, no retries. ' + 'Latency percentiles are intentionally absent here (see make load).'), + } + text = json.dumps(report, ensure_ascii=False, indent=2) + if args.output: + destination = Path(args.output) + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(text + '\n') + print(text) + check(not errors, 'some parallel writes failed') + check(len(acknowledged) == expected, 'not every write was acknowledged') + check(len(history) == expected, 'history holds %d messages instead of %d' + % (len(history), expected)) + check(not missing, 'messages missing from the history') + check(len(set(stored_ids)) == expected, 'duplicate message ids in the history') + check(ordered_correctly, 'history is not strictly ordered by (created_at, id)') + print('PASS: %d parallel messages from %d clients stored exactly once' % + (expected, args.clients)) + + +if __name__ == '__main__': + try: + main() + except (Failure, ValueError, OSError) as exc: + print('FAIL:', exc) + raise SystemExit(1) diff --git a/student-tests/run.py b/student-tests/run.py new file mode 100644 index 00000000..489c9e90 --- /dev/null +++ b/student-tests/run.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +"""Student-owned lab2 tests: sessions, validation, pagination. + +One command: make student-test +Direct call: python3 student-tests/run.py --base-url http://localhost:8080 + +The suite complements tests/smoke.py and never weakens it. Every run creates +synthetic users, chats and messages only; no public check is imported or patched. +""" +from __future__ import annotations + +import base64 +import datetime as dt +import http.client +import json +import sys +import urllib.request +import uuid +from pathlib import Path +from urllib.parse import quote, urlsplit + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) + +from client import Client, Failure, LAB, arguments, check, timestamp # noqa: E402 + +if LAB != 2: + print('FAIL: student tests target lab2, but lab.json declares lab%d' % LAB) + raise SystemExit(2) + +CHAT_PATH = '/api/v1/chats/%s' +MESSAGES_PATH = CHAT_PATH + '/messages' +MAX_PAGES = 200 + +TESTS = [] + + +def test(func): + TESTS.append(func) + return func + + +class Context: + """Shared synthetic fixtures: three users, one chat between two of them.""" + + def __init__(self, client): + self.client = client + self.alice, self.bob, self.eve, self.chat = client.bootstrap() + + def new_chat(self, title, members): + return self.client.call('POST', '/api/v1/chats', 201, actor=members[0], + body={'title': title, + 'member_ids': [member['id'] for member in members]}, + schema='Chat')['json'] + + def send(self, actor, chat, text): + return self.client.call('POST', MESSAGES_PATH % chat['id'], 201, actor=actor, + body={'text': text}, schema='Message')['json'] + + def page(self, actor, chat, limit, cursor=None): + query = '?limit=%d' % limit + if cursor is not None: + query += '&cursor=' + quote(cursor, safe='') + return self.client.call('GET', MESSAGES_PATH % chat['id'] + query, actor=actor, + schema='MessagePage')['json'] + + def traverse(self, actor, chat, limits): + """Walk the whole history, changing the page size between requests.""" + items, cursor, pages = [], None, 0 + while True: + limit = limits[min(pages, len(limits) - 1)] + page = self.page(actor, chat, limit, cursor) + items.extend(page['items']) + cursor = page['next_cursor'] + pages += 1 + check(pages < MAX_PAGES, 'pagination did not terminate') + if cursor is None: + break + return items, pages + + +def ordered(records): + """Contract order: (created_at, id) ascending, exactly as the client sorts.""" + return sorted(records, key=lambda item: (timestamp(item['created_at']), + uuid.UUID(item['id']).int)) + + +def traversal_order(records): + return [item['id'] for item in records] + + +# --------------------------------------------------------------------- health +@test +def health_and_request_id(ctx): + for path in ('/health/live', '/health/ready'): + result = ctx.client.call('GET', path, schema='Health') + check(result['json'] == {'status': 'ok'}, + '%s must return exactly {"status":"ok"}' % path) + check(len(result['headers']['X-Request-Id']) >= 8, 'request id looks too short') + marker = 'student-' + uuid.uuid4().hex + echoed = ctx.client.call('GET', '/health/live', headers={'X-Request-Id': marker}) + check(echoed['headers']['X-Request-Id'] == marker, 'client request id was not echoed') + error = ctx.client.call('GET', '/api/v1/users/me', 401, headers={'X-Request-Id': marker}) + check(error['headers']['X-Request-Id'] == marker, 'error response lost the request id') + check(error['json']['error']['request_id'] == marker, 'error body request_id mismatch') + + +# -------------------------------------------------------------- unknown route +@test +def unknown_route_and_method_are_json_404(ctx): + ctx.client.call('GET', '/api/v1/unknown', 404) + ctx.client.call('DELETE', '/api/v1/chats', 404, actor=ctx.alice) + ctx.client.call('GET', '/api/v1/chats/', 404, actor=ctx.alice) + + +# --------------------------------------------------------------- user payload +@test +def malformed_json_is_rejected(ctx): + json_headers = {'Content-Type': 'application/json'} + ctx.client.call('POST', '/api/v1/users', 400, raw=b'{"username": "broken",', headers=json_headers) + ctx.client.call('POST', '/api/v1/users', 400, raw=b'[1, 2, 3]', headers=json_headers) + ctx.client.call('POST', '/api/v1/users', 400, raw=b'"just a string"', headers=json_headers) + ctx.client.call('POST', '/api/v1/users', 400, raw=b'', headers=json_headers) + ctx.client.call('POST', '/api/v1/users', 415, raw=b'{"username": "x"}', + headers={'Content-Type': 'text/plain'}) + ctx.client.call('POST', '/api/v1/users', 415, raw=b'{"username": "x"}', + headers={'Content-Type': 'application/x-www-form-urlencoded'}) + + +@test +def duplicate_username_is_409(ctx): + name = 'dup_' + uuid.uuid4().hex[:12] + body = {'username': name, 'display_name': 'Дубликат', + 'password': 'T3st-' + uuid.uuid4().hex} + created = ctx.client.call('POST', '/api/v1/users', 201, body=body, schema='User')['json'] + check(created['username'] == body['username'], 'username changed on creation') + conflict = ctx.client.call('POST', '/api/v1/users', 409, body=body) + check(conflict['json']['error']['code'] == 'conflict', 'duplicate username must be conflict') + + +@test +def user_payload_validation(ctx): + name = 'valid_' + uuid.uuid4().hex[:8] + for bad in ( + {'username': name, 'display_name': 'Имя', 'password': 'ok', 'extra': 1}, + {'display_name': 'Имя', 'password': 'ok'}, + {'username': name, 'password': 'ok'}, + {'username': name, 'display_name': ' ', 'password': 'ok'}, + {'username': name, 'display_name': '', 'password': 'ok'}, + {'username': name, 'display_name': 42, 'password': 'ok'}, + {'username': name, 'display_name': None, 'password': 'ok'}, + {'username': name, 'display_name': 'д' * 101, 'password': 'ok'}, + {'username': 'AB', 'display_name': 'Имя', 'password': 'ok'}, + {'username': 'with space', 'display_name': 'Имя', 'password': 'ok'}, + {'username': 'юзер123', 'display_name': 'Имя', 'password': 'ok'}, + {'username': 'x' * 33, 'display_name': 'Имя', 'password': 'ok'}, + {'username': '', 'display_name': 'Имя', 'password': 'ok'}, + {'username': 123, 'display_name': 'Имя', 'password': 'ok'}, + # password too short + {'username': name, 'display_name': 'Имя', 'password': 'short'}, + # password too long + {'username': name, 'display_name': 'Имя', 'password': 'д' * 129}, + ): + ctx.client.call('POST', '/api/v1/users', 400, body=bad) + ok = ctx.client.call('POST', '/api/v1/users', 201, + body={'username': name, 'display_name': 'Студент Юникод 👋', + 'password': 'T3st-' + uuid.uuid4().hex}, + schema='User')['json'] + check(ok['display_name'] == 'Студент Юникод 👋', 'unicode display_name changed') + check('password' not in ok and 'token' not in ok, 'user projection leaks credentials') + + +# ------------------------------------------------------------- sessions +@test +def basic_login_issues_bearer_session(ctx): + reg = ctx.alice['registration'] + credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode() + session = ctx.client.call('POST', '/api/v1/auth/sessions', 201, + headers={'Authorization': 'Basic ' + credential}, + schema='Session')['json'] + check(session['user'] == ctx.alice['user'], 'session user mismatch') + check(session['token_type'] == 'Bearer', 'token type must be Bearer') + check(len(session['token']) >= 32, 'token must be opaque and long enough') + check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc), + 'session already expired') + ctx.alice['token'] = session['token'] + + # wrong password -> 401 + bad = base64.b64encode((reg['username'] + ':wrong-password').encode()).decode() + ctx.client.call('POST', '/api/v1/auth/sessions', 401, + headers={'Authorization': 'Basic ' + bad}) + # unknown user -> 401 + bad = base64.b64encode(('nonexistent-user:' + reg['password']).encode()).decode() + ctx.client.call('POST', '/api/v1/auth/sessions', 401, + headers={'Authorization': 'Basic ' + bad}) + + +@test +def bearer_required_for_business_endpoints(ctx): + # no auth -> 401 + ctx.client.call('GET', '/api/v1/users/me', 401) + # Basic on business endpoint -> 401 + reg = ctx.alice['registration'] + credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode() + ctx.client.call('GET', '/api/v1/users/me', 401, + headers={'Authorization': 'Basic ' + credential}) + # X-User-Id ignored -> 401 + ctx.client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': ctx.alice['id']}) + # Bearer works + me = ctx.client.call('GET', '/api/v1/users/me', 200, actor=ctx.alice, + schema='User')['json'] + check(me == ctx.alice['user'], 'current user mismatch') + + +@test +def logout_revokes_session(ctx): + reg = ctx.alice['registration'] + credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode() + session = ctx.client.call('POST', '/api/v1/auth/sessions', 201, + headers={'Authorization': 'Basic ' + credential}, + schema='Session')['json'] + token = session['token'] + ctx.client.call('DELETE', '/api/v1/auth/sessions/current', 204, + headers={'Authorization': 'Bearer ' + token}) + # revoked token -> 401 + ctx.client.call('GET', '/api/v1/users/me', 401, + headers={'Authorization': 'Bearer ' + token}) + ctx.client.call('DELETE', '/api/v1/auth/sessions/current', 401, + headers={'Authorization': 'Bearer ' + token}) + + +# -------------------------------------------------------------- message rules +@test +def foreign_chat_is_invisible(ctx): + other = ctx.new_chat('Чат без Евы', [ctx.alice, ctx.bob]) + ctx.client.call('GET', CHAT_PATH % other['id'], 404, actor=ctx.eve) + ctx.client.call('GET', MESSAGES_PATH % other['id'], 404, actor=ctx.eve) + ctx.client.call('POST', MESSAGES_PATH % other['id'], 404, actor=ctx.eve, + body={'text': 'чужое сообщение'}) + listed = ctx.client.call('GET', '/api/v1/chats?limit=100', actor=ctx.eve, + schema='ChatPage')['json'] + check(all(item['id'] != other['id'] for item in listed['items']), + 'foreign chat leaked into the list of another user') + member_view = ctx.client.call('GET', '/api/v1/chats?limit=100', actor=ctx.bob, + schema='ChatPage')['json'] + check(any(item['id'] == other['id'] for item in member_view['items']), + 'member cannot see the own chat') + ctx.client.call('GET', CHAT_PATH % uuid.uuid4(), 404, actor=ctx.alice) + ctx.client.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=ctx.alice) + ctx.client.call('GET', MESSAGES_PATH % 'not-a-uuid', 400, actor=ctx.alice) + + +@test +def message_payload_validation(ctx): + """Empty and whitespace-only text is rejected.""" + for bad in ( + {'text': ''}, + {'text': ' '}, + {'text': '\u00a0\t\n'}, + {}, + {'text': 123}, + {'text': None}, + {'text': []}, + {'text': 'ок', 'sender_id': ctx.bob['id']}, + {'text': 'д' * 4001}, + ): + ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 400, actor=ctx.alice, body=bad) + ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 401, body={'text': 'ок'}) + ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 400, actor=ctx.alice, raw=b'{"text":}', + headers={'Content-Type': 'application/json'}) + ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 415, actor=ctx.alice, + raw=b'{"text":"ok"}', headers={'Content-Type': 'application/xml'}) + + +@test +def text_limits_and_unicode_roundtrip(ctx): + longest = 'д' * 3999 + '👋' + check(len(longest) == 4000, 'fixture must be exactly 4000 code points') + created = ctx.send(ctx.bob, ctx.chat, longest) + check(created['text'] == longest, '4000 code points text changed') + check(created['sender_id'] == ctx.bob['id'], 'sender must be derived from actor') + mixed = 'Привет 👋🏽 \u00a0 e\u0301 — астральные 🚀🧪 и кавычки «»' + echo = ctx.send(ctx.alice, ctx.chat, mixed) + check(echo['text'] == mixed, 'mixed unicode text changed in transit') + check(echo['chat_id'] == ctx.chat['id'], 'message bound to the wrong chat') + + +# ---------------------------------------------------------------- pagination +@test +def multi_page_traversal_is_exact_and_stable(ctx): + chat = ctx.new_chat('Пагинация', [ctx.bob, ctx.alice]) + sent = [] + for index in range(7): + actor = ctx.alice if index % 2 == 0 else ctx.bob + sent.append(ctx.send(actor, chat, 'страница-%d-👋' % index)) + expected = traversal_order(ordered(sent)) + walked, pages = ctx.traverse(ctx.bob, chat, [2]) + check(pages >= 4, 'seven messages with limit=2 must take at least four pages') + check(traversal_order(walked) == expected, 'traversal lost, duplicated or reordered messages') + check(len(set(traversal_order(walked))) == len(walked), 'traversal returned duplicates') + switched, _ = ctx.traverse(ctx.alice, chat, [2, 3, 1]) + check(traversal_order(switched) == expected, 'changing limit mid-traversal changed the order') + singles, _ = ctx.traverse(ctx.alice, chat, [1]) + check(len(singles) == 7, 'limit=1 traversal must return all seven messages') + first = ctx.page(ctx.alice, chat, 2) + check(len(first['items']) == 2 and first['next_cursor'], 'first page shape is wrong') + check(first['items'][0]['id'] == expected[0], 'first page must start with the oldest message') + + +@test +def default_page_limit_is_fifty(ctx): + chat = ctx.new_chat('Предел страницы по умолчанию', [ctx.alice, ctx.bob]) + for index in range(51): + ctx.send(ctx.alice, chat, 'default-limit-%d' % index) + page = ctx.client.call('GET', MESSAGES_PATH % chat['id'], actor=ctx.alice, + schema='MessagePage')['json'] + check(len(page['items']) == 50, 'default limit must be 50, got %d' % len(page['items'])) + check(page['next_cursor'], 'a page that is not the last one needs a cursor') + tail = ctx.page(ctx.alice, chat, 100, page['next_cursor']) + check(len(tail['items']) == 1, 'the remaining page must hold exactly one message') + check(tail['next_cursor'] is None, 'last page must have next_cursor null') + merged = traversal_order(page['items']) + traversal_order(tail['items']) + check(len(set(merged)) == 51, 'page boundaries duplicated or dropped messages') + + +def main(): + args = arguments('Student-owned lab2 tests: sessions, validation, pagination') + client = Client(args.base_url, args.ca_file) + ctx = Context(client) + print('student-tests: %d cases against %s' % (len(TESTS), client.base)) + failures = [] + for func in TESTS: + try: + func(ctx) + except (Failure, ValueError, OSError) as exc: + failures.append(func.__name__) + print('FAIL %-42s %s' % (func.__name__, exc)) + else: + print('PASS %-42s' % func.__name__) + print('%d passed, %d failed' % (len(TESTS) - len(failures), len(failures))) + return 1 if failures else 0 + + +if __name__ == '__main__': + raise SystemExit(main())