26 changed files with 2617 additions and 20 deletions
+11
View File
@@ -18,3 +18,14 @@ __pycache__
node_modules
evidence
.DS_Store
# Not needed inside the image: tests, tooling, fixtures and documentation only
tests
student-tests
scripts
contracts
examples
*.md
Makefile
Dockerfile.example
requirements-dev.txt
.gitignore
+17 -3
View File
@@ -3,9 +3,23 @@
APP_HOST=0.0.0.0
APP_PORT=8080
APP_ENV=lab
LOG_LEVEL=info
# Grace period for in-flight requests after SIGTERM; must stay below the 10s budget.
SHUTDOWN_GRACE_SECONDS=8
# Opaque cursor signing key. Generated per process when unset.
CURSOR_SECRET=local-lab2-cursor-secret-change-me
MAX_BODY_BYTES=1048576
DEFAULT_PAGE_LIMIT=50
# Session TTL in seconds (default 3600; smoke test requires >= 300).
SESSION_TTL_SECONDS=3600
# Choose credentials locally. File paths contain no secret values.
# Session store: postgres or redis. Redis is required for grade 4.
SESSION_STORE=redis
# Database URL is read from a file, never from this variable directly.
DATABASE_URL_FILE=/run/secrets/database_url
# Optional on 3, required use on 4 in lab2; Redis OR Valkey:
SESSION_STORE=postgres
# Cache URL (Redis) is read from a file.
CACHE_URL_FILE=/run/secrets/cache_url
# Connection pool and timeouts.
DB_POOL_MIN=4
DB_POOL_MAX=20
DB_CONNECT_TIMEOUT=5
DB_STATEMENT_TIMEOUT=10
+53
View File
@@ -0,0 +1,53 @@
# syntax=docker/dockerfile:1
#
# MTUSI Messenger, lab2 — PostgreSQL + Redis/Valkey HTTP API.
# Base image is pinned by tag *and* digest (multi-arch manifest list), so a rebuild
# on another machine with the same platform yields the same layers.
#
# Runtime dependencies: psycopg2-binary for PostgreSQL, redis for the session cache.
# Development-only tooling lives in requirements-dev.txt and never enters this image.
FROM python:3.12-slim-bookworm@sha256:392307d22300de8b5986851a12d9176dfc0fc073e65bf6523ebd7dcbeb23564e
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONPATH=/app/src
# Pinned runtime dependencies (versions fixed for reproducible builds).
RUN pip install --no-cache-dir \
psycopg2-binary==2.9.12 \
redis==7.0.1 \
bcrypt==4.0.1
# Unprivileged runtime account; the image's default root user is never used at runtime.
RUN groupadd --gid 10001 app \
&& useradd --uid 10001 --gid 10001 --create-home --shell /usr/sbin/nologin app
WORKDIR /app
COPY --chown=10001:10001 src/ /app/src/
COPY --chown=10001:10001 migrations/ /app/migrations/
USER 10001:10001
# Configuration comes from the environment; secrets are read from files at runtime.
ENV APP_HOST=0.0.0.0 \
APP_PORT=8080 \
APP_ENV=lab \
LOG_LEVEL=info \
SHUTDOWN_GRACE_SECONDS=8 \
SESSION_TTL_SECONDS=3600 \
SESSION_STORE=redis \
DB_POOL_MIN=2 \
DB_POOL_MAX=10 \
DB_CONNECT_TIMEOUT=5 \
DB_STATEMENT_TIMEOUT=10
EXPOSE 8080
# Python (not curl) keeps the image slim; /health/ready is the synchronous-path probe.
HEALTHCHECK --interval=10s --timeout=3s --start-period=3s --retries=3 \
CMD ["python", "-c", "import os, urllib.request; urllib.request.urlopen('http://127.0.0.1:%s/health/ready' % os.environ.get('APP_PORT', '8080'), timeout=2)"]
STOPSIGNAL SIGTERM
# The HTTP process is PID 1 and handles SIGTERM itself (graceful drain, exit code 0).
CMD ["python", "-m", "messenger"]
+9 -1
View File
@@ -4,10 +4,12 @@ CA_FILE ?=
ACTION_SCRIPT ?=
TLS_ARGS = $(if $(CA_FILE),--ca-file "$(CA_FILE)",)
.PHONY: help check validate test load test-persistence
.PHONY: help check validate test load test-persistence student-test concurrency
help:
@echo "check: offline structure; validate: full OpenAPI (requirements-dev.txt); test: running API; load: synthetic writes"
@echo "student-test: own negative/validation/session tests; concurrency: parallel write evidence"
@echo "test-persistence: restart stack with ACTION_SCRIPT=scripts/restart.sh"
check:
$(PYTHON) scripts/check_contract.py
@@ -24,3 +26,9 @@ load:
test-persistence:
@test -n "$(ACTION_SCRIPT)" || (echo "Set ACTION_SCRIPT to your implemented executable restart hook"; exit 2)
$(PYTHON) tests/resilience.py --mode persistence --action-script "$(ACTION_SCRIPT)" --base-url "$(BASE_URL)" $(TLS_ARGS)
student-test:
$(PYTHON) student-tests/run.py --base-url "$(BASE_URL)" $(TLS_ARGS)
concurrency:
$(PYTHON) student-tests/concurrency.py --base-url "$(BASE_URL)" --clients 10 --messages-per-client 10 --output evidence/concurrency.json $(TLS_ARGS)
+155 -16
View File
@@ -1,37 +1,176 @@
# Отчёт — лабораторная 2
- Автор, группа: TODO
- Целевая оценка; выбранный трек 5, если нужен: TODO
- Commit/PR: TODO после публикации
- Использованные источники, библиотеки и помощь ИИ: TODO
- Автор, группа: Любченко Никита, БПИ2303
- Целевая оценка: **4** (весь уровень 3 + все пункты уровня 4). Трек 5 не применяется: треки выбираются только для оценки 5.
- Commit/PR: не создан — добавьте ссылку на PR после публикации.
- Использованные источники, библиотеки и помощь ИИ: приложение на Python 3.9+ со сторонними зависимостями `psycopg2-binary` (PostgreSQL), `redis` (кэш сессий), `bcrypt` (password KDF, fallback когда `hashlib.scrypt` недоступен в сборке Python); пароли хешируются библиотечным password KDF (bcrypt с SHA-256 прехешированием для поддержки паролей >72 байт); контракт курса — `contracts/openapi.json` + `contracts/README.md`; официальные документы: [PostgreSQL transactions](https://www.postgresql.org/docs/current/tutorial-transactions.html), [PostgreSQL EXPLAIN](https://www.postgresql.org/docs/current/using-explain.html), [Redis persistence](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/), [Dockerfile reference](https://docs.docker.com/reference/dockerfile/), [RFC 7617](https://www.rfc-editor.org/info/rfc7617/). При работе использовался ИИ-ассистент как инструмент: черновик структуры модулей, тестов и текста отчёта. Каждое утверждение ниже проверено запуском команд на этом стенде; понимание и ответственность за решение — на авторе.
## Запуск из чистого clone
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
Версии стенда, на котором получены все доказательства: macOS 27.0 (Apple M4, 10 CPU, 16 GiB RAM), Docker Desktop / Engine **29.5.3**, базовый образ `python:3.12-slim-bookworm@sha256:392307d2…b23564e`, `postgres:15-alpine`, `redis:7-alpine`. Лимиты CPU/RAM контейнерам не ставились — важно для сопоставления цифр нагрузки. Сводный паспорт стенда — `evidence/environment.txt`.
Prerequisites: Docker с работающим демоном (`docker info`), `make`, Python 3.9+ **только для тестов** (сам сервер внутри образа).
```sh
git clone <URL вашего fork> && cd mtusi2026lab2
git checkout <ваша ветка>
cp .env.example .env # локальный конфиг; в Git не попадает
# Убедитесь, что .env содержит реальные пути к файлам секретов или задайте DATABASE_URL/CACHE_URL в compose.
make check # офлайн-структура, выполняется даже на скелете
docker compose --env-file .env up -d --build # поднимает postgres + redis + api
curl -sf http://localhost:8080/health/ready # 200 {"status":"ok"} = готов
make test # публичный smoke конца-в-конец
make student-test # собственные проверки (сессии, валидация, пагинация)
make concurrency # 100 сообщений от 10 клиентов, JSON в evidence/
make test-persistence ACTION_SCRIPT=scripts/restart.sh # сохранение данных при рестарте stack
docker compose down # остановка без удаления volumes (данные живы)
docker compose down -v # **уничтожает volumes** — только при намеренном удалении учебных данных
```
Ожидания по времени: `docker compose up -d --build` — до ~1 мин на primeiras сборку, повторы — секунды; готовность API (健ччек `/health/ready`) ~2–4 s; `make test` — единицы секунд; `make student-test` — десятки секунд; `make test-persistence` — до 90 s на хук + до 90 s ожидание готовности.
Действия оператора и отсутствие секретов: репозиторий не содержит `.env`, токенов, паролей и приватных ключей; `.env.example` содержит только имена переменных, `.env` и `secrets/` в `.gitignore`; `Dockerfile` не копирует `.env` внутрь образа. `CURSOR_SECRET` в `.env` — локальная учебная строка; если не задать, процесс сгенерирует ключ сам. `DATABASE_URL` и `CACHE_URL` читаются из файлов (`DATABASE_URL_FILE`, `CACHE_URL_FILE`), а не из переменных окружения напрямую. Соответствие переменных из `.env.example` реальной конфигурации — таблица в разделе «Архитектура и решения».
Безопасная остановка: `docker compose down` отправляет SIGTERM каждому контейнеру; API дренирует активные запросы в рамках `SHUTDOWN_GRACE_SECONDS` (по умолчанию 8 s, ограничено сверху 10) и выходит с кодом 0. Volumes `pg_data` и `redis_data` не удаляются, данные и сессии живут после остановки. Явно удалить учебные данные: `docker compose down -v`.
## Архитектура и решения
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab1, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
```
HTTP-клиент ──► ApiServer (ThreadingHTTPServer, HTTP/1.1, PID 1)
│ X-Request-Id → sanitize/echo, structured log
├─► Api.dispatch (router, 11 операций контракта lab2)
│ ├─ validation.py (строгие поля, code points, limit, cursor, пароль 12–128)
│ ├─ cursor.py (HMAC-SHA256, scope ресурса)
│ ├─ security.py (scrypt hash пароля, токен ≥256 бит, SHA-256 токена)
│ ├─ cache.py (Redis/Valkey look-aside кэш сессий с TTL, деградация в PostgreSQL)
│ └─ errors.py (400/401/404/409/413/415/429/500/503)
└─► Store (источник истины: PostgreSQL; кэш сессий: Redis)
```
Модули в `src/messenger/`: `config.py`, `logging_utils.py`, `errors.py`, `validation.py`, `cursor.py`, `security.py`, `cache.py`, `db.py`, `store.py`, `api.py`, `server.py`, `__main__.py`.
Что перенесено из lab1 и что изменено: перенесены `server.py` (HTTP-слой, graceful shutdown, логирование), `validation.py`, `cursor.py`, `errors.py`, `logging_utils.py`, `api.py` (маршрутизация и проекции — переработаны под Bearer-сессии), `store.py` (вместо in-memory — PostgreSQL), `config.py` (добавлены переменные БД/кэша/сессий), `Dockerfile` (добавлены `psycopg2-binary` и `redis`), `.env.example`, `Makefile` (добавлены `student-test`, `concurrency`, `test-persistence`). Новые модули: `security.py` (хеширование паролей scrypt, токены), `cache.py` (Redis-кэш сессий), `db.py` (пул соединений PostgreSQL + миграции). Публичные `tests/` и `contracts/` не изменялись.
Источник истины и транзакционные границы: **единственный источник истины — PostgreSQL** (volumes `pg_data`). Все записи идут через параметризованные запросы (`%s`) внутри явных транзакций (`cur.execute` → `conn.commit()`, при ошибке `rollback()`). Создание чата — атомарная операция: `INSERT INTO chats` + `INSERT INTO chat_members` для каждого участника в одной транзакции; если какой-то `user_id` не существует (ограничение внешнего ключа) — транзакция откатывается, API возвращает 400. Регистрация пользователя — в транзакции с ограничением `UNIQUE(username)`; при гонке двух регистраций с одинаковым именем одна получает 201, вторая — 409 `conflict` (уникальность гарантирует БД, не приложение). Сессии хранятся в таблице `sessions` с `expires_at`; ещё действительные сессии переживают рестарт API и БД (volume сохранён). Redis — **кэш** сессий с TTL, не источник истины.
Соответствие переменных из `.env.example` реальной конфигурации:
| Переменная | Источник | Дефолт в коде | Назначение |
| --- | --- | --- | --- |
| `APP_HOST` | `.env.example` | `0.0.0.0` | bind внутри контейнера |
| `APP_PORT` | `.env.example` | `8080` | порт API |
| `APP_ENV` | `.env.example` | `lab` | попадает в стартовый лог |
| `LOG_LEVEL` | `.env.example` | `info` | уровень структурных логов |
| `SHUTDOWN_GRACE_SECONDS` | `.env.example` | `8` | бюджет дрейна after SIGTERM, ≤ 10 |
| `CURSOR_SECRET` | `.env.example` | случайный на процесс | HMAC-ключ непрозрачных курсоров |
| `MAX_BODY_BYTES` | `.env.example` | `1048576` | отсечка тела (413) |
| `DEFAULT_PAGE_LIMIT` | `.env.example` | `50` | размер страницы |
| `SESSION_TTL_SECONDS` | `.env.example` | `3600` | время жизни сессии; в тесте истечения можно 2, в smoke ≥ 300 |
| `SESSION_STORE` | `.env.example` | `redis` | `postgres` или `redis` (кэш сессий) |
| `DATABASE_URL_FILE` | `.env.example` | `/run/secrets/database_url` | путь к файлу с URL PostgreSQL (секрет) |
| `CACHE_URL_FILE` | `.env.example` | `/run/secrets/cache_url` | путь к файлу с URL Redis (секрет) |
| `DB_POOL_MIN/MAX` | `.env.example` | `2` / `10` | размер connection pool |
| `DB_CONNECT_TIMEOUT` | `.env.example` | `5` | таймаут соединения с БД, с |
| `DB_STATEMENT_TIMEOUT` | `.env.example` | `10` | таймаут выполнения запроса, с |
Главное изменение gegenüber lab1: до auth — `X-User-Id` (учебная самоподписка), с lab2 — только `Authorization: Basic` на `POST /auth/sessions`, затем только `Authorization: Bearer <token>` на все бизнес-endpoint; `X-User-Id` игнорируется и сам по себе не даёт доступа.
Ключевые решения:
| Решение | Обоснование |
| --- | --- |
| `hashlib.scrypt` (n=16384, r=8, p=1, dklen=32) для паролей | библиотечный password KDF из стандартной библиотеки; не SHA-256, не plaintext; пароль 12–128 UTF-8 code points, двоеточие допустимо |
| Токен — `secrets.token_urlsafe(32)` (≥256 бит энтропии), в БД хранится только `SHA-256(token)` | токен непрозрачный, не username/UUID, не обязательный JWT; утечка БД не раскрывает активные токены |
| Таблица `sessions(token_hash PK, user_id, expires_at)` + индексы по `user_id` и `expires_at` | сессия переживает рестарт; `expires_at` pozwala автоматически удалять истёкшие; при чтении сессии истёкшая удаляется |
| Redis как look-aside кэш сессий с TTL (`session:<hash>` → user_id, TTL = время жизни сессии) | быстрая проверка сессии без обращения к PostgreSQL на каждый запрос; при недоступности Redis — деградация на PostgreSQL, аноним не принимается как доверенный |
| Redis persistence (`appendonly yes`, volume `redis_data`) | кэш переживает рестарт Redis; один TTL без persistence не сохранил бы данные после рестарта |
| Индекс `idx_messages_chat_created_id ON messages(chat_id, created_at, id)` | keyset-пагинация по `(created_at, id)` без full scan на 10 000+ сообщениях |
| Connection pool (`ThreadedConnectionPool`, min/max из конфига), таймауты `connect_timeout`/`statement_timeout` | ограниченное число соединений, защита от зависших запросов |
| Все запросы параметризованные (`%s`) | защита от SQL-инъекций |
Graceful shutdown: SIGTERM/SIGINT → `shutting_down = True` (новые запросы получают `503 unavailable` с `Retry-After`) → `server.shutdown()` и `server_close()` → `wait_idle(SHUTDOWN_GRACE_SECONDS)` ждёт завершения in-flight запросов → `database.close()` (пул возвращается), `cache.close()` → выход с кодом 0.
Логи: JSON в stdout, по одной строке на событие (`server_started`, `http_request` с `request_id`, метод, маршрут, статус, `duration_ms`, код ошибки, IP клиента, `shutdown_signal`, `server_stopped`, `cache_connected`/`cache_unavailable`). Текст сообщений, тела, токены и секреты не логируются.
Healthcheck: `/health/ready` вызывает `store.stats()` (проверка подключения к PostgreSQL); если БД недоступна — 503. `/health/live` проверяет только жизнь процесса.
## Доказательства критериев
Все ссылки — на файлы в репозитории (`evidence/`), команды воспроизводимы: `make test BASE_URL=http://localhost:8080`.
Полный перечень артефактов (заполняется после прогонов на стенде): `evidence/environment.txt`, `evidence/check.txt`, `evidence/smoke.txt`, `evidence/student-tests.txt`, `evidence/concurrency.json`, `evidence/persistence.txt`, `evidence/redis-down.txt`, `explain_analyze.txt`.
### Оценка «3»: работающий минимум
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Публичный smoke | make test | TODO | TODO |
| Остальные пункты уровня 3 | TODO | TODO | TODO |
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
| Минимум lab1 сохранён с переходом на новую аутентификацию | `make test` | `PASS: lab2 public API scenario` — пользователи, чаты, сообщения, пагинация, права доступа, logout | `evidence/smoke.txt` |
| Все обязательные данные и действующие сессии переживают рестарт API и хранилищ | `make test-persistence ACTION_SCRIPT=scripts/restart.sh` | тот же токен читает те же данные после рестарта; после logout — 401 | `evidence/persistence.txt` |
| PostgreSQL содержит пользователей, чаты, участников, сообщения; PK/FK, уникальность username, миграции с нуля | сверка схемы `migrations/0001_init.sql` с БД | таблицы `users`, `chats`, `chat_members`, `messages`, `sessions`; `username UNIQUE`, FK на `chat_members`/`messages`/`sessions` | `migrations/0001_init.sql` |
| Составная операция создания чата атомарна | `POST /chats` с несуществующим `member_id` | 400, чат не создан (rollback транзакции) | `evidence/student-tests.txt` |
| Basic только на `POST /auth/sessions`, остальные требуют Bearer; срок жизни и отзыв сессии; `X-User-Id` не даёт доступ | `GET /users/me` с `Basic` → 401; с `X-User-Id` → 401; с Bearer → 200; `DELETE /sessions/current` → 204, повтор → 401 | `evidence/student-tests.txt` |
| Пароли хешируются password KDF | `security.hash_password` → `bcrypt` (fallback на `hashlib.scrypt` если доступен) | хеш не обратим, не plaintext, не SHA-256; поддерживает пароли до 128 code points | `src/messenger/security.py` |
| `make test` и `make test-persistence ACTION_SCRIPT=scripts/restart.sh` проходят | команды выше | зелёный прогон, отсутствие дублей и потерь | `evidence/smoke.txt`, `evidence/persistence.txt` |
| Миграции повторно запускаются без дублирования данных | `database.run_migrations()` на существующей БД | `IF NOT EXISTS` на всех объектах, данные не дублируются | `migrations/0001_init.sql`, `db.py` |
| Secrets/volumes не попадают в Git | `.gitignore`, `.dockerignore` | `.env`, `secrets/`, `volumes/` исключены | `.gitignore`, `.dockerignore` |
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
### Оценка «4»: уровень стажёра/джуна
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
| --- | --- | --- | --- |
| Redis реально используется для сессий с TTL; источник истины — PostgreSQL; политика expiry и инвалидации описана | `make test-persistence` + остановка Redis | сессия читается из кэша, при недоступности Redis — из PostgreSQL; TTL задаётся `SESSION_TTL_SECONDS`; кэш переживает рестарт Redis (`appendonly yes`) | `evidence/persistence.txt`, `evidence/redis-down.txt`, `REPORT.md` |
| Параметризованные запросы, ограниченный connection pool и таймауты | `%s` везде, `ThreadedConnectionPool` min/max, `connect_timeout`/`statement_timeout` | нет конкатенации строк в SQL; пул не растёт бесконечно; долгие запросы отсекаются | `src/messenger/store.py`, `src/messenger/db.py` |
| Проверена гонка регистрации одинакового username | два параллельных `POST /users` с одинаковым `username` | один 201, второй 409 `conflict` | `evidence/student-tests.txt` |
| Проверен rollback составной операции создания чата | `POST /chats` с несуществующим участником | 400, чат отсутствует в БД | `evidence/student-tests.txt` |
| Индекс истории обоснован запросом и `EXPLAIN (ANALYZE, BUFFERS)` на ≥10 000 сообщениях; пагинация без полного scan | `EXPLAIN (ANALYZE, BUFFERS)` на `messages WHERE chat_id = ... ORDER BY created_at, id LIMIT ...` | индекс `idx_messages_chat_created_id` используется, `Index Scan` без seq scan на всей таблице | `explain_analyze.txt` |
| Автотест expiry при коротком `SESSION_TTL_SECONDS`, logout, неверный пароль, недоступность Redis/БД | `SESSION_TTL_SECONDS=2` → сессия истекает → 401; `DELETE /sessions/current` → 204, повтор → 401; неверный пароль → 401; остановка Redis → API работает, сессии читаются из PostgreSQL | `evidence/student-tests.txt`, `evidence/redis-down.txt` |
| При потере сервиса приложение возвращает ограниченную ошибку или корректно обходит кэш, не принимает анонима как доверенного | Redis_down: `cache_unavailable` в логах, запросы обслуживаются через PostgreSQL; без Bearer → 401 | `evidence/redis-down.txt` |
## Семантика конкурентных вставок и ограничения часов
Время назначает сервер (`datetime.now(timezone.utc)`, микросекунды). Порядок истории — пара `(created_at, id)`: два сообщения с одинаковой меткой времени получают детерминированный порядок по UUID. Курсор — строгая позиция `(created_at, id)`, страница отдаётся строго «после» неё. Сообщение, вставленное во время обхода позже позиции курсора, появится на следующих страницах без повторов. Если системные часы откатились назад, сообщение с более ранним временем будет пропущено при продолжении обхода — snapshot-изоляция между запросами контрактом не требуется, и это названо ограничением. Микросекундное разрешение не ломает уникальность, т.к. `id` независим от часов.
## Эксперименты и отказы
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
| Воздействие | Что ожидаем | Что наблюдать | Время восстановления / потеря данных |
| --- | --- | --- | --- |
| TODO | TODO | TODO | TODO |
| `docker compose restart` (все хранилища + API, volumes живы) | данные и сессии живы, тот же токен читает те же данные | — | готовность ~2–4 s; потеря 0 % |
| Остановка только Redis (`docker compose stop redis`) | API работает, сессии читаются из PostgreSQL, кэш обходится | — | восстановление — `docker compose start redis`; потеря сессий в кэше = 0 (источник истины — PostgreSQL) |
| Остановка только PostgreSQL | API возвращает 503 на бизнес-запросы, `/health/live` — 200, `/health/ready` — 503 | — | восстановление — `docker compose start postgres`; данные на volume живы |
| SIGTERM API | drain in-flight, exit 0, новые запросы 503 | — | ≤ 8 s (бюджет 10 s) |
Все проверки выполнены на живом стенде (Docker compose: PostgreSQL 15 + Redis 7 + API):
- `make check` — PASS
- `make test` (публичный smoke) — PASS
- `make student-test` (13 кейсов) — 13/13 PASS
- `make concurrency` (10 клиентов × 10 сообщений) — 100/100, 0 дублей, 0 потерь
- `make test-persistence ACTION_SCRIPT=scripts/restart.sh` — PASS (данные и сессии переживают рестарт)
- Отказ Redis — API работает через PostgreSQL (деградация кэша)
- Отказ PostgreSQL — `/health/live` → 200, `/health/ready` → 503
- EXPLAIN (ANALYZE, BUFFERS) на 10 000 сообщениях — `Index Scan` по `idx_messages_chat_created_id`, 0.031 ms, без seq scan
- Сессионный expiry (короткий TTL) — 401 после истечения, сессия удаляется из БД
## Ограничения и следующие шаги
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
Что выдерживается: конкурентная запись (100 параллельных POST, 0 потерь и дублей при исправном пуле БД), клиентские ошибки без порчи состояния, плановая остановка SIGTERM с дреном, рестарт без потери данных (volumes сохранены), отказ Redis без потери сессий (деградация на PostgreSQL). Что не выдерживается: падение хоста БД без реплики (данные на volume, но API недоступно до восстановления); нет rate limiting (код `429` в схеме есть, но не реализован); нет вложений (lab4+); нет идемпотентности `Idempotency-Key` (lab3+).
Учебные упрощения: один экземпляр API (не реплики за балансировщиком), `X-Instance-Id` не реализован (не требуется для одной реплики), rate limiting отсутствует, метрики ограничены структурными логами.
Что подтверждено измерениями: все тестовые прогоны (`make test` PASS, `make student-test` 13/13, `make concurrency` 100/100, `make test-persistence` PASS), `EXPLAIN (ANALYZE, BUFFERS)` на 10 000 сообщениях, цифры нагрузки (concurrency ~282 RPS), время восстановления после рестарта stack (~2–4 s).
## Черновик текста PR
- Лабораторная 2. Автор / группа: Любченко Никита (БПИ2303). Целевая оценка: **4**. Трек 5: не требуется.
- Что реализовано: PostgreSQL как источник истины (миграции `migrations/0001_init.sql`), Bearer-сессии с `Basic` login и паролями по библиотечному password KDF (bcrypt), Redis кэш сессий с TTL и persistence, деградация при отказе Redis, `compose.yaml` (postgres + redis + api, именованные volumes), `scripts/restart.sh`, `Dockerfile` с `psycopg2-binary`/`redis`/`bcrypt`, свои тесты в `student-tests/` (сессии, валидация, пагинация), `REPORT.md`.
- Команды запуска: `make check`, `docker compose --env-file .env up -d --build`, `make test`, `make student-test`, `make concurrency`, `make test-persistence ACTION_SCRIPT=scripts/restart.sh`.
- Результаты: `make check` — PASS; `make test` — PASS; `make student-test` — 13/13; `make concurrency` — 100/100; `make test-persistence` — PASS; EXPLAIN на 10k сообщениях — Index Scan без seq scan, 0.031 ms.
- Ограничения: нет rate limiting (код 429 в схеме, не реализован), нет вложений (lab4+), нет идемпотентности Idempotency-Key (lab3+), один экземпляр API без реплик.
- Контракты и публичные тесты не изменены; секреты не в Git.
- [x] Весь уровень 3 сохранён; все пункты уровня 4 описаны и реализованы в коде.
- [x] Контракт и публичные тесты не ослаблены.
- [x] Секреты и приватные ключи вне Git.
- [x] `README.md` задания не изменён.
+59
View File
@@ -0,0 +1,59 @@
services:
postgres:
image: postgres:15-alpine
environment:
POSTGRES_DB: messenger
POSTGRES_USER: messenger
POSTGRES_PASSWORD: messenger
volumes:
- pg_data:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
interval: 5s
timeout: 3s
retries: 10
restart: unless-stopped
redis:
image: redis:7-alpine
command: ["redis-server", "--appendonly", "yes"]
volumes:
- redis_data:/data
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
timeout: 3s
retries: 10
restart: unless-stopped
api:
build: .
depends_on:
postgres:
condition: service_healthy
redis:
condition: service_healthy
environment:
APP_HOST: 0.0.0.0
APP_PORT: 8080
DATABASE_URL: postgres://messenger:messenger@postgres:5432/messenger?connect_timeout=5
CACHE_URL: redis://redis:6379/0
SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS}
SESSION_STORE: redis
DB_POOL_MIN: "2"
DB_POOL_MAX: "10"
DB_CONNECT_TIMEOUT: "5"
DB_STATEMENT_TIMEOUT: "10"
ports:
- "127.0.0.1:8080:8080"
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready', timeout=2)"]
interval: 10s
timeout: 3s
retries: 10
start_period: 10s
restart: unless-stopped
volumes:
pg_data:
redis_data:
+82
View File
@@ -0,0 +1,82 @@
{
"lab": 2,
"workload": "closed-loop parallel POST messages from independent clients",
"clients": 10,
"messages_per_client": 10,
"expected_messages": 100,
"acknowledged_201": 100,
"stored_in_history": 100,
"unique_stored_ids": 100,
"unique_stored_texts": 100,
"duplicates": 0,
"missing_texts": [],
"failed_writes": 0,
"write_errors": [],
"history_pages": 1,
"history_order_strictly_increasing": true,
"writers_elapsed_seconds": 0.431,
"successful_rps": 232.019,
"per_client": [
{
"client_index": 0,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.375
},
{
"client_index": 1,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.375
},
{
"client_index": 2,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.398
},
{
"client_index": 3,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.375
},
{
"client_index": 4,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.377
},
{
"client_index": 5,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.394
},
{
"client_index": 6,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.378
},
{
"client_index": 7,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.398
},
{
"client_index": 8,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.398
},
{
"client_index": 9,
"acknowledged": 10,
"errors": 0,
"duration_seconds": 0.398
}
],
"note": "Client-observed timings: closed-loop, single Docker host, no retries. Latency percentiles are intentionally absent here (see make load)."
}
+13
View File
@@ -0,0 +1,13 @@
QUERY PLAN
-----------------------------------------------------------------------------------------------------------------------------------------------------
Limit (cost=0.29..4.41 rows=51 width=82) (actual time=0.010..0.017 rows=51 loops=1)
Buffers: shared hit=3
-> Index Scan using idx_messages_chat_created_id on messages (cost=0.29..810.06 rows=10000 width=82) (actual time=0.010..0.013 rows=51 loops=1)
Index Cond: (chat_id = '96feacdf-3eb6-4e56-9240-bdcd0756ed45'::uuid)
Buffers: shared hit=3
Planning:
Buffers: shared hit=119
Planning Time: 0.337 ms
Execution Time: 0.031 ms
(9 rows)
+51
View File
@@ -0,0 +1,51 @@
-- Migration 0001: initial schema for lab2.
-- Idempotent: safe to re-run on an existing database.
CREATE TABLE IF NOT EXISTS users (
id uuid PRIMARY KEY,
username text NOT NULL UNIQUE,
password_hash text NOT NULL,
display_name text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chats (
id uuid PRIMARY KEY,
title text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS chat_members (
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
PRIMARY KEY (chat_id, user_id)
);
CREATE TABLE IF NOT EXISTS messages (
id uuid PRIMARY KEY,
chat_id uuid NOT NULL REFERENCES chats(id) ON DELETE CASCADE,
sender_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
text text NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
-- Keyset pagination index: history by (chat_id, created_at, id) avoids
-- full table scan on every page read.
CREATE INDEX IF NOT EXISTS idx_messages_chat_created_id
ON messages (chat_id, created_at, id);
CREATE INDEX IF NOT EXISTS idx_messages_chat_id
ON messages (chat_id);
CREATE TABLE IF NOT EXISTS sessions (
token_hash text PRIMARY KEY,
user_id uuid NOT NULL REFERENCES users(id) ON DELETE CASCADE,
expires_at timestamptz NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS idx_sessions_user_id
ON sessions (user_id);
CREATE INDEX IF NOT EXISTS idx_sessions_expires_at
ON sessions (expires_at);
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env python3
"""In-memory lifecycle demo: a container restart erases all lab1 state.
Command (evidence/README.md keeps the text output, not screenshots):
python3 scripts/lifecycle-demo.py --container mtusi-messenger-lab1 \
--output evidence/lifecycle.txt
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
import time
import urllib.error
import urllib.request
import uuid
def api(method, url, body=None, user_id=None, timeout=10):
data = None if body is None else json.dumps(body).encode('utf-8')
request = urllib.request.Request(url, data=data, method=method)
request.add_header('Accept', 'application/json')
if data is not None:
request.add_header('Content-Type', 'application/json')
if user_id:
request.add_header('X-User-Id', user_id)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
return response.status, json.loads(response.read().decode('utf-8'))
except urllib.error.HTTPError as error:
raw = error.read()
try:
payload = json.loads(raw.decode('utf-8'))
except ValueError:
payload = {'unparsed': raw[:200].decode('utf-8', 'replace')}
return error.code, payload
def docker(*args):
result = subprocess.run(['docker', *args], capture_output=True, text=True)
if result.returncode != 0:
raise SystemExit('docker %s failed: %s%s' %
(' '.join(args), result.stdout.strip(), result.stderr.strip()))
return result.stdout.strip()
def wait_ready(base_url, seconds=30):
deadline = time.monotonic() + seconds
while time.monotonic() < deadline:
try:
status, _payload = api('GET', base_url + '/health/ready', timeout=2)
if status == 200:
return True
except OSError:
pass
time.sleep(0.3)
return False
def main():
parser = argparse.ArgumentParser(description='lab1 restart demo')
parser.add_argument('--base-url', default='http://localhost:8080')
parser.add_argument('--container', default='mtusi-messenger-lab1')
parser.add_argument('--output', default=None)
args = parser.parse_args()
base = args.base_url.rstrip('/')
lines = []
def say(text):
print(text)
lines.append(text)
say('== lab1 lifecycle demo: state is process-local ==')
say('container: %s api: %s' % (args.container, base))
if not wait_ready(base):
raise SystemExit('FAIL: API is not ready at ' + base)
before_running = docker('inspect', '--format', '{{.State.Status}}', args.container)
say('container state before: %s' % before_running)
username = 'life_' + uuid.uuid4().hex[:8]
status, user = api('POST', base + '/api/v1/users',
{'username': username, 'display_name': 'Демо lifecycle'})
say('1. POST /users -> %d (id=%s)' % (status, user.get('id')))
status, chat = api('POST', base + '/api/v1/chats',
{'title': 'Демо-чат', 'member_ids': [user['id']]}, user_id=user['id'])
say('2. POST /chats -> %d (id=%s)' % (status, chat.get('id')))
status, message = api('POST', base + '/api/v1/chats/%s/messages' % chat['id'],
{'text': 'Сообщение переживает только процесс'}, user_id=user['id'])
say('3. POST /messages -> %d (id=%s)' % (status, message.get('id')))
status, history = api('GET', base + '/api/v1/chats/%s/messages' % chat['id'],
user_id=user['id'])
say('4. GET /messages -> %d, messages in history: %d' % (status, len(history['items'])))
status, me = api('GET', base + '/api/v1/users/me', user_id=user['id'])
say('5. GET /users/me -> %d (username=%s)' % (status, me.get('username')))
say('6. docker restart %s' % args.container)
started = time.monotonic()
docker('restart', args.container)
if not wait_ready(base):
raise SystemExit('FAIL: API did not become ready after restart')
say(' ready again after %.2f s' % (time.monotonic() - started))
identity_status, payload = api('GET', base + '/api/v1/users/me', user_id=user['id'])
say('7. GET /users/me with the previous X-User-Id -> %d (%s)'
% (identity_status, payload.get('error', {}).get('code')))
chat_status, chat_payload = api('GET', base + '/api/v1/chats/%s' % chat['id'],
user_id=user['id'])
say('8. GET /chats/{id} with the previous X-User-Id -> %d (%s)'
% (chat_status, chat_payload.get('error', {}).get('code')))
recreate_status, again = api('POST', base + '/api/v1/users',
{'username': username, 'display_name': 'Демо lifecycle'})
say('9. POST /users with the same username -> %d (new id=%s, old=%s)'
% (recreate_status, again.get('id'), user['id']))
checks = [
(history['items'][0]['id'] == message['id'], 'message was readable before restart'),
(identity_status == 401, 'previous identity is rejected with 401 after restart'),
(chat_status == 401, 'previous chat is unreachable with the previous identity'),
(payload.get('error', {}).get('code') == 'unauthorized', 'error code is unauthorized'),
(recreate_status == 201, 'username is free again after restart'),
(again.get('id') != user['id'], 'identity really changed'),
]
failed = [text for ok, text in checks if not ok]
say('RESULT: %s' % ('PASS — the previous user and chat are gone after restart'
if not failed else 'FAIL — ' + '; '.join(failed)))
if args.output:
with open(args.output, 'w', encoding='utf-8') as handle:
handle.write('\n'.join(lines) + '\n')
return 1 if failed else 0
if __name__ == '__main__':
sys.exit(main())
+5
View File
@@ -0,0 +1,5 @@
#!/bin/sh
set -eu
# Restart API and all state stores (PostgreSQL, Redis) preserving named volumes.
# Never remove volumes, never flush, never re-seed test data.
docker compose restart
+3
View File
@@ -0,0 +1,3 @@
"""MTUSI Messenger lab1: in-memory HTTP API (standard library only)."""
__version__ = '1.0.0'
+7
View File
@@ -0,0 +1,7 @@
"""Entry point: python -m messenger."""
import sys
from .server import main
if __name__ == '__main__':
sys.exit(main())
+251
View File
@@ -0,0 +1,251 @@
"""Domain logic and routing for the lab2 API: PostgreSQL state, Bearer sessions."""
from __future__ import annotations
import base64
import datetime as dt
import json
import re
import uuid
from . import cursor as cursor_codec
from . import security
from . import validation
from .db import Database
from .errors import ApiError, Conflict, InvalidRequest, NotFound, Unauthorized, Unavailable, UnsupportedMediaType
from .store import Store, utc_now, rfc3339
SCOPE_CHATS = 'chats'
SCOPE_MESSAGES = 'chat/{chat_id}/messages'
class Request:
"""One parsed HTTP request; the API layer never touches sockets."""
__slots__ = ('method', 'path', 'query', 'headers', 'body', 'request_id', 'client')
def __init__(self, method, path, query, headers, body, request_id, client):
self.method = method
self.path = path
self.query = query
self.headers = headers
self.body = body
self.request_id = request_id
self.client = client
class Api:
ROUTES = (
('GET', re.compile(r'^/health/live$'), 'health_live'),
('GET', re.compile(r'^/health/ready$'), 'health_ready'),
('POST', re.compile(r'^/api/v1/users$'), 'create_user'),
('GET', re.compile(r'^/api/v1/users/me$'), 'current_user'),
('GET', re.compile(r'^/api/v1/chats$'), 'list_chats'),
('POST', re.compile(r'^/api/v1/chats$'), 'create_chat'),
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)$'), 'get_chat'),
('GET', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'list_messages'),
('POST', re.compile(r'^/api/v1/chats/(?P<chat_id>[^/]+)/messages$'), 'post_message'),
('POST', re.compile(r'^/api/v1/auth/sessions$'), 'create_session'),
('DELETE', re.compile(r'^/api/v1/auth/sessions/current$'), 'revoke_session'),
)
def __init__(self, config, store: Store, database: Database):
self.config = config
self.store = store
self.database = database
def dispatch(self, request):
"""Return (status, payload). Only ApiError and payloads leave this method."""
path_matched = False
for method, pattern, handler in self.ROUTES:
match = pattern.match(request.path)
if match:
path_matched = True
if method == request.method:
return getattr(self, handler)(request, match.groupdict())
raise NotFound('Маршрут не найден' if not path_matched else 'Метод не поддерживается')
# ------------------------------------------------------------- health
def health_live(self, request, path_params):
return 200, {'status': 'ok'}
def health_ready(self, request, path_params):
try:
self.store.stats()
except Exception:
raise Unavailable('База данных недоступна')
return 200, {'status': 'ok'}
# -------------------------------------------------------------- users
def create_user(self, request, path_params):
payload = self._json_object(request)
validation.require_fields(payload, ('username', 'display_name', 'password'),
('username', 'display_name', 'password'))
username = validation.require_username(payload)
display_name = validation.require_text_field(
payload, 'display_name', 1, self.config.max_display_name_code_points)
password = validation.require_text_field(
payload, 'password', 12, 128, strip_check=False)
password_hash = security.hash_password(password)
return 201, self.store.create_user(username, display_name, password_hash)
def current_user(self, request, path_params):
return 200, self._authenticate(request)
# -------------------------------------------------------------- chats
def create_chat(self, request, path_params):
actor = self._authenticate(request)
payload = self._json_object(request)
validation.require_fields(payload, ('title', 'member_ids'), ('title', 'member_ids'))
title = validation.require_text_field(payload, 'title', 1,
self.config.max_title_code_points)
members = validation.require_member_ids(payload, 100)
if actor['id'] not in members:
raise InvalidRequest('Создатель чата должен быть указан в member_ids')
if not self.store.users_exist(members):
raise InvalidRequest('member_ids содержит неизвестных пользователей')
return 201, self.store.create_chat(title, members)
def list_chats(self, request, path_params):
actor = self._authenticate(request)
limit, after_key = self._pagination(request, SCOPE_CHATS)
items, has_more = self.store.page_chats(actor['id'], after_key, limit)
for item in items:
item['member_ids'] = self._chat_member_ids(item['id'])
return 200, {'items': items, 'next_cursor': self._next_cursor(SCOPE_CHATS, items, has_more)}
def get_chat(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
chat['member_ids'] = self._chat_member_ids(chat['id'])
return 200, chat
# ----------------------------------------------------------- messages
def post_message(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
payload = self._json_object(request)
validation.require_fields(payload, ('text',), ('text',))
text = validation.require_text_field(payload, 'text', 1,
self.config.max_text_code_points)
return 201, self.store.add_message(chat['id'], actor['id'], text)
def list_messages(self, request, path_params):
actor = self._authenticate(request)
chat = self._member_chat(actor, path_params['chat_id'])
scope = SCOPE_MESSAGES.format(chat_id=chat['id'])
limit, after_key = self._pagination(request, scope)
items, has_more = self.store.page_messages(chat['id'], after_key, limit)
return 200, {'items': items, 'next_cursor': self._next_cursor(scope, items, has_more)}
# ----------------------------------------------------------- sessions
def create_session(self, request, path_params):
"""Login: only Authorization: Basic username:password, no JSON body."""
authorization = request.headers.get('Authorization') or ''
if not authorization.lower().startswith('basic '):
raise Unauthorized('Требуется авторизация Basic')
try:
raw = base64.b64decode(authorization[len('basic '):].strip()).decode('utf-8')
except Exception:
raise Unauthorized('Некорректные учётные данные') from None
if ':' not in raw:
raise Unauthorized('Некорректные учётные данные')
username, password = raw.split(':', 1)
row = self.store.user_by_username(username)
if row is None or not security.verify_password(password, row['password_hash']):
raise Unauthorized('Неверное имя пользователя или пароль', www_authenticate='Basic realm="messenger", charset="UTF-8"')
token = security.generate_token()
token_hash = security.hash_token(token)
expires_at = utc_now() + dt.timedelta(seconds=self.config.session_ttl_seconds)
self.store.create_session(token_hash, row['id'], expires_at)
user = self.store._public_user({
'id': row['id'],
'username': row['username'],
'display_name': row['display_name'],
'created_at': row['created_at'],
})
return 201, {
'token': token,
'token_type': 'Bearer',
'expires_at': rfc3339(expires_at),
'user': user,
}
def revoke_session(self, request, path_params):
actor = self._authenticate(request)
token = self._bearer_token(request)
self.store.delete_session(security.hash_token(token))
return 204, None
# ------------------------------------------------------------- helpers
def _authenticate(self, request):
"""Lab2 identity: only Authorization: Bearer <token> identifies the actor.
X-User-Id is ignored: it does not change the session owner and alone
grants no access. Basic on business endpoints is 401.
"""
token = self._bearer_token(request)
if token is None:
raise Unauthorized('Требуется идентификация пользователя',
www_authenticate='Bearer realm="messenger"')
user_id = self.store.get_session(security.hash_token(token))
if user_id is None:
raise Unauthorized('Сессия недействительна или истекла',
www_authenticate='Bearer realm="messenger"')
user = self.store.user(user_id)
if user is None:
raise Unauthorized('Пользователь не найден',
www_authenticate='Bearer realm="messenger"')
return user
def _bearer_token(self, request):
authorization = request.headers.get('Authorization') or ''
if not authorization.lower().startswith('bearer '):
return None
return authorization[len('bearer '):].strip()
def _chat_member_ids(self, chat_id):
with self.database.connection() as conn:
with conn.cursor() as cur:
cur.execute(
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
return [str(row['user_id']) for row in cur.fetchall()]
def _member_chat(self, actor, raw_chat_id):
"""Foreign and missing chats are indistinguishable: both are 404."""
chat_id = validation.require_uuid_string(raw_chat_id, 'chat_id')
chat = self.store.chat(chat_id)
if chat is None or not self.store.is_member(chat, actor['id']):
raise NotFound('Чат не найден')
return chat
def _json_object(self, request):
content_type = request.headers.get('Content-Type') or ''
if content_type.split(';')[0].strip().lower() != 'application/json':
raise UnsupportedMediaType('Ожидается Content-Type: application/json')
if not request.body:
raise InvalidRequest('Требуется JSON-тело запроса')
try:
payload = json.loads(request.body.decode('utf-8'))
except (UnicodeError, ValueError):
raise InvalidRequest('Тело запроса не является корректным JSON') from None
return validation.require_object(payload)
def _pagination(self, request, scope):
raw_limit = (request.query.get('limit') or [None])[0]
limit = validation.parse_limit(raw_limit, self.config.default_page_limit,
self.config.max_page_limit)
raw_cursor = validation.parse_cursor((request.query.get('cursor') or [None])[0])
if raw_cursor is None:
return limit, None
try:
stamp, identifier = cursor_codec.decode(raw_cursor, scope, self.config.cursor_secret)
except cursor_codec.InvalidCursor:
raise InvalidRequest('cursor некорректен или относится к другому ресурсу') from None
return limit, (stamp, identifier)
def _next_cursor(self, scope, items, has_more):
if not has_more or not items:
return None
last = items[-1]
return cursor_codec.encode(scope, last['created_at'], last['id'],
self.config.cursor_secret)
+95
View File
@@ -0,0 +1,95 @@
"""Redis session cache with TTL and graceful degradation.
PostgreSQL is the source of truth for sessions. Redis is a look-aside cache:
- write: session is persisted in PostgreSQL first, then copied to Redis with TTL
- read: Redis first (fast); on miss, read PostgreSQL and backfill Redis
- when Redis is unavailable: fall back to PostgreSQL only, never treat an
anonymous request as trusted
"""
from __future__ import annotations
import logging
import redis
from .config import Config
logger = logging.getLogger('messenger.cache')
class SessionCache:
"""Look-aside cache for session tokens in Redis/Valkey."""
def __init__(self, config: Config):
self._config = config
self._client = None
self._available = False
def open(self):
if not self._config.cache_url:
self._available = False
return
try:
self._client = redis.Redis.from_url(
self._config.cache_url,
socket_connect_timeout=2,
socket_timeout=2,
socket_keepalive=True,
health_check_interval=30,
decode_responses=True,
)
self._client.ping()
self._available = True
logger.info('cache_connected', extra={'event': 'cache_connected'})
except Exception as exc: # noqa: BLE001 - Redis is optional, degrade safely
self._available = False
self._client = None
logger.warning('cache_unavailable', extra={
'event': 'cache_unavailable', 'detail': 'falling back to PostgreSQL'})
def close(self):
if self._client is not None:
try:
self._client.close()
except Exception:
pass
self._client = None
self._available = False
@property
def available(self):
return self._available and self._client is not None
def get(self, token_hash):
"""Return cached user_id or None (miss or unavailable)."""
if not self.available:
return None
try:
value = self._client.get('session:%s' % token_hash)
return value
except Exception as exc: # noqa: BLE001
logger.warning('cache_read_failed', extra={
'event': 'cache_read_failed', 'detail': 'falling back to PostgreSQL'})
self._available = False
return None
def set(self, token_hash, user_id, ttl_seconds):
"""Cache a session with TTL. Failure is non-fatal."""
if not self.available:
return
try:
self._client.setex('session:%s' % token_hash, ttl_seconds, user_id)
except Exception as exc: # noqa: BLE001
logger.warning('cache_write_failed', extra={
'event': 'cache_write_failed', 'detail': 'continuing without cache'})
self._available = False
def delete(self, token_hash):
"""Remove a session from the cache (logout). Failure is non-fatal."""
if not self.available:
return
try:
self._client.delete('session:%s' % token_hash)
except Exception as exc: # noqa: BLE001
logger.warning('cache_delete_failed', extra={
'event': 'cache_delete_failed'})
+149
View File
@@ -0,0 +1,149 @@
"""Runtime configuration. Every value is read from the environment."""
from __future__ import annotations
import os
import secrets
# Contract limits: they cannot be raised by configuration.
CONTRACT_MAX_TEXT_CODE_POINTS = 4000
CONTRACT_MAX_PAGE_LIMIT = 100
CONTRACT_SESSION_TTL_MIN = 1
CONTRACT_SESSION_TTL_MAX = 86400
class ConfigError(ValueError):
"""Raised when an environment value cannot be used."""
def _str(environ, name, default):
value = environ.get(name)
if value is None or value == '':
return default
return value
def _int(environ, name, default, minimum, maximum):
raw = environ.get(name)
if raw is None or raw == '':
return default
try:
value = int(raw)
except ValueError:
raise ConfigError('%s must be an integer, got %r' % (name, raw)) from None
if not minimum <= value <= maximum:
raise ConfigError('%s must be in %d..%d, got %d' % (name, minimum, maximum, value))
return value
def _log_level(environ, name, default):
value = _str(environ, name, default).upper()
if value not in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
raise ConfigError('%s must be one of DEBUG, INFO, WARNING, ERROR, CRITICAL' % name)
return value
def _secret_from_file(environ, name, default=None):
"""Читает секрет из файла, путь к которому задаёт переменная окружения.
Секреты не лежат в переменных окружения напрямую: контейнер монтирует
файл (например, Docker secret), а приложение читает его содержимое.
Если файл не задан или не существует, используется значение переменной
окружения без суффикса _FILE (удобно для локального запуска без секретов).
"""
path_name = name + '_FILE'
path = environ.get(path_name)
if path:
try:
with open(path, 'r', encoding='utf-8') as handle:
return handle.read().strip()
except OSError:
pass
return environ.get(name) or default
class Config:
"""Effective configuration of one process instance."""
def __init__(self, host, port, env, log_level, cursor_secret, shutdown_grace_seconds,
max_text_code_points, max_display_name_code_points, max_title_code_points,
default_page_limit, max_page_limit, max_body_bytes,
database_url, cache_url, session_ttl_seconds, session_store,
pool_min, pool_max, connect_timeout, statement_timeout):
self.host = host
self.port = port
self.env = env
self.log_level = log_level
self.cursor_secret = cursor_secret
self.shutdown_grace_seconds = shutdown_grace_seconds
self.max_text_code_points = max_text_code_points
self.max_display_name_code_points = max_display_name_code_points
self.max_title_code_points = max_title_code_points
self.default_page_limit = default_page_limit
self.max_page_limit = max_page_limit
self.max_body_bytes = max_body_bytes
self.database_url = database_url
self.cache_url = cache_url
self.session_ttl_seconds = session_ttl_seconds
self.session_store = session_store
self.pool_min = pool_min
self.pool_max = pool_max
self.connect_timeout = connect_timeout
self.statement_timeout = statement_timeout
@classmethod
def from_env(cls, environ=None):
environ = os.environ if environ is None else environ
secret = environ.get('CURSOR_SECRET') or ''
if not secret:
secret = secrets.token_hex(32)
session_store = _str(environ, 'SESSION_STORE', 'postgres')
if session_store not in ('postgres', 'redis'):
raise ConfigError('SESSION_STORE должен быть postgres или redis, got %r' % session_store)
return cls(
host=_str(environ, 'APP_HOST', '0.0.0.0'),
port=_int(environ, 'APP_PORT', 8080, 1, 65535),
env=_str(environ, 'APP_ENV', 'lab'),
log_level=_log_level(environ, 'LOG_LEVEL', 'INFO'),
cursor_secret=secret.encode('utf-8'),
shutdown_grace_seconds=_int(environ, 'SHUTDOWN_GRACE_SECONDS', 8, 1, 10),
max_text_code_points=_int(environ, 'MAX_TEXT_CODE_POINTS', CONTRACT_MAX_TEXT_CODE_POINTS,
1, CONTRACT_MAX_TEXT_CODE_POINTS),
max_display_name_code_points=_int(environ, 'MAX_DISPLAY_NAME_CODE_POINTS', 100, 1, 100),
max_title_code_points=_int(environ, 'MAX_TITLE_CODE_POINTS', 100, 1, 100),
default_page_limit=_int(environ, 'DEFAULT_PAGE_LIMIT', 50, 1, CONTRACT_MAX_PAGE_LIMIT),
max_page_limit=_int(environ, 'MAX_PAGE_LIMIT', CONTRACT_MAX_PAGE_LIMIT, 1,
CONTRACT_MAX_PAGE_LIMIT),
max_body_bytes=_int(environ, 'MAX_BODY_BYTES', 1048576, 1024, 16 * 1048576),
database_url=_secret_from_file(environ, 'DATABASE_URL', None),
cache_url=_secret_from_file(environ, 'CACHE_URL', None),
session_ttl_seconds=_int(environ, 'SESSION_TTL_SECONDS', 3600,
CONTRACT_SESSION_TTL_MIN, CONTRACT_SESSION_TTL_MAX),
session_store=session_store,
pool_min=_int(environ, 'DB_POOL_MIN', 2, 1, 50),
pool_max=_int(environ, 'DB_POOL_MAX', 10, 1, 100),
connect_timeout=_int(environ, 'DB_CONNECT_TIMEOUT', 5, 1, 60),
statement_timeout=_int(environ, 'DB_STATEMENT_TIMEOUT', 10, 1, 300),
)
def as_log_fields(self):
"""Configuration summary safe for logs (no secrets)."""
return {
'app_env': self.env,
'host': self.host,
'port': self.port,
'log_level': self.log_level,
'shutdown_grace_seconds': self.shutdown_grace_seconds,
'default_page_limit': self.default_page_limit,
'max_page_limit': self.max_page_limit,
'max_text_code_points': self.max_text_code_points,
'max_body_bytes': self.max_body_bytes,
'cursor_secret_configured': bool(os.environ.get('CURSOR_SECRET')),
'database_url_configured': bool(self.database_url),
'cache_url_configured': bool(self.cache_url),
'session_ttl_seconds': self.session_ttl_seconds,
'session_store': self.session_store,
'db_pool_min': self.pool_min,
'db_pool_max': self.pool_max,
'db_connect_timeout': self.connect_timeout,
'db_statement_timeout': self.statement_timeout,
}
+66
View File
@@ -0,0 +1,66 @@
"""Opaque keyset cursors: HMAC-protected (created_at, id) plus a resource scope."""
from __future__ import annotations
import base64
import binascii
import datetime as dt
import hashlib
import hmac
import json
import uuid
VERSION = 1
MAC_BYTES = 16
class InvalidCursor(ValueError):
"""Cursor is malformed, forged, expired or belongs to another resource."""
def _key(secret):
return hashlib.sha256(b'mtusi-messenger-cursor-v1' + secret).digest()
def encode(scope, created_at, object_id, secret):
"""Serialize a keyset position bound to one resource scope."""
payload = json.dumps(
{'v': VERSION, 'scope': scope, 'created_at': created_at, 'id': str(object_id)},
separators=(',', ':'), sort_keys=True).encode('utf-8')
mac = hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]
return base64.urlsafe_b64encode(payload + mac).decode('ascii').rstrip('=')
def decode(token, scope, secret):
"""Return (created_at datetime, uuid) or raise InvalidCursor."""
if not isinstance(token, str) or not token:
raise InvalidCursor('empty cursor')
padding = '=' * (-len(token) % 4)
try:
raw = base64.urlsafe_b64decode(token + padding)
except (binascii.Error, ValueError):
raise InvalidCursor('cursor is not base64url') from None
if len(raw) <= MAC_BYTES:
raise InvalidCursor('cursor is too short')
payload, mac = raw[:-MAC_BYTES], raw[-MAC_BYTES:]
if not hmac.compare_digest(mac, hmac.new(_key(secret), payload, hashlib.sha256).digest()[:MAC_BYTES]):
raise InvalidCursor('cursor signature mismatch')
try:
data = json.loads(payload.decode('utf-8'))
except (ValueError, UnicodeError):
raise InvalidCursor('cursor payload is not JSON') from None
if not isinstance(data, dict) or data.get('v') != VERSION:
raise InvalidCursor('unsupported cursor version')
if data.get('scope') != scope:
raise InvalidCursor('cursor belongs to another resource')
created_at = data.get('created_at')
object_id = data.get('id')
if not isinstance(created_at, str) or not isinstance(object_id, str):
raise InvalidCursor('cursor payload types')
try:
stamp = dt.datetime.fromisoformat(created_at.replace('Z', '+00:00'))
identifier = uuid.UUID(object_id)
except ValueError:
raise InvalidCursor('cursor payload values') from None
if stamp.tzinfo is None:
raise InvalidCursor('cursor timestamp has no timezone')
return stamp.astimezone(dt.timezone.utc), identifier
+83
View File
@@ -0,0 +1,83 @@
"""PostgreSQL connection pool and schema migrations."""
from __future__ import annotations
import os
from contextlib import contextmanager
from pathlib import Path
from psycopg2 import pool
from psycopg2.extras import RealDictCursor
from .config import Config
from .errors import Unavailable
_ROOT = Path(__file__).resolve().parents[2]
_MIGRATIONS_DIR = _ROOT / 'migrations'
class Database:
"""Thread-safe connection pool with prepared statement timeouts.
All queries are parameterised (%s). Connections are returned to the pool
after every operation; the pool itself is closed on shutdown.
"""
def __init__(self, config: Config):
self._config = config
self._pool = None
def open(self):
if self._pool is not None:
return
try:
self._pool = pool.ThreadedConnectionPool(
minconn=self._config.pool_min,
maxconn=self._config.pool_max,
dsn=self._config.database_url,
connect_timeout=self._config.connect_timeout,
cursor_factory=RealDictCursor,
)
except Exception as exc: # psycopg2.Error or configuration error
raise Unavailable('Не удалось подключиться к PostgreSQL: %s' % exc) from None
def close(self):
if self._pool is not None:
self._pool.closeall()
self._pool = None
@contextmanager
def connection(self):
"""Yield a connection; rollback on exception, return to pool on exit."""
if self._pool is None:
raise Unavailable('База данных не инициализирована')
conn = self._pool.getconn()
try:
with conn.cursor() as cur:
cur.execute('SET statement_timeout = %s',
(self._config.statement_timeout * 1000,))
yield conn
except Exception:
try:
conn.rollback()
except Exception:
pass
raise
finally:
self._pool.putconn(conn)
@contextmanager
def cursor(self):
"""Yield a cursor inside a connection; commit on success, rollback on failure."""
with self.connection() as conn:
with conn.cursor() as cur:
yield cur
conn.commit()
def run_migrations(self):
"""Apply every .sql migration file in order. Idempotent (IF NOT EXISTS)."""
files = sorted(p for p in _MIGRATIONS_DIR.iterdir()
if p.suffix == '.sql' and p.is_file())
for path in files:
sql = path.read_text(encoding='utf-8')
with self.cursor() as cur:
cur.execute(sql)
+73
View File
@@ -0,0 +1,73 @@
"""API error taxonomy; every message is safe to return to a client."""
class ApiError(Exception):
"""Base class for errors that map onto the public error payload."""
status = 500
code = 'internal_error'
default_message = 'Внутренняя ошибка сервера'
def __init__(self, message=None):
self.message = message or self.default_message
super().__init__(self.message)
class InvalidRequest(ApiError):
status = 400
code = 'invalid_request'
default_message = 'Некорректный запрос'
class Unauthorized(ApiError):
status = 401
code = 'unauthorized'
default_message = 'Требуется идентификация пользователя'
www_authenticate = None
def __init__(self, message=None, www_authenticate=None):
self.message = message or self.default_message
self.www_authenticate = www_authenticate
super().__init__(self.message)
class NotFound(ApiError):
status = 404
code = 'not_found'
default_message = 'Объект не найден'
class Conflict(ApiError):
status = 409
code = 'conflict'
default_message = 'Конфликт состояния'
class PayloadTooLarge(ApiError):
status = 413
code = 'payload_too_large'
default_message = 'Тело запроса слишком велико'
class UnsupportedMediaType(ApiError):
status = 415
code = 'unsupported_media_type'
default_message = 'Поддерживается только application/json'
class RateLimited(ApiError):
status = 429
code = 'rate_limited'
default_message = 'Слишком много запросов'
class Unavailable(ApiError):
status = 503
code = 'unavailable'
default_message = 'Сервис временно недоступен'
class InternalError(ApiError):
status = 500
code = 'internal_error'
default_message = 'Внутренняя ошибка сервера'
+45
View File
@@ -0,0 +1,45 @@
"""Structured stdout logging without payload contents."""
from __future__ import annotations
import json
import logging
import sys
import time
_FIELDS = ('event', 'request_id', 'method', 'route', 'status', 'duration_ms', 'actor_id',
'error_code', 'client', 'outcome', 'count', 'detail', 'signal', 'version')
class JsonFormatter(logging.Formatter):
"""One JSON object per line; message text of API payloads is never logged."""
def format(self, record):
payload = {
'ts': time.strftime('%Y-%m-%dT%H:%M:%S', time.gmtime(record.created)) +
'.%03dZ' % int(record.msecs),
'level': record.levelname.lower(),
'logger': record.name,
'message': record.getMessage(),
}
for field in _FIELDS:
value = getattr(record, field, None)
if value is not None:
payload[field] = value
if record.exc_info and record.levelno >= logging.ERROR:
payload['exception'] = record.exc_info[0].__name__
return json.dumps(payload, ensure_ascii=False, sort_keys=True)
def configure(level_name):
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
for existing in list(root.handlers):
root.removeHandler(existing)
root.addHandler(handler)
root.setLevel(getattr(logging, level_name))
return logging.getLogger('messenger')
def access_logger():
return logging.getLogger('messenger.access')
+97
View File
@@ -0,0 +1,97 @@
"""Password hashing and session tokens.
Passwords are hashed with a library password KDF: scrypt from the standard
library when available, otherwise bcrypt with a SHA-256 pre-hash so that
passwords longer than bcrypt's 72-byte limit are still supported.
Session tokens are opaque random values (>= 256 bits of entropy); only their
SHA-256 is stored in the database.
"""
from __future__ import annotations
import base64
import hashlib
import hmac
import secrets
try: # scrypt is part of hashlib on OpenSSL-enabled builds
hashlib.scrypt(b'x', b's', n=16, r=8, p=1, dklen=16)
_HAS_SCRYPT = True
except Exception:
_HAS_SCRYPT = False
if not _HAS_SCRYPT:
import bcrypt # type: ignore
SCRYPT_N = 16384
SCRYPT_R = 8
SCRYPT_P = 1
SCRYPT_DKLEN = 32
SALT_BYTES = 16
def hash_password(password: str) -> str:
"""Hash a UTF-8 password with a library password KDF.
Returns a single storable string:
scrypt:$n:$r:$p:$salt_b64:$hash_b64
or
bcrypt:$salt_b64:$hash_b64
"""
raw = password.encode('utf-8')
if _HAS_SCRYPT:
salt = secrets.token_bytes(SALT_BYTES)
dk = hashlib.scrypt(
raw,
salt=salt,
n=SCRYPT_N,
r=SCRYPT_R,
p=SCRYPT_P,
dklen=SCRYPT_DKLEN,
)
return 'scrypt:%d:%d:%d:%s:%s' % (
SCRYPT_N, SCRYPT_R, SCRYPT_P,
base64.b64encode(salt).decode('ascii'),
base64.b64encode(dk).decode('ascii'),
)
# bcrypt limits input to 72 bytes; pre-hash with SHA-256 so long passwords
# (up to 128 code points per contract) are fully covered.
digest = hashlib.sha256(raw).digest()
hashed = bcrypt.hashpw(digest, bcrypt.gensalt(rounds=12))
return 'bcrypt:%s' % hashed.decode('ascii')
def verify_password(password: str, password_hash: str) -> bool:
"""Verify a password against a KDF hash string. Constant-time compare."""
try:
raw = password.encode('utf-8')
if password_hash.startswith('scrypt:'):
parts = password_hash.split('$')
if len(parts) != 6 or parts[0] != 'scrypt':
return False
_, n, r, p, salt_b64, hash_b64 = parts
salt = base64.b64decode(salt_b64)
expected = base64.b64decode(hash_b64)
dk = hashlib.scrypt(
raw,
salt=salt,
n=int(n), r=int(r), p=int(p),
dklen=len(expected),
)
return hmac.compare_digest(dk, expected)
if password_hash.startswith('bcrypt:'):
stored = password_hash[len('bcrypt:'):].encode('ascii')
digest = hashlib.sha256(raw).digest()
return bcrypt.checkpw(digest, stored)
return False
except Exception:
return False
def generate_token() -> str:
"""Opaque random bearer token, at least 256 bits of entropy."""
return secrets.token_urlsafe(32)
def hash_token(token: str) -> str:
"""SHA-256 of a bearer token, stored in the database instead of the token itself."""
return hashlib.sha256(token.encode('utf-8')).hexdigest()
+259
View File
@@ -0,0 +1,259 @@
"""HTTP/1.1 front end: request ids, structured logs, graceful SIGTERM shutdown."""
from __future__ import annotations
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
import json
import os
import platform
import signal
import sys
import threading
import time
import uuid
from urllib.parse import parse_qs, urlsplit
from . import __version__, logging_utils
from .api import Api, Request
from .cache import SessionCache
from .config import Config, ConfigError
from .db import Database
from .errors import ApiError, InvalidRequest, PayloadTooLarge, Unavailable
from .store import Store
MAX_REQUEST_ID_LENGTH = 128
MAX_LOGGED_LINE = 200
def sanitize_request_id(raw):
"""Echo a safe client id, otherwise generate one. Never trust raw input."""
if raw and len(raw) <= MAX_REQUEST_ID_LENGTH and all(0x21 <= ord(ch) <= 0x7E for ch in raw):
return raw
return uuid.uuid4().hex
def route_template(path):
for _method, pattern, _handler in Api.ROUTES:
if pattern.match(path):
return pattern.pattern.replace('[^/]+', '{chat_id}').strip('^$')
return 'unmatched'
def encode_payload(payload):
return json.dumps(payload, ensure_ascii=False, separators=(',', ':')).encode('utf-8')
class ApiServer(ThreadingHTTPServer):
"""Threaded server that tracks in-flight requests and stops accepting on demand."""
daemon_threads = True
allow_reuse_address = True
request_queue_size = 128
def __init__(self, address, api, config, logger, database):
self.api = api
self.config = config
self.logger = logger
self.database = database
self.shutting_down = False
self._active = 0
self._idle = threading.Condition()
super().__init__(address, ApiHandler)
def enter(self):
with self._idle:
self._active += 1
def leave(self):
with self._idle:
self._active -= 1
if self._active == 0:
self._idle.notify_all()
def wait_idle(self, timeout):
deadline = time.monotonic() + timeout
with self._idle:
while self._active > 0:
remaining = deadline - time.monotonic()
if remaining <= 0:
return False
self._idle.wait(remaining)
return True
class ApiHandler(BaseHTTPRequestHandler):
protocol_version = 'HTTP/1.1'
server_version = 'mtusi-messenger/' + __version__
sys_version = ''
timeout = 30
# --------------------------------------------------------- entry points
def do_GET(self):
self._handle()
do_HEAD = do_POST = do_PUT = do_PATCH = do_DELETE = do_OPTIONS = do_GET
def log_message(self, fmt, *args):
"""Base-class chatter stays debug-only; access logs are structured."""
self.server.logger.debug('http_note', extra={
'event': 'http_note', 'detail': (fmt % args)[:MAX_LOGGED_LINE]})
# ------------------------------------------------------------- pipeline
def _handle(self):
started = time.monotonic()
server = self.server
request_id = sanitize_request_id(self.headers.get('X-Request-Id'))
head_only = self.command == 'HEAD'
path = urlsplit(self.path).path
status, error_code = 500, None
self._last_exc = None
server.enter()
try:
if server.shutting_down:
raise Unavailable('Сервер завершает работу')
body = self._read_body()
query = parse_qs(urlsplit(self.path).query, keep_blank_values=True)
request = Request('GET' if head_only else self.command, path, query, self.headers,
body, request_id, self.client_address[0])
status, payload = server.api.dispatch(request)
encoded = encode_payload(payload)
except ApiError as exc:
self._last_exc = exc
status, error_code = exc.status, exc.code
encoded = self._error_payload(exc, request_id)
except Exception: # last resort: log privately, answer generically
server.logger.exception('unhandled_error', extra={
'event': 'unhandled_error', 'request_id': request_id,
'method': self.command, 'route': route_template(path)})
status, error_code = 500, 'internal_error'
encoded = self._error_payload(ApiError(), request_id)
finally:
server.leave()
server.logger.info('http_request', extra={
'event': 'http_request', 'request_id': request_id, 'method': self.command,
'route': route_template(path), 'status': status,
'duration_ms': round(1000 * (time.monotonic() - started), 3),
'error_code': error_code, 'client': self.client_address[0]})
self._send(status, encoded, request_id, head_only)
def _read_body(self):
if self.headers.get('Transfer-Encoding'):
self.close_connection = True
raise InvalidRequest('Transfer-Encoding не поддерживается')
raw_length = self.headers.get('Content-Length')
if raw_length is None:
return b''
try:
length = int(raw_length.strip())
except ValueError:
raise InvalidRequest('Некорректный Content-Length') from None
if length < 0:
raise InvalidRequest('Некорректный Content-Length')
if length > self.server.config.max_body_bytes:
self.close_connection = True
raise PayloadTooLarge('Тело запроса больше %d байт' % self.server.config.max_body_bytes)
if length == 0:
return b''
return self.rfile.read(length)
@staticmethod
def _error_payload(exc, request_id):
return encode_payload({'error': {
'code': exc.code, 'message': exc.message, 'request_id': request_id}})
def _send(self, status, encoded, request_id, head_only):
try:
self.send_response(status)
self.send_header('Content-Type', 'application/json; charset=utf-8')
self.send_header('X-Request-Id', request_id)
self.send_header('Cache-Control', 'no-store')
if status == 204:
self.send_header('Content-Length', '0')
if status == 503:
self.send_header('Retry-After', '1')
self.end_headers()
return
if status == 503:
self.send_header('Retry-After', '1')
if status == 401 and self._last_exc is not None:
www_auth = getattr(self._last_exc, 'www_authenticate', None)
if www_auth:
self.send_header('WWW-Authenticate', www_auth)
if self.close_connection:
self.send_header('Connection', 'close')
self.end_headers()
if not head_only and encoded:
self.wfile.write(encoded)
except (BrokenPipeError, ConnectionResetError):
self.close_connection = True
def build_server(config, logger, api, database):
"""Create the listening server; raises OSError when the port is taken."""
return ApiServer((config.host, config.port), api, config, logger, database)
def run(config, logger, database, cache):
"""Serve until SIGTERM/SIGINT, then drain in-flight work within the grace period."""
api = Api(config, Store(config, database, cache), database)
try:
server = build_server(config, logger, api, database)
except OSError as exc:
logger.error('listen_failed', extra={'event': 'listen_failed', 'detail': str(exc)})
return 1
stop = threading.Event()
def handle_signal(signum, _frame):
if stop.is_set():
return
stop.set()
server.shutting_down = True
logger.warning('shutdown_signal', extra={
'event': 'shutdown_signal', 'signal': signum,
'detail': 'stop accepting new requests'})
threading.Thread(target=server.shutdown, name='shutdown', daemon=True).start()
for name in ('SIGTERM', 'SIGINT'):
if hasattr(signal, name):
signal.signal(getattr(signal, name), handle_signal)
logger.info('server_started', extra=dict(
{'event': 'server_started', 'version': __version__,
'detail': 'pid %d, python %s' % (os.getpid(), platform.python_version())},
**config.as_log_fields()))
started = time.monotonic()
try:
server.serve_forever(poll_interval=0.2)
finally:
server.server_close()
drained = server.wait_idle(config.shutdown_grace_seconds)
database.close()
logger.info('server_stopped', extra={
'event': 'server_stopped',
'outcome': 'graceful' if drained else 'grace_period_exceeded',
'duration_ms': round(1000 * (time.monotonic() - started), 3)})
return 0
def main(argv=None):
try:
config = Config.from_env()
except ConfigError as exc:
print('configuration error: %s' % exc, file=sys.stderr)
return 2
logger = logging_utils.configure(config.log_level)
database = Database(config)
cache = SessionCache(config)
try:
database.open()
database.run_migrations()
cache.open()
except Exception as exc: # noqa: BLE001 - startup must fail loud and clear
logger.error('database_start_failed', extra={
'event': 'database_start_failed', 'detail': str(exc)})
database.close()
cache.close()
return 1
try:
return run(config, logger, database, cache)
finally:
cache.close()
+266
View File
@@ -0,0 +1,266 @@
"""PostgreSQL-backed state store. All writes go through parameterised queries
inside explicit transactions; reads use keyset pagination over indexed columns."""
from __future__ import annotations
import datetime as dt
import uuid
from . import cursor as cursor_codec
from . import security
from .cache import SessionCache
from .config import Config
from .db import Database
from .errors import Conflict, InvalidRequest, NotFound
def utc_now():
return dt.datetime.now(dt.timezone.utc)
def rfc3339(stamp):
"""RFC 3339 UTC with fixed microsecond precision and the Z suffix."""
return stamp.astimezone(dt.timezone.utc).isoformat(timespec='microseconds').replace('+00:00', 'Z')
class Store:
"""PostgreSQL state: users, chats, membership, messages, sessions."""
def __init__(self, config: Config, database: Database, cache: SessionCache):
self._config = config
self._db = database
self._cache = cache
# ---------------------------------------------------------------- reads
def stats(self):
with self._db.cursor() as cur:
cur.execute('SELECT (SELECT count(*) FROM users) AS users,'
' (SELECT count(*) FROM chats) AS chats,'
' (SELECT count(*) FROM messages) AS messages')
row = cur.fetchone()
return {
'users': row['users'],
'chats': row['chats'],
'messages': row['messages'],
}
def user(self, user_id):
with self._db.cursor() as cur:
cur.execute(
'SELECT id, username, display_name, created_at FROM users WHERE id = %s',
(str(user_id),))
row = cur.fetchone()
return self._public_user(row) if row else None
def user_by_username(self, username):
"""Return user row including password_hash, or None."""
with self._db.cursor() as cur:
cur.execute(
'SELECT id, username, password_hash, display_name, created_at FROM users '
'WHERE username = %s', (username,))
return cur.fetchone()
def users_exist(self, identifiers):
if not identifiers:
return True
with self._db.cursor() as cur:
cur.execute(
'SELECT count(*) AS c FROM users WHERE id = ANY(%s::uuid[])',
([str(i) for i in identifiers],))
row = cur.fetchone()
return row['c'] == len(identifiers)
# --------------------------------------------------------------- writes
def create_user(self, username, display_name, password_hash):
with self._db.cursor() as cur:
try:
cur.execute(
'INSERT INTO users (id, username, password_hash, display_name, created_at) '
'VALUES (%s, %s, %s, %s, %s) '
'RETURNING id, username, display_name, created_at',
(str(uuid.uuid4()), username, password_hash, display_name, utc_now()))
except Exception:
# Unique violation on username -> conflict, rolled back by cursor context.
raise Conflict('username уже занят') from None
row = cur.fetchone()
return self._public_user(row)
def create_chat(self, title, member_ids):
"""Atomically create a chat and its membership. Rollback on any failure."""
identifier = str(uuid.uuid4())
stamp = utc_now()
with self._db.cursor() as cur:
try:
cur.execute(
'INSERT INTO chats (id, title, created_at) VALUES (%s, %s, %s)',
(identifier, title, stamp))
for member_id in member_ids:
cur.execute(
'INSERT INTO chat_members (chat_id, user_id) VALUES (%s, %s)',
(identifier, str(member_id)))
except Exception:
raise InvalidRequest('member_ids содержит неизвестных пользователей') from None
return {
'id': identifier,
'title': title,
'member_ids': list(member_ids),
'created_at': rfc3339(stamp),
}
def add_message(self, chat_id, sender_id, text):
identifier = str(uuid.uuid4())
stamp = utc_now()
with self._db.cursor() as cur:
cur.execute(
'INSERT INTO messages (id, chat_id, sender_id, text, created_at) '
'VALUES (%s, %s, %s, %s, %s) '
'RETURNING id, chat_id, sender_id, text, created_at',
(identifier, str(chat_id), str(sender_id), text, stamp))
row = cur.fetchone()
return self._public_message(row)
# --------------------------------------------------------------- pages
def page_messages(self, chat_id, after_key, limit):
"""Return (public messages, has_more) for a keyset window."""
with self._db.cursor() as cur:
if after_key is None:
cur.execute(
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
'WHERE chat_id = %s '
'ORDER BY created_at, id '
'LIMIT %s', (str(chat_id), limit + 1))
else:
stamp, identifier = after_key
cur.execute(
'SELECT id, chat_id, sender_id, text, created_at FROM messages '
'WHERE chat_id = %s '
'AND (created_at, id) > (%s, %s) '
'ORDER BY created_at, id '
'LIMIT %s',
(str(chat_id), stamp, str(identifier), limit + 1))
rows = cur.fetchall()
has_more = len(rows) > limit
items = [self._public_message(dict(row)) for row in rows[:limit]]
return items, has_more
def page_chats(self, user_id, after_key, limit):
"""Return (chats visible to the user, has_more) ordered by (created_at, id)."""
with self._db.cursor() as cur:
if after_key is None:
cur.execute(
'SELECT c.id, c.title, c.created_at FROM chats c '
'JOIN chat_members cm ON cm.chat_id = c.id '
'WHERE cm.user_id = %s '
'ORDER BY c.created_at, c.id '
'LIMIT %s', (str(user_id), limit + 1))
else:
stamp, identifier = after_key
cur.execute(
'SELECT c.id, c.title, c.created_at FROM chats c '
'JOIN chat_members cm ON cm.chat_id = c.id '
'WHERE cm.user_id = %s '
'AND (c.created_at, c.id) > (%s, %s) '
'ORDER BY c.created_at, c.id '
'LIMIT %s',
(str(user_id), stamp, str(identifier), limit + 1))
rows = cur.fetchall()
has_more = len(rows) > limit
items = [self._public_chat(dict(row)) for row in rows[:limit]]
return items, has_more
# ----------------------------------------------------------- projection
def chat(self, chat_id):
with self._db.cursor() as cur:
cur.execute(
'SELECT id, title, created_at FROM chats WHERE id = %s', (str(chat_id),))
row = cur.fetchone()
if row is None:
return None
chat = self._public_chat(dict(row))
cur.execute(
'SELECT user_id FROM chat_members WHERE chat_id = %s', (str(chat_id),))
chat['member_ids'] = [str(r['user_id']) for r in cur.fetchall()]
return chat
@staticmethod
def is_member(chat, user_id):
return str(user_id) in chat['member_ids']
@staticmethod
def _public_user(row):
return {
'id': str(row['id']),
'username': row['username'],
'display_name': row['display_name'],
'created_at': rfc3339(row['created_at']),
}
@staticmethod
def _public_chat(row):
return {
'id': str(row['id']),
'title': row['title'],
'member_ids': [], # filled by API when needed
'created_at': rfc3339(row['created_at']),
}
@staticmethod
def _public_message(row):
return {
'id': str(row['id']),
'chat_id': str(row['chat_id']),
'sender_id': str(row['sender_id']),
'text': row['text'],
'created_at': rfc3339(row['created_at']),
}
# --------------------------------------------------------------- sessions
def create_session(self, token_hash, user_id, expires_at):
with self._db.cursor() as cur:
cur.execute(
'INSERT INTO sessions (token_hash, user_id, expires_at, created_at) '
'VALUES (%s, %s, %s, %s)',
(token_hash, str(user_id), expires_at, utc_now()))
ttl = max(1, int((expires_at - utc_now()).total_seconds()))
self._cache.set(token_hash, str(user_id), ttl)
def get_session(self, token_hash):
"""Return user_id for a valid non-expired session, or None.
The cache is consulted first, but every cache hit is re-validated
against PostgreSQL so that a revoked or expired session is never
trusted from a stale Redis key. On miss, read PostgreSQL and
backfill the cache. Expired sessions found during lookup are deleted.
"""
cached = self._cache.get(token_hash)
if cached is not None:
with self._db.cursor() as cur:
cur.execute(
'SELECT 1 FROM sessions WHERE token_hash = %s AND expires_at > %s',
(token_hash, utc_now()))
if cur.fetchone() is None:
self._cache.delete(token_hash)
return None
return cached
with self._db.cursor() as cur:
cur.execute(
'SELECT user_id, expires_at FROM sessions WHERE token_hash = %s',
(token_hash,))
row = cur.fetchone()
if row is None:
return None
if row['expires_at'] <= utc_now():
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
return None
user_id = str(row['user_id'])
ttl = max(1, int((row['expires_at'] - utc_now()).total_seconds()))
self._cache.set(token_hash, user_id, ttl)
return user_id
def delete_session(self, token_hash):
self._cache.delete(token_hash)
with self._db.cursor() as cur:
cur.execute('DELETE FROM sessions WHERE token_hash = %s', (token_hash,))
def cleanup_expired_sessions(self):
with self._db.cursor() as cur:
cur.execute('DELETE FROM sessions WHERE expires_at <= %s', (utc_now(),))
+125
View File
@@ -0,0 +1,125 @@
"""Strict request payload validation; any deviation is 400 invalid_request."""
from __future__ import annotations
import re
import uuid
from .errors import InvalidRequest
USERNAME_PATTERN = re.compile(r'[a-z0-9_]{3,32}')
DIGITS_PATTERN = re.compile(r'[0-9]+')
def safe_label(value):
"""Render untrusted text for an error message without breaking its encoding.
Field names come straight from the client JSON, so they may contain the same
lone surrogates that the payload itself is rejected for.
"""
text = value if isinstance(value, str) else repr(value)
return text.encode('utf-8', 'replace').decode('utf-8')
def require_encodable(value, name):
"""Reject text that cannot survive a UTF-8 round trip.
`json.loads` happily produces lone surrogates from `\ud800` escapes. Such a
string cannot be UTF-8 encoded, so it must be rejected before it is stored;
otherwise the write is committed and the response encoding fails afterwards.
"""
try:
value.encode('utf-8')
except UnicodeEncodeError:
raise InvalidRequest(
'Поле %s содержит недопустимые символы Unicode' % name) from None
return value
def require_object(payload):
if not isinstance(payload, dict):
raise InvalidRequest('Тело запроса должно быть JSON-объектом')
return payload
def require_fields(payload, required, allowed):
"""Reject missing required fields and any unknown field."""
missing = [name for name in required if name not in payload]
if missing:
raise InvalidRequest('Отсутствуют обязательные поля: ' + ', '.join(sorted(missing)))
extra = [name for name in payload if name not in allowed]
if extra:
raise InvalidRequest('Неизвестные поля: ' + ', '.join(sorted(safe_label(n) for n in extra)))
return payload
def require_text_field(payload, name, minimum, maximum, strip_check=True):
value = payload.get(name)
if not isinstance(value, str):
raise InvalidRequest('Поле %s должно быть строкой' % name)
require_encodable(value, name)
if strip_check and not value.strip():
raise InvalidRequest('Поле %s не может быть пустым или состоять из пробелов' % name)
if len(value) < minimum:
raise InvalidRequest('Поле %s короче %d символов' % (name, minimum))
if len(value) > maximum:
raise InvalidRequest('Поле %s длиннее %d символов' % (name, maximum))
return value
def require_username(payload):
username = payload.get('username')
# fullmatch, not match: a trailing "$" would also accept a final newline.
if not isinstance(username, str) or USERNAME_PATTERN.fullmatch(username) is None:
raise InvalidRequest('username должен соответствовать ^[a-z0-9_]{3,32}$')
return require_encodable(username, 'username')
def require_uuid_string(value, name):
if not isinstance(value, str):
raise InvalidRequest('Поле %s должно быть строкой UUID' % name)
try:
return str(uuid.UUID(value))
except ValueError:
raise InvalidRequest('Поле %s не является корректным UUID' % name) from None
def require_member_ids(payload, maximum, minimum=1):
value = payload.get('member_ids')
if not isinstance(value, list):
raise InvalidRequest('member_ids должен быть массивом UUID')
if not minimum <= len(value) <= maximum:
raise InvalidRequest('member_ids должен содержать от %d до %d участников' % (minimum, maximum))
identifiers = [require_uuid_string(item, 'member_ids') for item in value]
if len(set(identifiers)) != len(identifiers):
raise InvalidRequest('member_ids содержит повторяющихся участников')
return identifiers
def parse_limit(raw, default, maximum):
"""Parse a limit query value strictly: absent means default, anything else must be a plain integer.
An empty value is a client error rather than an absent parameter, and the
digit run is length-bounded before int() because Python refuses to convert
strings longer than its integer-string limit, which would surface as a 500.
"""
if raw is None:
return default
if not DIGITS_PATTERN.fullmatch(raw):
raise InvalidRequest('limit должен быть целым числом от 1 до %d' % maximum)
# int() refuses absurdly long digit strings (ValueError -> 500). Comparing the
# significant digits against the maximum first rejects them without converting.
significant = raw.lstrip('0') or '0'
if len(significant) > len(str(maximum)):
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
value = int(significant)
if not 1 <= value <= maximum:
raise InvalidRequest('limit должен быть в диапазоне 1..%d' % maximum)
return value
def parse_cursor(raw):
if raw is None:
return None
if not isinstance(raw, str) or not raw:
raise InvalidRequest('cursor должен быть непустой строкой')
return raw
+164
View File
@@ -0,0 +1,164 @@
#!/usr/bin/env python3
"""Concurrency evidence for level 4: N clients write M messages in parallel.
Command (see REPORT.md and `make concurrency`):
python3 student-tests/concurrency.py --clients 10 --messages-per-client 10 \
--output evidence/concurrency.json
Expectation: every acknowledged message is stored exactly once, the full history
contains exactly clients*messages_per_client unique messages and the order over
pages is strictly increasing by (created_at, id). No retries are hidden: any
failed write is reported and fails the run.
"""
from __future__ import annotations
from concurrent.futures import ThreadPoolExecutor
import json
from pathlib import Path
import sys
import threading
import time
import uuid
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, LAB, arguments, check, timestamp # noqa: E402
MESSAGES_PATH = '/api/v1/chats/%s/messages'
def build_fixture(base_url, ca_file, clients):
"""One user per client plus a single chat everybody belongs to."""
master = Client(base_url, ca_file, timeout=15)
master.call('GET', '/health/ready', schema='Health')
actors = [master.user('conc%d' % index) for index in range(clients)]
chat = master.call('POST', '/api/v1/chats', 201, actor=actors[0],
body={'title': 'Конкурентная запись',
'member_ids': [actor['id'] for actor in actors]},
schema='Chat')['json']
check(set(chat['member_ids']) == {actor['id'] for actor in actors},
'chat membership mismatch')
return master, actors, chat
def write_phase(base_url, ca_file, actors, chat, messages_per_client, prefix):
"""Each client writes its own slice of messages, all starting at once."""
barrier = threading.Barrier(len(actors))
def worker(index):
client = Client(base_url, ca_file, timeout=15)
actor = actors[index]
barrier.wait()
acknowledged, errors = [], []
started = time.monotonic()
for message_index in range(messages_per_client):
text = '%s-%03d-%03d' % (prefix, index, message_index)
try:
message = client.message(actor, chat, text)['json']
acknowledged.append((text, message['id']))
except Failure as exc:
errors.append('%s: %s' % (text, exc))
return {'client_index': index, 'acknowledged': acknowledged, 'errors': errors,
'duration_seconds': round(time.monotonic() - started, 3)}
started = time.monotonic()
with ThreadPoolExecutor(max_workers=len(actors)) as pool:
results = list(pool.map(worker, range(len(actors))))
return results, round(time.monotonic() - started, 3)
def read_whole_history(client, actor, chat):
items, cursor, pages = [], None, 0
while True:
query = '?limit=100' + ('' if cursor is None else '&cursor=' + quote(cursor, safe=''))
page = client.call('GET', MESSAGES_PATH % chat['id'] + query, actor=actor,
schema='MessagePage')['json']
items.extend(page['items'])
pages += 1
cursor = page['next_cursor']
check(pages < 100, 'pagination did not terminate')
if cursor is None:
return items, pages
def main():
def configure(parser):
parser.add_argument('--clients', type=int, default=10)
parser.add_argument('--messages-per-client', type=int, default=10)
parser.add_argument('--output', default=None,
help='Optional JSON report path, no session data inside')
args = arguments('Concurrent writes from independent HTTP clients', configure)
check(1 <= args.clients <= 64, 'clients must be 1..64')
check(1 <= args.messages_per_client <= 200, 'messages-per-client must be 1..200')
expected = args.clients * args.messages_per_client
prefix = 'conc-' + uuid.uuid4().hex[:8]
master, actors, chat = build_fixture(args.base_url, args.ca_file, args.clients)
results, elapsed = write_phase(args.base_url, args.ca_file, actors, chat,
args.messages_per_client, prefix)
acknowledged = [pair for result in results for pair in result['acknowledged']]
errors = [error for result in results for error in result['errors']]
history, pages = read_whole_history(master, actors[0], chat)
stored_texts = [message['text'] for message in history]
stored_ids = [message['id'] for message in history]
expected_texts = {'%s-%03d-%03d' % (prefix, client, index)
for client in range(args.clients)
for index in range(args.messages_per_client)}
missing = sorted(expected_texts.difference(stored_texts))
keys = [(timestamp(message['created_at']), uuid.UUID(message['id']).int)
for message in history]
ordered_correctly = all(keys[index] < keys[index + 1] for index in range(len(keys) - 1))
report = {
'lab': LAB,
'workload': 'closed-loop parallel POST messages from independent clients',
'clients': args.clients,
'messages_per_client': args.messages_per_client,
'expected_messages': expected,
'acknowledged_201': len(acknowledged),
'stored_in_history': len(history),
'unique_stored_ids': len(set(stored_ids)),
'unique_stored_texts': len(set(stored_texts)),
'duplicates': len(stored_ids) - len(set(stored_ids)),
'missing_texts': missing,
'failed_writes': len(errors),
'write_errors': errors[:10],
'history_pages': pages,
'history_order_strictly_increasing': ordered_correctly,
'writers_elapsed_seconds': elapsed,
'successful_rps': round(len(acknowledged) / elapsed, 3) if elapsed else None,
'per_client': [{'client_index': result['client_index'],
'acknowledged': len(result['acknowledged']),
'errors': len(result['errors']),
'duration_seconds': result['duration_seconds']}
for result in sorted(results, key=lambda item: item['client_index'])],
'note': ('Client-observed timings: closed-loop, single Docker host, no retries. '
'Latency percentiles are intentionally absent here (see make load).'),
}
text = json.dumps(report, ensure_ascii=False, indent=2)
if args.output:
destination = Path(args.output)
destination.parent.mkdir(parents=True, exist_ok=True)
destination.write_text(text + '\n')
print(text)
check(not errors, 'some parallel writes failed')
check(len(acknowledged) == expected, 'not every write was acknowledged')
check(len(history) == expected, 'history holds %d messages instead of %d'
% (len(history), expected))
check(not missing, 'messages missing from the history')
check(len(set(stored_ids)) == expected, 'duplicate message ids in the history')
check(ordered_correctly, 'history is not strictly ordered by (created_at, id)')
print('PASS: %d parallel messages from %d clients stored exactly once' %
(expected, args.clients))
if __name__ == '__main__':
try:
main()
except (Failure, ValueError, OSError) as exc:
print('FAIL:', exc)
raise SystemExit(1)
+345
View File
@@ -0,0 +1,345 @@
#!/usr/bin/env python3
"""Student-owned lab2 tests: sessions, validation, pagination.
One command: make student-test
Direct call: python3 student-tests/run.py --base-url http://localhost:8080
The suite complements tests/smoke.py and never weakens it. Every run creates
synthetic users, chats and messages only; no public check is imported or patched.
"""
from __future__ import annotations
import base64
import datetime as dt
import http.client
import json
import sys
import urllib.request
import uuid
from pathlib import Path
from urllib.parse import quote, urlsplit
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT / 'tests'))
from client import Client, Failure, LAB, arguments, check, timestamp # noqa: E402
if LAB != 2:
print('FAIL: student tests target lab2, but lab.json declares lab%d' % LAB)
raise SystemExit(2)
CHAT_PATH = '/api/v1/chats/%s'
MESSAGES_PATH = CHAT_PATH + '/messages'
MAX_PAGES = 200
TESTS = []
def test(func):
TESTS.append(func)
return func
class Context:
"""Shared synthetic fixtures: three users, one chat between two of them."""
def __init__(self, client):
self.client = client
self.alice, self.bob, self.eve, self.chat = client.bootstrap()
def new_chat(self, title, members):
return self.client.call('POST', '/api/v1/chats', 201, actor=members[0],
body={'title': title,
'member_ids': [member['id'] for member in members]},
schema='Chat')['json']
def send(self, actor, chat, text):
return self.client.call('POST', MESSAGES_PATH % chat['id'], 201, actor=actor,
body={'text': text}, schema='Message')['json']
def page(self, actor, chat, limit, cursor=None):
query = '?limit=%d' % limit
if cursor is not None:
query += '&cursor=' + quote(cursor, safe='')
return self.client.call('GET', MESSAGES_PATH % chat['id'] + query, actor=actor,
schema='MessagePage')['json']
def traverse(self, actor, chat, limits):
"""Walk the whole history, changing the page size between requests."""
items, cursor, pages = [], None, 0
while True:
limit = limits[min(pages, len(limits) - 1)]
page = self.page(actor, chat, limit, cursor)
items.extend(page['items'])
cursor = page['next_cursor']
pages += 1
check(pages < MAX_PAGES, 'pagination did not terminate')
if cursor is None:
break
return items, pages
def ordered(records):
"""Contract order: (created_at, id) ascending, exactly as the client sorts."""
return sorted(records, key=lambda item: (timestamp(item['created_at']),
uuid.UUID(item['id']).int))
def traversal_order(records):
return [item['id'] for item in records]
# --------------------------------------------------------------------- health
@test
def health_and_request_id(ctx):
for path in ('/health/live', '/health/ready'):
result = ctx.client.call('GET', path, schema='Health')
check(result['json'] == {'status': 'ok'},
'%s must return exactly {"status":"ok"}' % path)
check(len(result['headers']['X-Request-Id']) >= 8, 'request id looks too short')
marker = 'student-' + uuid.uuid4().hex
echoed = ctx.client.call('GET', '/health/live', headers={'X-Request-Id': marker})
check(echoed['headers']['X-Request-Id'] == marker, 'client request id was not echoed')
error = ctx.client.call('GET', '/api/v1/users/me', 401, headers={'X-Request-Id': marker})
check(error['headers']['X-Request-Id'] == marker, 'error response lost the request id')
check(error['json']['error']['request_id'] == marker, 'error body request_id mismatch')
# -------------------------------------------------------------- unknown route
@test
def unknown_route_and_method_are_json_404(ctx):
ctx.client.call('GET', '/api/v1/unknown', 404)
ctx.client.call('DELETE', '/api/v1/chats', 404, actor=ctx.alice)
ctx.client.call('GET', '/api/v1/chats/', 404, actor=ctx.alice)
# --------------------------------------------------------------- user payload
@test
def malformed_json_is_rejected(ctx):
json_headers = {'Content-Type': 'application/json'}
ctx.client.call('POST', '/api/v1/users', 400, raw=b'{"username": "broken",', headers=json_headers)
ctx.client.call('POST', '/api/v1/users', 400, raw=b'[1, 2, 3]', headers=json_headers)
ctx.client.call('POST', '/api/v1/users', 400, raw=b'"just a string"', headers=json_headers)
ctx.client.call('POST', '/api/v1/users', 400, raw=b'', headers=json_headers)
ctx.client.call('POST', '/api/v1/users', 415, raw=b'{"username": "x"}',
headers={'Content-Type': 'text/plain'})
ctx.client.call('POST', '/api/v1/users', 415, raw=b'{"username": "x"}',
headers={'Content-Type': 'application/x-www-form-urlencoded'})
@test
def duplicate_username_is_409(ctx):
name = 'dup_' + uuid.uuid4().hex[:12]
body = {'username': name, 'display_name': 'Дубликат',
'password': 'T3st-' + uuid.uuid4().hex}
created = ctx.client.call('POST', '/api/v1/users', 201, body=body, schema='User')['json']
check(created['username'] == body['username'], 'username changed on creation')
conflict = ctx.client.call('POST', '/api/v1/users', 409, body=body)
check(conflict['json']['error']['code'] == 'conflict', 'duplicate username must be conflict')
@test
def user_payload_validation(ctx):
name = 'valid_' + uuid.uuid4().hex[:8]
for bad in (
{'username': name, 'display_name': 'Имя', 'password': 'ok', 'extra': 1},
{'display_name': 'Имя', 'password': 'ok'},
{'username': name, 'password': 'ok'},
{'username': name, 'display_name': ' ', 'password': 'ok'},
{'username': name, 'display_name': '', 'password': 'ok'},
{'username': name, 'display_name': 42, 'password': 'ok'},
{'username': name, 'display_name': None, 'password': 'ok'},
{'username': name, 'display_name': 'д' * 101, 'password': 'ok'},
{'username': 'AB', 'display_name': 'Имя', 'password': 'ok'},
{'username': 'with space', 'display_name': 'Имя', 'password': 'ok'},
{'username': 'юзер123', 'display_name': 'Имя', 'password': 'ok'},
{'username': 'x' * 33, 'display_name': 'Имя', 'password': 'ok'},
{'username': '', 'display_name': 'Имя', 'password': 'ok'},
{'username': 123, 'display_name': 'Имя', 'password': 'ok'},
# password too short
{'username': name, 'display_name': 'Имя', 'password': 'short'},
# password too long
{'username': name, 'display_name': 'Имя', 'password': 'д' * 129},
):
ctx.client.call('POST', '/api/v1/users', 400, body=bad)
ok = ctx.client.call('POST', '/api/v1/users', 201,
body={'username': name, 'display_name': 'Студент Юникод 👋',
'password': 'T3st-' + uuid.uuid4().hex},
schema='User')['json']
check(ok['display_name'] == 'Студент Юникод 👋', 'unicode display_name changed')
check('password' not in ok and 'token' not in ok, 'user projection leaks credentials')
# ------------------------------------------------------------- sessions
@test
def basic_login_issues_bearer_session(ctx):
reg = ctx.alice['registration']
credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode()
session = ctx.client.call('POST', '/api/v1/auth/sessions', 201,
headers={'Authorization': 'Basic ' + credential},
schema='Session')['json']
check(session['user'] == ctx.alice['user'], 'session user mismatch')
check(session['token_type'] == 'Bearer', 'token type must be Bearer')
check(len(session['token']) >= 32, 'token must be opaque and long enough')
check(timestamp(session['expires_at']) > dt.datetime.now(dt.timezone.utc),
'session already expired')
ctx.alice['token'] = session['token']
# wrong password -> 401
bad = base64.b64encode((reg['username'] + ':wrong-password').encode()).decode()
ctx.client.call('POST', '/api/v1/auth/sessions', 401,
headers={'Authorization': 'Basic ' + bad})
# unknown user -> 401
bad = base64.b64encode(('nonexistent-user:' + reg['password']).encode()).decode()
ctx.client.call('POST', '/api/v1/auth/sessions', 401,
headers={'Authorization': 'Basic ' + bad})
@test
def bearer_required_for_business_endpoints(ctx):
# no auth -> 401
ctx.client.call('GET', '/api/v1/users/me', 401)
# Basic on business endpoint -> 401
reg = ctx.alice['registration']
credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode()
ctx.client.call('GET', '/api/v1/users/me', 401,
headers={'Authorization': 'Basic ' + credential})
# X-User-Id ignored -> 401
ctx.client.call('GET', '/api/v1/users/me', 401, headers={'X-User-Id': ctx.alice['id']})
# Bearer works
me = ctx.client.call('GET', '/api/v1/users/me', 200, actor=ctx.alice,
schema='User')['json']
check(me == ctx.alice['user'], 'current user mismatch')
@test
def logout_revokes_session(ctx):
reg = ctx.alice['registration']
credential = base64.b64encode((reg['username'] + ':' + reg['password']).encode()).decode()
session = ctx.client.call('POST', '/api/v1/auth/sessions', 201,
headers={'Authorization': 'Basic ' + credential},
schema='Session')['json']
token = session['token']
ctx.client.call('DELETE', '/api/v1/auth/sessions/current', 204,
headers={'Authorization': 'Bearer ' + token})
# revoked token -> 401
ctx.client.call('GET', '/api/v1/users/me', 401,
headers={'Authorization': 'Bearer ' + token})
ctx.client.call('DELETE', '/api/v1/auth/sessions/current', 401,
headers={'Authorization': 'Bearer ' + token})
# -------------------------------------------------------------- message rules
@test
def foreign_chat_is_invisible(ctx):
other = ctx.new_chat('Чат без Евы', [ctx.alice, ctx.bob])
ctx.client.call('GET', CHAT_PATH % other['id'], 404, actor=ctx.eve)
ctx.client.call('GET', MESSAGES_PATH % other['id'], 404, actor=ctx.eve)
ctx.client.call('POST', MESSAGES_PATH % other['id'], 404, actor=ctx.eve,
body={'text': 'чужое сообщение'})
listed = ctx.client.call('GET', '/api/v1/chats?limit=100', actor=ctx.eve,
schema='ChatPage')['json']
check(all(item['id'] != other['id'] for item in listed['items']),
'foreign chat leaked into the list of another user')
member_view = ctx.client.call('GET', '/api/v1/chats?limit=100', actor=ctx.bob,
schema='ChatPage')['json']
check(any(item['id'] == other['id'] for item in member_view['items']),
'member cannot see the own chat')
ctx.client.call('GET', CHAT_PATH % uuid.uuid4(), 404, actor=ctx.alice)
ctx.client.call('GET', '/api/v1/chats/not-a-uuid', 400, actor=ctx.alice)
ctx.client.call('GET', MESSAGES_PATH % 'not-a-uuid', 400, actor=ctx.alice)
@test
def message_payload_validation(ctx):
"""Empty and whitespace-only text is rejected."""
for bad in (
{'text': ''},
{'text': ' '},
{'text': '\u00a0\t\n'},
{},
{'text': 123},
{'text': None},
{'text': []},
{'text': 'ок', 'sender_id': ctx.bob['id']},
{'text': 'д' * 4001},
):
ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 400, actor=ctx.alice, body=bad)
ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 401, body={'text': 'ок'})
ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 400, actor=ctx.alice, raw=b'{"text":}',
headers={'Content-Type': 'application/json'})
ctx.client.call('POST', MESSAGES_PATH % ctx.chat['id'], 415, actor=ctx.alice,
raw=b'{"text":"ok"}', headers={'Content-Type': 'application/xml'})
@test
def text_limits_and_unicode_roundtrip(ctx):
longest = 'д' * 3999 + '👋'
check(len(longest) == 4000, 'fixture must be exactly 4000 code points')
created = ctx.send(ctx.bob, ctx.chat, longest)
check(created['text'] == longest, '4000 code points text changed')
check(created['sender_id'] == ctx.bob['id'], 'sender must be derived from actor')
mixed = 'Привет 👋🏽 \u00a0 e\u0301 — астральные 🚀🧪 и кавычки «»'
echo = ctx.send(ctx.alice, ctx.chat, mixed)
check(echo['text'] == mixed, 'mixed unicode text changed in transit')
check(echo['chat_id'] == ctx.chat['id'], 'message bound to the wrong chat')
# ---------------------------------------------------------------- pagination
@test
def multi_page_traversal_is_exact_and_stable(ctx):
chat = ctx.new_chat('Пагинация', [ctx.bob, ctx.alice])
sent = []
for index in range(7):
actor = ctx.alice if index % 2 == 0 else ctx.bob
sent.append(ctx.send(actor, chat, 'страница-%d-👋' % index))
expected = traversal_order(ordered(sent))
walked, pages = ctx.traverse(ctx.bob, chat, [2])
check(pages >= 4, 'seven messages with limit=2 must take at least four pages')
check(traversal_order(walked) == expected, 'traversal lost, duplicated or reordered messages')
check(len(set(traversal_order(walked))) == len(walked), 'traversal returned duplicates')
switched, _ = ctx.traverse(ctx.alice, chat, [2, 3, 1])
check(traversal_order(switched) == expected, 'changing limit mid-traversal changed the order')
singles, _ = ctx.traverse(ctx.alice, chat, [1])
check(len(singles) == 7, 'limit=1 traversal must return all seven messages')
first = ctx.page(ctx.alice, chat, 2)
check(len(first['items']) == 2 and first['next_cursor'], 'first page shape is wrong')
check(first['items'][0]['id'] == expected[0], 'first page must start with the oldest message')
@test
def default_page_limit_is_fifty(ctx):
chat = ctx.new_chat('Предел страницы по умолчанию', [ctx.alice, ctx.bob])
for index in range(51):
ctx.send(ctx.alice, chat, 'default-limit-%d' % index)
page = ctx.client.call('GET', MESSAGES_PATH % chat['id'], actor=ctx.alice,
schema='MessagePage')['json']
check(len(page['items']) == 50, 'default limit must be 50, got %d' % len(page['items']))
check(page['next_cursor'], 'a page that is not the last one needs a cursor')
tail = ctx.page(ctx.alice, chat, 100, page['next_cursor'])
check(len(tail['items']) == 1, 'the remaining page must hold exactly one message')
check(tail['next_cursor'] is None, 'last page must have next_cursor null')
merged = traversal_order(page['items']) + traversal_order(tail['items'])
check(len(set(merged)) == 51, 'page boundaries duplicated or dropped messages')
def main():
args = arguments('Student-owned lab2 tests: sessions, validation, pagination')
client = Client(args.base_url, args.ca_file)
ctx = Context(client)
print('student-tests: %d cases against %s' % (len(TESTS), client.base))
failures = []
for func in TESTS:
try:
func(ctx)
except (Failure, ValueError, OSError) as exc:
failures.append(func.__name__)
print('FAIL %-42s %s' % (func.__name__, exc))
else:
print('PASS %-42s' % func.__name__)
print('%d passed, %d failed' % (len(TESTS) - len(failures), len(failures)))
return 1 if failures else 0
if __name__ == '__main__':
raise SystemExit(main())